Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified
📁 ..	-	2025-05-25 12:26
📄 __init__.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Google API Core. This package contains common code and utilities used by Google client libraries. """ from google.api_core import version as api_core_version __version__ = api_core_version.__version__	782 B	2025-05-25 12:26
📁 __pycache__	-	2025-05-25 12:26
📄 _rest_streaming_base.py # Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for server-side streaming in REST.""" from collections import deque import string from typing import Deque, Union import types import proto import google.protobuf.message from google.protobuf.json_format import Parse class BaseResponseIterator: """Base Iterator over REST API responses. This class should not be used directly. Args: response_message_cls (Union[proto.Message, google.protobuf.message.Message]): A response class expected to be returned from an API. Raises: ValueError: If `response_message_cls` is not a subclass of `proto.Message` or `google.protobuf.message.Message`. """ def __init__( self, response_message_cls: Union[proto.Message, google.protobuf.message.Message], ): self._response_message_cls = response_message_cls # Contains a list of JSON responses ready to be sent to user. self._ready_objs: Deque[str] = deque() # Current JSON response being built. self._obj = "" # Keeps track of the nesting level within a JSON object. self._level = 0 # Keeps track whether HTTP response is currently sending values # inside of a string value. self._in_string = False # Whether an escape symbol "\" was encountered. self._escape_next = False self._grab = types.MethodType(self._create_grab(), self) def _process_chunk(self, chunk: str): if self._level == 0: if chunk[0] != "[": raise ValueError( "Can only parse array of JSON objects, instead got %s" % chunk ) for char in chunk: if char == "{": if self._level == 1: # Level 1 corresponds to the outermost JSON object # (i.e. the one we care about). self._obj = "" if not self._in_string: self._level += 1 self._obj += char elif char == "}": self._obj += char if not self._in_string: self._level -= 1 if not self._in_string and self._level == 1: self._ready_objs.append(self._obj) elif char == '"': # Helps to deal with an escaped quotes inside of a string. if not self._escape_next: self._in_string = not self._in_string self._obj += char elif char in string.whitespace: if self._in_string: self._obj += char elif char == "[": if self._level == 0: self._level += 1 else: self._obj += char elif char == "]": if self._level == 1: self._level -= 1 else: self._obj += char else: self._obj += char self._escape_next = not self._escape_next if char == "\\" else False def _create_grab(self): if issubclass(self._response_message_cls, proto.Message): def grab(this): return this._response_message_cls.from_json( this._ready_objs.popleft(), ignore_unknown_fields=True ) return grab elif issubclass(self._response_message_cls, google.protobuf.message.Message): def grab(this): return Parse(this._ready_objs.popleft(), this._response_message_cls()) return grab else: raise ValueError( "Response message class must be a subclass of proto.Message or google.protobuf.message.Message." )	4.25 KB	2025-05-25 12:26
📄 bidi.py # Copyright 2017, Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Bi-directional streaming RPC helpers.""" import collections import datetime import logging import queue as queue_module import threading import time from google.api_core import exceptions _LOGGER = logging.getLogger(__name__) _BIDIRECTIONAL_CONSUMER_NAME = "Thread-ConsumeBidirectionalStream" class _RequestQueueGenerator(object): """A helper for sending requests to a gRPC stream from a Queue. This generator takes requests off a given queue and yields them to gRPC. This helper is useful when you have an indeterminate, indefinite, or otherwise open-ended set of requests to send through a request-streaming (or bidirectional) RPC. The reason this is necessary is because gRPC takes an iterator as the request for request-streaming RPCs. gRPC consumes this iterator in another thread to allow it to block while generating requests for the stream. However, if the generator blocks indefinitely gRPC will not be able to clean up the thread as it'll be blocked on `next(iterator)` and not be able to check the channel status to stop iterating. This helper mitigates that by waiting on the queue with a timeout and checking the RPC state before yielding. Finally, it allows for retrying without swapping queues because if it does pull an item off the queue when the RPC is inactive, it'll immediately put it back and then exit. This is necessary because yielding the item in this case will cause gRPC to discard it. In practice, this means that the order of messages is not guaranteed. If such a thing is necessary it would be easy to use a priority queue. Example:: requests = request_queue_generator(q) call = stub.StreamingRequest(iter(requests)) requests.call = call for response in call: print(response) q.put(...) Note that it is possible to accomplish this behavior without "spinning" (using a queue timeout). One possible way would be to use more threads to multiplex the grpc end event with the queue, another possible way is to use selectors and a custom event/queue object. Both of these approaches are significant from an engineering perspective for small benefit - the CPU consumed by spinning is pretty minuscule. Args: queue (queue_module.Queue): The request queue. period (float): The number of seconds to wait for items from the queue before checking if the RPC is cancelled. In practice, this determines the maximum amount of time the request consumption thread will live after the RPC is cancelled. initial_request (Union[protobuf.Message, Callable[None, protobuf.Message]]): The initial request to yield. This is done independently of the request queue to allow fo easily restarting streams that require some initial configuration request. """ def __init__(self, queue, period=1, initial_request=None): self._queue = queue self._period = period self._initial_request = initial_request self.call = None def _is_active(self): # Note: there is a possibility that this starts before the call # property is set. So we have to check if self.call is set before # seeing if it's active. We need to return True if self.call is None. # See https://github.com/googleapis/python-api-core/issues/560. return self.call is None or self.call.is_active() def __iter__(self): if self._initial_request is not None: if callable(self._initial_request): yield self._initial_request() else: yield self._initial_request while True: try: item = self._queue.get(timeout=self._period) except queue_module.Empty: if not self._is_active(): _LOGGER.debug( "Empty queue and inactive call, exiting request " "generator." ) return else: # call is still active, keep waiting for queue items. continue # The consumer explicitly sent "None", indicating that the request # should end. if item is None: _LOGGER.debug("Cleanly exiting request generator.") return if not self._is_active(): # We have an item, but the call is closed. We should put the # item back on the queue so that the next call can consume it. self._queue.put(item) _LOGGER.debug( "Inactive call, replacing item on queue and exiting " "request generator." ) return yield item class _Throttle(object): """A context manager limiting the total entries in a sliding time window. If more than ``access_limit`` attempts are made to enter the context manager instance in the last ``time window`` interval, the exceeding requests block until enough time elapses. The context manager instances are thread-safe and can be shared between multiple threads. If multiple requests are blocked and waiting to enter, the exact order in which they are allowed to proceed is not determined. Example:: max_three_per_second = _Throttle( access_limit=3, time_window=datetime.timedelta(seconds=1) ) for i in range(5): with max_three_per_second as time_waited: print("{}: Waited {} seconds to enter".format(i, time_waited)) Args: access_limit (int): the maximum number of entries allowed in the time window time_window (datetime.timedelta): the width of the sliding time window """ def __init__(self, access_limit, time_window): if access_limit < 1: raise ValueError("access_limit argument must be positive") if time_window <= datetime.timedelta(0): raise ValueError("time_window argument must be a positive timedelta") self._time_window = time_window self._access_limit = access_limit self._past_entries = collections.deque( maxlen=access_limit ) # least recent first self._entry_lock = threading.Lock() def __enter__(self): with self._entry_lock: cutoff_time = datetime.datetime.now() - self._time_window # drop the entries that are too old, as they are no longer relevant while self._past_entries and self._past_entries[0] < cutoff_time: self._past_entries.popleft() if len(self._past_entries) < self._access_limit: self._past_entries.append(datetime.datetime.now()) return 0.0 # no waiting was needed to_wait = (self._past_entries[0] - cutoff_time).total_seconds() time.sleep(to_wait) self._past_entries.append(datetime.datetime.now()) return to_wait def __exit__(self, _): pass def __repr__(self): return "{}(access_limit={}, time_window={})".format( self.__class__.__name__, self._access_limit, repr(self._time_window) ) class BidiRpc(object): """A helper for consuming a bi-directional streaming RPC. This maps gRPC's built-in interface which uses a request iterator and a response iterator into a socket-like :func:`send` and :func:`recv`. This is a more useful pattern for long-running or asymmetric streams (streams where there is not a direct correlation between the requests and responses). Example:: initial_request = example_pb2.StreamingRpcRequest( setting='example') rpc = BidiRpc( stub.StreamingRpc, initial_request=initial_request, metadata=[('name', 'value')] ) rpc.open() while rpc.is_active(): print(rpc.recv()) rpc.send(example_pb2.StreamingRpcRequest( data='example')) This does not* retry the stream on errors. See :class:`ResumableBidiRpc`. Args: start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to start the RPC. initial_request (Union[protobuf.Message, Callable[None, protobuf.Message]]): The initial request to yield. This is useful if an initial request is needed to start the stream. metadata (Sequence[Tuple(str, str)]): RPC metadata to include in the request. """ def __init__(self, start_rpc, initial_request=None, metadata=None): self._start_rpc = start_rpc self._initial_request = initial_request self._rpc_metadata = metadata self._request_queue = queue_module.Queue() self._request_generator = None self._is_active = False self._callbacks = [] self.call = None def add_done_callback(self, callback): """Adds a callback that will be called when the RPC terminates. This occurs when the RPC errors or is successfully terminated. Args: callback (Callable[[grpc.Future], None]): The callback to execute. It will be provided with the same gRPC future as the underlying stream which will also be a :class:`grpc.Call`. """ self._callbacks.append(callback) def _on_call_done(self, future): # This occurs when the RPC errors or is successfully terminated. # Note that grpc's "future" here can also be a grpc.RpcError. # See note in https://github.com/grpc/grpc/issues/10885#issuecomment-302651331 # that `grpc.RpcError` is also `grpc.call`. for callback in self._callbacks: callback(future) def open(self): """Opens the stream.""" if self.is_active: raise ValueError("Can not open an already open stream.") request_generator = _RequestQueueGenerator( self._request_queue, initial_request=self._initial_request ) try: call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata) except exceptions.GoogleAPICallError as exc: # The original `grpc.RpcError` (which is usually also a `grpc.Call`) is # available from the ``response`` property on the mapped exception. self._on_call_done(exc.response) raise request_generator.call = call # TODO: api_core should expose the future interface for wrapped # callables as well. if hasattr(call, "_wrapped"): # pragma: NO COVER call._wrapped.add_done_callback(self._on_call_done) else: call.add_done_callback(self._on_call_done) self._request_generator = request_generator self.call = call def close(self): """Closes the stream.""" if self.call is None: return self._request_queue.put(None) self.call.cancel() self._request_generator = None self._initial_request = None self._callbacks = [] # Don't set self.call to None. Keep it around so that send/recv can # raise the error. def send(self, request): """Queue a message to be sent on the stream. Send is non-blocking. If the underlying RPC has been closed, this will raise. Args: request (protobuf.Message): The request to send. """ if self.call is None: raise ValueError("Can not send() on an RPC that has never been open()ed.") # Don't use self.is_active(), as ResumableBidiRpc will overload it # to mean something semantically different. if self.call.is_active(): self._request_queue.put(request) else: # calling next should cause the call to raise. next(self.call) def recv(self): """Wait for a message to be returned from the stream. Recv is blocking. If the underlying RPC has been closed, this will raise. Returns: protobuf.Message: The received message. """ if self.call is None: raise ValueError("Can not recv() on an RPC that has never been open()ed.") return next(self.call) @property def is_active(self): """bool: True if this stream is currently open and active.""" return self.call is not None and self.call.is_active() @property def pending_requests(self): """int: Returns an estimate of the number of queued requests.""" return self._request_queue.qsize() def _never_terminate(future_or_error): """By default, no errors cause BiDi termination.""" return False class ResumableBidiRpc(BidiRpc): """A :class:`BidiRpc` that can automatically resume the stream on errors. It uses the ``should_recover`` arg to determine if it should re-establish the stream on error. Example:: def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') metadata = [('header_name', 'value')] rpc = ResumableBidiRpc( stub.StreamingRpc, should_recover=should_recover, initial_request=initial_request, metadata=metadata ) rpc.open() while rpc.is_active(): print(rpc.recv()) rpc.send(example_pb2.StreamingRpcRequest( data='example')) Args: start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to start the RPC. initial_request (Union[protobuf.Message, Callable[None, protobuf.Message]]): The initial request to yield. This is useful if an initial request is needed to start the stream. should_recover (Callable[[Exception], bool]): A function that returns True if the stream should be recovered. This will be called whenever an error is encountered on the stream. should_terminate (Callable[[Exception], bool]): A function that returns True if the stream should be terminated. This will be called whenever an error is encountered on the stream. metadata Sequence[Tuple(str, str)]: RPC metadata to include in the request. throttle_reopen (bool): If ``True``, throttling will be applied to stream reopen calls. Defaults to ``False``. """ def __init__( self, start_rpc, should_recover, should_terminate=_never_terminate, initial_request=None, metadata=None, throttle_reopen=False, ): super(ResumableBidiRpc, self).__init__(start_rpc, initial_request, metadata) self._should_recover = should_recover self._should_terminate = should_terminate self._operational_lock = threading.RLock() self._finalized = False self._finalize_lock = threading.Lock() if throttle_reopen: self._reopen_throttle = _Throttle( access_limit=5, time_window=datetime.timedelta(seconds=10) ) else: self._reopen_throttle = None def _finalize(self, result): with self._finalize_lock: if self._finalized: return for callback in self._callbacks: callback(result) self._finalized = True def _on_call_done(self, future): # Unlike the base class, we only execute the callbacks on a terminal # error, not for errors that we can recover from. Note that grpc's # "future" here is also a grpc.RpcError. with self._operational_lock: if self._should_terminate(future): self._finalize(future) elif not self._should_recover(future): self._finalize(future) else: _LOGGER.debug("Re-opening stream from gRPC callback.") self._reopen() def _reopen(self): with self._operational_lock: # Another thread already managed to re-open this stream. if self.call is not None and self.call.is_active(): _LOGGER.debug("Stream was already re-established.") return self.call = None # Request generator should exit cleanly since the RPC its bound to # has exited. self._request_generator = None # Note: we do not currently do any sort of backoff here. The # assumption is that re-establishing the stream under normal # circumstances will happen in intervals greater than 60s. # However, it is possible in a degenerative case that the server # closes the stream rapidly which would lead to thrashing here, # but hopefully in those cases the server would return a non- # retryable error. try: if self._reopen_throttle: with self._reopen_throttle: self.open() else: self.open() # If re-opening or re-calling the method fails for any reason, # consider it a terminal error and finalize the stream. except Exception as exc: _LOGGER.debug("Failed to re-open stream due to %s", exc) self._finalize(exc) raise _LOGGER.info("Re-established stream") def _recoverable(self, method, args, kwargs): """Wraps a method to recover the stream and retry on error. If a retryable error occurs while making the call, then the stream will be re-opened and the method will be retried. This happens indefinitely so long as the error is a retryable one. If an error occurs while re-opening the stream, then this method will raise immediately and trigger finalization of this object. Args: method (Callable[..., Any]): The method to call. args: The args to pass to the method. kwargs: The kwargs to pass to the method. """ while True: try: return method(args, *kwargs) except Exception as exc: with self._operational_lock: _LOGGER.debug("Call to retryable %r caused %s.", method, exc) if self._should_terminate(exc): self.close() _LOGGER.debug("Terminating %r due to %s.", method, exc) self._finalize(exc) break if not self._should_recover(exc): self.close() _LOGGER.debug("Not retrying %r due to %s.", method, exc) self._finalize(exc) raise exc _LOGGER.debug("Re-opening stream from retryable %r.", method) self._reopen() def _send(self, request): # Grab a reference to the RPC call. Because another thread (notably # the gRPC error thread) can modify self.call (by invoking reopen), # we should ensure our reference can not change underneath us. # If self.call is modified (such as replaced with a new RPC call) then # this will use the "old" RPC, which should result in the same # exception passed into gRPC's error handler being raised here, which # will be handled by the usual error handling in retryable. with self._operational_lock: call = self.call if call is None: raise ValueError("Can not send() on an RPC that has never been open()ed.") # Don't use self.is_active(), as ResumableBidiRpc will overload it # to mean something semantically different. if call.is_active(): self._request_queue.put(request) pass else: # calling next should cause the call to raise. next(call) def send(self, request): return self._recoverable(self._send, request) def _recv(self): with self._operational_lock: call = self.call if call is None: raise ValueError("Can not recv() on an RPC that has never been open()ed.") return next(call) def recv(self): return self._recoverable(self._recv) def close(self): self._finalize(None) super(ResumableBidiRpc, self).close() @property def is_active(self): """bool: True if this stream is currently open and active.""" # Use the operational lock. It's entirely possible for something # to check the active state while* the RPC is being retried. # Also, use finalized to track the actual terminal state here. # This is because if the stream is re-established by the gRPC thread # it's technically possible to check this between when gRPC marks the # RPC as inactive and when gRPC executes our callback that re-opens # the stream. with self._operational_lock: return self.call is not None and not self._finalized class BackgroundConsumer(object): """A bi-directional stream consumer that runs in a separate thread. This maps the consumption of a stream into a callback-based model. It also provides :func:`pause` and :func:`resume` to allow for flow-control. Example:: def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') rpc = ResumeableBidiRpc( stub.StreamingRpc, initial_request=initial_request, should_recover=should_recover) def on_response(response): print(response) consumer = BackgroundConsumer(rpc, on_response) consumer.start() Note that error handling must be done by using the provided ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit whenever the RPC itself exits and will not provide any error details. Args: bidi_rpc (BidiRpc): The RPC to consume. Should not have been ``open()``ed yet. on_response (Callable[[protobuf.Message], None]): The callback to be called for every response on the stream. """ def __init__(self, bidi_rpc, on_response): self._bidi_rpc = bidi_rpc self._on_response = on_response self._paused = False self._wake = threading.Condition() self._thread = None self._operational_lock = threading.Lock() def _on_call_done(self, future): # Resume the thread if it's paused, this prevents blocking forever # when the RPC has terminated. self.resume() def _thread_main(self, ready): try: ready.set() self._bidi_rpc.add_done_callback(self._on_call_done) self._bidi_rpc.open() while self._bidi_rpc.is_active: # Do not allow the paused status to change at all during this # section. There is a condition where we could be resumed # between checking if we are paused and calling wake.wait(), # which means that we will miss the notification to wake up # (oops!) and wait for a notification that will never come. # Keeping the lock throughout avoids that. # In the future, we could use `Condition.wait_for` if we drop # Python 2.7. # See: https://github.com/googleapis/python-api-core/issues/211 with self._wake: while self._paused: _LOGGER.debug("paused, waiting for waking.") self._wake.wait() _LOGGER.debug("woken.") _LOGGER.debug("waiting for recv.") response = self._bidi_rpc.recv() _LOGGER.debug("recved response.") self._on_response(response) except exceptions.GoogleAPICallError as exc: _LOGGER.debug( "%s caught error %s and will exit. Generally this is due to " "the RPC itself being cancelled and the error will be " "surfaced to the calling code.", _BIDIRECTIONAL_CONSUMER_NAME, exc, exc_info=True, ) except Exception as exc: _LOGGER.exception( "%s caught unexpected exception %s and will exit.", _BIDIRECTIONAL_CONSUMER_NAME, exc, ) _LOGGER.info("%s exiting", _BIDIRECTIONAL_CONSUMER_NAME) def start(self): """Start the background thread and begin consuming the thread.""" with self._operational_lock: ready = threading.Event() thread = threading.Thread( name=_BIDIRECTIONAL_CONSUMER_NAME, target=self._thread_main, args=(ready,), ) thread.daemon = True thread.start() # Other parts of the code rely on `thread.is_alive` which # isn't sufficient to know if a thread is active, just that it may # soon be active. This can cause races. Further protect # against races by using a ready event and wait on it to be set. ready.wait() self._thread = thread _LOGGER.debug("Started helper thread %s", thread.name) def stop(self): """Stop consuming the stream and shutdown the background thread.""" with self._operational_lock: self._bidi_rpc.close() if self._thread is not None: # Resume the thread to wake it up in case it is sleeping. self.resume() # The daemonized thread may itself block, so don't wait # for it longer than a second. self._thread.join(1.0) if self._thread.is_alive(): # pragma: NO COVER _LOGGER.warning("Background thread did not exit.") self._thread = None self._on_response = None @property def is_active(self): """bool: True if the background thread is active.""" return self._thread is not None and self._thread.is_alive() def pause(self): """Pauses the response stream. This does not pause the request stream. """ with self._wake: self._paused = True def resume(self): """Resumes the response stream.""" with self._wake: self._paused = False self._wake.notify_all() @property def is_paused(self): """bool: True if the response stream is paused.""" return self._paused	27.39 KB	2025-05-25 12:26
📄 client_info.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for providing client information. Client information is used to send information about the calling client, such as the library and Python version, to API services. """ import platform from typing import Union from google.api_core import version as api_core_version _PY_VERSION = platform.python_version() _API_CORE_VERSION = api_core_version.__version__ _GRPC_VERSION: Union[str, None] try: import grpc _GRPC_VERSION = grpc.__version__ except ImportError: # pragma: NO COVER _GRPC_VERSION = None class ClientInfo(object): """Client information used to generate a user-agent for API calls. This user-agent information is sent along with API calls to allow the receiving service to do analytics on which versions of Python and Google libraries are being used. Args: python_version (str): The Python interpreter version, for example, ``'3.9.6'``. grpc_version (Optional[str]): The gRPC library version. api_core_version (str): The google-api-core library version. gapic_version (Optional[str]): The version of gapic-generated client library, if the library was generated by gapic. client_library_version (Optional[str]): The version of the client library, generally used if the client library was not generated by gapic or if additional functionality was built on top of a gapic client library. user_agent (Optional[str]): Prefix to the user agent header. This is used to supply information such as application name or partner tool. Recommended format: ``application-or-tool-ID/major.minor.version``. rest_version (Optional[str]): A string with labeled versions of the dependencies used for REST transport. """ def __init__( self, python_version=_PY_VERSION, grpc_version=_GRPC_VERSION, api_core_version=_API_CORE_VERSION, gapic_version=None, client_library_version=None, user_agent=None, rest_version=None, ): self.python_version = python_version self.grpc_version = grpc_version self.api_core_version = api_core_version self.gapic_version = gapic_version self.client_library_version = client_library_version self.user_agent = user_agent self.rest_version = rest_version def to_user_agent(self): """Returns the user-agent string for this client info.""" # Note: the order here is important as the internal metrics system # expects these items to be in specific locations. ua = "" if self.user_agent is not None: ua += "{user_agent} " ua += "gl-python/{python_version} " if self.grpc_version is not None: ua += "grpc/{grpc_version} " if self.rest_version is not None: ua += "rest/{rest_version} " ua += "gax/{api_core_version} " if self.gapic_version is not None: ua += "gapic/{gapic_version} " if self.client_library_version is not None: ua += "gccl/{client_library_version} " return ua.format(**self.__dict__).strip()	3.71 KB	2025-05-25 12:26
📄 client_logging.py import logging import json import os from typing import List, Optional _LOGGING_INITIALIZED = False _BASE_LOGGER_NAME = "google" # Fields to be included in the StructuredLogFormatter. # # TODO(https://github.com/googleapis/python-api-core/issues/761): Update this list to support additional logging fields. _recognized_logging_fields = [ "httpRequest", "rpcName", "serviceName", "credentialsType", "credentialsInfo", "universeDomain", "request", "response", "metadata", "retryAttempt", "httpResponse", ] # Additional fields to be Logged. def logger_configured(logger) -> bool: """Determines whether `logger` has non-default configuration Args: logger: The logger to check. Returns: bool: Whether the logger has any non-default configuration. """ return ( logger.handlers != [] or logger.level != logging.NOTSET or not logger.propagate ) def initialize_logging(): """Initializes "google" loggers, partly based on the environment variable Initializes the "google" logger and any loggers (at the "google" level or lower) specified by the environment variable GOOGLE_SDK_PYTHON_LOGGING_SCOPE, as long as none of these loggers were previously configured. If any such loggers (including the "google" logger) are initialized, they are set to NOT propagate log events up to their parent loggers. This initialization is executed only once, and hence the environment variable is only processed the first time this function is called. """ global _LOGGING_INITIALIZED if _LOGGING_INITIALIZED: return scopes = os.getenv("GOOGLE_SDK_PYTHON_LOGGING_SCOPE", "") setup_logging(scopes) _LOGGING_INITIALIZED = True def parse_logging_scopes(scopes: Optional[str] = None) -> List[str]: """Returns a list of logger names. Splits the single string of comma-separated logger names into a list of individual logger name strings. Args: scopes: The name of a single logger. (In the future, this will be a comma-separated list of multiple loggers.) Returns: A list of all the logger names in scopes. """ if not scopes: return [] # TODO(https://github.com/googleapis/python-api-core/issues/759): check if the namespace is a valid namespace. # TODO(b/380481951): Support logging multiple scopes. # TODO(b/380483756): Raise or log a warning for an invalid scope. namespaces = [scopes] return namespaces def configure_defaults(logger): """Configures `logger` to emit structured info to stdout.""" if not logger_configured(logger): console_handler = logging.StreamHandler() logger.setLevel("DEBUG") logger.propagate = False formatter = StructuredLogFormatter() console_handler.setFormatter(formatter) logger.addHandler(console_handler) def setup_logging(scopes: str = ""): """Sets up logging for the specified `scopes`. If the loggers specified in `scopes` have not been previously configured, this will configure them to emit structured log entries to stdout, and to not propagate their log events to their parent loggers. Additionally, if the "google" logger (whether it was specified in `scopes` or not) was not previously configured, it will also configure it to not propagate log events to the root logger. Args: scopes: The name of a single logger. (In the future, this will be a comma-separated list of multiple loggers.) """ # only returns valid logger scopes (namespaces) # this list has at most one element. logger_names = parse_logging_scopes(scopes) for namespace in logger_names: # This will either create a module level logger or get the reference of the base logger instantiated above. logger = logging.getLogger(namespace) # Configure default settings. configure_defaults(logger) # disable log propagation at base logger level to the root logger only if a base logger is not already configured via code changes. base_logger = logging.getLogger(_BASE_LOGGER_NAME) if not logger_configured(base_logger): base_logger.propagate = False # TODO(https://github.com/googleapis/python-api-core/issues/763): Expand documentation. class StructuredLogFormatter(logging.Formatter): # TODO(https://github.com/googleapis/python-api-core/issues/761): ensure that additional fields such as # function name, file name, and line no. appear in a log output. def format(self, record: logging.LogRecord): log_obj = { "timestamp": self.formatTime(record), "severity": record.levelname, "name": record.name, "message": record.getMessage(), } for field_name in _recognized_logging_fields: value = getattr(record, field_name, None) if value is not None: log_obj[field_name] = value return json.dumps(log_obj)	4.91 KB	2025-05-25 12:26
📄 client_options.py # Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Client options class. Client options provide a consistent interface for user options to be defined across clients. You can pass a client options object to a client. .. code-block:: python from google.api_core.client_options import ClientOptions from google.cloud.vision_v1 import ImageAnnotatorClient def get_client_cert(): # code to load client certificate and private key. return client_cert_bytes, client_private_key_bytes options = ClientOptions(api_endpoint="foo.googleapis.com", client_cert_source=get_client_cert) client = ImageAnnotatorClient(client_options=options) You can also pass a mapping object. .. code-block:: python from google.cloud.vision_v1 import ImageAnnotatorClient client = ImageAnnotatorClient( client_options={ "api_endpoint": "foo.googleapis.com", "client_cert_source" : get_client_cert }) """ from typing import Callable, Mapping, Optional, Sequence, Tuple class ClientOptions(object): """Client Options used to set options on clients. Args: api_endpoint (Optional[str]): The desired API endpoint, e.g., compute.googleapis.com client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): A callback which returns client certificate bytes and private key bytes both in PEM format. ``client_cert_source`` and ``client_encrypted_cert_source`` are mutually exclusive. client_encrypted_cert_source (Optional[Callable[[], Tuple[str, str, bytes]]]): A callback which returns client certificate file path, encrypted private key file path, and the passphrase bytes.``client_cert_source`` and ``client_encrypted_cert_source`` are mutually exclusive. quota_project_id (Optional[str]): A project name that a client's quota belongs to. credentials_file (Optional[str]): A path to a file storing credentials. ``credentials_file` and ``api_key`` are mutually exclusive. .. warning:: Important: If you accept a credential configuration (credential JSON/File/Stream) from an external source for authentication to Google Cloud Platform, you must validate it before providing it to any Google API or client library. Providing an unvalidated credential configuration to Google APIs or libraries can compromise the security of your systems and data. For more information, refer to `Validate credential configurations from external sources`_. .. _Validate credential configurations from external sources: https://cloud.google.com/docs/authentication/external/externally-sourced-credentials scopes (Optional[Sequence[str]]): OAuth access token override scopes. api_key (Optional[str]): Google API key. ``credentials_file`` and ``api_key`` are mutually exclusive. api_audience (Optional[str]): The intended audience for the API calls to the service that will be set when using certain 3rd party authentication flows. Audience is typically a resource identifier. If not set, the service endpoint value will be used as a default. An example of a valid ``api_audience`` is: "https://language.googleapis.com". universe_domain (Optional[str]): The desired universe domain. This must match the one in credentials. If not set, the default universe domain is `googleapis.com`. If both `api_endpoint` and `universe_domain` are set, then `api_endpoint` is used as the service endpoint. If `api_endpoint` is not specified, the format will be `{service}.{universe_domain}`. Raises: ValueError: If both ``client_cert_source`` and ``client_encrypted_cert_source`` are provided, or both ``credentials_file`` and ``api_key`` are provided. """ def __init__( self, api_endpoint: Optional[str] = None, client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, client_encrypted_cert_source: Optional[ Callable[[], Tuple[str, str, bytes]] ] = None, quota_project_id: Optional[str] = None, credentials_file: Optional[str] = None, scopes: Optional[Sequence[str]] = None, api_key: Optional[str] = None, api_audience: Optional[str] = None, universe_domain: Optional[str] = None, ): if client_cert_source and client_encrypted_cert_source: raise ValueError( "client_cert_source and client_encrypted_cert_source are mutually exclusive" ) if api_key and credentials_file: raise ValueError("api_key and credentials_file are mutually exclusive") self.api_endpoint = api_endpoint self.client_cert_source = client_cert_source self.client_encrypted_cert_source = client_encrypted_cert_source self.quota_project_id = quota_project_id self.credentials_file = credentials_file self.scopes = scopes self.api_key = api_key self.api_audience = api_audience self.universe_domain = universe_domain def __repr__(self) -> str: return "ClientOptions: " + repr(self.__dict__) def from_dict(options: Mapping[str, object]) -> ClientOptions: """Construct a client options object from a mapping object. Args: options (collections.abc.Mapping): A mapping object with client options. See the docstring for ClientOptions for details on valid arguments. """ client_options = ClientOptions() for key, value in options.items(): if hasattr(client_options, key): setattr(client_options, key, value) else: raise ValueError("ClientOptions does not accept an option '" + key + "'") return client_options	6.41 KB	2025-05-25 12:26
📄 datetime_helpers.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for :mod:`datetime`.""" import calendar import datetime import re from google.protobuf import timestamp_pb2 _UTC_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) _RFC3339_MICROS = "%Y-%m-%dT%H:%M:%S.%fZ" _RFC3339_NO_FRACTION = "%Y-%m-%dT%H:%M:%S" # datetime.strptime cannot handle nanosecond precision: parse w/ regex _RFC3339_NANOS = re.compile( r""" (?P<no_fraction> \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} # YYYY-MM-DDTHH:MM:SS ) ( # Optional decimal part \. # decimal point (?P<nanos>\d{1,9}) # nanoseconds, maybe truncated )? Z # Zulu """, re.VERBOSE, ) def utcnow(): """A :meth:`datetime.datetime.utcnow()` alias to allow mocking in tests.""" return datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None) def to_milliseconds(value): """Convert a zone-aware datetime to milliseconds since the unix epoch. Args: value (datetime.datetime): The datetime to covert. Returns: int: Milliseconds since the unix epoch. """ micros = to_microseconds(value) return micros // 1000 def from_microseconds(value): """Convert timestamp in microseconds since the unix epoch to datetime. Args: value (float): The timestamp to convert, in microseconds. Returns: datetime.datetime: The datetime object equivalent to the timestamp in UTC. """ return _UTC_EPOCH + datetime.timedelta(microseconds=value) def to_microseconds(value): """Convert a datetime to microseconds since the unix epoch. Args: value (datetime.datetime): The datetime to covert. Returns: int: Microseconds since the unix epoch. """ if not value.tzinfo: value = value.replace(tzinfo=datetime.timezone.utc) # Regardless of what timezone is on the value, convert it to UTC. value = value.astimezone(datetime.timezone.utc) # Convert the datetime to a microsecond timestamp. return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond def from_iso8601_date(value): """Convert a ISO8601 date string to a date. Args: value (str): The ISO8601 date string. Returns: datetime.date: A date equivalent to the date string. """ return datetime.datetime.strptime(value, "%Y-%m-%d").date() def from_iso8601_time(value): """Convert a zoneless ISO8601 time string to a time. Args: value (str): The ISO8601 time string. Returns: datetime.time: A time equivalent to the time string. """ return datetime.datetime.strptime(value, "%H:%M:%S").time() def from_rfc3339(value): """Convert an RFC3339-format timestamp to a native datetime. Supported formats include those without fractional seconds, or with any fraction up to nanosecond precision. .. note:: Python datetimes do not support nanosecond precision; this function therefore truncates such values to microseconds. Args: value (str): The RFC3339 string to convert. Returns: datetime.datetime: The datetime object equivalent to the timestamp in UTC. Raises: ValueError: If the timestamp does not match the RFC3339 regular expression. """ with_nanos = _RFC3339_NANOS.match(value) if with_nanos is None: raise ValueError( "Timestamp: {!r}, does not match pattern: {!r}".format( value, _RFC3339_NANOS.pattern ) ) bare_seconds = datetime.datetime.strptime( with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION ) fraction = with_nanos.group("nanos") if fraction is None: micros = 0 else: scale = 9 - len(fraction) nanos = int(fraction) * (10*scale) micros = nanos // 1000 return bare_seconds.replace(microsecond=micros, tzinfo=datetime.timezone.utc) from_rfc3339_nanos = from_rfc3339 # from_rfc3339_nanos method was deprecated. def to_rfc3339(value, ignore_zone=True): """Convert a datetime to an RFC3339 timestamp string. Args: value (datetime.datetime): The datetime object to be converted to a string. ignore_zone (bool): If True, then the timezone (if any) of the datetime object is ignored and the datetime is treated as UTC. Returns: str: The RFC3339 formatted string representing the datetime. """ if not ignore_zone and value.tzinfo is not None: # Convert to UTC and remove the time zone info. value = value.replace(tzinfo=None) - value.utcoffset() return value.strftime(_RFC3339_MICROS) class DatetimeWithNanoseconds(datetime.datetime): """Track nanosecond in addition to normal datetime attrs. Nanosecond can be passed only as a keyword argument. """ __slots__ = ("_nanosecond",) # pylint: disable=arguments-differ def __new__(cls, args, *kw): nanos = kw.pop("nanosecond", 0) if nanos > 0: if "microsecond" in kw: raise TypeError("Specify only one of 'microsecond' or 'nanosecond'") kw["microsecond"] = nanos // 1000 inst = datetime.datetime.__new__(cls, args, *kw) inst._nanosecond = nanos or 0 return inst # pylint: disable=arguments-differ @property def nanosecond(self): """Read-only: nanosecond precision.""" return self._nanosecond def rfc3339(self): """Return an RFC3339-compliant timestamp. Returns: (str): Timestamp string according to RFC3339 spec. """ if self._nanosecond == 0: return to_rfc3339(self) nanos = str(self._nanosecond).rjust(9, "0").rstrip("0") return "{}.{}Z".format(self.strftime(_RFC3339_NO_FRACTION), nanos) @classmethod def from_rfc3339(cls, stamp): """Parse RFC3339-compliant timestamp, preserving nanoseconds. Args: stamp (str): RFC3339 stamp, with up to nanosecond precision Returns: :class:`DatetimeWithNanoseconds`: an instance matching the timestamp string Raises: ValueError: if `stamp` does not match the expected format """ with_nanos = _RFC3339_NANOS.match(stamp) if with_nanos is None: raise ValueError( "Timestamp: {}, does not match pattern: {}".format( stamp, _RFC3339_NANOS.pattern ) ) bare = datetime.datetime.strptime( with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION ) fraction = with_nanos.group("nanos") if fraction is None: nanos = 0 else: scale = 9 - len(fraction) nanos = int(fraction) (10*scale) return cls( bare.year, bare.month, bare.day, bare.hour, bare.minute, bare.second, nanosecond=nanos, tzinfo=datetime.timezone.utc, ) def timestamp_pb(self): """Return a timestamp message. Returns: (:class:`~google.protobuf.timestamp_pb2.Timestamp`): Timestamp message """ inst = ( self if self.tzinfo is not None else self.replace(tzinfo=datetime.timezone.utc) ) delta = inst - _UTC_EPOCH seconds = int(delta.total_seconds()) nanos = self._nanosecond or self.microsecond 1000 return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) @classmethod def from_timestamp_pb(cls, stamp): """Parse RFC3339-compliant timestamp, preserving nanoseconds. Args: stamp (:class:`~google.protobuf.timestamp_pb2.Timestamp`): timestamp message Returns: :class:`DatetimeWithNanoseconds`: an instance matching the timestamp message """ microseconds = int(stamp.seconds * 1e6) bare = from_microseconds(microseconds) return cls( bare.year, bare.month, bare.day, bare.hour, bare.minute, bare.second, nanosecond=stamp.nanos, tzinfo=datetime.timezone.utc, )	8.82 KB	2025-05-25 12:26
📄 exceptions.py # Copyright 2014 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Exceptions raised by Google API core & clients. This module provides base classes for all errors raised by libraries based on :mod:`google.api_core`, including both HTTP and gRPC clients. """ from __future__ import absolute_import from __future__ import unicode_literals import http.client from typing import Optional, Dict from typing import Union import warnings from google.rpc import error_details_pb2 def _warn_could_not_import_grpcio_status(): warnings.warn( "Please install grpcio-status to obtain helpful grpc error messages.", ImportWarning, ) # pragma: NO COVER try: import grpc try: from grpc_status import rpc_status except ImportError: # pragma: NO COVER _warn_could_not_import_grpcio_status() rpc_status = None except ImportError: # pragma: NO COVER grpc = None # Lookup tables for mapping exceptions from HTTP and gRPC transports. # Populated by _GoogleAPICallErrorMeta _HTTP_CODE_TO_EXCEPTION: Dict[int, Exception] = {} _GRPC_CODE_TO_EXCEPTION: Dict[int, Exception] = {} # Additional lookup table to map integer status codes to grpc status code # grpc does not currently support initializing enums from ints # i.e., grpc.StatusCode(5) raises an error _INT_TO_GRPC_CODE = {} if grpc is not None: # pragma: no branch for x in grpc.StatusCode: _INT_TO_GRPC_CODE[x.value[0]] = x class GoogleAPIError(Exception): """Base class for all exceptions raised by Google API Clients.""" pass class DuplicateCredentialArgs(GoogleAPIError): """Raised when multiple credentials are passed.""" pass class RetryError(GoogleAPIError): """Raised when a function has exhausted all of its available retries. Args: message (str): The exception message. cause (Exception): The last exception raised when retrying the function. """ def __init__(self, message, cause): super(RetryError, self).__init__(message) self.message = message self._cause = cause @property def cause(self): """The last exception raised when retrying the function.""" return self._cause def __str__(self): return "{}, last exception: {}".format(self.message, self.cause) class _GoogleAPICallErrorMeta(type): """Metaclass for registering GoogleAPICallError subclasses.""" def __new__(mcs, name, bases, class_dict): cls = type.__new__(mcs, name, bases, class_dict) if cls.code is not None: _HTTP_CODE_TO_EXCEPTION.setdefault(cls.code, cls) if cls.grpc_status_code is not None: _GRPC_CODE_TO_EXCEPTION.setdefault(cls.grpc_status_code, cls) return cls class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): """Base class for exceptions raised by calling API methods. Args: message (str): The exception message. errors (Sequence[Any]): An optional list of error details. details (Sequence[Any]): An optional list of objects defined in google.rpc.error_details. response (Union[requests.Request, grpc.Call]): The response or gRPC call metadata. error_info (Union[error_details_pb2.ErrorInfo, None]): An optional object containing error info (google.rpc.error_details.ErrorInfo). """ code: Union[int, None] = None """Optional[int]: The HTTP status code associated with this error. This may be ``None`` if the exception does not have a direct mapping to an HTTP error. See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html """ grpc_status_code = None """Optional[grpc.StatusCode]: The gRPC status code associated with this error. This may be ``None`` if the exception does not match up to a gRPC error. """ def __init__(self, message, errors=(), details=(), response=None, error_info=None): super(GoogleAPICallError, self).__init__(message) self.message = message """str: The exception message.""" self._errors = errors self._details = details self._response = response self._error_info = error_info def __str__(self): error_msg = "{} {}".format(self.code, self.message) if self.details: error_msg = "{} {}".format(error_msg, self.details) # Note: This else condition can be removed once proposal A from # b/284179390 is implemented. else: if self.errors: errors = [ f"{error.code}: {error.message}" for error in self.errors if hasattr(error, "code") and hasattr(error, "message") ] if errors: error_msg = "{} {}".format(error_msg, "\n".join(errors)) return error_msg @property def reason(self): """The reason of the error. Reference: https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 Returns: Union[str, None]: An optional string containing reason of the error. """ return self._error_info.reason if self._error_info else None @property def domain(self): """The logical grouping to which the "reason" belongs. Reference: https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 Returns: Union[str, None]: An optional string containing a logical grouping to which the "reason" belongs. """ return self._error_info.domain if self._error_info else None @property def metadata(self): """Additional structured details about this error. Reference: https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 Returns: Union[Dict[str, str], None]: An optional object containing structured details about the error. """ return self._error_info.metadata if self._error_info else None @property def errors(self): """Detailed error information. Returns: Sequence[Any]: A list of additional error details. """ return list(self._errors) @property def details(self): """Information contained in google.rpc.status.details. Reference: https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto Returns: Sequence[Any]: A list of structured objects from error_details.proto """ return list(self._details) @property def response(self): """Optional[Union[requests.Request, grpc.Call]]: The response or gRPC call metadata.""" return self._response class Redirection(GoogleAPICallError): """Base class for for all redirection (HTTP 3xx) responses.""" class MovedPermanently(Redirection): """Exception mapping a ``301 Moved Permanently`` response.""" code = http.client.MOVED_PERMANENTLY class NotModified(Redirection): """Exception mapping a ``304 Not Modified`` response.""" code = http.client.NOT_MODIFIED class TemporaryRedirect(Redirection): """Exception mapping a ``307 Temporary Redirect`` response.""" code = http.client.TEMPORARY_REDIRECT class ResumeIncomplete(Redirection): """Exception mapping a ``308 Resume Incomplete`` response. .. note:: :attr:`http.client.PERMANENT_REDIRECT` is ``308``, but Google APIs differ in their use of this status code. """ code = 308 class ClientError(GoogleAPICallError): """Base class for all client error (HTTP 4xx) responses.""" class BadRequest(ClientError): """Exception mapping a ``400 Bad Request`` response.""" code = http.client.BAD_REQUEST class InvalidArgument(BadRequest): """Exception mapping a :attr:`grpc.StatusCode.INVALID_ARGUMENT` error.""" grpc_status_code = grpc.StatusCode.INVALID_ARGUMENT if grpc is not None else None class FailedPrecondition(BadRequest): """Exception mapping a :attr:`grpc.StatusCode.FAILED_PRECONDITION` error.""" grpc_status_code = grpc.StatusCode.FAILED_PRECONDITION if grpc is not None else None class OutOfRange(BadRequest): """Exception mapping a :attr:`grpc.StatusCode.OUT_OF_RANGE` error.""" grpc_status_code = grpc.StatusCode.OUT_OF_RANGE if grpc is not None else None class Unauthorized(ClientError): """Exception mapping a ``401 Unauthorized`` response.""" code = http.client.UNAUTHORIZED class Unauthenticated(Unauthorized): """Exception mapping a :attr:`grpc.StatusCode.UNAUTHENTICATED` error.""" grpc_status_code = grpc.StatusCode.UNAUTHENTICATED if grpc is not None else None class Forbidden(ClientError): """Exception mapping a ``403 Forbidden`` response.""" code = http.client.FORBIDDEN class PermissionDenied(Forbidden): """Exception mapping a :attr:`grpc.StatusCode.PERMISSION_DENIED` error.""" grpc_status_code = grpc.StatusCode.PERMISSION_DENIED if grpc is not None else None class NotFound(ClientError): """Exception mapping a ``404 Not Found`` response or a :attr:`grpc.StatusCode.NOT_FOUND` error.""" code = http.client.NOT_FOUND grpc_status_code = grpc.StatusCode.NOT_FOUND if grpc is not None else None class MethodNotAllowed(ClientError): """Exception mapping a ``405 Method Not Allowed`` response.""" code = http.client.METHOD_NOT_ALLOWED class Conflict(ClientError): """Exception mapping a ``409 Conflict`` response.""" code = http.client.CONFLICT class AlreadyExists(Conflict): """Exception mapping a :attr:`grpc.StatusCode.ALREADY_EXISTS` error.""" grpc_status_code = grpc.StatusCode.ALREADY_EXISTS if grpc is not None else None class Aborted(Conflict): """Exception mapping a :attr:`grpc.StatusCode.ABORTED` error.""" grpc_status_code = grpc.StatusCode.ABORTED if grpc is not None else None class LengthRequired(ClientError): """Exception mapping a ``411 Length Required`` response.""" code = http.client.LENGTH_REQUIRED class PreconditionFailed(ClientError): """Exception mapping a ``412 Precondition Failed`` response.""" code = http.client.PRECONDITION_FAILED class RequestRangeNotSatisfiable(ClientError): """Exception mapping a ``416 Request Range Not Satisfiable`` response.""" code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE class TooManyRequests(ClientError): """Exception mapping a ``429 Too Many Requests`` response.""" code = http.client.TOO_MANY_REQUESTS class ResourceExhausted(TooManyRequests): """Exception mapping a :attr:`grpc.StatusCode.RESOURCE_EXHAUSTED` error.""" grpc_status_code = grpc.StatusCode.RESOURCE_EXHAUSTED if grpc is not None else None class Cancelled(ClientError): """Exception mapping a :attr:`grpc.StatusCode.CANCELLED` error.""" # This maps to HTTP status code 499. See # https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto code = 499 grpc_status_code = grpc.StatusCode.CANCELLED if grpc is not None else None class ServerError(GoogleAPICallError): """Base for 5xx responses.""" class InternalServerError(ServerError): """Exception mapping a ``500 Internal Server Error`` response. or a :attr:`grpc.StatusCode.INTERNAL` error.""" code = http.client.INTERNAL_SERVER_ERROR grpc_status_code = grpc.StatusCode.INTERNAL if grpc is not None else None class Unknown(ServerError): """Exception mapping a :attr:`grpc.StatusCode.UNKNOWN` error.""" grpc_status_code = grpc.StatusCode.UNKNOWN if grpc is not None else None class DataLoss(ServerError): """Exception mapping a :attr:`grpc.StatusCode.DATA_LOSS` error.""" grpc_status_code = grpc.StatusCode.DATA_LOSS if grpc is not None else None class MethodNotImplemented(ServerError): """Exception mapping a ``501 Not Implemented`` response or a :attr:`grpc.StatusCode.UNIMPLEMENTED` error.""" code = http.client.NOT_IMPLEMENTED grpc_status_code = grpc.StatusCode.UNIMPLEMENTED if grpc is not None else None class BadGateway(ServerError): """Exception mapping a ``502 Bad Gateway`` response.""" code = http.client.BAD_GATEWAY class ServiceUnavailable(ServerError): """Exception mapping a ``503 Service Unavailable`` response or a :attr:`grpc.StatusCode.UNAVAILABLE` error.""" code = http.client.SERVICE_UNAVAILABLE grpc_status_code = grpc.StatusCode.UNAVAILABLE if grpc is not None else None class GatewayTimeout(ServerError): """Exception mapping a ``504 Gateway Timeout`` response.""" code = http.client.GATEWAY_TIMEOUT class DeadlineExceeded(GatewayTimeout): """Exception mapping a :attr:`grpc.StatusCode.DEADLINE_EXCEEDED` error.""" grpc_status_code = grpc.StatusCode.DEADLINE_EXCEEDED if grpc is not None else None class AsyncRestUnsupportedParameterError(NotImplementedError): """Raised when an unsupported parameter is configured against async rest transport.""" pass def exception_class_for_http_status(status_code): """Return the exception class for a specific HTTP status code. Args: status_code (int): The HTTP status code. Returns: :func:`type`: the appropriate subclass of :class:`GoogleAPICallError`. """ return _HTTP_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError) def from_http_status(status_code, message, kwargs): """Create a :class:`GoogleAPICallError` from an HTTP status code. Args: status_code (int): The HTTP status code. message (str): The exception message. kwargs: Additional arguments passed to the :class:`GoogleAPICallError` constructor. Returns: GoogleAPICallError: An instance of the appropriate subclass of :class:`GoogleAPICallError`. """ error_class = exception_class_for_http_status(status_code) error = error_class(message, kwargs) if error.code is None: error.code = status_code return error def _format_rest_error_message(error, method, url): method = method.upper() if method else None message = "{method} {url}: {error}".format( method=method, url=url, error=error, ) return message # NOTE: We're moving away from `from_http_status` because it expects an aiohttp response compared # to `format_http_response_error` which expects a more abstract response from google.auth and is # compatible with both sync and async response types. # TODO(https://github.com/googleapis/python-api-core/issues/691): Add type hint for response. def format_http_response_error( response, method: str, url: str, payload: Optional[Dict] = None ): """Create a :class:`GoogleAPICallError` from a google auth rest response. Args: response Union[google.auth.transport.Response, google.auth.aio.transport.Response]: The HTTP response. method Optional(str): The HTTP request method. url Optional(str): The HTTP request url. payload Optional(dict): The HTTP response payload. If not passed in, it is read from response for a response type of google.auth.transport.Response. Returns: GoogleAPICallError: An instance of the appropriate subclass of :class:`GoogleAPICallError`, with the message and errors populated from the response. """ payload = {} if not payload else payload error_message = payload.get("error", {}).get("message", "unknown error") errors = payload.get("error", {}).get("errors", ()) # In JSON, details are already formatted in developer-friendly way. details = payload.get("error", {}).get("details", ()) error_info_list = list( filter( lambda detail: detail.get("@type", "") == "type.googleapis.com/google.rpc.ErrorInfo", details, ) ) error_info = error_info_list[0] if error_info_list else None message = _format_rest_error_message(error_message, method, url) exception = from_http_status( response.status_code, message, errors=errors, details=details, response=response, error_info=error_info, ) return exception def from_http_response(response): """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`. Args: response (requests.Response): The HTTP response. Returns: GoogleAPICallError: An instance of the appropriate subclass of :class:`GoogleAPICallError`, with the message and errors populated from the response. """ try: payload = response.json() except ValueError: payload = {"error": {"message": response.text or "unknown error"}} return format_http_response_error( response, response.request.method, response.request.url, payload ) def exception_class_for_grpc_status(status_code): """Return the exception class for a specific :class:`grpc.StatusCode`. Args: status_code (grpc.StatusCode): The gRPC status code. Returns: :func:`type`: the appropriate subclass of :class:`GoogleAPICallError`. """ return _GRPC_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError) def from_grpc_status(status_code, message, kwargs): """Create a :class:`GoogleAPICallError` from a :class:`grpc.StatusCode`. Args: status_code (Union[grpc.StatusCode, int]): The gRPC status code. message (str): The exception message. kwargs: Additional arguments passed to the :class:`GoogleAPICallError` constructor. Returns: GoogleAPICallError: An instance of the appropriate subclass of :class:`GoogleAPICallError`. """ if isinstance(status_code, int): status_code = _INT_TO_GRPC_CODE.get(status_code, status_code) error_class = exception_class_for_grpc_status(status_code) error = error_class(message, kwargs) if error.grpc_status_code is None: error.grpc_status_code = status_code return error def _is_informative_grpc_error(rpc_exc): return hasattr(rpc_exc, "code") and hasattr(rpc_exc, "details") def _parse_grpc_error_details(rpc_exc): if not rpc_status: # pragma: NO COVER _warn_could_not_import_grpcio_status() return [], None try: status = rpc_status.from_call(rpc_exc) except NotImplementedError: # workaround return [], None if not status: return [], None possible_errors = [ error_details_pb2.BadRequest, error_details_pb2.PreconditionFailure, error_details_pb2.QuotaFailure, error_details_pb2.ErrorInfo, error_details_pb2.RetryInfo, error_details_pb2.ResourceInfo, error_details_pb2.RequestInfo, error_details_pb2.DebugInfo, error_details_pb2.Help, error_details_pb2.LocalizedMessage, ] error_info = None error_details = [] for detail in status.details: matched_detail_cls = list( filter(lambda x: detail.Is(x.DESCRIPTOR), possible_errors) ) # If nothing matched, use detail directly. if len(matched_detail_cls) == 0: info = detail else: info = matched_detail_cls[0]() detail.Unpack(info) error_details.append(info) if isinstance(info, error_details_pb2.ErrorInfo): error_info = info return error_details, error_info def from_grpc_error(rpc_exc): """Create a :class:`GoogleAPICallError` from a :class:`grpc.RpcError`. Args: rpc_exc (grpc.RpcError): The gRPC error. Returns: GoogleAPICallError: An instance of the appropriate subclass of :class:`GoogleAPICallError`. """ # NOTE(lidiz) All gRPC error shares the parent class grpc.RpcError. # However, check for grpc.RpcError breaks backward compatibility. if ( grpc is not None and isinstance(rpc_exc, grpc.Call) ) or _is_informative_grpc_error(rpc_exc): details, err_info = _parse_grpc_error_details(rpc_exc) return from_grpc_status( rpc_exc.code(), rpc_exc.details(), errors=(rpc_exc,), details=details, response=rpc_exc, error_info=err_info, ) else: return GoogleAPICallError(str(rpc_exc), errors=(rpc_exc,), response=rpc_exc)	20.65 KB	2025-05-25 12:26
📄 extended_operation.py # Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Futures for extended long-running operations returned from Google Cloud APIs. These futures can be used to synchronously wait for the result of a long-running operations using :meth:`ExtendedOperation.result`: .. code-block:: python extended_operation = my_api_client.long_running_method() extended_operation.result() Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: .. code-block:: python extended_operation = my_api_client.long_running_method() def my_callback(ex_op): print(f"Operation {ex_op.name} completed") extended_operation.add_done_callback(my_callback) """ import threading from google.api_core import exceptions from google.api_core.future import polling class ExtendedOperation(polling.PollingFuture): """An ExtendedOperation future for interacting with a Google API Long-Running Operation. Args: extended_operation (proto.Message): The initial operation. refresh (Callable[[], type(extended_operation)]): A callable that returns the latest state of the operation. cancel (Callable[[], None]): A callable that tries to cancel the operation. polling Optional(google.api_core.retry.Retry): The configuration used for polling. This can be used to control how often :meth:`done` is polled. If the ``timeout`` argument to :meth:`result` is specified it will override the ``polling.timeout`` property. retry Optional(google.api_core.retry.Retry): DEPRECATED use ``polling`` instead. If specified it will override ``polling`` parameter to maintain backward compatibility. Note: Most long-running API methods use google.api_core.operation.Operation This class is a wrapper for a subset of methods that use alternative Long-Running Operation (LRO) semantics. Note: there is not a concrete type the extended operation must be. It MUST have fields that correspond to the following, POSSIBLY WITH DIFFERENT NAMES: * name: str * status: Union[str, bool, enum.Enum] * error_code: int * error_message: str """ def __init__( self, extended_operation, refresh, cancel, polling=polling.DEFAULT_POLLING, kwargs, ): super().__init__(polling=polling, kwargs) self._extended_operation = extended_operation self._refresh = refresh self._cancel = cancel # Note: the extended operation does not give a good way to indicate cancellation. # We make do with manually tracking cancellation and checking for doneness. self._cancelled = False self._completion_lock = threading.Lock() # Invoke in case the operation came back already complete. self._handle_refreshed_operation() # Note: the following four properties MUST be overridden in a subclass # if, and only if, the fields in the corresponding extended operation message # have different names. # # E.g. we have an extended operation class that looks like # # class MyOperation(proto.Message): # moniker = proto.Field(proto.STRING, number=1) # status_msg = proto.Field(proto.STRING, number=2) # optional http_error_code = proto.Field(proto.INT32, number=3) # optional http_error_msg = proto.Field(proto.STRING, number=4) # # the ExtendedOperation subclass would provide property overrides that map # to these (poorly named) fields. @property def name(self): return self._extended_operation.name @property def status(self): return self._extended_operation.status @property def error_code(self): return self._extended_operation.error_code @property def error_message(self): return self._extended_operation.error_message def __getattr__(self, name): return getattr(self._extended_operation, name) def done(self, retry=None): self._refresh_and_update(retry) return self._extended_operation.done def cancel(self): if self.done(): return False self._cancel() self._cancelled = True return True def cancelled(self): # TODO(dovs): there is not currently a good way to determine whether the # operation has been cancelled. # The best we can do is manually keep track of cancellation # and check for doneness. if not self._cancelled: return False self._refresh_and_update() return self._extended_operation.done def _refresh_and_update(self, retry=None): if not self._extended_operation.done: self._extended_operation = ( self._refresh(retry=retry) if retry else self._refresh() ) self._handle_refreshed_operation() def _handle_refreshed_operation(self): with self._completion_lock: if not self._extended_operation.done: return if self.error_code and self.error_message: # Note: `errors` can be removed once proposal A from # b/284179390 is implemented. errors = [] if hasattr(self, "error") and hasattr(self.error, "errors"): errors = self.error.errors exception = exceptions.from_http_status( status_code=self.error_code, message=self.error_message, response=self._extended_operation, errors=errors, ) self.set_exception(exception) elif self.error_code or self.error_message: exception = exceptions.GoogleAPICallError( f"Unexpected error {self.error_code}: {self.error_message}" ) self.set_exception(exception) else: # Extended operations have no payload. self.set_result(None) @classmethod def make(cls, refresh, cancel, extended_operation, *kwargs): """ Return an instantiated ExtendedOperation (or child) that wraps a refresh callable * a cancel callable (can be a no-op) * an initial result .. note:: It is the caller's responsibility to set up refresh and cancel with their correct request argument. The reason for this is that the services that use Extended Operations have rpcs that look something like the following: // service.proto service MyLongService { rpc StartLongTask(StartLongTaskRequest) returns (ExtendedOperation) { option (google.cloud.operation_service) = "CustomOperationService"; } } service CustomOperationService { rpc Get(GetOperationRequest) returns (ExtendedOperation) { option (google.cloud.operation_polling_method) = true; } } Any info needed for the poll, e.g. a name, path params, etc. is held in the request, which the initial client method is in a much better position to make made because the caller made the initial request. TL;DR: the caller sets up closures for refresh and cancel that carry the properly configured requests. Args: refresh (Callable[Optional[Retry]][type(extended_operation)]): A callable that returns the latest state of the operation. cancel (Callable[][Any]): A callable that tries to cancel the operation on a best effort basis. extended_operation (Any): The initial response of the long running method. See the docstring for ExtendedOperation.__init__ for requirements on the type and fields of extended_operation """ return cls(extended_operation, refresh, cancel, **kwargs)	8.43 KB	2025-05-25 12:26
📁 future	-	2025-05-25 12:26
📁 gapic_v1	-	2025-05-25 12:26
📄 general_helpers.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This import for backward compatibility only. from functools import wraps # noqa: F401 pragma: NO COVER	681 B	2025-05-25 12:26
📄 grpc_helpers.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for :mod:`grpc`.""" from typing import Generic, Iterator, Optional, TypeVar import collections import functools import warnings import grpc from google.api_core import exceptions import google.auth import google.auth.credentials import google.auth.transport.grpc import google.auth.transport.requests import google.protobuf PROTOBUF_VERSION = google.protobuf.__version__ # The grpcio-gcp package only has support for protobuf < 4 if PROTOBUF_VERSION[0:2] == "3.": # pragma: NO COVER try: import grpc_gcp warnings.warn( """Support for grpcio-gcp is deprecated. This feature will be removed from `google-api-core` after January 1, 2024. If you need to continue to use this feature, please pin to a specific version of `google-api-core`.""", DeprecationWarning, ) HAS_GRPC_GCP = True except ImportError: HAS_GRPC_GCP = False else: HAS_GRPC_GCP = False # The list of gRPC Callable interfaces that return iterators. _STREAM_WRAP_CLASSES = (grpc.UnaryStreamMultiCallable, grpc.StreamStreamMultiCallable) # denotes the proto response type for grpc calls P = TypeVar("P") def _patch_callable_name(callable_): """Fix-up gRPC callable attributes. gRPC callable lack the ``__name__`` attribute which causes :func:`functools.wraps` to error. This adds the attribute if needed. """ if not hasattr(callable_, "__name__"): callable_.__name__ = callable_.__class__.__name__ def _wrap_unary_errors(callable_): """Map errors for Unary-Unary and Stream-Unary gRPC callables.""" _patch_callable_name(callable_) @functools.wraps(callable_) def error_remapped_callable(args, kwargs): try: return callable_(args, *kwargs) except grpc.RpcError as exc: raise exceptions.from_grpc_error(exc) from exc return error_remapped_callable class _StreamingResponseIterator(Generic[P], grpc.Call): def __init__(self, wrapped, prefetch_first_result=True): self._wrapped = wrapped # This iterator is used in a retry context, and returned outside after init. # gRPC will not throw an exception until the stream is consumed, so we need # to retrieve the first result, in order to fail, in order to trigger a retry. try: if prefetch_first_result: self._stored_first_result = next(self._wrapped) except TypeError: # It is possible the wrapped method isn't an iterable (a grpc.Call # for instance). If this happens don't store the first result. pass except StopIteration: # ignore stop iteration at this time. This should be handled outside of retry. pass def __iter__(self) -> Iterator[P]: """This iterator is also an iterable that returns itself.""" return self def __next__(self) -> P: """Get the next response from the stream. Returns: protobuf.Message: A single response from the stream. """ try: if hasattr(self, "_stored_first_result"): result = self._stored_first_result del self._stored_first_result return result return next(self._wrapped) except grpc.RpcError as exc: # If the stream has already returned data, we cannot recover here. raise exceptions.from_grpc_error(exc) from exc # grpc.Call & grpc.RpcContext interface def add_callback(self, callback): return self._wrapped.add_callback(callback) def cancel(self): return self._wrapped.cancel() def code(self): return self._wrapped.code() def details(self): return self._wrapped.details() def initial_metadata(self): return self._wrapped.initial_metadata() def is_active(self): return self._wrapped.is_active() def time_remaining(self): return self._wrapped.time_remaining() def trailing_metadata(self): return self._wrapped.trailing_metadata() # public type alias denoting the return type of streaming gapic calls GrpcStream = _StreamingResponseIterator[P] def _wrap_stream_errors(callable_): """Wrap errors for Unary-Stream and Stream-Stream gRPC callables. The callables that return iterators require a bit more logic to re-map errors when iterating. This wraps both the initial invocation and the iterator of the return value to re-map errors. """ _patch_callable_name(callable_) @functools.wraps(callable_) def error_remapped_callable(args, *kwargs): try: result = callable_(args, kwargs) # Auto-fetching the first result causes PubSub client's streaming pull # to hang when re-opening the stream, thus we need examine the hacky # hidden flag to see if pre-fetching is disabled. # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257 prefetch_first = getattr(callable_, "_prefetch_first_result_", True) return _StreamingResponseIterator( result, prefetch_first_result=prefetch_first ) except grpc.RpcError as exc: raise exceptions.from_grpc_error(exc) from exc return error_remapped_callable def wrap_errors(callable_): """Wrap a gRPC callable and map :class:`grpc.RpcErrors` to friendly error classes. Errors raised by the gRPC callable are mapped to the appropriate :class:`google.api_core.exceptions.GoogleAPICallError` subclasses. The original `grpc.RpcError` (which is usually also a `grpc.Call`) is available from the ``response`` property on the mapped exception. This is useful for extracting metadata from the original error. Args: callable_ (Callable): A gRPC callable. Returns: Callable: The wrapped gRPC callable. """ if isinstance(callable_, _STREAM_WRAP_CLASSES): return _wrap_stream_errors(callable_) else: return _wrap_unary_errors(callable_) def _create_composite_credentials( credentials=None, credentials_file=None, default_scopes=None, scopes=None, ssl_credentials=None, quota_project_id=None, default_host=None, ): """Create the composite credentials for secure channels. Args: credentials (google.auth.credentials.Credentials): The credentials. If not specified, then this function will attempt to ascertain the credentials from the environment using :func:`google.auth.default`. credentials_file (str): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is mutually exclusive with credentials. .. warning:: Important: If you accept a credential configuration (credential JSON/File/Stream) from an external source for authentication to Google Cloud Platform, you must validate it before providing it to any Google API or client library. Providing an unvalidated credential configuration to Google APIs or libraries can compromise the security of your systems and data. For more information, refer to `Validate credential configurations from external sources`_. .. _Validate credential configurations from external sources: https://cloud.google.com/docs/authentication/external/externally-sourced-credentials default_scopes (Sequence[str]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. scopes (Sequence[str]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. ssl_credentials (grpc.ChannelCredentials): Optional SSL channel credentials. This can be used to specify different certificates. quota_project_id (str): An optional project to use for billing and quota. default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". Returns: grpc.ChannelCredentials: The composed channel credentials object. Raises: google.api_core.DuplicateCredentialArgs: If both a credentials object and credentials_file are passed. """ if credentials and credentials_file: raise exceptions.DuplicateCredentialArgs( "'credentials' and 'credentials_file' are mutually exclusive." ) if credentials_file: credentials, _ = google.auth.load_credentials_from_file( credentials_file, scopes=scopes, default_scopes=default_scopes ) elif credentials: credentials = google.auth.credentials.with_scopes_if_required( credentials, scopes=scopes, default_scopes=default_scopes ) else: credentials, _ = google.auth.default( scopes=scopes, default_scopes=default_scopes ) if quota_project_id and isinstance( credentials, google.auth.credentials.CredentialsWithQuotaProject ): credentials = credentials.with_quota_project(quota_project_id) request = google.auth.transport.requests.Request() # Create the metadata plugin for inserting the authorization header. metadata_plugin = google.auth.transport.grpc.AuthMetadataPlugin( credentials, request, default_host=default_host, ) # Create a set of grpc.CallCredentials using the metadata plugin. google_auth_credentials = grpc.metadata_call_credentials(metadata_plugin) # if `ssl_credentials` is set, use `grpc.composite_channel_credentials` instead of # `grpc.compute_engine_channel_credentials` as the former supports passing # `ssl_credentials` via `channel_credentials` which is needed for mTLS. if ssl_credentials: # Combine the ssl credentials and the authorization credentials. # See https://grpc.github.io/grpc/python/grpc.html#grpc.composite_channel_credentials return grpc.composite_channel_credentials( ssl_credentials, google_auth_credentials ) else: # Use grpc.compute_engine_channel_credentials in order to support Direct Path. # See https://grpc.github.io/grpc/python/grpc.html#grpc.compute_engine_channel_credentials # TODO(https://github.com/googleapis/python-api-core/issues/598): # Although `grpc.compute_engine_channel_credentials` returns channel credentials # outside of a Google Compute Engine environment (GCE), we should determine if # there is a way to reliably detect a GCE environment so that # `grpc.compute_engine_channel_credentials` is not called outside of GCE. return grpc.compute_engine_channel_credentials(google_auth_credentials) def create_channel( target, credentials=None, scopes=None, ssl_credentials=None, credentials_file=None, quota_project_id=None, default_scopes=None, default_host=None, compression=None, attempt_direct_path: Optional[bool] = False, kwargs, ): """Create a secure channel with credentials. Args: target (str): The target service address in the format 'hostname:port'. credentials (google.auth.credentials.Credentials): The credentials. If not specified, then this function will attempt to ascertain the credentials from the environment using :func:`google.auth.default`. scopes (Sequence[str]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. ssl_credentials (grpc.ChannelCredentials): Optional SSL channel credentials. This can be used to specify different certificates. credentials_file (str): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is mutually exclusive with credentials. .. warning:: Important: If you accept a credential configuration (credential JSON/File/Stream) from an external source for authentication to Google Cloud Platform, you must validate it before providing it to any Google API or client library. Providing an unvalidated credential configuration to Google APIs or libraries can compromise the security of your systems and data. For more information, refer to `Validate credential configurations from external sources`_. .. _Validate credential configurations from external sources: https://cloud.google.com/docs/authentication/external/externally-sourced-credentials quota_project_id (str): An optional project to use for billing and quota. default_scopes (Sequence[str]): Default scopes passed by a Google client library. Use 'scopes' for user-defined scopes. default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". compression (grpc.Compression): An optional value indicating the compression method to be used over the lifetime of the channel. attempt_direct_path (Optional[bool]): If set, Direct Path will be attempted when the request is made. Direct Path is only available within a Google Compute Engine (GCE) environment and provides a proxyless connection which increases the available throughput, reduces latency, and increases reliability. Note: - This argument should only be set in a GCE environment and for Services that are known to support Direct Path. - If this argument is set outside of GCE, then this request will fail unless the back-end service happens to have configured fall-back to DNS. - If the request causes a `ServiceUnavailable` response, it is recommended that the client repeat the request with `attempt_direct_path` set to `False` as the Service may not support Direct Path. - Using `ssl_credentials` with `attempt_direct_path` set to `True` will result in `ValueError` as this combination is not yet supported. kwargs: Additional key-word args passed to :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. Note: `grpc_gcp` is only supported in environments with protobuf < 4.0.0. Returns: grpc.Channel: The created channel. Raises: google.api_core.DuplicateCredentialArgs: If both a credentials object and credentials_file are passed. ValueError: If `ssl_credentials` is set and `attempt_direct_path` is set to `True`. """ # If `ssl_credentials` is set and `attempt_direct_path` is set to `True`, # raise ValueError as this is not yet supported. # See https://github.com/googleapis/python-api-core/issues/590 if ssl_credentials and attempt_direct_path: raise ValueError("Using ssl_credentials with Direct Path is not supported") composite_credentials = _create_composite_credentials( credentials=credentials, credentials_file=credentials_file, default_scopes=default_scopes, scopes=scopes, ssl_credentials=ssl_credentials, quota_project_id=quota_project_id, default_host=default_host, ) # Note that grpcio-gcp is deprecated if HAS_GRPC_GCP: # pragma: NO COVER if compression is not None and compression != grpc.Compression.NoCompression: warnings.warn( "The `compression` argument is ignored for grpc_gcp.secure_channel creation.", DeprecationWarning, ) if attempt_direct_path: warnings.warn( """The `attempt_direct_path` argument is ignored for grpc_gcp.secure_channel creation.""", DeprecationWarning, ) return grpc_gcp.secure_channel(target, composite_credentials, kwargs) if attempt_direct_path: target = _modify_target_for_direct_path(target) return grpc.secure_channel( target, composite_credentials, compression=compression, kwargs ) def _modify_target_for_direct_path(target: str) -> str: """ Given a target, return a modified version which is compatible with Direct Path. Args: target (str): The target service address in the format 'hostname[:port]' or 'dns://hostname[:port]'. Returns: target (str): The target service address which is converted into a format compatible with Direct Path. If the target contains `dns:///` or does not contain `:///`, the target will be converted in a format compatible with Direct Path; otherwise the original target will be returned as the original target may already denote Direct Path. """ # A DNS prefix may be included with the target to indicate the endpoint is living in the Internet, # outside of Google Cloud Platform. dns_prefix = "dns:///" # Remove "dns:///" if `attempt_direct_path` is set to True as # the Direct Path prefix `google-c2p:///` will be used instead. target = target.replace(dns_prefix, "") direct_path_separator = ":///" if direct_path_separator not in target: target_without_port = target.split(":")[0] # Modify the target to use Direct Path by adding the `google-c2p:///` prefix target = f"google-c2p{direct_path_separator}{target_without_port}" return target _MethodCall = collections.namedtuple( "_MethodCall", ("request", "timeout", "metadata", "credentials", "compression") ) _ChannelRequest = collections.namedtuple("_ChannelRequest", ("method", "request")) class _CallableStub(object): """Stub for the grpc.*MultiCallable interfaces.""" def __init__(self, method, channel): self._method = method self._channel = channel self.response = None """Union[protobuf.Message, Callable[protobuf.Message], exception]: The response to give when invoking this callable. If this is a callable, it will be invoked with the request protobuf. If it's an exception, the exception will be raised when this is invoked. """ self.responses = None """Iterator[ Union[protobuf.Message, Callable[protobuf.Message], exception]]: An iterator of responses. If specified, self.response will be populated on each invocation by calling ``next(self.responses)``.""" self.requests = [] """List[protobuf.Message]: All requests sent to this callable.""" self.calls = [] """List[Tuple]: All invocations of this callable. Each tuple is the request, timeout, metadata, compression, and credentials.""" def __call__( self, request, timeout=None, metadata=None, credentials=None, compression=None ): self._channel.requests.append(_ChannelRequest(self._method, request)) self.calls.append( _MethodCall(request, timeout, metadata, credentials, compression) ) self.requests.append(request) response = self.response if self.responses is not None: if response is None: response = next(self.responses) else: raise ValueError( "{method}.response and {method}.responses are mutually " "exclusive.".format(method=self._method) ) if callable(response): return response(request) if isinstance(response, Exception): raise response if response is not None: return response raise ValueError('Method stub for "{}" has no response.'.format(self._method)) def _simplify_method_name(method): """Simplifies a gRPC method name. When gRPC invokes the channel to create a callable, it gives a full method name like "/google.pubsub.v1.Publisher/CreateTopic". This returns just the name of the method, in this case "CreateTopic". Args: method (str): The name of the method. Returns: str: The simplified name of the method. """ return method.rsplit("/", 1).pop() class ChannelStub(grpc.Channel): """A testing stub for the grpc.Channel interface. This can be used to test any client that eventually uses a gRPC channel to communicate. By passing in a channel stub, you can configure which responses are returned and track which requests are made. For example: .. code-block:: python channel_stub = grpc_helpers.ChannelStub() client = FooClient(channel=channel_stub) channel_stub.GetFoo.response = foo_pb2.Foo(name='bar') foo = client.get_foo(labels=['baz']) assert foo.name == 'bar' assert channel_stub.GetFoo.requests[0].labels = ['baz'] Each method on the stub can be accessed and configured on the channel. Here's some examples of various configurations: .. code-block:: python # Return a basic response: channel_stub.GetFoo.response = foo_pb2.Foo(name='bar') assert client.get_foo().name == 'bar' # Raise an exception: channel_stub.GetFoo.response = NotFound('...') with pytest.raises(NotFound): client.get_foo() # Use a sequence of responses: channel_stub.GetFoo.responses = iter([ foo_pb2.Foo(name='bar'), foo_pb2.Foo(name='baz'), ]) assert client.get_foo().name == 'bar' assert client.get_foo().name == 'baz' # Use a callable def on_get_foo(request): return foo_pb2.Foo(name='bar' + request.id) channel_stub.GetFoo.response = on_get_foo assert client.get_foo(id='123').name == 'bar123' """ def __init__(self, responses=[]): self.requests = [] """Sequence[Tuple[str, protobuf.Message]]: A list of all requests made on this channel in order. The tuple is of method name, request message.""" self._method_stubs = {} def _stub_for_method(self, method): method = _simplify_method_name(method) self._method_stubs[method] = _CallableStub(method, self) return self._method_stubs[method] def __getattr__(self, key): try: return self._method_stubs[key] except KeyError: raise AttributeError def unary_unary( self, method, request_serializer=None, response_deserializer=None, _registered_method=False, ): """grpc.Channel.unary_unary implementation.""" return self._stub_for_method(method) def unary_stream( self, method, request_serializer=None, response_deserializer=None, _registered_method=False, ): """grpc.Channel.unary_stream implementation.""" return self._stub_for_method(method) def stream_unary( self, method, request_serializer=None, response_deserializer=None, _registered_method=False, ): """grpc.Channel.stream_unary implementation.""" return self._stub_for_method(method) def stream_stream( self, method, request_serializer=None, response_deserializer=None, _registered_method=False, ): """grpc.Channel.stream_stream implementation.""" return self._stub_for_method(method) def subscribe(self, callback, try_to_connect=False): """grpc.Channel.subscribe implementation.""" pass def unsubscribe(self, callback): """grpc.Channel.unsubscribe implementation.""" pass def close(self): """grpc.Channel.close implementation.""" pass	24.21 KB	2025-05-25 12:26
📄 grpc_helpers_async.py # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """AsyncIO helpers for :mod:`grpc` supporting 3.7+. Please combine more detailed docstring in grpc_helpers.py to use following functions. This module is implementing the same surface with AsyncIO semantics. """ import asyncio import functools from typing import AsyncGenerator, Generic, Iterator, Optional, TypeVar import grpc from grpc import aio from google.api_core import exceptions, grpc_helpers # denotes the proto response type for grpc calls P = TypeVar("P") # NOTE(lidiz) Alternatively, we can hack "__getattribute__" to perform # automatic patching for us. But that means the overhead of creating an # extra Python function spreads to every single send and receive. class _WrappedCall(aio.Call): def __init__(self): self._call = None def with_call(self, call): """Supplies the call object separately to keep __init__ clean.""" self._call = call return self async def initial_metadata(self): return await self._call.initial_metadata() async def trailing_metadata(self): return await self._call.trailing_metadata() async def code(self): return await self._call.code() async def details(self): return await self._call.details() def cancelled(self): return self._call.cancelled() def done(self): return self._call.done() def time_remaining(self): return self._call.time_remaining() def cancel(self): return self._call.cancel() def add_done_callback(self, callback): self._call.add_done_callback(callback) async def wait_for_connection(self): try: await self._call.wait_for_connection() except grpc.RpcError as rpc_error: raise exceptions.from_grpc_error(rpc_error) from rpc_error class _WrappedUnaryResponseMixin(Generic[P], _WrappedCall): def __await__(self) -> Iterator[P]: try: response = yield from self._call.__await__() return response except grpc.RpcError as rpc_error: raise exceptions.from_grpc_error(rpc_error) from rpc_error class _WrappedStreamResponseMixin(Generic[P], _WrappedCall): def __init__(self): self._wrapped_async_generator = None async def read(self) -> P: try: return await self._call.read() except grpc.RpcError as rpc_error: raise exceptions.from_grpc_error(rpc_error) from rpc_error async def _wrapped_aiter(self) -> AsyncGenerator[P, None]: try: # NOTE(lidiz) coverage doesn't understand the exception raised from # __anext__ method. It is covered by test case: # test_wrap_stream_errors_aiter_non_rpc_error async for response in self._call: # pragma: no branch yield response except grpc.RpcError as rpc_error: raise exceptions.from_grpc_error(rpc_error) from rpc_error def __aiter__(self) -> AsyncGenerator[P, None]: if not self._wrapped_async_generator: self._wrapped_async_generator = self._wrapped_aiter() return self._wrapped_async_generator class _WrappedStreamRequestMixin(_WrappedCall): async def write(self, request): try: await self._call.write(request) except grpc.RpcError as rpc_error: raise exceptions.from_grpc_error(rpc_error) from rpc_error async def done_writing(self): try: await self._call.done_writing() except grpc.RpcError as rpc_error: raise exceptions.from_grpc_error(rpc_error) from rpc_error # NOTE(lidiz) Implementing each individual class separately, so we don't # expose any API that should not be seen. E.g., __aiter__ in unary-unary # RPC, or __await__ in stream-stream RPC. class _WrappedUnaryUnaryCall(_WrappedUnaryResponseMixin[P], aio.UnaryUnaryCall): """Wrapped UnaryUnaryCall to map exceptions.""" class _WrappedUnaryStreamCall(_WrappedStreamResponseMixin[P], aio.UnaryStreamCall): """Wrapped UnaryStreamCall to map exceptions.""" class _WrappedStreamUnaryCall( _WrappedUnaryResponseMixin[P], _WrappedStreamRequestMixin, aio.StreamUnaryCall ): """Wrapped StreamUnaryCall to map exceptions.""" class _WrappedStreamStreamCall( _WrappedStreamRequestMixin, _WrappedStreamResponseMixin[P], aio.StreamStreamCall ): """Wrapped StreamStreamCall to map exceptions.""" # public type alias denoting the return type of async streaming gapic calls GrpcAsyncStream = _WrappedStreamResponseMixin # public type alias denoting the return type of unary gapic calls AwaitableGrpcCall = _WrappedUnaryResponseMixin def _wrap_unary_errors(callable_): """Map errors for Unary-Unary async callables.""" @functools.wraps(callable_) def error_remapped_callable(args, kwargs): call = callable_(args, *kwargs) return _WrappedUnaryUnaryCall().with_call(call) return error_remapped_callable def _wrap_stream_errors(callable_, wrapper_type): """Map errors for streaming RPC async callables.""" @functools.wraps(callable_) async def error_remapped_callable(args, *kwargs): call = callable_(args, kwargs) call = wrapper_type().with_call(call) await call.wait_for_connection() return call return error_remapped_callable def wrap_errors(callable_): """Wrap a gRPC async callable and map :class:`grpc.RpcErrors` to friendly error classes. Errors raised by the gRPC callable are mapped to the appropriate :class:`google.api_core.exceptions.GoogleAPICallError` subclasses. The original `grpc.RpcError` (which is usually also a `grpc.Call`) is available from the ``response`` property on the mapped exception. This is useful for extracting metadata from the original error. Args: callable_ (Callable): A gRPC callable. Returns: Callable: The wrapped gRPC callable. """ grpc_helpers._patch_callable_name(callable_) if isinstance(callable_, aio.UnaryStreamMultiCallable): return _wrap_stream_errors(callable_, _WrappedUnaryStreamCall) elif isinstance(callable_, aio.StreamUnaryMultiCallable): return _wrap_stream_errors(callable_, _WrappedStreamUnaryCall) elif isinstance(callable_, aio.StreamStreamMultiCallable): return _wrap_stream_errors(callable_, _WrappedStreamStreamCall) else: return _wrap_unary_errors(callable_) def create_channel( target, credentials=None, scopes=None, ssl_credentials=None, credentials_file=None, quota_project_id=None, default_scopes=None, default_host=None, compression=None, attempt_direct_path: Optional[bool] = False, kwargs ): """Create an AsyncIO secure channel with credentials. Args: target (str): The target service address in the format 'hostname:port'. credentials (google.auth.credentials.Credentials): The credentials. If not specified, then this function will attempt to ascertain the credentials from the environment using :func:`google.auth.default`. scopes (Sequence[str]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. ssl_credentials (grpc.ChannelCredentials): Optional SSL channel credentials. This can be used to specify different certificates. credentials_file (str): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is mutually exclusive with credentials. .. warning:: Important: If you accept a credential configuration (credential JSON/File/Stream) from an external source for authentication to Google Cloud Platform, you must validate it before providing it to any Google API or client library. Providing an unvalidated credential configuration to Google APIs or libraries can compromise the security of your systems and data. For more information, refer to `Validate credential configurations from external sources`_. .. _Validate credential configurations from external sources: https://cloud.google.com/docs/authentication/external/externally-sourced-credentials quota_project_id (str): An optional project to use for billing and quota. default_scopes (Sequence[str]): Default scopes passed by a Google client library. Use 'scopes' for user-defined scopes. default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". compression (grpc.Compression): An optional value indicating the compression method to be used over the lifetime of the channel. attempt_direct_path (Optional[bool]): If set, Direct Path will be attempted when the request is made. Direct Path is only available within a Google Compute Engine (GCE) environment and provides a proxyless connection which increases the available throughput, reduces latency, and increases reliability. Note: - This argument should only be set in a GCE environment and for Services that are known to support Direct Path. - If this argument is set outside of GCE, then this request will fail unless the back-end service happens to have configured fall-back to DNS. - If the request causes a `ServiceUnavailable` response, it is recommended that the client repeat the request with `attempt_direct_path` set to `False` as the Service may not support Direct Path. - Using `ssl_credentials` with `attempt_direct_path` set to `True` will result in `ValueError` as this combination is not yet supported. kwargs: Additional key-word args passed to :func:`aio.secure_channel`. Returns: aio.Channel: The created channel. Raises: google.api_core.DuplicateCredentialArgs: If both a credentials object and credentials_file are passed. ValueError: If `ssl_credentials` is set and `attempt_direct_path` is set to `True`. """ # If `ssl_credentials` is set and `attempt_direct_path` is set to `True`, # raise ValueError as this is not yet supported. # See https://github.com/googleapis/python-api-core/issues/590 if ssl_credentials and attempt_direct_path: raise ValueError("Using ssl_credentials with Direct Path is not supported") composite_credentials = grpc_helpers._create_composite_credentials( credentials=credentials, credentials_file=credentials_file, scopes=scopes, default_scopes=default_scopes, ssl_credentials=ssl_credentials, quota_project_id=quota_project_id, default_host=default_host, ) if attempt_direct_path: target = grpc_helpers._modify_target_for_direct_path(target) return aio.secure_channel( target, composite_credentials, compression=compression, **kwargs ) class FakeUnaryUnaryCall(_WrappedUnaryUnaryCall): """Fake implementation for unary-unary RPCs. It is a dummy object for response message. Supply the intended response upon the initialization, and the coroutine will return the exact response message. """ def __init__(self, response=object()): self.response = response self._future = asyncio.get_event_loop().create_future() self._future.set_result(self.response) def __await__(self): response = yield from self._future.__await__() return response class FakeStreamUnaryCall(_WrappedStreamUnaryCall): """Fake implementation for stream-unary RPCs. It is a dummy object for response message. Supply the intended response upon the initialization, and the coroutine will return the exact response message. """ def __init__(self, response=object()): self.response = response self._future = asyncio.get_event_loop().create_future() self._future.set_result(self.response) def __await__(self): response = yield from self._future.__await__() return response async def wait_for_connection(self): pass	12.66 KB	2025-05-25 12:26
📄 iam.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Non-API-specific IAM policy definitions For allowed roles / permissions, see: https://cloud.google.com/iam/docs/understanding-roles Example usage: .. code-block:: python # ``get_iam_policy`` returns a :class:'~google.api_core.iam.Policy`. policy = resource.get_iam_policy(requested_policy_version=3) phred = "user:phred@example.com" admin_group = "group:admins@groups.example.com" account = "serviceAccount:account-1234@accounts.example.com" policy.version = 3 policy.bindings = [ { "role": "roles/owner", "members": {phred, admin_group, account} }, { "role": "roles/editor", "members": {"allAuthenticatedUsers"} }, { "role": "roles/viewer", "members": {"allUsers"} "condition": { "title": "request_time", "description": "Requests made before 2021-01-01T00:00:00Z", "expression": "request.time < timestamp(\"2021-01-01T00:00:00Z\")" } } ] resource.set_iam_policy(policy) """ import collections import collections.abc import operator import warnings # Generic IAM roles OWNER_ROLE = "roles/owner" """Generic role implying all rights to an object.""" EDITOR_ROLE = "roles/editor" """Generic role implying rights to modify an object.""" VIEWER_ROLE = "roles/viewer" """Generic role implying rights to access an object.""" _ASSIGNMENT_DEPRECATED_MSG = """\ Assigning to '{}' is deprecated. Use the `policy.bindings` property to modify bindings instead.""" _DICT_ACCESS_MSG = """\ Dict access is not supported on policies with version > 1 or with conditional bindings.""" class InvalidOperationException(Exception): """Raised when trying to use Policy class as a dict.""" pass class Policy(collections.abc.MutableMapping): """IAM Policy Args: etag (Optional[str]): ETag used to identify a unique of the policy version (Optional[int]): The syntax schema version of the policy. Note: Using conditions in bindings requires the policy's version to be set to `3` or greater, depending on the versions that are currently supported. Accessing the policy using dict operations will raise InvalidOperationException when the policy's version is set to 3. Use the policy.bindings getter/setter to retrieve and modify the policy's bindings. See: IAM Policy https://cloud.google.com/iam/reference/rest/v1/Policy Policy versions https://cloud.google.com/iam/docs/policies#versions Conditions overview https://cloud.google.com/iam/docs/conditions-overview. """ _OWNER_ROLES = (OWNER_ROLE,) """Roles mapped onto our ``owners`` attribute.""" _EDITOR_ROLES = (EDITOR_ROLE,) """Roles mapped onto our ``editors`` attribute.""" _VIEWER_ROLES = (VIEWER_ROLE,) """Roles mapped onto our ``viewers`` attribute.""" def __init__(self, etag=None, version=None): self.etag = etag self.version = version self._bindings = [] def __iter__(self): self.__check_version__() # Exclude bindings with no members return (binding["role"] for binding in self._bindings if binding["members"]) def __len__(self): self.__check_version__() # Exclude bindings with no members return len(list(self.__iter__())) def __getitem__(self, key): self.__check_version__() for b in self._bindings: if b["role"] == key: return b["members"] # If the binding does not yet exist, create one # NOTE: This will create bindings with no members # which are ignored by __iter__ and __len__ new_binding = {"role": key, "members": set()} self._bindings.append(new_binding) return new_binding["members"] def __setitem__(self, key, value): self.__check_version__() value = set(value) for binding in self._bindings: if binding["role"] == key: binding["members"] = value return self._bindings.append({"role": key, "members": value}) def __delitem__(self, key): self.__check_version__() for b in self._bindings: if b["role"] == key: self._bindings.remove(b) return raise KeyError(key) def __check_version__(self): """Raise InvalidOperationException if version is greater than 1 or policy contains conditions.""" raise_version = self.version is not None and self.version > 1 if raise_version or self._contains_conditions(): raise InvalidOperationException(_DICT_ACCESS_MSG) def _contains_conditions(self): for b in self._bindings: if b.get("condition") is not None: return True return False @property def bindings(self): """The policy's list of bindings. A binding is specified by a dictionary with keys: * role (str): Role that is assigned to `members`. * members (:obj:`set` of str): Specifies the identities associated to this binding. * condition (:obj:`dict` of str:str): Specifies a condition under which this binding will apply. * title (str): Title for the condition. * description (:obj:str, optional): Description of the condition. * expression: A CEL expression. Type: :obj:`list` of :obj:`dict` See: Policy versions https://cloud.google.com/iam/docs/policies#versions Conditions overview https://cloud.google.com/iam/docs/conditions-overview. Example: .. code-block:: python USER = "user:phred@example.com" ADMIN_GROUP = "group:admins@groups.example.com" SERVICE_ACCOUNT = "serviceAccount:account-1234@accounts.example.com" CONDITION = { "title": "request_time", "description": "Requests made before 2021-01-01T00:00:00Z", # Optional "expression": "request.time < timestamp(\"2021-01-01T00:00:00Z\")" } # Set policy's version to 3 before setting bindings containing conditions. policy.version = 3 policy.bindings = [ { "role": "roles/viewer", "members": {USER, ADMIN_GROUP, SERVICE_ACCOUNT}, "condition": CONDITION }, ... ] """ return self._bindings @bindings.setter def bindings(self, bindings): self._bindings = bindings @property def owners(self): """Legacy access to owner role. Raise InvalidOperationException if version is greater than 1 or policy contains conditions. DEPRECATED: use `policy.bindings` to access bindings instead. """ result = set() for role in self._OWNER_ROLES: for member in self.get(role, ()): result.add(member) return frozenset(result) @owners.setter def owners(self, value): """Update owners. Raise InvalidOperationException if version is greater than 1 or policy contains conditions. DEPRECATED: use `policy.bindings` to access bindings instead. """ warnings.warn( _ASSIGNMENT_DEPRECATED_MSG.format("owners", OWNER_ROLE), DeprecationWarning ) self[OWNER_ROLE] = value @property def editors(self): """Legacy access to editor role. Raise InvalidOperationException if version is greater than 1 or policy contains conditions. DEPRECATED: use `policy.bindings` to access bindings instead. """ result = set() for role in self._EDITOR_ROLES: for member in self.get(role, ()): result.add(member) return frozenset(result) @editors.setter def editors(self, value): """Update editors. Raise InvalidOperationException if version is greater than 1 or policy contains conditions. DEPRECATED: use `policy.bindings` to modify bindings instead. """ warnings.warn( _ASSIGNMENT_DEPRECATED_MSG.format("editors", EDITOR_ROLE), DeprecationWarning, ) self[EDITOR_ROLE] = value @property def viewers(self): """Legacy access to viewer role. Raise InvalidOperationException if version is greater than 1 or policy contains conditions. DEPRECATED: use `policy.bindings` to modify bindings instead. """ result = set() for role in self._VIEWER_ROLES: for member in self.get(role, ()): result.add(member) return frozenset(result) @viewers.setter def viewers(self, value): """Update viewers. Raise InvalidOperationException if version is greater than 1 or policy contains conditions. DEPRECATED: use `policy.bindings` to modify bindings instead. """ warnings.warn( _ASSIGNMENT_DEPRECATED_MSG.format("viewers", VIEWER_ROLE), DeprecationWarning, ) self[VIEWER_ROLE] = value @staticmethod def user(email): """Factory method for a user member. Args: email (str): E-mail for this particular user. Returns: str: A member string corresponding to the given user. """ return "user:%s" % (email,) @staticmethod def service_account(email): """Factory method for a service account member. Args: email (str): E-mail for this particular service account. Returns: str: A member string corresponding to the given service account. """ return "serviceAccount:%s" % (email,) @staticmethod def group(email): """Factory method for a group member. Args: email (str): An id or e-mail for this particular group. Returns: str: A member string corresponding to the given group. """ return "group:%s" % (email,) @staticmethod def domain(domain): """Factory method for a domain member. Args: domain (str): The domain for this member. Returns: str: A member string corresponding to the given domain. """ return "domain:%s" % (domain,) @staticmethod def all_users(): """Factory method for a member representing all users. Returns: str: A member string representing all users. """ return "allUsers" @staticmethod def authenticated_users(): """Factory method for a member representing all authenticated users. Returns: str: A member string representing all authenticated users. """ return "allAuthenticatedUsers" @classmethod def from_api_repr(cls, resource): """Factory: create a policy from a JSON resource. Args: resource (dict): policy resource returned by ``getIamPolicy`` API. Returns: :class:`Policy`: the parsed policy """ version = resource.get("version") etag = resource.get("etag") policy = cls(etag, version) policy.bindings = resource.get("bindings", []) for binding in policy.bindings: binding["members"] = set(binding.get("members", ())) return policy def to_api_repr(self): """Render a JSON policy resource. Returns: dict: a resource to be passed to the ``setIamPolicy`` API. """ resource = {} if self.etag is not None: resource["etag"] = self.etag if self.version is not None: resource["version"] = self.version if self._bindings and len(self._bindings) > 0: bindings = [] for binding in self._bindings: members = binding.get("members") if members: new_binding = {"role": binding["role"], "members": sorted(members)} condition = binding.get("condition") if condition: new_binding["condition"] = condition bindings.append(new_binding) if bindings: # Sort bindings by role key = operator.itemgetter("role") resource["bindings"] = sorted(bindings, key=key) return resource	12.9 KB	2025-05-25 12:26
📄 operation.py # Copyright 2016 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Futures for long-running operations returned from Google Cloud APIs. These futures can be used to synchronously wait for the result of a long-running operation using :meth:`Operation.result`: .. code-block:: python operation = my_api_client.long_running_method() result = operation.result() Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: .. code-block:: python operation = my_api_client.long_running_method() def my_callback(future): result = future.result() operation.add_done_callback(my_callback) """ import functools import threading from google.api_core import exceptions from google.api_core import protobuf_helpers from google.api_core.future import polling from google.longrunning import operations_pb2 from google.protobuf import json_format from google.rpc import code_pb2 class Operation(polling.PollingFuture): """A Future for interacting with a Google API Long-Running Operation. Args: operation (google.longrunning.operations_pb2.Operation): The initial operation. refresh (Callable[[], ~.api_core.operation.Operation]): A callable that returns the latest state of the operation. cancel (Callable[[], None]): A callable that tries to cancel the operation. result_type (func:`type`): The protobuf type for the operation's result. metadata_type (func:`type`): The protobuf type for the operation's metadata. polling (google.api_core.retry.Retry): The configuration used for polling. This parameter controls how often :meth:`done` is polled. If the ``timeout`` argument is specified in the :meth:`result` method, it will override the ``polling.timeout`` property. retry (google.api_core.retry.Retry): DEPRECATED: use ``polling`` instead. If specified it will override ``polling`` parameter to maintain backward compatibility. """ def __init__( self, operation, refresh, cancel, result_type, metadata_type=None, polling=polling.DEFAULT_POLLING, kwargs ): super(Operation, self).__init__(polling=polling, kwargs) self._operation = operation self._refresh = refresh self._cancel = cancel self._result_type = result_type self._metadata_type = metadata_type self._completion_lock = threading.Lock() # Invoke this in case the operation came back already complete. self._set_result_from_operation() @property def operation(self): """google.longrunning.Operation: The current long-running operation.""" return self._operation @property def metadata(self): """google.protobuf.Message: the current operation metadata.""" if not self._operation.HasField("metadata"): return None return protobuf_helpers.from_any_pb( self._metadata_type, self._operation.metadata ) @classmethod def deserialize(self, payload): """Deserialize a ``google.longrunning.Operation`` protocol buffer. Args: payload (bytes): A serialized operation protocol buffer. Returns: ~.operations_pb2.Operation: An Operation protobuf object. """ return operations_pb2.Operation.FromString(payload) def _set_result_from_operation(self): """Set the result or exception from the operation if it is complete.""" # This must be done in a lock to prevent the polling thread # and main thread from both executing the completion logic # at the same time. with self._completion_lock: # If the operation isn't complete or if the result has already been # set, do not call set_result/set_exception again. # Note: self._result_set is set to True in set_result and # set_exception, in case those methods are invoked directly. if not self._operation.done or self._result_set: return if self._operation.HasField("response"): response = protobuf_helpers.from_any_pb( self._result_type, self._operation.response ) self.set_result(response) elif self._operation.HasField("error"): exception = exceptions.from_grpc_status( status_code=self._operation.error.code, message=self._operation.error.message, errors=(self._operation.error,), response=self._operation, ) self.set_exception(exception) else: exception = exceptions.GoogleAPICallError( "Unexpected state: Long-running operation had neither " "response nor error set." ) self.set_exception(exception) def _refresh_and_update(self, retry=None): """Refresh the operation and update the result if needed. Args: retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. """ # If the currently cached operation is done, no need to make another # RPC as it will not change once done. if not self._operation.done: self._operation = self._refresh(retry=retry) if retry else self._refresh() self._set_result_from_operation() def done(self, retry=None): """Checks to see if the operation is complete. Args: retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. Returns: bool: True if the operation is complete, False otherwise. """ self._refresh_and_update(retry) return self._operation.done def cancel(self): """Attempt to cancel the operation. Returns: bool: True if the cancel RPC was made, False if the operation is already complete. """ if self.done(): return False self._cancel() return True def cancelled(self): """True if the operation was cancelled.""" self._refresh_and_update() return ( self._operation.HasField("error") and self._operation.error.code == code_pb2.CANCELLED ) def _refresh_http(api_request, operation_name, retry=None): """Refresh an operation using a JSON/HTTP client. Args: api_request (Callable): A callable used to make an API request. This should generally be :meth:`google.cloud._http.Connection.api_request`. operation_name (str): The name of the operation. retry (google.api_core.retry.Retry): (Optional) retry policy Returns: google.longrunning.operations_pb2.Operation: The operation. """ path = "operations/{}".format(operation_name) if retry is not None: api_request = retry(api_request) api_response = api_request(method="GET", path=path) return json_format.ParseDict(api_response, operations_pb2.Operation()) def _cancel_http(api_request, operation_name): """Cancel an operation using a JSON/HTTP client. Args: api_request (Callable): A callable used to make an API request. This should generally be :meth:`google.cloud._http.Connection.api_request`. operation_name (str): The name of the operation. """ path = "operations/{}:cancel".format(operation_name) api_request(method="POST", path=path) def from_http_json(operation, api_request, result_type, kwargs): """Create an operation future using a HTTP/JSON client. This interacts with the long-running operations `service`_ (specific to a given API) via `HTTP/JSON`_. .. _HTTP/JSON: https://cloud.google.com/speech/reference/rest/\ v1beta1/operations#Operation Args: operation (dict): Operation as a dictionary. api_request (Callable): A callable used to make an API request. This should generally be :meth:`google.cloud._http.Connection.api_request`. result_type (:func:`type`): The protobuf result type. kwargs: Keyword args passed into the :class:`Operation` constructor. Returns: ~.api_core.operation.Operation: The operation future to track the given operation. """ operation_proto = json_format.ParseDict(operation, operations_pb2.Operation()) refresh = functools.partial(_refresh_http, api_request, operation_proto.name) cancel = functools.partial(_cancel_http, api_request, operation_proto.name) return Operation(operation_proto, refresh, cancel, result_type, kwargs) def _refresh_grpc(operations_stub, operation_name, retry=None): """Refresh an operation using a gRPC client. Args: operations_stub (google.longrunning.operations_pb2.OperationsStub): The gRPC operations stub. operation_name (str): The name of the operation. retry (google.api_core.retry.Retry): (Optional) retry policy Returns: google.longrunning.operations_pb2.Operation: The operation. """ request_pb = operations_pb2.GetOperationRequest(name=operation_name) rpc = operations_stub.GetOperation if retry is not None: rpc = retry(rpc) return rpc(request_pb) def _cancel_grpc(operations_stub, operation_name): """Cancel an operation using a gRPC client. Args: operations_stub (google.longrunning.operations_pb2.OperationsStub): The gRPC operations stub. operation_name (str): The name of the operation. """ request_pb = operations_pb2.CancelOperationRequest(name=operation_name) operations_stub.CancelOperation(request_pb) def from_grpc(operation, operations_stub, result_type, grpc_metadata=None, kwargs): """Create an operation future using a gRPC client. This interacts with the long-running operations `service`_ (specific to a given API) via gRPC. .. _service: https://github.com/googleapis/googleapis/blob/\ 050400df0fdb16f63b63e9dee53819044bffc857/\ google/longrunning/operations.proto#L38 Args: operation (google.longrunning.operations_pb2.Operation): The operation. operations_stub (google.longrunning.operations_pb2.OperationsStub): The operations stub. result_type (:func:`type`): The protobuf result type. grpc_metadata (Optional[List[Tuple[str, str]]]): Additional metadata to pass to the rpc. kwargs: Keyword args passed into the :class:`Operation` constructor. Returns: ~.api_core.operation.Operation: The operation future to track the given operation. """ refresh = functools.partial( _refresh_grpc, operations_stub, operation.name, metadata=grpc_metadata, ) cancel = functools.partial( _cancel_grpc, operations_stub, operation.name, metadata=grpc_metadata, ) return Operation(operation, refresh, cancel, result_type, kwargs) def from_gapic(operation, operations_client, result_type, grpc_metadata=None, kwargs): """Create an operation future from a gapic client. This interacts with the long-running operations `service`_ (specific to a given API) via a gapic client. .. _service: https://github.com/googleapis/googleapis/blob/\ 050400df0fdb16f63b63e9dee53819044bffc857/\ google/longrunning/operations.proto#L38 Args: operation (google.longrunning.operations_pb2.Operation): The operation. operations_client (google.api_core.operations_v1.OperationsClient): The operations client. result_type (:func:`type`): The protobuf result type. grpc_metadata (Optional[List[Tuple[str, str]]]): Additional metadata to pass to the rpc. kwargs: Keyword args passed into the :class:`Operation` constructor. Returns: ~.api_core.operation.Operation: The operation future to track the given operation. """ refresh = functools.partial( operations_client.get_operation, operation.name, metadata=grpc_metadata, ) cancel = functools.partial( operations_client.cancel_operation, operation.name, metadata=grpc_metadata, ) return Operation(operation, refresh, cancel, result_type, kwargs)	12.89 KB	2025-05-25 12:26
📄 operation_async.py # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """AsyncIO futures for long-running operations returned from Google Cloud APIs. These futures can be used to await for the result of a long-running operation using :meth:`AsyncOperation.result`: .. code-block:: python operation = my_api_client.long_running_method() result = await operation.result() Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: .. code-block:: python operation = my_api_client.long_running_method() def my_callback(future): result = await future.result() operation.add_done_callback(my_callback) """ import functools import threading from google.api_core import exceptions from google.api_core import protobuf_helpers from google.api_core.future import async_future from google.longrunning import operations_pb2 from google.rpc import code_pb2 class AsyncOperation(async_future.AsyncFuture): """A Future for interacting with a Google API Long-Running Operation. Args: operation (google.longrunning.operations_pb2.Operation): The initial operation. refresh (Callable[[], ~.api_core.operation.Operation]): A callable that returns the latest state of the operation. cancel (Callable[[], None]): A callable that tries to cancel the operation. result_type (func:`type`): The protobuf type for the operation's result. metadata_type (func:`type`): The protobuf type for the operation's metadata. retry (google.api_core.retry.Retry): The retry configuration used when polling. This can be used to control how often :meth:`done` is polled. Regardless of the retry's ``deadline``, it will be overridden by the ``timeout`` argument to :meth:`result`. """ def __init__( self, operation, refresh, cancel, result_type, metadata_type=None, retry=async_future.DEFAULT_RETRY, ): super().__init__(retry=retry) self._operation = operation self._refresh = refresh self._cancel = cancel self._result_type = result_type self._metadata_type = metadata_type self._completion_lock = threading.Lock() # Invoke this in case the operation came back already complete. self._set_result_from_operation() @property def operation(self): """google.longrunning.Operation: The current long-running operation.""" return self._operation @property def metadata(self): """google.protobuf.Message: the current operation metadata.""" if not self._operation.HasField("metadata"): return None return protobuf_helpers.from_any_pb( self._metadata_type, self._operation.metadata ) @classmethod def deserialize(cls, payload): """Deserialize a ``google.longrunning.Operation`` protocol buffer. Args: payload (bytes): A serialized operation protocol buffer. Returns: ~.operations_pb2.Operation: An Operation protobuf object. """ return operations_pb2.Operation.FromString(payload) def _set_result_from_operation(self): """Set the result or exception from the operation if it is complete.""" # This must be done in a lock to prevent the async_future thread # and main thread from both executing the completion logic # at the same time. with self._completion_lock: # If the operation isn't complete or if the result has already been # set, do not call set_result/set_exception again. if not self._operation.done or self._future.done(): return if self._operation.HasField("response"): response = protobuf_helpers.from_any_pb( self._result_type, self._operation.response ) self.set_result(response) elif self._operation.HasField("error"): exception = exceptions.GoogleAPICallError( self._operation.error.message, errors=(self._operation.error,), response=self._operation, ) self.set_exception(exception) else: exception = exceptions.GoogleAPICallError( "Unexpected state: Long-running operation had neither " "response nor error set." ) self.set_exception(exception) async def _refresh_and_update(self, retry=async_future.DEFAULT_RETRY): """Refresh the operation and update the result if needed. Args: retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. """ # If the currently cached operation is done, no need to make another # RPC as it will not change once done. if not self._operation.done: self._operation = await self._refresh(retry=retry) self._set_result_from_operation() async def done(self, retry=async_future.DEFAULT_RETRY): """Checks to see if the operation is complete. Args: retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. Returns: bool: True if the operation is complete, False otherwise. """ await self._refresh_and_update(retry) return self._operation.done async def cancel(self): """Attempt to cancel the operation. Returns: bool: True if the cancel RPC was made, False if the operation is already complete. """ result = await self.done() if result: return False else: await self._cancel() return True async def cancelled(self): """True if the operation was cancelled.""" await self._refresh_and_update() return ( self._operation.HasField("error") and self._operation.error.code == code_pb2.CANCELLED ) def from_gapic(operation, operations_client, result_type, grpc_metadata=None, kwargs): """Create an operation future from a gapic client. This interacts with the long-running operations `service`_ (specific to a given API) via a gapic client. .. _service: https://github.com/googleapis/googleapis/blob/\ 050400df0fdb16f63b63e9dee53819044bffc857/\ google/longrunning/operations.proto#L38 Args: operation (google.longrunning.operations_pb2.Operation): The operation. operations_client (google.api_core.operations_v1.OperationsClient): The operations client. result_type (:func:`type`): The protobuf result type. grpc_metadata (Optional[List[Tuple[str, str]]]): Additional metadata to pass to the rpc. kwargs: Keyword args passed into the :class:`Operation` constructor. Returns: ~.api_core.operation.Operation: The operation future to track the given operation. """ refresh = functools.partial( operations_client.get_operation, operation.name, metadata=grpc_metadata, ) cancel = functools.partial( operations_client.cancel_operation, operation.name, metadata=grpc_metadata, ) return AsyncOperation(operation, refresh, cancel, result_type, kwargs)	7.86 KB	2025-05-25 12:26
📁 operations_v1	-	2025-05-25 12:26
📄 page_iterator.py # Copyright 2015 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Iterators for paging through paged API methods. These iterators simplify the process of paging through API responses where the request takes a page token and the response is a list of results with a token for the next page. See `list pagination`_ in the Google API Style Guide for more details. .. _list pagination: https://cloud.google.com/apis/design/design_patterns#list_pagination API clients that have methods that follow the list pagination pattern can return an :class:`.Iterator`. You can use this iterator to get all of the results across all pages:: >>> results_iterator = client.list_resources() >>> list(results_iterator) # Convert to a list (consumes all values). Or you can walk your way through items and call off the search early if you find what you're looking for (resulting in possibly fewer requests):: >>> for resource in results_iterator: ... print(resource.name) ... if not resource.is_valid: ... break At any point, you may check the number of items consumed by referencing the ``num_results`` property of the iterator:: >>> for my_item in results_iterator: ... if results_iterator.num_results >= 10: ... break When iterating, not every new item will send a request to the server. To iterate based on each page of items (where a page corresponds to a request):: >>> for page in results_iterator.pages: ... print('=' * 20) ... print(' Page number: {:d}'.format(iterator.page_number)) ... print(' Items in page: {:d}'.format(page.num_items)) ... print(' First item: {!r}'.format(next(page))) ... print('Items remaining: {:d}'.format(page.remaining)) ... print('Next page token: {}'.format(iterator.next_page_token)) ==================== Page number: 1 Items in page: 1 First item: <MyItemClass at 0x7f1d3cccf690> Items remaining: 0 Next page token: eav1OzQB0OM8rLdGXOEsyQWSG ==================== Page number: 2 Items in page: 19 First item: <MyItemClass at 0x7f1d3cccffd0> Items remaining: 18 Next page token: None Then, for each page you can get all the resources on that page by iterating through it or using :func:`list`:: >>> list(page) [ <MyItemClass at 0x7fd64a098ad0>, <MyItemClass at 0x7fd64a098ed0>, <MyItemClass at 0x7fd64a098e90>, ] """ import abc class Page(object): """Single page of results in an iterator. Args: parent (google.api_core.page_iterator.Iterator): The iterator that owns the current page. items (Sequence[Any]): An iterable (that also defines __len__) of items from a raw API response. item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): Callable to convert an item from the type in the raw API response into the native object. Will be called with the iterator and a single item. raw_page Optional[google.protobuf.message.Message]: The raw page response. """ def __init__(self, parent, items, item_to_value, raw_page=None): self._parent = parent self._num_items = len(items) self._remaining = self._num_items self._item_iter = iter(items) self._item_to_value = item_to_value self._raw_page = raw_page @property def raw_page(self): """google.protobuf.message.Message""" return self._raw_page @property def num_items(self): """int: Total items in the page.""" return self._num_items @property def remaining(self): """int: Remaining items in the page.""" return self._remaining def __iter__(self): """The :class:`Page` is an iterator of items.""" return self def __next__(self): """Get the next value in the page.""" item = next(self._item_iter) result = self._item_to_value(self._parent, item) # Since we've successfully got the next value from the # iterator, we update the number of remaining. self._remaining -= 1 return result def _item_to_value_identity(iterator, item): """An item to value transformer that returns the item un-changed.""" # pylint: disable=unused-argument # We are conforming to the interface defined by Iterator. return item class Iterator(object, metaclass=abc.ABCMeta): """A generic class for iterating through API list responses. Args: client(google.cloud.client.Client): The API client. item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): Callable to convert an item from the type in the raw API response into the native object. Will be called with the iterator and a single item. page_token (str): A token identifying a page in a result set to start fetching results from. max_results (int): The maximum number of results to fetch. """ def __init__( self, client, item_to_value=_item_to_value_identity, page_token=None, max_results=None, ): self._started = False self.__active_iterator = None self.client = client """Optional[Any]: The client that created this iterator.""" self.item_to_value = item_to_value """Callable[Iterator, Any]: Callable to convert an item from the type in the raw API response into the native object. Will be called with the iterator and a single item. """ self.max_results = max_results """int: The maximum number of results to fetch""" # The attributes below will change over the life of the iterator. self.page_number = 0 """int: The current page of results.""" self.next_page_token = page_token """str: The token for the next page of results. If this is set before the iterator starts, it effectively offsets the iterator to a specific starting point.""" self.num_results = 0 """int: The total number of results fetched so far.""" @property def pages(self): """Iterator of pages in the response. returns: types.GeneratorType[google.api_core.page_iterator.Page]: A generator of page instances. raises: ValueError: If the iterator has already been started. """ if self._started: raise ValueError("Iterator has already started", self) self._started = True return self._page_iter(increment=True) def _items_iter(self): """Iterator for each item returned.""" for page in self._page_iter(increment=False): for item in page: self.num_results += 1 yield item def __iter__(self): """Iterator for each item returned. Returns: types.GeneratorType[Any]: A generator of items from the API. Raises: ValueError: If the iterator has already been started. """ if self._started: raise ValueError("Iterator has already started", self) self._started = True return self._items_iter() def __next__(self): if self.__active_iterator is None: self.__active_iterator = iter(self) return next(self.__active_iterator) def _page_iter(self, increment): """Generator of pages of API responses. Args: increment (bool): Flag indicating if the total number of results should be incremented on each page. This is useful since a page iterator will want to increment by results per page while an items iterator will want to increment per item. Yields: Page: each page of items from the API. """ page = self._next_page() while page is not None: self.page_number += 1 if increment: self.num_results += page.num_items yield page page = self._next_page() @abc.abstractmethod def _next_page(self): """Get the next page in the iterator. This does nothing and is intended to be over-ridden by subclasses to return the next :class:`Page`. Raises: NotImplementedError: Always, this method is abstract. """ raise NotImplementedError def _do_nothing_page_start(iterator, page, response): """Helper to provide custom behavior after a :class:`Page` is started. This is a do-nothing stand-in as the default value. Args: iterator (Iterator): An iterator that holds some request info. page (Page): The page that was just created. response (Any): The API response for a page. """ # pylint: disable=unused-argument pass class HTTPIterator(Iterator): """A generic class for iterating through HTTP/JSON API list responses. To make an iterator work, you'll need to provide a way to convert a JSON item returned from the API into the object of your choice (via ``item_to_value``). You also may need to specify a custom ``items_key`` so that a given response (containing a page of results) can be parsed into an iterable page of the actual objects you want. Args: client (google.cloud.client.Client): The API client. api_request (Callable): The function to use to make API requests. Generally, this will be :meth:`google.cloud._http.JSONConnection.api_request`. path (str): The method path to query for the list of items. item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): Callable to convert an item from the type in the JSON response into a native object. Will be called with the iterator and a single item. items_key (str): The key in the API response where the list of items can be found. page_token (str): A token identifying a page in a result set to start fetching results from. page_size (int): The maximum number of results to fetch per page max_results (int): The maximum number of results to fetch extra_params (dict): Extra query string parameters for the API call. page_start (Callable[ google.api_core.page_iterator.Iterator, google.api_core.page_iterator.Page, dict]): Callable to provide any special behavior after a new page has been created. Assumed signature takes the :class:`.Iterator` that started the page, the :class:`.Page` that was started and the dictionary containing the page response. next_token (str): The name of the field used in the response for page tokens. .. autoattribute:: pages """ _DEFAULT_ITEMS_KEY = "items" _PAGE_TOKEN = "pageToken" _MAX_RESULTS = "maxResults" _NEXT_TOKEN = "nextPageToken" _RESERVED_PARAMS = frozenset([_PAGE_TOKEN]) _HTTP_METHOD = "GET" def __init__( self, client, api_request, path, item_to_value, items_key=_DEFAULT_ITEMS_KEY, page_token=None, page_size=None, max_results=None, extra_params=None, page_start=_do_nothing_page_start, next_token=_NEXT_TOKEN, ): super(HTTPIterator, self).__init__( client, item_to_value, page_token=page_token, max_results=max_results ) self.api_request = api_request self.path = path self._items_key = items_key self.extra_params = extra_params self._page_size = page_size self._page_start = page_start self._next_token = next_token # Verify inputs / provide defaults. if self.extra_params is None: self.extra_params = {} self._verify_params() def _verify_params(self): """Verifies the parameters don't use any reserved parameter. Raises: ValueError: If a reserved parameter is used. """ reserved_in_use = self._RESERVED_PARAMS.intersection(self.extra_params) if reserved_in_use: raise ValueError("Using a reserved parameter", reserved_in_use) def _next_page(self): """Get the next page in the iterator. Returns: Optional[Page]: The next page in the iterator or :data:`None` if there are no pages left. """ if self._has_next_page(): response = self._get_next_page_response() items = response.get(self._items_key, ()) page = Page(self, items, self.item_to_value, raw_page=response) self._page_start(self, page, response) self.next_page_token = response.get(self._next_token) return page else: return None def _has_next_page(self): """Determines whether or not there are more pages with results. Returns: bool: Whether the iterator has more pages. """ if self.page_number == 0: return True if self.max_results is not None: if self.num_results >= self.max_results: return False return self.next_page_token is not None def _get_query_params(self): """Getter for query parameters for the next request. Returns: dict: A dictionary of query parameters. """ result = {} if self.next_page_token is not None: result[self._PAGE_TOKEN] = self.next_page_token page_size = None if self.max_results is not None: page_size = self.max_results - self.num_results if self._page_size is not None: page_size = min(page_size, self._page_size) elif self._page_size is not None: page_size = self._page_size if page_size is not None: result[self._MAX_RESULTS] = page_size result.update(self.extra_params) return result def _get_next_page_response(self): """Requests the next page from the path provided. Returns: dict: The parsed JSON response of the next page's contents. Raises: ValueError: If the HTTP method is not ``GET`` or ``POST``. """ params = self._get_query_params() if self._HTTP_METHOD == "GET": return self.api_request( method=self._HTTP_METHOD, path=self.path, query_params=params ) elif self._HTTP_METHOD == "POST": return self.api_request( method=self._HTTP_METHOD, path=self.path, data=params ) else: raise ValueError("Unexpected HTTP method", self._HTTP_METHOD) class _GAXIterator(Iterator): """A generic class for iterating through Cloud gRPC APIs list responses. Any: client (google.cloud.client.Client): The API client. page_iter (google.gax.PageIterator): A GAX page iterator to be wrapped to conform to the :class:`Iterator` interface. item_to_value (Callable[Iterator, Any]): Callable to convert an item from the protobuf response into a native object. Will be called with the iterator and a single item. max_results (int): The maximum number of results to fetch. .. autoattribute:: pages """ def __init__(self, client, page_iter, item_to_value, max_results=None): super(_GAXIterator, self).__init__( client, item_to_value, page_token=page_iter.page_token, max_results=max_results, ) self._gax_page_iter = page_iter def _next_page(self): """Get the next page in the iterator. Wraps the response from the :class:`~google.gax.PageIterator` in a :class:`Page` instance and captures some state at each page. Returns: Optional[Page]: The next page in the iterator or :data:`None` if there are no pages left. """ try: items = next(self._gax_page_iter) page = Page(self, items, self.item_to_value) self.next_page_token = self._gax_page_iter.page_token or None return page except StopIteration: return None class GRPCIterator(Iterator): """A generic class for iterating through gRPC list responses. .. note:: The class does not take a ``page_token`` argument because it can just be specified in the ``request``. Args: client (google.cloud.client.Client): The API client. This unused by this class, but kept to satisfy the :class:`Iterator` interface. method (Callable[protobuf.Message]): A bound gRPC method that should take a single message for the request. request (protobuf.Message): The request message. items_field (str): The field in the response message that has the items for the page. item_to_value (Callable[GRPCIterator, Any]): Callable to convert an item from the type in the JSON response into a native object. Will be called with the iterator and a single item. request_token_field (str): The field in the request message used to specify the page token. response_token_field (str): The field in the response message that has the token for the next page. max_results (int): The maximum number of results to fetch. .. autoattribute:: pages """ _DEFAULT_REQUEST_TOKEN_FIELD = "page_token" _DEFAULT_RESPONSE_TOKEN_FIELD = "next_page_token" def __init__( self, client, method, request, items_field, item_to_value=_item_to_value_identity, request_token_field=_DEFAULT_REQUEST_TOKEN_FIELD, response_token_field=_DEFAULT_RESPONSE_TOKEN_FIELD, max_results=None, ): super(GRPCIterator, self).__init__( client, item_to_value, max_results=max_results ) self._method = method self._request = request self._items_field = items_field self._request_token_field = request_token_field self._response_token_field = response_token_field def _next_page(self): """Get the next page in the iterator. Returns: Page: The next page in the iterator or :data:`None` if there are no pages left. """ if not self._has_next_page(): return None if self.next_page_token is not None: setattr(self._request, self._request_token_field, self.next_page_token) response = self._method(self._request) self.next_page_token = getattr(response, self._response_token_field) items = getattr(response, self._items_field) page = Page(self, items, self.item_to_value, raw_page=response) return page def _has_next_page(self): """Determines whether or not there are more pages with results. Returns: bool: Whether the iterator has more pages. """ if self.page_number == 0: return True if self.max_results is not None: if self.num_results >= self.max_results: return False # Note: intentionally a falsy check instead of a None check. The RPC # can return an empty string indicating no more pages. return True if self.next_page_token else False	19.85 KB	2025-05-25 12:26
📄 page_iterator_async.py # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """AsyncIO iterators for paging through paged API methods. These iterators simplify the process of paging through API responses where the request takes a page token and the response is a list of results with a token for the next page. See `list pagination`_ in the Google API Style Guide for more details. .. _list pagination: https://cloud.google.com/apis/design/design_patterns#list_pagination API clients that have methods that follow the list pagination pattern can return an :class:`.AsyncIterator`: >>> results_iterator = await client.list_resources() Or you can walk your way through items and call off the search early if you find what you're looking for (resulting in possibly fewer requests):: >>> async for resource in results_iterator: ... print(resource.name) ... if not resource.is_valid: ... break At any point, you may check the number of items consumed by referencing the ``num_results`` property of the iterator:: >>> async for my_item in results_iterator: ... if results_iterator.num_results >= 10: ... break When iterating, not every new item will send a request to the server. To iterate based on each page of items (where a page corresponds to a request):: >>> async for page in results_iterator.pages: ... print('=' * 20) ... print(' Page number: {:d}'.format(iterator.page_number)) ... print(' Items in page: {:d}'.format(page.num_items)) ... print(' First item: {!r}'.format(next(page))) ... print('Items remaining: {:d}'.format(page.remaining)) ... print('Next page token: {}'.format(iterator.next_page_token)) ==================== Page number: 1 Items in page: 1 First item: <MyItemClass at 0x7f1d3cccf690> Items remaining: 0 Next page token: eav1OzQB0OM8rLdGXOEsyQWSG ==================== Page number: 2 Items in page: 19 First item: <MyItemClass at 0x7f1d3cccffd0> Items remaining: 18 Next page token: None """ import abc from google.api_core.page_iterator import Page def _item_to_value_identity(iterator, item): """An item to value transformer that returns the item un-changed.""" # pylint: disable=unused-argument # We are conforming to the interface defined by Iterator. return item class AsyncIterator(abc.ABC): """A generic class for iterating through API list responses. Args: client(google.cloud.client.Client): The API client. item_to_value (Callable[google.api_core.page_iterator_async.AsyncIterator, Any]): Callable to convert an item from the type in the raw API response into the native object. Will be called with the iterator and a single item. page_token (str): A token identifying a page in a result set to start fetching results from. max_results (int): The maximum number of results to fetch. """ def __init__( self, client, item_to_value=_item_to_value_identity, page_token=None, max_results=None, ): self._started = False self.__active_aiterator = None self.client = client """Optional[Any]: The client that created this iterator.""" self.item_to_value = item_to_value """Callable[Iterator, Any]: Callable to convert an item from the type in the raw API response into the native object. Will be called with the iterator and a single item. """ self.max_results = max_results """int: The maximum number of results to fetch.""" # The attributes below will change over the life of the iterator. self.page_number = 0 """int: The current page of results.""" self.next_page_token = page_token """str: The token for the next page of results. If this is set before the iterator starts, it effectively offsets the iterator to a specific starting point.""" self.num_results = 0 """int: The total number of results fetched so far.""" @property def pages(self): """Iterator of pages in the response. returns: types.GeneratorType[google.api_core.page_iterator.Page]: A generator of page instances. raises: ValueError: If the iterator has already been started. """ if self._started: raise ValueError("Iterator has already started", self) self._started = True return self._page_aiter(increment=True) async def _items_aiter(self): """Iterator for each item returned.""" async for page in self._page_aiter(increment=False): for item in page: self.num_results += 1 yield item def __aiter__(self): """Iterator for each item returned. Returns: types.GeneratorType[Any]: A generator of items from the API. Raises: ValueError: If the iterator has already been started. """ if self._started: raise ValueError("Iterator has already started", self) self._started = True return self._items_aiter() async def __anext__(self): if self.__active_aiterator is None: self.__active_aiterator = self.__aiter__() return await self.__active_aiterator.__anext__() async def _page_aiter(self, increment): """Generator of pages of API responses. Args: increment (bool): Flag indicating if the total number of results should be incremented on each page. This is useful since a page iterator will want to increment by results per page while an items iterator will want to increment per item. Yields: Page: each page of items from the API. """ page = await self._next_page() while page is not None: self.page_number += 1 if increment: self.num_results += page.num_items yield page page = await self._next_page() @abc.abstractmethod async def _next_page(self): """Get the next page in the iterator. This does nothing and is intended to be over-ridden by subclasses to return the next :class:`Page`. Raises: NotImplementedError: Always, this method is abstract. """ raise NotImplementedError class AsyncGRPCIterator(AsyncIterator): """A generic class for iterating through gRPC list responses. .. note:: The class does not take a ``page_token`` argument because it can just be specified in the ``request``. Args: client (google.cloud.client.Client): The API client. This unused by this class, but kept to satisfy the :class:`Iterator` interface. method (Callable[protobuf.Message]): A bound gRPC method that should take a single message for the request. request (protobuf.Message): The request message. items_field (str): The field in the response message that has the items for the page. item_to_value (Callable[GRPCIterator, Any]): Callable to convert an item from the type in the JSON response into a native object. Will be called with the iterator and a single item. request_token_field (str): The field in the request message used to specify the page token. response_token_field (str): The field in the response message that has the token for the next page. max_results (int): The maximum number of results to fetch. .. autoattribute:: pages """ _DEFAULT_REQUEST_TOKEN_FIELD = "page_token" _DEFAULT_RESPONSE_TOKEN_FIELD = "next_page_token" def __init__( self, client, method, request, items_field, item_to_value=_item_to_value_identity, request_token_field=_DEFAULT_REQUEST_TOKEN_FIELD, response_token_field=_DEFAULT_RESPONSE_TOKEN_FIELD, max_results=None, ): super().__init__(client, item_to_value, max_results=max_results) self._method = method self._request = request self._items_field = items_field self._request_token_field = request_token_field self._response_token_field = response_token_field async def _next_page(self): """Get the next page in the iterator. Returns: Page: The next page in the iterator or :data:`None` if there are no pages left. """ if not self._has_next_page(): return None if self.next_page_token is not None: setattr(self._request, self._request_token_field, self.next_page_token) response = await self._method(self._request) self.next_page_token = getattr(response, self._response_token_field) items = getattr(response, self._items_field) page = Page(self, items, self.item_to_value, raw_page=response) return page def _has_next_page(self): """Determines whether or not there are more pages with results. Returns: bool: Whether the iterator has more pages. """ if self.page_number == 0: return True # Note: intentionally a falsy check instead of a None check. The RPC # can return an empty string indicating no more pages. if self.max_results is not None: if self.num_results >= self.max_results: return False return True if self.next_page_token else False	10.05 KB	2025-05-25 12:26
📄 path_template.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Expand and validate URL path templates. This module provides the :func:`expand` and :func:`validate` functions for interacting with Google-style URL `path templates`_ which are commonly used in Google APIs for `resource names`_. .. _path templates: https://github.com/googleapis/googleapis/blob /57e2d376ac7ef48681554204a3ba78a414f2c533/google/api/http.proto#L212 .. _resource names: https://cloud.google.com/apis/design/resource_names """ from __future__ import unicode_literals from collections import deque import copy import functools import re # Regular expression for extracting variable parts from a path template. # The variables can be expressed as: # # - "": a single-segment positional variable, for example: "books/" # - "": a multi-segment positional variable, for example: "shelf//book/" # - "{name}": a single-segment wildcard named variable, for example # "books/{name}" # - "{name=}: same as above. # - "{name=}": a multi-segment wildcard named variable, for example # "shelf/{name=}" # - "{name=/path//}": a multi-segment named variable with a sub-template. _VARIABLE_RE = re.compile( r""" ( # Capture the entire variable expression (?P<positional>\\?) # Match & capture and ** positional variables. \| # Match & capture named variables {name} { (?P<name>[^/]+?) # Optionally match and capture the named variable's template. (?:=(?P<template>.+?))? } ) """, re.VERBOSE, ) # Segment expressions used for validating paths against a template. _SINGLE_SEGMENT_PATTERN = r"([^/]+)" _MULTI_SEGMENT_PATTERN = r"(.+)" def _expand_variable_match(positional_vars, named_vars, match): """Expand a matched variable with its value. Args: positional_vars (list): A list of positional variables. This list will be modified. named_vars (dict): A dictionary of named variables. match (re.Match): A regular expression match. Returns: str: The expanded variable to replace the match. Raises: ValueError: If a positional or named variable is required by the template but not specified or if an unexpected template expression is encountered. """ positional = match.group("positional") name = match.group("name") if name is not None: try: return str(named_vars[name]) except KeyError: raise ValueError( "Named variable '{}' not specified and needed by template " "`{}` at position {}".format(name, match.string, match.start()) ) elif positional is not None: try: return str(positional_vars.pop(0)) except IndexError: raise ValueError( "Positional variable not specified and needed by template " "`{}` at position {}".format(match.string, match.start()) ) else: raise ValueError("Unknown template expression {}".format(match.group(0))) def expand(tmpl, args, kwargs): """Expand a path template with the given variables. .. code-block:: python >>> expand('users//messages/', 'me', '123') users/me/messages/123 >>> expand('/v1/{name=shelves//books/}', name='shelves/1/books/3') /v1/shelves/1/books/3 Args: tmpl (str): The path template. args: The positional variables for the path. kwargs: The named variables for the path. Returns: str: The expanded path Raises: ValueError: If a positional or named variable is required by the template but not specified or if an unexpected template expression is encountered. """ replacer = functools.partial(_expand_variable_match, list(args), kwargs) return _VARIABLE_RE.sub(replacer, tmpl) def _replace_variable_with_pattern(match): """Replace a variable match with a pattern that can be used to validate it. Args: match (re.Match): A regular expression match Returns: str: A regular expression pattern that can be used to validate the variable in an expanded path. Raises: ValueError: If an unexpected template expression is encountered. """ positional = match.group("positional") name = match.group("name") template = match.group("template") if name is not None: if not template: return _SINGLE_SEGMENT_PATTERN.format(name) elif template == "": return _MULTI_SEGMENT_PATTERN.format(name) else: return _generate_pattern_for_template(template) elif positional == "": return _SINGLE_SEGMENT_PATTERN elif positional == "*": return _MULTI_SEGMENT_PATTERN else: raise ValueError("Unknown template expression {}".format(match.group(0))) def _generate_pattern_for_template(tmpl): """Generate a pattern that can validate a path template. Args: tmpl (str): The path template Returns: str: A regular expression pattern that can be used to validate an expanded path template. """ return _VARIABLE_RE.sub(_replace_variable_with_pattern, tmpl) def get_field(request, field): """Get the value of a field from a given dictionary. Args: request (dict \| Message): A dictionary or a Message object. field (str): The key to the request in dot notation. Returns: The value of the field. """ parts = field.split(".") value = request for part in parts: if not isinstance(value, dict): value = getattr(value, part, None) else: value = value.get(part) if isinstance(value, dict): return return value def delete_field(request, field): """Delete the value of a field from a given dictionary. Args: request (dict \| Message): A dictionary object or a Message. field (str): The key to the request in dot notation. """ parts = deque(field.split(".")) while len(parts) > 1: part = parts.popleft() if not isinstance(request, dict): if hasattr(request, part): request = getattr(request, part, None) else: return else: request = request.get(part) part = parts.popleft() if not isinstance(request, dict): if hasattr(request, part): request.ClearField(part) else: return else: request.pop(part, None) def validate(tmpl, path): """Validate a path against the path template. .. code-block:: python >>> validate('users//messages/', 'users/me/messages/123') True >>> validate('users//messages/', 'users/me/drafts/123') False >>> validate('/v1/{name=shelves//books/}', /v1/shelves/1/books/3) True >>> validate('/v1/{name=shelves//books/}', /v1/shelves/1/tapes/3) False Args: tmpl (str): The path template. path (str): The expanded path. Returns: bool: True if the path matches. """ pattern = _generate_pattern_for_template(tmpl) + "$" return True if re.match(pattern, path) is not None else False def transcode(http_options, message=None, request_kwargs): """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here, https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 Args: http_options (list(dict)): A list of dicts which consist of these keys, 'method' (str): The http method 'uri' (str): The path template 'body' (str): The body field name (optional) (This is a simplified representation of the proto option `google.api.http`) message (Message) : A request object (optional) request_kwargs (dict) : A dict representing the request object Returns: dict: The transcoded request with these keys, 'method' (str) : The http method 'uri' (str) : The expanded uri 'body' (dict \| Message) : A dict or a Message representing the body (optional) 'query_params' (dict \| Message) : A dict or Message mapping query parameter variables and values Raises: ValueError: If the request does not match the given template. """ transcoded_value = message or request_kwargs bindings = [] for http_option in http_options: request = {} # Assign path uri_template = http_option["uri"] fields = [ (m.group("name"), m.group("template")) for m in _VARIABLE_RE.finditer(uri_template) ] bindings.append((uri_template, fields)) path_args = {field: get_field(transcoded_value, field) for field, _ in fields} request["uri"] = expand(uri_template, path_args) if not validate(uri_template, request["uri"]) or not all(path_args.values()): continue # Remove fields used in uri path from request leftovers = copy.deepcopy(transcoded_value) for path_field, _ in fields: delete_field(leftovers, path_field) # Assign body and query params body = http_option.get("body") if body: if body == "": request["body"] = leftovers if message: request["query_params"] = message.__class__() else: request["query_params"] = {} else: try: if message: request["body"] = getattr(leftovers, body) delete_field(leftovers, body) else: request["body"] = leftovers.pop(body) except (KeyError, AttributeError): continue request["query_params"] = leftovers else: request["query_params"] = leftovers request["method"] = http_option["method"] return request bindings_description = [ '\n\tURI: "{}"' "\n\tRequired request fields:\n\t\t{}".format( uri, "\n\t\t".join( [ 'field: "{}", pattern: "{}"'.format(n, p if p else "*") for n, p in fields ] ), ) for uri, fields in bindings ] raise ValueError( "Invalid request." "\nSome of the fields of the request message are either not initialized or " "initialized with an invalid value." "\nPlease make sure your request matches at least one accepted HTTP binding." "\nTo match a binding the request message must have all the required fields " "initialized with values matching their patterns as listed below:{}".format( "\n".join(bindings_description) ) )	11.41 KB	2025-05-25 12:26
📄 protobuf_helpers.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for :mod:`protobuf`.""" import collections import collections.abc import copy import inspect from google.protobuf import field_mask_pb2 from google.protobuf import message from google.protobuf import wrappers_pb2 _SENTINEL = object() _WRAPPER_TYPES = ( wrappers_pb2.BoolValue, wrappers_pb2.BytesValue, wrappers_pb2.DoubleValue, wrappers_pb2.FloatValue, wrappers_pb2.Int32Value, wrappers_pb2.Int64Value, wrappers_pb2.StringValue, wrappers_pb2.UInt32Value, wrappers_pb2.UInt64Value, ) def from_any_pb(pb_type, any_pb): """Converts an ``Any`` protobuf to the specified message type. Args: pb_type (type): the type of the message that any_pb stores an instance of. any_pb (google.protobuf.any_pb2.Any): the object to be converted. Returns: pb_type: An instance of the pb_type message. Raises: TypeError: if the message could not be converted. """ msg = pb_type() # Unwrap proto-plus wrapped messages. if callable(getattr(pb_type, "pb", None)): msg_pb = pb_type.pb(msg) else: msg_pb = msg # Unpack the Any object and populate the protobuf message instance. if not any_pb.Unpack(msg_pb): raise TypeError( f"Could not convert `{any_pb.TypeName()}` with underlying type `google.protobuf.any_pb2.Any` to `{msg_pb.DESCRIPTOR.full_name}`" ) # Done; return the message. return msg def check_oneof(kwargs): """Raise ValueError if more than one keyword argument is not ``None``. Args: kwargs (dict): The keyword arguments sent to the function. Raises: ValueError: If more than one entry in ``kwargs`` is not ``None``. """ # Sanity check: If no keyword arguments were sent, this is fine. if not kwargs: return not_nones = [val for val in kwargs.values() if val is not None] if len(not_nones) > 1: raise ValueError( "Only one of {fields} should be set.".format( fields=", ".join(sorted(kwargs.keys())) ) ) def get_messages(module): """Discovers all protobuf Message classes in a given import module. Args: module (module): A Python module; :func:`dir` will be run against this module to find Message subclasses. Returns: dict[str, google.protobuf.message.Message]: A dictionary with the Message class names as keys, and the Message subclasses themselves as values. """ answer = collections.OrderedDict() for name in dir(module): candidate = getattr(module, name) if inspect.isclass(candidate) and issubclass(candidate, message.Message): answer[name] = candidate return answer def _resolve_subkeys(key, separator="."): """Resolve a potentially nested key. If the key contains the ``separator`` (e.g. ``.``) then the key will be split on the first instance of the subkey:: >>> _resolve_subkeys('a.b.c') ('a', 'b.c') >>> _resolve_subkeys('d\|e\|f', separator='\|') ('d', 'e\|f') If not, the subkey will be :data:`None`:: >>> _resolve_subkeys('foo') ('foo', None) Args: key (str): A string that may or may not contain the separator. separator (str): The namespace separator. Defaults to `.`. Returns: Tuple[str, str]: The key and subkey(s). """ parts = key.split(separator, 1) if len(parts) > 1: return parts else: return parts[0], None def get(msg_or_dict, key, default=_SENTINEL): """Retrieve a key's value from a protobuf Message or dictionary. Args: mdg_or_dict (Union[~google.protobuf.message.Message, Mapping]): the object. key (str): The key to retrieve from the object. default (Any): If the key is not present on the object, and a default is set, returns that default instead. A type-appropriate falsy default is generally recommended, as protobuf messages almost always have default values for unset values and it is not always possible to tell the difference between a falsy value and an unset one. If no default is set then :class:`KeyError` will be raised if the key is not present in the object. Returns: Any: The return value from the underlying Message or dict. Raises: KeyError: If the key is not found. Note that, for unset values, messages and dictionaries may not have consistent behavior. TypeError: If ``msg_or_dict`` is not a Message or Mapping. """ # We may need to get a nested key. Resolve this. key, subkey = _resolve_subkeys(key) # Attempt to get the value from the two types of objects we know about. # If we get something else, complain. if isinstance(msg_or_dict, message.Message): answer = getattr(msg_or_dict, key, default) elif isinstance(msg_or_dict, collections.abc.Mapping): answer = msg_or_dict.get(key, default) else: raise TypeError( "get() expected a dict or protobuf message, got {!r}.".format( type(msg_or_dict) ) ) # If the object we got back is our sentinel, raise KeyError; this is # a "not found" case. if answer is _SENTINEL: raise KeyError(key) # If a subkey exists, call this method recursively against the answer. if subkey is not None and answer is not default: return get(answer, subkey, default=default) return answer def _set_field_on_message(msg, key, value): """Set helper for protobuf Messages.""" # Attempt to set the value on the types of objects we know how to deal # with. if isinstance(value, (collections.abc.MutableSequence, tuple)): # Clear the existing repeated protobuf message of any elements # currently inside it. while getattr(msg, key): getattr(msg, key).pop() # Write our new elements to the repeated field. for item in value: if isinstance(item, collections.abc.Mapping): getattr(msg, key).add(item) else: # protobuf's RepeatedCompositeContainer doesn't support # append. getattr(msg, key).extend([item]) elif isinstance(value, collections.abc.Mapping): # Assign the dictionary values to the protobuf message. for item_key, item_value in value.items(): set(getattr(msg, key), item_key, item_value) elif isinstance(value, message.Message): getattr(msg, key).CopyFrom(value) else: setattr(msg, key, value) def set(msg_or_dict, key, value): """Set a key's value on a protobuf Message or dictionary. Args: msg_or_dict (Union[~google.protobuf.message.Message, Mapping]): the object. key (str): The key to set. value (Any): The value to set. Raises: TypeError: If ``msg_or_dict`` is not a Message or dictionary. """ # Sanity check: Is our target object valid? if not isinstance(msg_or_dict, (collections.abc.MutableMapping, message.Message)): raise TypeError( "set() expected a dict or protobuf message, got {!r}.".format( type(msg_or_dict) ) ) # We may be setting a nested key. Resolve this. basekey, subkey = _resolve_subkeys(key) # If a subkey exists, then get that object and call this method # recursively against it using the subkey. if subkey is not None: if isinstance(msg_or_dict, collections.abc.MutableMapping): msg_or_dict.setdefault(basekey, {}) set(get(msg_or_dict, basekey), subkey, value) return if isinstance(msg_or_dict, collections.abc.MutableMapping): msg_or_dict[key] = value else: _set_field_on_message(msg_or_dict, key, value) def setdefault(msg_or_dict, key, value): """Set the key on a protobuf Message or dictionary to a given value if the current value is falsy. Because protobuf Messages do not distinguish between unset values and falsy ones particularly well (by design), this method treats any falsy value (e.g. 0, empty list) as a target to be overwritten, on both Messages and dictionaries. Args: msg_or_dict (Union[~google.protobuf.message.Message, Mapping]): the object. key (str): The key on the object in question. value (Any): The value to set. Raises: TypeError: If ``msg_or_dict`` is not a Message or dictionary. """ if not get(msg_or_dict, key, default=None): set(msg_or_dict, key, value) def field_mask(original, modified): """Create a field mask by comparing two messages. Args: original (~google.protobuf.message.Message): the original message. If set to None, this field will be interpreted as an empty message. modified (~google.protobuf.message.Message): the modified message. If set to None, this field will be interpreted as an empty message. Returns: google.protobuf.field_mask_pb2.FieldMask: field mask that contains the list of field names that have different values between the two messages. If the messages are equivalent, then the field mask is empty. Raises: ValueError: If the ``original`` or ``modified`` are not the same type. """ if original is None and modified is None: return field_mask_pb2.FieldMask() if original is None and modified is not None: original = copy.deepcopy(modified) original.Clear() if modified is None and original is not None: modified = copy.deepcopy(original) modified.Clear() if not isinstance(original, type(modified)): raise ValueError( "expected that both original and modified should be of the " 'same type, received "{!r}" and "{!r}".'.format( type(original), type(modified) ) ) return field_mask_pb2.FieldMask(paths=_field_mask_helper(original, modified)) def _field_mask_helper(original, modified, current=""): answer = [] for name in original.DESCRIPTOR.fields_by_name: field_path = _get_path(current, name) original_val = getattr(original, name) modified_val = getattr(modified, name) if _is_message(original_val) or _is_message(modified_val): if original_val != modified_val: # Wrapper types do not need to include the .value part of the # path. if _is_wrapper(original_val) or _is_wrapper(modified_val): answer.append(field_path) elif not modified_val.ListFields(): answer.append(field_path) else: answer.extend( _field_mask_helper(original_val, modified_val, field_path) ) else: if original_val != modified_val: answer.append(field_path) return answer def _get_path(current, name): # gapic-generator-python appends underscores to field names # that collide with python keywords. # `_` is stripped away as it is not possible to # natively define a field with a trailing underscore in protobuf. # APIs will reject field masks if fields have trailing underscores. # See https://github.com/googleapis/python-api-core/issues/227 name = name.rstrip("_") if not current: return name return "%s.%s" % (current, name) def _is_message(value): return isinstance(value, message.Message) def _is_wrapper(value): return type(value) in _WRAPPER_TYPES	12.16 KB	2025-05-25 12:26
📄 py.typed # Marker file for PEP 561. # The google-api-core package uses inline types.	78 B	2025-05-25 12:26
📄 rest_helpers.py # Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for rest transports.""" import functools import operator def flatten_query_params(obj, strict=False): """Flatten a dict into a list of (name,value) tuples. The result is suitable for setting query params on an http request. .. code-block:: python >>> obj = {'a': ... {'b': ... {'c': ['x', 'y', 'z']} }, ... 'd': 'uvw', ... 'e': True, } >>> flatten_query_params(obj, strict=True) [('a.b.c', 'x'), ('a.b.c', 'y'), ('a.b.c', 'z'), ('d', 'uvw'), ('e', 'true')] Note that, as described in https://github.com/googleapis/googleapis/blob/48d9fb8c8e287c472af500221c6450ecd45d7d39/google/api/http.proto#L117, repeated fields (i.e. list-valued fields) may only contain primitive types (not lists or dicts). This is enforced in this function. Args: obj: a possibly nested dictionary (from json), or None strict: a bool, defaulting to False, to enforce that all values in the result tuples be strings and, if boolean, lower-cased. Returns: a list of tuples, with each tuple having a (possibly) multi-part name and a scalar value. Raises: TypeError if obj is not a dict or None ValueError if obj contains a list of non-primitive values. """ if obj is not None and not isinstance(obj, dict): raise TypeError("flatten_query_params must be called with dict object") return _flatten(obj, key_path=[], strict=strict) def _flatten(obj, key_path, strict=False): if obj is None: return [] if isinstance(obj, dict): return _flatten_dict(obj, key_path=key_path, strict=strict) if isinstance(obj, list): return _flatten_list(obj, key_path=key_path, strict=strict) return _flatten_value(obj, key_path=key_path, strict=strict) def _is_primitive_value(obj): if obj is None: return False if isinstance(obj, (list, dict)): raise ValueError("query params may not contain repeated dicts or lists") return True def _flatten_value(obj, key_path, strict=False): return [(".".join(key_path), _canonicalize(obj, strict=strict))] def _flatten_dict(obj, key_path, strict=False): items = ( _flatten(value, key_path=key_path + [key], strict=strict) for key, value in obj.items() ) return functools.reduce(operator.concat, items, []) def _flatten_list(elems, key_path, strict=False): # Only lists of scalar values are supported. # The name (key_path) is repeated for each value. items = ( _flatten_value(elem, key_path=key_path, strict=strict) for elem in elems if _is_primitive_value(elem) ) return functools.reduce(operator.concat, items, []) def _canonicalize(obj, strict=False): if strict: value = str(obj) if isinstance(obj, bool): value = value.lower() return value return obj	3.45 KB	2025-05-25 12:26
📄 rest_streaming.py # Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for server-side streaming in REST.""" from typing import Union import proto import requests import google.protobuf.message from google.api_core._rest_streaming_base import BaseResponseIterator class ResponseIterator(BaseResponseIterator): """Iterator over REST API responses. Args: response (requests.Response): An API response object. response_message_cls (Union[proto.Message, google.protobuf.message.Message]): A response class expected to be returned from an API. Raises: ValueError: - If `response_message_cls` is not a subclass of `proto.Message` or `google.protobuf.message.Message`. """ def __init__( self, response: requests.Response, response_message_cls: Union[proto.Message, google.protobuf.message.Message], ): self._response = response # Inner iterator over HTTP response's content. self._response_itr = self._response.iter_content(decode_unicode=True) super(ResponseIterator, self).__init__( response_message_cls=response_message_cls ) def cancel(self): """Cancel existing streaming operation.""" self._response.close() def __next__(self): while not self._ready_objs: try: chunk = next(self._response_itr) self._process_chunk(chunk) except StopIteration as e: if self._level > 0: raise ValueError("Unfinished stream: %s" % self._obj) raise e return self._grab() def __iter__(self): return self	2.16 KB	2025-05-25 12:26
📄 rest_streaming_async.py # Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for asynchronous server-side streaming in REST.""" from typing import Union import proto try: import google.auth.aio.transport except ImportError as e: # pragma: NO COVER raise ImportError( "`google-api-core[async_rest]` is required to use asynchronous rest streaming. " "Install the `async_rest` extra of `google-api-core` using " "`pip install google-api-core[async_rest]`." ) from e import google.protobuf.message from google.api_core._rest_streaming_base import BaseResponseIterator class AsyncResponseIterator(BaseResponseIterator): """Asynchronous Iterator over REST API responses. Args: response (google.auth.aio.transport.Response): An API response object. response_message_cls (Union[proto.Message, google.protobuf.message.Message]): A response class expected to be returned from an API. Raises: ValueError: - If `response_message_cls` is not a subclass of `proto.Message` or `google.protobuf.message.Message`. """ def __init__( self, response: google.auth.aio.transport.Response, response_message_cls: Union[proto.Message, google.protobuf.message.Message], ): self._response = response self._chunk_size = 1024 # TODO(https://github.com/googleapis/python-api-core/issues/703): mypy does not recognize the abstract content # method as an async generator as it looks for the `yield` keyword in the implementation. # Given that the abstract method is not implemented, mypy fails to recognize it as an async generator. # mypy warnings are silenced until the linked issue is resolved. self._response_itr = self._response.content(self._chunk_size).__aiter__() # type: ignore super(AsyncResponseIterator, self).__init__( response_message_cls=response_message_cls ) async def __aenter__(self): return self async def cancel(self): """Cancel existing streaming operation.""" await self._response.close() async def __anext__(self): while not self._ready_objs: try: chunk = await self._response_itr.__anext__() chunk = chunk.decode("utf-8") self._process_chunk(chunk) except StopAsyncIteration as e: if self._level > 0: raise ValueError("i Unfinished stream: %s" % self._obj) raise e except ValueError as e: raise e return self._grab() def __aiter__(self): return self async def __aexit__(self, exc_type, exc, tb): """Cancel existing async streaming operation.""" await self._response.close()	3.26 KB	2025-05-25 12:26
📁 retry	-	2025-05-25 12:26
📄 retry_async.py # Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # The following imports are for backwards compatibility with https://github.com/googleapis/python-api-core/blob/4d7d2edee2c108d43deb151e6e0fdceb56b73275/google/api_core/retry_async.py # # TODO: Revert these imports on the next major version release (https://github.com/googleapis/python-api-core/issues/576) from google.api_core import datetime_helpers # noqa: F401 from google.api_core import exceptions # noqa: F401 from google.api_core.retry import exponential_sleep_generator # noqa: F401 from google.api_core.retry import if_exception_type # noqa: F401 from google.api_core.retry import if_transient_error # noqa: F401 from google.api_core.retry.retry_unary_async import AsyncRetry from google.api_core.retry.retry_unary_async import retry_target __all__ = ( "AsyncRetry", "datetime_helpers", "exceptions", "exponential_sleep_generator", "if_exception_type", "if_transient_error", "retry_target", )	1.48 KB	2025-05-25 12:26
📄 timeout.py # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Decorators for applying timeout arguments to functions. These decorators are used to wrap API methods to apply either a Deadline-dependent (recommended), constant (DEPRECATED) or exponential (DEPRECATED) timeout argument. For example, imagine an API method that can take a while to return results, such as one that might block until a resource is ready: .. code-block:: python def is_thing_ready(timeout=None): response = requests.get('https://example.com/is_thing_ready') response.raise_for_status() return response.json() This module allows a function like this to be wrapped so that timeouts are automatically determined, for example: .. code-block:: python timeout_ = timeout.ExponentialTimeout() is_thing_ready_with_timeout = timeout_(is_thing_ready) for n in range(10): try: is_thing_ready_with_timeout({'example': 'data'}) except: pass In this example the first call to ``is_thing_ready`` will have a relatively small timeout (like 1 second). If the resource is available and the request completes quickly, the loop exits. But, if the resource isn't yet available and the request times out, it'll be retried - this time with a larger timeout. In the broader context these decorators are typically combined with :mod:`google.api_core.retry` to implement API methods with a signature that matches ``api_method(request, timeout=None, retry=None)``. """ from __future__ import unicode_literals import datetime import functools from google.api_core import datetime_helpers _DEFAULT_INITIAL_TIMEOUT = 5.0 # seconds _DEFAULT_MAXIMUM_TIMEOUT = 30.0 # seconds _DEFAULT_TIMEOUT_MULTIPLIER = 2.0 # If specified, must be in seconds. If none, deadline is not used in the # timeout calculation. _DEFAULT_DEADLINE = None class TimeToDeadlineTimeout(object): """A decorator that decreases timeout set for an RPC based on how much time has left till its deadline. The deadline is calculated as ``now + initial_timeout`` when this decorator is first called for an rpc. In other words this decorator implements deadline semantics in terms of a sequence of decreasing timeouts t0 > t1 > t2 ... tn >= 0. Args: timeout (Optional[float]): the timeout (in seconds) to applied to the wrapped function. If `None`, the target function is expected to never timeout. """ def __init__(self, timeout=None, clock=datetime_helpers.utcnow): self._timeout = timeout self._clock = clock def __call__(self, func): """Apply the timeout decorator. Args: func (Callable): The function to apply the timeout argument to. This function must accept a timeout keyword argument. Returns: Callable: The wrapped function. """ first_attempt_timestamp = self._clock().timestamp() @functools.wraps(func) def func_with_timeout(args, kwargs): """Wrapped function that adds timeout.""" if self._timeout is not None: # All calculations are in seconds now_timestamp = self._clock().timestamp() # To avoid usage of nonlocal but still have round timeout # numbers for first attempt (in most cases the only attempt made # for an RPC. if now_timestamp - first_attempt_timestamp < 0.001: now_timestamp = first_attempt_timestamp time_since_first_attempt = now_timestamp - first_attempt_timestamp remaining_timeout = self._timeout - time_since_first_attempt # Although the `deadline` parameter in `google.api_core.retry.Retry` # is deprecated, and should be treated the same as the `timeout`, # it is still possible for the `deadline` argument in # `google.api_core.retry.Retry` to be larger than the `timeout`. # See https://github.com/googleapis/python-api-core/issues/654 # Only positive non-zero timeouts are supported. # Revert back to the initial timeout for negative or 0 timeout values. if remaining_timeout < 1: remaining_timeout = self._timeout kwargs["timeout"] = remaining_timeout return func(args, *kwargs) return func_with_timeout def __str__(self): return "<TimeToDeadlineTimeout timeout={:.1f}>".format(self._timeout) class ConstantTimeout(object): """A decorator that adds a constant timeout argument. DEPRECATED: use ``TimeToDeadlineTimeout`` instead. This is effectively equivalent to ``functools.partial(func, timeout=timeout)``. Args: timeout (Optional[float]): the timeout (in seconds) to applied to the wrapped function. If `None`, the target function is expected to never timeout. """ def __init__(self, timeout=None): self._timeout = timeout def __call__(self, func): """Apply the timeout decorator. Args: func (Callable): The function to apply the timeout argument to. This function must accept a timeout keyword argument. Returns: Callable: The wrapped function. """ @functools.wraps(func) def func_with_timeout(args, *kwargs): """Wrapped function that adds timeout.""" kwargs["timeout"] = self._timeout return func(args, *kwargs) return func_with_timeout def __str__(self): return "<ConstantTimeout timeout={:.1f}>".format(self._timeout) def _exponential_timeout_generator(initial, maximum, multiplier, deadline): """A generator that yields exponential timeout values. Args: initial (float): The initial timeout. maximum (float): The maximum timeout. multiplier (float): The multiplier applied to the timeout. deadline (float): The overall deadline across all invocations. Yields: float: A timeout value. """ if deadline is not None: deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta( seconds=deadline ) else: deadline_datetime = datetime.datetime.max timeout = initial while True: now = datetime_helpers.utcnow() yield min( # The calculated timeout based on invocations. timeout, # The set maximum timeout. maximum, # The remaining time before the deadline is reached. float((deadline_datetime - now).seconds), ) timeout = timeout multiplier class ExponentialTimeout(object): """A decorator that adds an exponentially increasing timeout argument. DEPRECATED: the concept of incrementing timeout exponentially has been deprecated. Use ``TimeToDeadlineTimeout`` instead. This is useful if a function is called multiple times. Each time the function is called this decorator will calculate a new timeout parameter based on the the number of times the function has been called. For example .. code-block:: python Args: initial (float): The initial timeout to pass. maximum (float): The maximum timeout for any one call. multiplier (float): The multiplier applied to the timeout for each invocation. deadline (Optional[float]): The overall deadline across all invocations. This is used to prevent a very large calculated timeout from pushing the overall execution time over the deadline. This is especially useful in conjunction with :mod:`google.api_core.retry`. If ``None``, the timeouts will not be adjusted to accommodate an overall deadline. """ def __init__( self, initial=_DEFAULT_INITIAL_TIMEOUT, maximum=_DEFAULT_MAXIMUM_TIMEOUT, multiplier=_DEFAULT_TIMEOUT_MULTIPLIER, deadline=_DEFAULT_DEADLINE, ): self._initial = initial self._maximum = maximum self._multiplier = multiplier self._deadline = deadline def with_deadline(self, deadline): """Return a copy of this timeout with the given deadline. Args: deadline (float): The overall deadline across all invocations. Returns: ExponentialTimeout: A new instance with the given deadline. """ return ExponentialTimeout( initial=self._initial, maximum=self._maximum, multiplier=self._multiplier, deadline=deadline, ) def __call__(self, func): """Apply the timeout decorator. Args: func (Callable): The function to apply the timeout argument to. This function must accept a timeout keyword argument. Returns: Callable: The wrapped function. """ timeouts = _exponential_timeout_generator( self._initial, self._maximum, self._multiplier, self._deadline ) @functools.wraps(func) def func_with_timeout(args, kwargs): """Wrapped function that adds timeout.""" kwargs["timeout"] = next(timeouts) return func(args, **kwargs) return func_with_timeout def __str__(self): return ( "<ExponentialTimeout initial={:.1f}, maximum={:.1f}, " "multiplier={:.1f}, deadline={:.1f}>".format( self._initial, self._maximum, self._multiplier, self._deadline ) )	10.04 KB	2025-05-25 12:26
📄 universe.py # Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helpers for universe domain.""" from typing import Any, Optional DEFAULT_UNIVERSE = "googleapis.com" class EmptyUniverseError(ValueError): def __init__(self): message = "Universe Domain cannot be an empty string." super().__init__(message) class UniverseMismatchError(ValueError): def __init__(self, client_universe, credentials_universe): message = ( f"The configured universe domain ({client_universe}) does not match the universe domain " f"found in the credentials ({credentials_universe}). " "If you haven't configured the universe domain explicitly, " f"`{DEFAULT_UNIVERSE}` is the default." ) super().__init__(message) def determine_domain( client_universe_domain: Optional[str], universe_domain_env: Optional[str] ) -> str: """Return the universe domain used by the client. Args: client_universe_domain (Optional[str]): The universe domain configured via the client options. universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. Returns: str: The universe domain to be used by the client. Raises: ValueError: If the universe domain is an empty string. """ universe_domain = DEFAULT_UNIVERSE if client_universe_domain is not None: universe_domain = client_universe_domain elif universe_domain_env is not None: universe_domain = universe_domain_env if len(universe_domain.strip()) == 0: raise EmptyUniverseError return universe_domain def compare_domains(client_universe: str, credentials: Any) -> bool: """Returns True iff the universe domains used by the client and credentials match. Args: client_universe (str): The universe domain configured via the client options. credentials Any: The credentials being used in the client. Returns: bool: True iff client_universe matches the universe in credentials. Raises: ValueError: when client_universe does not match the universe in credentials. """ credentials_universe = getattr(credentials, "universe_domain", DEFAULT_UNIVERSE) if client_universe != credentials_universe: raise UniverseMismatchError(client_universe, credentials_universe) return True	2.88 KB	2025-05-25 12:26
📄 version.py # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. __version__ = "2.24.2"	598 B	2025-05-25 12:26
📄 version_header.py # Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. API_VERSION_METADATA_KEY = "x-goog-api-version" def to_api_version_header(version_identifier): """Returns data for the API Version header for the given `version_identifier`. Args: version_identifier (str): The version identifier to be used in the tuple returned. Returns: Tuple(str, str): A tuple containing the API Version metadata key and value. """ return (API_VERSION_METADATA_KEY, version_identifier)	1.02 KB	2025-05-25 12:26

Edit File:

Terminal