Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified
📁 ..	-	2025-10-13 08:00
📄 __init__.py # Copyright (C) 2001-2007 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org """A package for parsing, handling, and generating email messages.""" __all__ = [ 'base64mime', 'charset', 'encoders', 'errors', 'feedparser', 'generator', 'header', 'iterators', 'message', 'message_from_file', 'message_from_binary_file', 'message_from_string', 'message_from_bytes', 'mime', 'parser', 'quoprimime', 'utils', ] # Some convenience routines. Don't import Parser and Message as side-effects # of importing email since those cascadingly import most of the rest of the # email package. def message_from_string(s, args, kws): """Parse a string into a Message object model. Optional _class and strict are passed to the Parser constructor. """ from email.parser import Parser return Parser(args, *kws).parsestr(s) def message_from_bytes(s, args, *kws): """Parse a bytes string into a Message object model. Optional _class and strict are passed to the Parser constructor. """ from email.parser import BytesParser return BytesParser(args, *kws).parsebytes(s) def message_from_file(fp, args, *kws): """Read a file and parse its contents into a Message object model. Optional _class and strict are passed to the Parser constructor. """ from email.parser import Parser return Parser(args, *kws).parse(fp) def message_from_binary_file(fp, args, *kws): """Read a binary file and parse its contents into a Message object model. Optional _class and strict are passed to the Parser constructor. """ from email.parser import BytesParser return BytesParser(args, **kws).parse(fp)	1.72 KB	2024-04-17 17:55
📁 __pycache__	-	2025-10-13 08:00
📄 _encoded_words.py """ Routines for manipulating RFC2047 encoded words. This is currently a package-private API, but will be considered for promotion to a public API if there is demand. """ # An ecoded word looks like this: # # =?charset[lang]?cte?encoded_string?= # # for more information about charset see the charset module. Here it is one # of the preferred MIME charset names (hopefully; you never know when parsing). # cte (Content Transfer Encoding) is either 'q' or 'b' (ignoring case). In # theory other letters could be used for other encodings, but in practice this # (almost?) never happens. There could be a public API for adding entries # to the CTE tables, but YAGNI for now. 'q' is Quoted Printable, 'b' is # Base64. The meaning of encoded_string should be obvious. 'lang' is optional # as indicated by the brackets (they are not part of the syntax) but is almost # never encountered in practice. # # The general interface for a CTE decoder is that it takes the encoded_string # as its argument, and returns a tuple (cte_decoded_string, defects). The # cte_decoded_string is the original binary that was encoded using the # specified cte. 'defects' is a list of MessageDefect instances indicating any # problems encountered during conversion. 'charset' and 'lang' are the # corresponding strings extracted from the EW, case preserved. # # The general interface for a CTE encoder is that it takes a binary sequence # as input and returns the cte_encoded_string, which is an ascii-only string. # # Each decoder must also supply a length function that takes the binary # sequence as its argument and returns the length of the resulting encoded # string. # # The main API functions for the module are decode, which calls the decoder # referenced by the cte specifier, and encode, which adds the appropriate # RFC 2047 "chrome" to the encoded string, and can optionally automatically # select the shortest possible encoding. See their docstrings below for # details. import re import base64 import binascii import functools from string import ascii_letters, digits from email import errors __all__ = ['decode_q', 'encode_q', 'decode_b', 'encode_b', 'len_q', 'len_b', 'decode', 'encode', ] # # Quoted Printable # # regex based decoder. _q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub, lambda m: bytes.fromhex(m.group(1).decode())) def decode_q(encoded): encoded = encoded.replace(b'_', b' ') return _q_byte_subber(encoded), [] # dict mapping bytes to their encoded form class _QByteMap(dict): safe = b'-!+/' + ascii_letters.encode('ascii') + digits.encode('ascii') def __missing__(self, key): if key in self.safe: self[key] = chr(key) else: self[key] = "={:02X}".format(key) return self[key] _q_byte_map = _QByteMap() # In headers spaces are mapped to '_'. _q_byte_map[ord(' ')] = '_' def encode_q(bstring): return ''.join(_q_byte_map[x] for x in bstring) def len_q(bstring): return sum(len(_q_byte_map[x]) for x in bstring) # # Base64 # def decode_b(encoded): # First try encoding with validate=True, fixing the padding if needed. # This will succeed only if encoded includes no invalid characters. pad_err = len(encoded) % 4 missing_padding = b'==='[:4-pad_err] if pad_err else b'' try: return ( base64.b64decode(encoded + missing_padding, validate=True), [errors.InvalidBase64PaddingDefect()] if pad_err else [], ) except binascii.Error: # Since we had correct padding, this is likely an invalid char error. # # The non-alphabet characters are ignored as far as padding # goes, but we don't know how many there are. So try without adding # padding to see if it works. try: return ( base64.b64decode(encoded, validate=False), [errors.InvalidBase64CharactersDefect()], ) except binascii.Error: # Add as much padding as could possibly be necessary (extra padding # is ignored). try: return ( base64.b64decode(encoded + b'==', validate=False), [errors.InvalidBase64CharactersDefect(), errors.InvalidBase64PaddingDefect()], ) except binascii.Error: # This only happens when the encoded string's length is 1 more # than a multiple of 4, which is invalid. # # bpo-27397: Just return the encoded string since there's no # way to decode. return encoded, [errors.InvalidBase64LengthDefect()] def encode_b(bstring): return base64.b64encode(bstring).decode('ascii') def len_b(bstring): groups_of_3, leftover = divmod(len(bstring), 3) # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. return groups_of_3 * 4 + (4 if leftover else 0) _cte_decoders = { 'q': decode_q, 'b': decode_b, } def decode(ew): """Decode encoded word and return (string, charset, lang, defects) tuple. An RFC 2047/2243 encoded word has the form: =?charsetlang?cte?encoded_string?= where 'lang' may be omitted but the other parts may not be. This function expects exactly such a string (that is, it does not check the syntax and may raise errors if the string is not well formed), and returns the encoded_string decoded first from its Content Transfer Encoding and then from the resulting bytes into unicode using the specified charset. If the cte-decoded string does not successfully decode using the specified character set, a defect is added to the defects list and the unknown octets are replaced by the unicode 'unknown' character \\uFDFF. The specified charset and language are returned. The default for language, which is rarely if ever encountered, is the empty string. """ _, charset, cte, cte_string, _ = ew.split('?') charset, _, lang = charset.partition('') cte = cte.lower() # Recover the original bytes and do CTE decoding. bstring = cte_string.encode('ascii', 'surrogateescape') bstring, defects = _cte_decoders[cte](bstring) # Turn the CTE decoded bytes into unicode. try: string = bstring.decode(charset) except UnicodeError: defects.append(errors.UndecodableBytesDefect("Encoded word " "contains bytes not decodable using {} charset".format(charset))) string = bstring.decode(charset, 'surrogateescape') except LookupError: string = bstring.decode('ascii', 'surrogateescape') if charset.lower() != 'unknown-8bit': defects.append(errors.CharsetError("Unknown charset {} " "in encoded word; decoded as unknown bytes".format(charset))) return string, charset, lang, defects _cte_encoders = { 'q': encode_q, 'b': encode_b, } _cte_encode_length = { 'q': len_q, 'b': len_b, } def encode(string, charset='utf-8', encoding=None, lang=''): """Encode string using the CTE encoding that produces the shorter result. Produces an RFC 2047/2243 encoded word of the form: =?charsetlang?cte?encoded_string?= where 'lang' is omitted unless the 'lang' parameter is given a value. Optional argument charset (defaults to utf-8) specifies the charset to use to encode the string to binary before CTE encoding it. Optional argument 'encoding' is the cte specifier for the encoding that should be used ('q' or 'b'); if it is None (the default) the encoding which produces the shortest encoded sequence is used, except that 'q' is preferred if it is up to five characters longer. Optional argument 'lang' (default '') gives the RFC 2243 language string to specify in the encoded word. """ if charset == 'unknown-8bit': bstring = string.encode('ascii', 'surrogateescape') else: bstring = string.encode(charset) if encoding is None: qlen = _cte_encode_length['q'](bstring) blen = _cte_encode_length['b'](bstring) # Bias toward q. 5 is arbitrary. encoding = 'q' if qlen - blen < 5 else 'b' encoded = _cte_encoders[encoding](bstring) if lang: lang = '' + lang return "=?{}{}?{}?{}?=".format(charset, lang, encoding, encoded)	8.32 KB	2024-04-17 17:55
📄 _header_value_parser.py """Header value parser implementing various email-related RFC parsing rules. The parsing methods defined in this module implement various email related parsing rules. Principal among them is RFC 5322, which is the followon to RFC 2822 and primarily a clarification of the former. It also implements RFC 2047 encoded word decoding. RFC 5322 goes to considerable trouble to maintain backward compatibility with RFC 822 in the parse phase, while cleaning up the structure on the generation phase. This parser supports correct RFC 5322 generation by tagging white space as folding white space only when folding is allowed in the non-obsolete rule sets. Actually, the parser is even more generous when accepting input than RFC 5322 mandates, following the spirit of Postel's Law, which RFC 5322 encourages. Where possible deviations from the standard are annotated on the 'defects' attribute of tokens that deviate. The general structure of the parser follows RFC 5322, and uses its terminology where there is a direct correspondence. Where the implementation requires a somewhat different structure than that used by the formal grammar, new terms that mimic the closest existing terms are used. Thus, it really helps to have a copy of RFC 5322 handy when studying this code. Input to the parser is a string that has already been unfolded according to RFC 5322 rules. According to the RFC this unfolding is the very first step, and this parser leaves the unfolding step to a higher level message parser, which will have already detected the line breaks that need unfolding while determining the beginning and end of each header. The output of the parser is a TokenList object, which is a list subclass. A TokenList is a recursive data structure. The terminal nodes of the structure are Terminal objects, which are subclasses of str. These do not correspond directly to terminal objects in the formal grammar, but are instead more practical higher level combinations of true terminals. All TokenList and Terminal objects have a 'value' attribute, which produces the semantically meaningful value of that part of the parse subtree. The value of all whitespace tokens (no matter how many sub-tokens they may contain) is a single space, as per the RFC rules. This includes 'CFWS', which is herein included in the general class of whitespace tokens. There is one exception to the rule that whitespace tokens are collapsed into single spaces in values: in the value of a 'bare-quoted-string' (a quoted-string with no leading or trailing whitespace), any whitespace that appeared between the quotation marks is preserved in the returned value. Note that in all Terminal strings quoted pairs are turned into their unquoted values. All TokenList and Terminal objects also have a string value, which attempts to be a "canonical" representation of the RFC-compliant form of the substring that produced the parsed subtree, including minimal use of quoted pair quoting. Whitespace runs are not collapsed. Comment tokens also have a 'content' attribute providing the string found between the parens (including any nested comments) with whitespace preserved. All TokenList and Terminal objects have a 'defects' attribute which is a possibly empty list all of the defects found while creating the token. Defects may appear on any token in the tree, and a composite list of all defects in the subtree is available through the 'all_defects' attribute of any node. (For Terminal notes x.defects == x.all_defects.) Each object in a parse tree is called a 'token', and each has a 'token_type' attribute that gives the name from the RFC 5322 grammar that it represents. Not all RFC 5322 nodes are produced, and there is one non-RFC 5322 node that may be produced: 'ptext'. A 'ptext' is a string of printable ascii characters. It is returned in place of lists of (ctext/quoted-pair) and (qtext/quoted-pair). XXX: provide complete list of token types. """ import re import sys import urllib # For urllib.parse.unquote from string import hexdigits from collections import OrderedDict from operator import itemgetter from email import _encoded_words as _ew from email import errors from email import utils # # Useful constants and functions # WSP = set(' \t') CFWS_LEADER = WSP \| set('(') SPECIALS = set(r'()<>@,:;.\"[]') ATOM_ENDS = SPECIALS \| WSP DOT_ATOM_ENDS = ATOM_ENDS - set('.') # '.', '"', and '(' do not end phrases in order to support obs-phrase PHRASE_ENDS = SPECIALS - set('."(') TSPECIALS = (SPECIALS \| set('/?=')) - set('.') TOKEN_ENDS = TSPECIALS \| WSP ASPECIALS = TSPECIALS \| set("'%") ATTRIBUTE_ENDS = ASPECIALS \| WSP EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' # Match a RFC 2047 word, looks like =?utf-8?q?someword?= rfc2047_matcher = re.compile(r''' =\? # literal =? [^?] # charset \? # literal ? [qQbB] # literal 'q' or 'b', case insensitive \? # literal ? .? # encoded word \?= # literal ?= ''', re.VERBOSE \| re.MULTILINE) # # TokenList and its subclasses # class TokenList(list): token_type = None syntactic_break = True ew_combine_allowed = True def __init__(self, args, *kw): super().__init__(args, *kw) self.defects = [] def __str__(self): return ''.join(str(x) for x in self) def __repr__(self): return '{}({})'.format(self.__class__.__name__, super().__repr__()) @property def value(self): return ''.join(x.value for x in self if x.value) @property def all_defects(self): return sum((x.all_defects for x in self), self.defects) def startswith_fws(self): return self[0].startswith_fws() @property def as_ew_allowed(self): """True if all top level tokens of this part may be RFC2047 encoded.""" return all(part.as_ew_allowed for part in self) @property def comments(self): comments = [] for token in self: comments.extend(token.comments) return comments def fold(self, , policy): return _refold_parse_tree(self, policy=policy) def pprint(self, indent=''): print(self.ppstr(indent=indent)) def ppstr(self, indent=''): return '\n'.join(self._pp(indent=indent)) def _pp(self, indent=''): yield '{}{}/{}('.format( indent, self.__class__.__name__, self.token_type) for token in self: if not hasattr(token, '_pp'): yield (indent + ' !! invalid element in token ' 'list: {!r}'.format(token)) else: yield from token._pp(indent+' ') if self.defects: extra = ' Defects: {}'.format(self.defects) else: extra = '' yield '{}){}'.format(indent, extra) class WhiteSpaceTokenList(TokenList): @property def value(self): return ' ' @property def comments(self): return [x.content for x in self if x.token_type=='comment'] class UnstructuredTokenList(TokenList): token_type = 'unstructured' class Phrase(TokenList): token_type = 'phrase' class Word(TokenList): token_type = 'word' class CFWSList(WhiteSpaceTokenList): token_type = 'cfws' class Atom(TokenList): token_type = 'atom' class Token(TokenList): token_type = 'token' encode_as_ew = False class EncodedWord(TokenList): token_type = 'encoded-word' cte = None charset = None lang = None class QuotedString(TokenList): token_type = 'quoted-string' @property def content(self): for x in self: if x.token_type == 'bare-quoted-string': return x.value @property def quoted_value(self): res = [] for x in self: if x.token_type == 'bare-quoted-string': res.append(str(x)) else: res.append(x.value) return ''.join(res) @property def stripped_value(self): for token in self: if token.token_type == 'bare-quoted-string': return token.value class BareQuotedString(QuotedString): token_type = 'bare-quoted-string' def __str__(self): return quote_string(''.join(str(x) for x in self)) @property def value(self): return ''.join(str(x) for x in self) class Comment(WhiteSpaceTokenList): token_type = 'comment' def __str__(self): return ''.join(sum([ ["("], [self.quote(x) for x in self], [")"], ], [])) def quote(self, value): if value.token_type == 'comment': return str(value) return str(value).replace('\\', '\\\\').replace( '(', r'$').replace( ')', r'$') @property def content(self): return ''.join(str(x) for x in self) @property def comments(self): return [self.content] class AddressList(TokenList): token_type = 'address-list' @property def addresses(self): return [x for x in self if x.token_type=='address'] @property def mailboxes(self): return sum((x.mailboxes for x in self if x.token_type=='address'), []) @property def all_mailboxes(self): return sum((x.all_mailboxes for x in self if x.token_type=='address'), []) class Address(TokenList): token_type = 'address' @property def display_name(self): if self[0].token_type == 'group': return self[0].display_name @property def mailboxes(self): if self[0].token_type == 'mailbox': return [self[0]] elif self[0].token_type == 'invalid-mailbox': return [] return self[0].mailboxes @property def all_mailboxes(self): if self[0].token_type == 'mailbox': return [self[0]] elif self[0].token_type == 'invalid-mailbox': return [self[0]] return self[0].all_mailboxes class MailboxList(TokenList): token_type = 'mailbox-list' @property def mailboxes(self): return [x for x in self if x.token_type=='mailbox'] @property def all_mailboxes(self): return [x for x in self if x.token_type in ('mailbox', 'invalid-mailbox')] class GroupList(TokenList): token_type = 'group-list' @property def mailboxes(self): if not self or self[0].token_type != 'mailbox-list': return [] return self[0].mailboxes @property def all_mailboxes(self): if not self or self[0].token_type != 'mailbox-list': return [] return self[0].all_mailboxes class Group(TokenList): token_type = "group" @property def mailboxes(self): if self[2].token_type != 'group-list': return [] return self[2].mailboxes @property def all_mailboxes(self): if self[2].token_type != 'group-list': return [] return self[2].all_mailboxes @property def display_name(self): return self[0].display_name class NameAddr(TokenList): token_type = 'name-addr' @property def display_name(self): if len(self) == 1: return None return self[0].display_name @property def local_part(self): return self[-1].local_part @property def domain(self): return self[-1].domain @property def route(self): return self[-1].route @property def addr_spec(self): return self[-1].addr_spec class AngleAddr(TokenList): token_type = 'angle-addr' @property def local_part(self): for x in self: if x.token_type == 'addr-spec': return x.local_part @property def domain(self): for x in self: if x.token_type == 'addr-spec': return x.domain @property def route(self): for x in self: if x.token_type == 'obs-route': return x.domains @property def addr_spec(self): for x in self: if x.token_type == 'addr-spec': if x.local_part: return x.addr_spec else: return quote_string(x.local_part) + x.addr_spec else: return '<>' class ObsRoute(TokenList): token_type = 'obs-route' @property def domains(self): return [x.domain for x in self if x.token_type == 'domain'] class Mailbox(TokenList): token_type = 'mailbox' @property def display_name(self): if self[0].token_type == 'name-addr': return self[0].display_name @property def local_part(self): return self[0].local_part @property def domain(self): return self[0].domain @property def route(self): if self[0].token_type == 'name-addr': return self[0].route @property def addr_spec(self): return self[0].addr_spec class InvalidMailbox(TokenList): token_type = 'invalid-mailbox' @property def display_name(self): return None local_part = domain = route = addr_spec = display_name class Domain(TokenList): token_type = 'domain' as_ew_allowed = False @property def domain(self): return ''.join(super().value.split()) class DotAtom(TokenList): token_type = 'dot-atom' class DotAtomText(TokenList): token_type = 'dot-atom-text' as_ew_allowed = True class AddrSpec(TokenList): token_type = 'addr-spec' as_ew_allowed = False @property def local_part(self): return self[0].local_part @property def domain(self): if len(self) < 3: return None return self[-1].domain @property def value(self): if len(self) < 3: return self[0].value return self[0].value.rstrip()+self[1].value+self[2].value.lstrip() @property def addr_spec(self): nameset = set(self.local_part) if len(nameset) > len(nameset-DOT_ATOM_ENDS): lp = quote_string(self.local_part) else: lp = self.local_part if self.domain is not None: return lp + '@' + self.domain return lp class ObsLocalPart(TokenList): token_type = 'obs-local-part' as_ew_allowed = False class DisplayName(Phrase): token_type = 'display-name' ew_combine_allowed = False @property def display_name(self): res = TokenList(self) if len(res) == 0: return res.value if res[0].token_type == 'cfws': res.pop(0) else: if res[0][0].token_type == 'cfws': res[0] = TokenList(res[0][1:]) if res[-1].token_type == 'cfws': res.pop() else: if res[-1][-1].token_type == 'cfws': res[-1] = TokenList(res[-1][:-1]) return res.value @property def value(self): quote = False if self.defects: quote = True else: for x in self: if x.token_type == 'quoted-string': quote = True if len(self) != 0 and quote: pre = post = '' if self[0].token_type=='cfws' or self[0][0].token_type=='cfws': pre = ' ' if self[-1].token_type=='cfws' or self[-1][-1].token_type=='cfws': post = ' ' return pre+quote_string(self.display_name)+post else: return super().value class LocalPart(TokenList): token_type = 'local-part' as_ew_allowed = False @property def value(self): if self[0].token_type == "quoted-string": return self[0].quoted_value else: return self[0].value @property def local_part(self): # Strip whitespace from front, back, and around dots. res = [DOT] last = DOT last_is_tl = False for tok in self[0] + [DOT]: if tok.token_type == 'cfws': continue if (last_is_tl and tok.token_type == 'dot' and last[-1].token_type == 'cfws'): res[-1] = TokenList(last[:-1]) is_tl = isinstance(tok, TokenList) if (is_tl and last.token_type == 'dot' and tok[0].token_type == 'cfws'): res.append(TokenList(tok[1:])) else: res.append(tok) last = res[-1] last_is_tl = is_tl res = TokenList(res[1:-1]) return res.value class DomainLiteral(TokenList): token_type = 'domain-literal' as_ew_allowed = False @property def domain(self): return ''.join(super().value.split()) @property def ip(self): for x in self: if x.token_type == 'ptext': return x.value class MIMEVersion(TokenList): token_type = 'mime-version' major = None minor = None class Parameter(TokenList): token_type = 'parameter' sectioned = False extended = False charset = 'us-ascii' @property def section_number(self): # Because the first token, the attribute (name) eats CFWS, the second # token is always the section if there is one. return self[1].number if self.sectioned else 0 @property def param_value(self): # This is part of the "handle quoted extended parameters" hack. for token in self: if token.token_type == 'value': return token.stripped_value if token.token_type == 'quoted-string': for token in token: if token.token_type == 'bare-quoted-string': for token in token: if token.token_type == 'value': return token.stripped_value return '' class InvalidParameter(Parameter): token_type = 'invalid-parameter' class Attribute(TokenList): token_type = 'attribute' @property def stripped_value(self): for token in self: if token.token_type.endswith('attrtext'): return token.value class Section(TokenList): token_type = 'section' number = None class Value(TokenList): token_type = 'value' @property def stripped_value(self): token = self[0] if token.token_type == 'cfws': token = self[1] if token.token_type.endswith( ('quoted-string', 'attribute', 'extended-attribute')): return token.stripped_value return self.value class MimeParameters(TokenList): token_type = 'mime-parameters' syntactic_break = False @property def params(self): # The RFC specifically states that the ordering of parameters is not # guaranteed and may be reordered by the transport layer. So we have # to assume the RFC 2231 pieces can come in any order. However, we # output them in the order that we first see a given name, which gives # us a stable __str__. params = OrderedDict() for token in self: if not token.token_type.endswith('parameter'): continue if token[0].token_type != 'attribute': continue name = token[0].value.strip() if name not in params: params[name] = [] params[name].append((token.section_number, token)) for name, parts in params.items(): parts = sorted(parts, key=itemgetter(0)) first_param = parts[0][1] charset = first_param.charset # Our arbitrary error recovery is to ignore duplicate parameters, # to use appearance order if there are duplicate rfc 2231 parts, # and to ignore gaps. This mimics the error recovery of get_param. if not first_param.extended and len(parts) > 1: if parts[1][0] == 0: parts[1][1].defects.append(errors.InvalidHeaderDefect( 'duplicate parameter name; duplicate(s) ignored')) parts = parts[:1] # Else assume the 0 was missing...note that this is different # from get_param, but we registered a defect for this earlier. value_parts = [] i = 0 for section_number, param in parts: if section_number != i: # We could get fancier here and look for a complete # duplicate extended parameter and ignore the second one # seen. But we're not doing that. The old code didn't. if not param.extended: param.defects.append(errors.InvalidHeaderDefect( 'duplicate parameter name; duplicate ignored')) continue else: param.defects.append(errors.InvalidHeaderDefect( "inconsistent RFC2231 parameter numbering")) i += 1 value = param.param_value if param.extended: try: value = urllib.parse.unquote_to_bytes(value) except UnicodeEncodeError: # source had surrogate escaped bytes. What we do now # is a bit of an open question. I'm not sure this is # the best choice, but it is what the old algorithm did value = urllib.parse.unquote(value, encoding='latin-1') else: try: value = value.decode(charset, 'surrogateescape') except LookupError: # XXX: there should really be a custom defect for # unknown character set to make it easy to find, # because otherwise unknown charset is a silent # failure. value = value.decode('us-ascii', 'surrogateescape') if utils._has_surrogates(value): param.defects.append(errors.UndecodableBytesDefect()) value_parts.append(value) value = ''.join(value_parts) yield name, value def __str__(self): params = [] for name, value in self.params: if value: params.append('{}={}'.format(name, quote_string(value))) else: params.append(name) params = '; '.join(params) return ' ' + params if params else '' class ParameterizedHeaderValue(TokenList): # Set this false so that the value doesn't wind up on a new line even # if it and the parameters would fit there but not on the first line. syntactic_break = False @property def params(self): for token in reversed(self): if token.token_type == 'mime-parameters': return token.params return {} class ContentType(ParameterizedHeaderValue): token_type = 'content-type' as_ew_allowed = False maintype = 'text' subtype = 'plain' class ContentDisposition(ParameterizedHeaderValue): token_type = 'content-disposition' as_ew_allowed = False content_disposition = None class ContentTransferEncoding(TokenList): token_type = 'content-transfer-encoding' as_ew_allowed = False cte = '7bit' class HeaderLabel(TokenList): token_type = 'header-label' as_ew_allowed = False class Header(TokenList): token_type = 'header' # # Terminal classes and instances # class Terminal(str): as_ew_allowed = True ew_combine_allowed = True syntactic_break = True def __new__(cls, value, token_type): self = super().__new__(cls, value) self.token_type = token_type self.defects = [] return self def __repr__(self): return "{}({})".format(self.__class__.__name__, super().__repr__()) def pprint(self): print(self.__class__.__name__ + '/' + self.token_type) @property def all_defects(self): return list(self.defects) def _pp(self, indent=''): return ["{}{}/{}({}){}".format( indent, self.__class__.__name__, self.token_type, super().__repr__(), '' if not self.defects else ' {}'.format(self.defects), )] def pop_trailing_ws(self): # This terminates the recursion. return None @property def comments(self): return [] def __getnewargs__(self): return(str(self), self.token_type) class WhiteSpaceTerminal(Terminal): @property def value(self): return ' ' def startswith_fws(self): return True class ValueTerminal(Terminal): @property def value(self): return self def startswith_fws(self): return False class EWWhiteSpaceTerminal(WhiteSpaceTerminal): @property def value(self): return '' def __str__(self): return '' class _InvalidEwError(errors.HeaderParseError): """Invalid encoded word found while parsing headers.""" # XXX these need to become classes and used as instances so # that a program can't change them in a parse tree and screw # up other parse trees. Maybe should have tests for that, too. DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # # Parser # # Parse strings according to RFC822/2047/2822/5322 rules. # # This is a stateless parser. Each get_XXX function accepts a string and # returns either a Terminal or a TokenList representing the RFC object named # by the method and a string containing the remaining unparsed characters # from the input. Thus a parser method consumes the next syntactic construct # of a given type and returns a token representing the construct plus the # unparsed remainder of the input string. # # For example, if the first element of a structured header is a 'phrase', # then: # # phrase, value = get_phrase(value) # # returns the complete phrase from the start of the string value, plus any # characters left in the string after the phrase is removed. _wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split _non_atom_end_matcher = re.compile(r"[^{}]+".format( re.escape(''.join(ATOM_ENDS)))).match _non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall _non_token_end_matcher = re.compile(r"[^{}]+".format( re.escape(''.join(TOKEN_ENDS)))).match _non_attribute_end_matcher = re.compile(r"[^{}]+".format( re.escape(''.join(ATTRIBUTE_ENDS)))).match _non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format( re.escape(''.join(EXTENDED_ATTRIBUTE_ENDS)))).match def _validate_xtext(xtext): """If input token contains ASCII non-printables, register a defect.""" non_printables = _non_printable_finder(xtext) if non_printables: xtext.defects.append(errors.NonPrintableDefect(non_printables)) if utils._has_surrogates(xtext): xtext.defects.append(errors.UndecodableBytesDefect( "Non-ASCII characters found in header token")) def _get_ptext_to_endchars(value, endchars): """Scan printables/quoted-pairs until endchars and return unquoted ptext. This function turns a run of qcontent, ccontent-without-comments, or dtext-with-quoted-printables into a single string by unquoting any quoted printables. It returns the string, the remaining value, and a flag that is True iff there were any quoted printables decoded. """ fragment, remainder = _wsp_splitter(value, 1) vchars = [] escape = False had_qp = False for pos in range(len(fragment)): if fragment[pos] == '\\': if escape: escape = False had_qp = True else: escape = True continue if escape: escape = False elif fragment[pos] in endchars: break vchars.append(fragment[pos]) else: pos = pos + 1 return ''.join(vchars), ''.join([fragment[pos:]] + remainder), had_qp def get_fws(value): """FWS = 1WSP This isn't the RFC definition. We're using fws to represent tokens where folding can be done, but when we are parsing the unfolding has already been done so we don't need to watch out for CRLF. """ newvalue = value.lstrip() fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws') return fws, newvalue def get_encoded_word(value): """ encoded-word = "=?" charset "?" encoding "?" encoded-text "?=" """ ew = EncodedWord() if not value.startswith('=?'): raise errors.HeaderParseError( "expected encoded word but found {}".format(value)) tok, remainder = value[2:].split('?=', 1) if tok == value[2:]: raise errors.HeaderParseError( "expected encoded word but found {}".format(value)) remstr = ''.join(remainder) if (len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits and tok.count('?') < 2): # The ? after the CTE was followed by an encoded word escape (=XX). rest, remainder = remstr.split('?=', 1) tok = tok + '?=' + rest if len(tok.split()) > 1: ew.defects.append(errors.InvalidHeaderDefect( "whitespace inside encoded word")) ew.cte = value value = ''.join(remainder) try: text, charset, lang, defects = _ew.decode('=?' + tok + '?=') except (ValueError, KeyError): raise _InvalidEwError( "encoded word format invalid: '{}'".format(ew.cte)) ew.charset = charset ew.lang = lang ew.defects.extend(defects) while text: if text[0] in WSP: token, text = get_fws(text) ew.append(token) continue chars, remainder = _wsp_splitter(text, 1) vtext = ValueTerminal(chars, 'vtext') _validate_xtext(vtext) ew.append(vtext) text = ''.join(remainder) # Encoded words should be followed by a WS if value and value[0] not in WSP: ew.defects.append(errors.InvalidHeaderDefect( "missing trailing whitespace after encoded-word")) return ew, value def get_unstructured(value): """unstructured = (([FWS] vchar) WSP) / obs-unstruct obs-unstruct = ((LF CR (obs-utext) LF CR)) / FWS) obs-utext = %d0 / obs-NO-WS-CTL / LF / CR obs-NO-WS-CTL is control characters except WSP/CR/LF. So, basically, we have printable runs, plus control characters or nulls in the obsolete syntax, separated by whitespace. Since RFC 2047 uses the obsolete syntax in its specification, but requires whitespace on either side of the encoded words, I can see no reason to need to separate the non-printable-non-whitespace from the printable runs if they occur, so we parse this into xtext tokens separated by WSP tokens. Because an 'unstructured' value must by definition constitute the entire value, this 'get' routine does not return a remaining value, only the parsed TokenList. """ # XXX: but what about bare CR and LF? They might signal the start or # end of an encoded word. YAGNI for now, since our current parsers # will never send us strings with bare CR or LF. unstructured = UnstructuredTokenList() while value: if value[0] in WSP: token, value = get_fws(value) unstructured.append(token) continue valid_ew = True if value.startswith('=?'): try: token, value = get_encoded_word(value) except _InvalidEwError: valid_ew = False except errors.HeaderParseError: # XXX: Need to figure out how to register defects when # appropriate here. pass else: have_ws = True if len(unstructured) > 0: if unstructured[-1].token_type != 'fws': unstructured.defects.append(errors.InvalidHeaderDefect( "missing whitespace before encoded word")) have_ws = False if have_ws and len(unstructured) > 1: if unstructured[-2].token_type == 'encoded-word': unstructured[-1] = EWWhiteSpaceTerminal( unstructured[-1], 'fws') unstructured.append(token) continue tok, remainder = _wsp_splitter(value, 1) # Split in the middle of an atom if there is a rfc2047 encoded word # which does not have WSP on both sides. The defect will be registered # the next time through the loop. # This needs to only be performed when the encoded word is valid; # otherwise, performing it on an invalid encoded word can cause # the parser to go in an infinite loop. if valid_ew and rfc2047_matcher.search(tok): tok, remainder = value.partition('=?') vtext = ValueTerminal(tok, 'vtext') _validate_xtext(vtext) unstructured.append(vtext) value = ''.join(remainder) return unstructured def get_qp_ctext(value): r"""ctext = <printable ascii except \ ( )> This is not the RFC ctext, since we are handling nested comments in comment and unquoting quoted-pairs here. We allow anything except the '()' characters, but if we find any ASCII other than the RFC defined printable ASCII, a NonPrintableDefect is added to the token's defects list. Since quoted pairs are converted to their unquoted values, what is returned is a 'ptext' token. In this case it is a WhiteSpaceTerminal, so it's value is ' '. """ ptext, value, _ = _get_ptext_to_endchars(value, '()') ptext = WhiteSpaceTerminal(ptext, 'ptext') _validate_xtext(ptext) return ptext, value def get_qcontent(value): """qcontent = qtext / quoted-pair We allow anything except the DQUOTE character, but if we find any ASCII other than the RFC defined printable ASCII, a NonPrintableDefect is added to the token's defects list. Any quoted pairs are converted to their unquoted values, so what is returned is a 'ptext' token. In this case it is a ValueTerminal. """ ptext, value, _ = _get_ptext_to_endchars(value, '"') ptext = ValueTerminal(ptext, 'ptext') _validate_xtext(ptext) return ptext, value def get_atext(value): """atext = <matches _atext_matcher> We allow any non-ATOM_ENDS in atext, but add an InvalidATextDefect to the token's defects list if we find non-atext characters. """ m = _non_atom_end_matcher(value) if not m: raise errors.HeaderParseError( "expected atext but found '{}'".format(value)) atext = m.group() value = value[len(atext):] atext = ValueTerminal(atext, 'atext') _validate_xtext(atext) return atext, value def get_bare_quoted_string(value): """bare-quoted-string = DQUOTE ([FWS] qcontent) [FWS] DQUOTE A quoted-string without the leading or trailing white space. Its value is the text between the quote marks, with whitespace preserved and quoted pairs decoded. """ if value[0] != '"': raise errors.HeaderParseError( "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() value = value[1:] if value and value[0] == '"': token, value = get_qcontent(value) bare_quoted_string.append(token) while value and value[0] != '"': if value[0] in WSP: token, value = get_fws(value) elif value[:2] == '=?': valid_ew = False try: token, value = get_encoded_word(value) bare_quoted_string.defects.append(errors.InvalidHeaderDefect( "encoded word inside quoted string")) valid_ew = True except errors.HeaderParseError: token, value = get_qcontent(value) # Collapse the whitespace between two encoded words that occur in a # bare-quoted-string. if valid_ew and len(bare_quoted_string) > 1: if (bare_quoted_string[-1].token_type == 'fws' and bare_quoted_string[-2].token_type == 'encoded-word'): bare_quoted_string[-1] = EWWhiteSpaceTerminal( bare_quoted_string[-1], 'fws') else: token, value = get_qcontent(value) bare_quoted_string.append(token) if not value: bare_quoted_string.defects.append(errors.InvalidHeaderDefect( "end of header inside quoted string")) return bare_quoted_string, value return bare_quoted_string, value[1:] def get_comment(value): """comment = "(" ([FWS] ccontent) [FWS] ")" ccontent = ctext / quoted-pair / comment We handle nested comments here, and quoted-pair in our qp-ctext routine. """ if value and value[0] != '(': raise errors.HeaderParseError( "expected '(' but found '{}'".format(value)) comment = Comment() value = value[1:] while value and value[0] != ")": if value[0] in WSP: token, value = get_fws(value) elif value[0] == '(': token, value = get_comment(value) else: token, value = get_qp_ctext(value) comment.append(token) if not value: comment.defects.append(errors.InvalidHeaderDefect( "end of header inside comment")) return comment, value return comment, value[1:] def get_cfws(value): """CFWS = (1([FWS] comment) [FWS]) / FWS """ cfws = CFWSList() while value and value[0] in CFWS_LEADER: if value[0] in WSP: token, value = get_fws(value) else: token, value = get_comment(value) cfws.append(token) return cfws, value def get_quoted_string(value): """quoted-string = [CFWS] <bare-quoted-string> [CFWS] 'bare-quoted-string' is an intermediate class defined by this parser and not by the RFC grammar. It is the quoted string without any attached CFWS. """ quoted_string = QuotedString() if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) quoted_string.append(token) token, value = get_bare_quoted_string(value) quoted_string.append(token) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) quoted_string.append(token) return quoted_string, value def get_atom(value): """atom = [CFWS] 1atext [CFWS] An atom could be an rfc2047 encoded word. """ atom = Atom() if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) atom.append(token) if value and value[0] in ATOM_ENDS: raise errors.HeaderParseError( "expected atom but found '{}'".format(value)) if value.startswith('=?'): try: token, value = get_encoded_word(value) except errors.HeaderParseError: # XXX: need to figure out how to register defects when # appropriate here. token, value = get_atext(value) else: token, value = get_atext(value) atom.append(token) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) atom.append(token) return atom, value def get_dot_atom_text(value): """ dot-text = 1atext ("." 1atext) """ dot_atom_text = DotAtomText() if not value or value[0] in ATOM_ENDS: raise errors.HeaderParseError("expected atom at a start of " "dot-atom-text but found '{}'".format(value)) while value and value[0] not in ATOM_ENDS: token, value = get_atext(value) dot_atom_text.append(token) if value and value[0] == '.': dot_atom_text.append(DOT) value = value[1:] if dot_atom_text[-1] is DOT: raise errors.HeaderParseError("expected atom at end of dot-atom-text " "but found '{}'".format('.'+value)) return dot_atom_text, value def get_dot_atom(value): """ dot-atom = [CFWS] dot-atom-text [CFWS] Any place we can have a dot atom, we could instead have an rfc2047 encoded word. """ dot_atom = DotAtom() if value[0] in CFWS_LEADER: token, value = get_cfws(value) dot_atom.append(token) if value.startswith('=?'): try: token, value = get_encoded_word(value) except errors.HeaderParseError: # XXX: need to figure out how to register defects when # appropriate here. token, value = get_dot_atom_text(value) else: token, value = get_dot_atom_text(value) dot_atom.append(token) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) dot_atom.append(token) return dot_atom, value def get_word(value): """word = atom / quoted-string Either atom or quoted-string may start with CFWS. We have to peel off this CFWS first to determine which type of word to parse. Afterward we splice the leading CFWS, if any, into the parsed sub-token. If neither an atom or a quoted-string is found before the next special, a HeaderParseError is raised. The token returned is either an Atom or a QuotedString, as appropriate. This means the 'word' level of the formal grammar is not represented in the parse tree; this is because having that extra layer when manipulating the parse tree is more confusing than it is helpful. """ if value[0] in CFWS_LEADER: leader, value = get_cfws(value) else: leader = None if not value: raise errors.HeaderParseError( "Expected 'atom' or 'quoted-string' but found nothing.") if value[0]=='"': token, value = get_quoted_string(value) elif value[0] in SPECIALS: raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " "but found '{}'".format(value)) else: token, value = get_atom(value) if leader is not None: token[:0] = [leader] return token, value def get_phrase(value): """ phrase = 1word / obs-phrase obs-phrase = word (word / "." / CFWS) This means a phrase can be a sequence of words, periods, and CFWS in any order as long as it starts with at least one word. If anything other than words is detected, an ObsoleteHeaderDefect is added to the token's defect list. We also accept a phrase that starts with CFWS followed by a dot; this is registered as an InvalidHeaderDefect, since it is not supported by even the obsolete grammar. """ phrase = Phrase() try: token, value = get_word(value) phrase.append(token) except errors.HeaderParseError: phrase.defects.append(errors.InvalidHeaderDefect( "phrase does not start with word")) while value and value[0] not in PHRASE_ENDS: if value[0]=='.': phrase.append(DOT) phrase.defects.append(errors.ObsoleteHeaderDefect( "period in 'phrase'")) value = value[1:] else: try: token, value = get_word(value) except errors.HeaderParseError: if value[0] in CFWS_LEADER: token, value = get_cfws(value) phrase.defects.append(errors.ObsoleteHeaderDefect( "comment found without atom")) else: raise phrase.append(token) return phrase, value def get_local_part(value): """ local-part = dot-atom / quoted-string / obs-local-part """ local_part = LocalPart() leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( "expected local-part but found '{}'".format(value)) try: token, value = get_dot_atom(value) except errors.HeaderParseError: try: token, value = get_word(value) except errors.HeaderParseError: if value[0] != '\\' and value[0] in PHRASE_ENDS: raise token = TokenList() if leader is not None: token[:0] = [leader] local_part.append(token) if value and (value[0]=='\\' or value[0] not in PHRASE_ENDS): obs_local_part, value = get_obs_local_part(str(local_part) + value) if obs_local_part.token_type == 'invalid-obs-local-part': local_part.defects.append(errors.InvalidHeaderDefect( "local-part is not dot-atom, quoted-string, or obs-local-part")) else: local_part.defects.append(errors.ObsoleteHeaderDefect( "local-part is not a dot-atom (contains CFWS)")) local_part[0] = obs_local_part try: local_part.value.encode('ascii') except UnicodeEncodeError: local_part.defects.append(errors.NonASCIILocalPartDefect( "local-part contains non-ASCII characters)")) return local_part, value def get_obs_local_part(value): """ obs-local-part = word ("." word) """ obs_local_part = ObsLocalPart() last_non_ws_was_dot = False while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS): if value[0] == '.': if last_non_ws_was_dot: obs_local_part.defects.append(errors.InvalidHeaderDefect( "invalid repeated '.'")) obs_local_part.append(DOT) last_non_ws_was_dot = True value = value[1:] continue elif value[0]=='\\': obs_local_part.append(ValueTerminal(value[0], 'misplaced-special')) value = value[1:] obs_local_part.defects.append(errors.InvalidHeaderDefect( "'\\' character outside of quoted-string/ccontent")) last_non_ws_was_dot = False continue if obs_local_part and obs_local_part[-1].token_type != 'dot': obs_local_part.defects.append(errors.InvalidHeaderDefect( "missing '.' between words")) try: token, value = get_word(value) last_non_ws_was_dot = False except errors.HeaderParseError: if value[0] not in CFWS_LEADER: raise token, value = get_cfws(value) obs_local_part.append(token) if (obs_local_part[0].token_type == 'dot' or obs_local_part[0].token_type=='cfws' and obs_local_part[1].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid leading '.' in local part")) if (obs_local_part[-1].token_type == 'dot' or obs_local_part[-1].token_type=='cfws' and obs_local_part[-2].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid trailing '.' in local part")) if obs_local_part.defects: obs_local_part.token_type = 'invalid-obs-local-part' return obs_local_part, value def get_dtext(value): r""" dtext = <printable ascii except \ [ ]> / obs-dtext obs-dtext = obs-NO-WS-CTL / quoted-pair We allow anything except the excluded characters, but if we find any ASCII other than the RFC defined printable ASCII, a NonPrintableDefect is added to the token's defects list. Quoted pairs are converted to their unquoted values, so what is returned is a ptext token, in this case a ValueTerminal. If there were quoted-printables, an ObsoleteHeaderDefect is added to the returned token's defect list. """ ptext, value, had_qp = _get_ptext_to_endchars(value, '[]') ptext = ValueTerminal(ptext, 'ptext') if had_qp: ptext.defects.append(errors.ObsoleteHeaderDefect( "quoted printable found in domain-literal")) _validate_xtext(ptext) return ptext, value def _check_for_early_dl_end(value, domain_literal): if value: return False domain_literal.append(errors.InvalidHeaderDefect( "end of input inside domain-literal")) domain_literal.append(ValueTerminal(']', 'domain-literal-end')) return True def get_domain_literal(value): """ domain-literal = [CFWS] "[" ([FWS] dtext) [FWS] "]" [CFWS] """ domain_literal = DomainLiteral() if value[0] in CFWS_LEADER: token, value = get_cfws(value) domain_literal.append(token) if not value: raise errors.HeaderParseError("expected domain-literal") if value[0] != '[': raise errors.HeaderParseError("expected '[' at start of domain-literal " "but found '{}'".format(value)) value = value[1:] if _check_for_early_dl_end(value, domain_literal): return domain_literal, value domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if value[0] in WSP: token, value = get_fws(value) domain_literal.append(token) token, value = get_dtext(value) domain_literal.append(token) if _check_for_early_dl_end(value, domain_literal): return domain_literal, value if value[0] in WSP: token, value = get_fws(value) domain_literal.append(token) if _check_for_early_dl_end(value, domain_literal): return domain_literal, value if value[0] != ']': raise errors.HeaderParseError("expected ']' at end of domain-literal " "but found '{}'".format(value)) domain_literal.append(ValueTerminal(']', 'domain-literal-end')) value = value[1:] if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) domain_literal.append(token) return domain_literal, value def get_domain(value): """ domain = dot-atom / domain-literal / obs-domain obs-domain = atom ("." atom)) """ domain = Domain() leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( "expected domain but found '{}'".format(value)) if value[0] == '[': token, value = get_domain_literal(value) if leader is not None: token[:0] = [leader] domain.append(token) return domain, value try: token, value = get_dot_atom(value) except errors.HeaderParseError: token, value = get_atom(value) if value and value[0] == '@': raise errors.HeaderParseError('Invalid Domain') if leader is not None: token[:0] = [leader] domain.append(token) if value and value[0] == '.': domain.defects.append(errors.ObsoleteHeaderDefect( "domain is not a dot-atom (contains CFWS)")) if domain[0].token_type == 'dot-atom': domain[:] = domain[0] while value and value[0] == '.': domain.append(DOT) token, value = get_atom(value[1:]) domain.append(token) return domain, value def get_addr_spec(value): """ addr-spec = local-part "@" domain """ addr_spec = AddrSpec() token, value = get_local_part(value) addr_spec.append(token) if not value or value[0] != '@': addr_spec.defects.append(errors.InvalidHeaderDefect( "add-spec local part with no domain")) return addr_spec, value addr_spec.append(ValueTerminal('@', 'address-at-symbol')) token, value = get_domain(value[1:]) addr_spec.append(token) return addr_spec, value def get_obs_route(value): """ obs-route = obs-domain-list ":" obs-domain-list = (CFWS / ",") "@" domain ("," [CFWS] ["@" domain]) Returns an obs-route token with the appropriate sub-tokens (that is, there is no obs-domain-list in the parse tree). """ obs_route = ObsRoute() while value and (value[0]==',' or value[0] in CFWS_LEADER): if value[0] in CFWS_LEADER: token, value = get_cfws(value) obs_route.append(token) elif value[0] == ',': obs_route.append(ListSeparator) value = value[1:] if not value or value[0] != '@': raise errors.HeaderParseError( "expected obs-route domain but found '{}'".format(value)) obs_route.append(RouteComponentMarker) token, value = get_domain(value[1:]) obs_route.append(token) while value and value[0]==',': obs_route.append(ListSeparator) value = value[1:] if not value: break if value[0] in CFWS_LEADER: token, value = get_cfws(value) obs_route.append(token) if value[0] == '@': obs_route.append(RouteComponentMarker) token, value = get_domain(value[1:]) obs_route.append(token) if not value: raise errors.HeaderParseError("end of header while parsing obs-route") if value[0] != ':': raise errors.HeaderParseError( "expected ':' marking end of " "obs-route but found '{}'".format(value)) obs_route.append(ValueTerminal(':', 'end-of-obs-route-marker')) return obs_route, value[1:] def get_angle_addr(value): """ angle-addr = [CFWS] "<" addr-spec ">" [CFWS] / obs-angle-addr obs-angle-addr = [CFWS] "<" obs-route addr-spec ">" [CFWS] """ angle_addr = AngleAddr() if value[0] in CFWS_LEADER: token, value = get_cfws(value) angle_addr.append(token) if not value or value[0] != '<': raise errors.HeaderParseError( "expected angle-addr but found '{}'".format(value)) angle_addr.append(ValueTerminal('<', 'angle-addr-start')) value = value[1:] # Although it is not legal per RFC5322, SMTP uses '<>' in certain # circumstances. if value[0] == '>': angle_addr.append(ValueTerminal('>', 'angle-addr-end')) angle_addr.defects.append(errors.InvalidHeaderDefect( "null addr-spec in angle-addr")) value = value[1:] return angle_addr, value try: token, value = get_addr_spec(value) except errors.HeaderParseError: try: token, value = get_obs_route(value) angle_addr.defects.append(errors.ObsoleteHeaderDefect( "obsolete route specification in angle-addr")) except errors.HeaderParseError: raise errors.HeaderParseError( "expected addr-spec or obs-route but found '{}'".format(value)) angle_addr.append(token) token, value = get_addr_spec(value) angle_addr.append(token) if value and value[0] == '>': value = value[1:] else: angle_addr.defects.append(errors.InvalidHeaderDefect( "missing trailing '>' on angle-addr")) angle_addr.append(ValueTerminal('>', 'angle-addr-end')) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) angle_addr.append(token) return angle_addr, value def get_display_name(value): """ display-name = phrase Because this is simply a name-rule, we don't return a display-name token containing a phrase, but rather a display-name token with the content of the phrase. """ display_name = DisplayName() token, value = get_phrase(value) display_name.extend(token[:]) display_name.defects = token.defects[:] return display_name, value def get_name_addr(value): """ name-addr = [display-name] angle-addr """ name_addr = NameAddr() # Both the optional display name and the angle-addr can start with cfws. leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( "expected name-addr but found '{}'".format(leader)) if value[0] != '<': if value[0] in PHRASE_ENDS: raise errors.HeaderParseError( "expected name-addr but found '{}'".format(value)) token, value = get_display_name(value) if not value: raise errors.HeaderParseError( "expected name-addr but found '{}'".format(token)) if leader is not None: token[0][:0] = [leader] leader = None name_addr.append(token) token, value = get_angle_addr(value) if leader is not None: token[:0] = [leader] name_addr.append(token) return name_addr, value def get_mailbox(value): """ mailbox = name-addr / addr-spec """ # The only way to figure out if we are dealing with a name-addr or an # addr-spec is to try parsing each one. mailbox = Mailbox() try: token, value = get_name_addr(value) except errors.HeaderParseError: try: token, value = get_addr_spec(value) except errors.HeaderParseError: raise errors.HeaderParseError( "expected mailbox but found '{}'".format(value)) if any(isinstance(x, errors.InvalidHeaderDefect) for x in token.all_defects): mailbox.token_type = 'invalid-mailbox' mailbox.append(token) return mailbox, value def get_invalid_mailbox(value, endchars): """ Read everything up to one of the chars in endchars. This is outside the formal grammar. The InvalidMailbox TokenList that is returned acts like a Mailbox, but the data attributes are None. """ invalid_mailbox = InvalidMailbox() while value and value[0] not in endchars: if value[0] in PHRASE_ENDS: invalid_mailbox.append(ValueTerminal(value[0], 'misplaced-special')) value = value[1:] else: token, value = get_phrase(value) invalid_mailbox.append(token) return invalid_mailbox, value def get_mailbox_list(value): """ mailbox-list = (mailbox ("," mailbox)) / obs-mbox-list obs-mbox-list = ([CFWS] ",") mailbox ("," [mailbox / CFWS]) For this routine we go outside the formal grammar in order to improve error handling. We recognize the end of the mailbox list only at the end of the value or at a ';' (the group terminator). This is so that we can turn invalid mailboxes into InvalidMailbox tokens and continue parsing any remaining valid mailboxes. We also allow all mailbox entries to be null, and this condition is handled appropriately at a higher level. """ mailbox_list = MailboxList() while value and value[0] != ';': try: token, value = get_mailbox(value) mailbox_list.append(token) except errors.HeaderParseError: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value or value[0] in ',;': mailbox_list.append(leader) mailbox_list.defects.append(errors.ObsoleteHeaderDefect( "empty element in mailbox-list")) else: token, value = get_invalid_mailbox(value, ',;') if leader is not None: token[:0] = [leader] mailbox_list.append(token) mailbox_list.defects.append(errors.InvalidHeaderDefect( "invalid mailbox in mailbox-list")) elif value[0] == ',': mailbox_list.defects.append(errors.ObsoleteHeaderDefect( "empty element in mailbox-list")) else: token, value = get_invalid_mailbox(value, ',;') if leader is not None: token[:0] = [leader] mailbox_list.append(token) mailbox_list.defects.append(errors.InvalidHeaderDefect( "invalid mailbox in mailbox-list")) if value and value[0] not in ',;': # Crap after mailbox; treat it as an invalid mailbox. # The mailbox info will still be available. mailbox = mailbox_list[-1] mailbox.token_type = 'invalid-mailbox' token, value = get_invalid_mailbox(value, ',;') mailbox.extend(token) mailbox_list.defects.append(errors.InvalidHeaderDefect( "invalid mailbox in mailbox-list")) if value and value[0] == ',': mailbox_list.append(ListSeparator) value = value[1:] return mailbox_list, value def get_group_list(value): """ group-list = mailbox-list / CFWS / obs-group-list obs-group-list = 1([CFWS] ",") [CFWS] """ group_list = GroupList() if not value: group_list.defects.append(errors.InvalidHeaderDefect( "end of header before group-list")) return group_list, value leader = None if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: # This should never happen in email parsing, since CFWS-only is a # legal alternative to group-list in a group, which is the only # place group-list appears. group_list.defects.append(errors.InvalidHeaderDefect( "end of header in group-list")) group_list.append(leader) return group_list, value if value[0] == ';': group_list.append(leader) return group_list, value token, value = get_mailbox_list(value) if len(token.all_mailboxes)==0: if leader is not None: group_list.append(leader) group_list.extend(token) group_list.defects.append(errors.ObsoleteHeaderDefect( "group-list with empty entries")) return group_list, value if leader is not None: token[:0] = [leader] group_list.append(token) return group_list, value def get_group(value): """ group = display-name ":" [group-list] ";" [CFWS] """ group = Group() token, value = get_display_name(value) if not value or value[0] != ':': raise errors.HeaderParseError("expected ':' at end of group " "display name but found '{}'".format(value)) group.append(token) group.append(ValueTerminal(':', 'group-display-name-terminator')) value = value[1:] if value and value[0] == ';': group.append(ValueTerminal(';', 'group-terminator')) return group, value[1:] token, value = get_group_list(value) group.append(token) if not value: group.defects.append(errors.InvalidHeaderDefect( "end of header in group")) elif value[0] != ';': raise errors.HeaderParseError( "expected ';' at end of group but found {}".format(value)) group.append(ValueTerminal(';', 'group-terminator')) value = value[1:] if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) group.append(token) return group, value def get_address(value): """ address = mailbox / group Note that counter-intuitively, an address can be either a single address or a list of addresses (a group). This is why the returned Address object has a 'mailboxes' attribute which treats a single address as a list of length one. When you need to differentiate between to two cases, extract the single element, which is either a mailbox or a group token. """ # The formal grammar isn't very helpful when parsing an address. mailbox # and group, especially when allowing for obsolete forms, start off very # similarly. It is only when you reach one of @, <, or : that you know # what you've got. So, we try each one in turn, starting with the more # likely of the two. We could perhaps make this more efficient by looking # for a phrase and then branching based on the next character, but that # would be a premature optimization. address = Address() try: token, value = get_group(value) except errors.HeaderParseError: try: token, value = get_mailbox(value) except errors.HeaderParseError: raise errors.HeaderParseError( "expected address but found '{}'".format(value)) address.append(token) return address, value def get_address_list(value): """ address_list = (address ("," address)) / obs-addr-list obs-addr-list = ([CFWS] ",") address ("," [address / CFWS]) We depart from the formal grammar here by continuing to parse until the end of the input, assuming the input to be entirely composed of an address-list. This is always true in email parsing, and allows us to skip invalid addresses to parse additional valid ones. """ address_list = AddressList() while value: try: token, value = get_address(value) address_list.append(token) except errors.HeaderParseError as err: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value or value[0] == ',': address_list.append(leader) address_list.defects.append(errors.ObsoleteHeaderDefect( "address-list entry with no content")) else: token, value = get_invalid_mailbox(value, ',') if leader is not None: token[:0] = [leader] address_list.append(Address([token])) address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) elif value[0] == ',': address_list.defects.append(errors.ObsoleteHeaderDefect( "empty element in address-list")) else: token, value = get_invalid_mailbox(value, ',') if leader is not None: token[:0] = [leader] address_list.append(Address([token])) address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value and value[0] != ',': # Crap after address; treat it as an invalid mailbox. # The mailbox info will still be available. mailbox = address_list[-1][0] mailbox.token_type = 'invalid-mailbox' token, value = get_invalid_mailbox(value, ',') mailbox.extend(token) address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value: # Must be a , at this point. address_list.append(ValueTerminal(',', 'list-separator')) value = value[1:] return address_list, value # # XXX: As I begin to add additional header parsers, I'm realizing we probably # have two level of parser routines: the get_XXX methods that get a token in # the grammar, and parse_XXX methods that parse an entire field value. So # get_address_list above should really be a parse_ method, as probably should # be get_unstructured. # def parse_mime_version(value): """ mime-version = [CFWS] 1digit [CFWS] "." [CFWS] 1digit [CFWS] """ # The [CFWS] is implicit in the RFC 2045 BNF. # XXX: This routine is a bit verbose, should factor out a get_int method. mime_version = MIMEVersion() if not value: mime_version.defects.append(errors.HeaderMissingRequiredValue( "Missing MIME version number (eg: 1.0)")) return mime_version if value[0] in CFWS_LEADER: token, value = get_cfws(value) mime_version.append(token) if not value: mime_version.defects.append(errors.HeaderMissingRequiredValue( "Expected MIME version number but found only CFWS")) digits = '' while value and value[0] != '.' and value[0] not in CFWS_LEADER: digits += value[0] value = value[1:] if not digits.isdigit(): mime_version.defects.append(errors.InvalidHeaderDefect( "Expected MIME major version number but found {!r}".format(digits))) mime_version.append(ValueTerminal(digits, 'xtext')) else: mime_version.major = int(digits) mime_version.append(ValueTerminal(digits, 'digits')) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) mime_version.append(token) if not value or value[0] != '.': if mime_version.major is not None: mime_version.defects.append(errors.InvalidHeaderDefect( "Incomplete MIME version; found only major number")) if value: mime_version.append(ValueTerminal(value, 'xtext')) return mime_version mime_version.append(ValueTerminal('.', 'version-separator')) value = value[1:] if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) mime_version.append(token) if not value: if mime_version.major is not None: mime_version.defects.append(errors.InvalidHeaderDefect( "Incomplete MIME version; found only major number")) return mime_version digits = '' while value and value[0] not in CFWS_LEADER: digits += value[0] value = value[1:] if not digits.isdigit(): mime_version.defects.append(errors.InvalidHeaderDefect( "Expected MIME minor version number but found {!r}".format(digits))) mime_version.append(ValueTerminal(digits, 'xtext')) else: mime_version.minor = int(digits) mime_version.append(ValueTerminal(digits, 'digits')) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) mime_version.append(token) if value: mime_version.defects.append(errors.InvalidHeaderDefect( "Excess non-CFWS text after MIME version")) mime_version.append(ValueTerminal(value, 'xtext')) return mime_version def get_invalid_parameter(value): """ Read everything up to the next ';'. This is outside the formal grammar. The InvalidParameter TokenList that is returned acts like a Parameter, but the data attributes are None. """ invalid_parameter = InvalidParameter() while value and value[0] != ';': if value[0] in PHRASE_ENDS: invalid_parameter.append(ValueTerminal(value[0], 'misplaced-special')) value = value[1:] else: token, value = get_phrase(value) invalid_parameter.append(token) return invalid_parameter, value def get_ttext(value): """ttext = <matches _ttext_matcher> We allow any non-TOKEN_ENDS in ttext, but add defects to the token's defects list if we find non-ttext characters. We also register defects for any non-printables even though the RFC doesn't exclude all of them, because we follow the spirit of RFC 5322. """ m = _non_token_end_matcher(value) if not m: raise errors.HeaderParseError( "expected ttext but found '{}'".format(value)) ttext = m.group() value = value[len(ttext):] ttext = ValueTerminal(ttext, 'ttext') _validate_xtext(ttext) return ttext, value def get_token(value): """token = [CFWS] 1ttext [CFWS] The RFC equivalent of ttext is any US-ASCII chars except space, ctls, or tspecials. We also exclude tabs even though the RFC doesn't. The RFC implies the CFWS but is not explicit about it in the BNF. """ mtoken = Token() if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) mtoken.append(token) if value and value[0] in TOKEN_ENDS: raise errors.HeaderParseError( "expected token but found '{}'".format(value)) token, value = get_ttext(value) mtoken.append(token) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) mtoken.append(token) return mtoken, value def get_attrtext(value): """attrtext = 1(any non-ATTRIBUTE_ENDS character) We allow any non-ATTRIBUTE_ENDS in attrtext, but add defects to the token's defects list if we find non-attrtext characters. We also register defects for any non-printables even though the RFC doesn't exclude all of them, because we follow the spirit of RFC 5322. """ m = _non_attribute_end_matcher(value) if not m: raise errors.HeaderParseError( "expected attrtext but found {!r}".format(value)) attrtext = m.group() value = value[len(attrtext):] attrtext = ValueTerminal(attrtext, 'attrtext') _validate_xtext(attrtext) return attrtext, value def get_attribute(value): """ [CFWS] 1attrtext [CFWS] This version of the BNF makes the CFWS explicit, and as usual we use a value terminal for the actual run of characters. The RFC equivalent of attrtext is the token characters, with the subtraction of '', "'", and '%'. We include tab in the excluded set just as we do for token. """ attribute = Attribute() if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) attribute.append(token) if value and value[0] in ATTRIBUTE_ENDS: raise errors.HeaderParseError( "expected token but found '{}'".format(value)) token, value = get_attrtext(value) attribute.append(token) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) attribute.append(token) return attribute, value def get_extended_attrtext(value): """attrtext = 1(any non-ATTRIBUTE_ENDS character plus '%') This is a special parsing routine so that we get a value that includes % escapes as a single string (which we decode as a single string later). """ m = _non_extended_attribute_end_matcher(value) if not m: raise errors.HeaderParseError( "expected extended attrtext but found {!r}".format(value)) attrtext = m.group() value = value[len(attrtext):] attrtext = ValueTerminal(attrtext, 'extended-attrtext') _validate_xtext(attrtext) return attrtext, value def get_extended_attribute(value): """ [CFWS] 1extended_attrtext [CFWS] This is like the non-extended version except we allow % characters, so that we can pick up an encoded value as a single string. """ # XXX: should we have an ExtendedAttribute TokenList? attribute = Attribute() if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) attribute.append(token) if value and value[0] in EXTENDED_ATTRIBUTE_ENDS: raise errors.HeaderParseError( "expected token but found '{}'".format(value)) token, value = get_extended_attrtext(value) attribute.append(token) if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) attribute.append(token) return attribute, value def get_section(value): """ '' digits The formal BNF is more complicated because leading 0s are not allowed. We check for that and add a defect. We also assume no CFWS is allowed between the '' and the digits, though the RFC is not crystal clear on that. The caller should already have dealt with leading CFWS. """ section = Section() if not value or value[0] != '': raise errors.HeaderParseError("Expected section but found {}".format( value)) section.append(ValueTerminal('', 'section-marker')) value = value[1:] if not value or not value[0].isdigit(): raise errors.HeaderParseError("Expected section number but " "found {}".format(value)) digits = '' while value and value[0].isdigit(): digits += value[0] value = value[1:] if digits[0] == '0' and digits != '0': section.defects.append(errors.InvalidHeaderError( "section number has an invalid leading 0")) section.number = int(digits) section.append(ValueTerminal(digits, 'digits')) return section, value def get_value(value): """ quoted-string / attribute """ v = Value() if not value: raise errors.HeaderParseError("Expected value but found end of string") leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError("Expected value but found " "only {}".format(leader)) if value[0] == '"': token, value = get_quoted_string(value) else: token, value = get_extended_attribute(value) if leader is not None: token[:0] = [leader] v.append(token) return v, value def get_parameter(value): """ attribute [section] [""] [CFWS] "=" value The CFWS is implied by the RFC but not made explicit in the BNF. This simplified form of the BNF from the RFC is made to conform with the RFC BNF through some extra checks. We do it this way because it makes both error recovery and working with the resulting parse tree easier. """ # It is possible CFWS would also be implicitly allowed between the section # and the 'extended-attribute' marker (the '') , but we've never seen that # in the wild and we will therefore ignore the possibility. param = Parameter() token, value = get_attribute(value) param.append(token) if not value or value[0] == ';': param.defects.append(errors.InvalidHeaderDefect("Parameter contains " "name ({}) but no value".format(token))) return param, value if value[0] == '': try: token, value = get_section(value) param.sectioned = True param.append(token) except errors.HeaderParseError: pass if not value: raise errors.HeaderParseError("Incomplete parameter") if value[0] == '': param.append(ValueTerminal('', 'extended-parameter-marker')) value = value[1:] param.extended = True if value[0] != '=': raise errors.HeaderParseError("Parameter not followed by '='") param.append(ValueTerminal('=', 'parameter-separator')) value = value[1:] leader = None if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) param.append(token) remainder = None appendto = param if param.extended and value and value[0] == '"': # Now for some serious hackery to handle the common invalid case of # double quotes around an extended value. We also accept (with defect) # a value marked as encoded that isn't really. qstring, remainder = get_quoted_string(value) inner_value = qstring.stripped_value semi_valid = False if param.section_number == 0: if inner_value and inner_value[0] == "'": semi_valid = True else: token, rest = get_attrtext(inner_value) if rest and rest[0] == "'": semi_valid = True else: try: token, rest = get_extended_attrtext(inner_value) except: pass else: if not rest: semi_valid = True if semi_valid: param.defects.append(errors.InvalidHeaderDefect( "Quoted string value for extended parameter is invalid")) param.append(qstring) for t in qstring: if t.token_type == 'bare-quoted-string': t[:] = [] appendto = t break value = inner_value else: remainder = None param.defects.append(errors.InvalidHeaderDefect( "Parameter marked as extended but appears to have a " "quoted string value that is non-encoded")) if value and value[0] == "'": token = None else: token, value = get_value(value) if not param.extended or param.section_number > 0: if not value or value[0] != "'": appendto.append(token) if remainder is not None: assert not value, value value = remainder return param, value param.defects.append(errors.InvalidHeaderDefect( "Apparent initial-extended-value but attribute " "was not marked as extended or was not initial section")) if not value: # Assume the charset/lang is missing and the token is the value. param.defects.append(errors.InvalidHeaderDefect( "Missing required charset/lang delimiters")) appendto.append(token) if remainder is None: return param, value else: if token is not None: for t in token: if t.token_type == 'extended-attrtext': break t.token_type == 'attrtext' appendto.append(t) param.charset = t.value if value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {!r}".format(value)) appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if value and value[0] != "'": token, value = get_attrtext(value) appendto.append(token) param.lang = token.value if not value or value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {}".format(value)) appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if remainder is not None: # Treat the rest of value as bare quoted string content. v = Value() while value: if value[0] in WSP: token, value = get_fws(value) elif value[0] == '"': token = ValueTerminal('"', 'DQUOTE') value = value[1:] else: token, value = get_qcontent(value) v.append(token) token = v else: token, value = get_value(value) appendto.append(token) if remainder is not None: assert not value, value value = remainder return param, value def parse_mime_parameters(value): """ parameter ( ";" parameter ) That BNF is meant to indicate this routine should only be called after finding and handling the leading ';'. There is no corresponding rule in the formal RFC grammar, but it is more convenient for us for the set of parameters to be treated as its own TokenList. This is 'parse' routine because it consumes the reminaing value, but it would never be called to parse a full header. Instead it is called to parse everything after the non-parameter value of a specific MIME header. """ mime_parameters = MimeParameters() while value: try: token, value = get_parameter(value) mime_parameters.append(token) except errors.HeaderParseError as err: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: mime_parameters.append(leader) return mime_parameters if value[0] == ';': if leader is not None: mime_parameters.append(leader) mime_parameters.defects.append(errors.InvalidHeaderDefect( "parameter entry with no content")) else: token, value = get_invalid_parameter(value) if leader: token[:0] = [leader] mime_parameters.append(token) mime_parameters.defects.append(errors.InvalidHeaderDefect( "invalid parameter {!r}".format(token))) if value and value[0] != ';': # Junk after the otherwise valid parameter. Mark it as # invalid, but it will have a value. param = mime_parameters[-1] param.token_type = 'invalid-parameter' token, value = get_invalid_parameter(value) param.extend(token) mime_parameters.defects.append(errors.InvalidHeaderDefect( "parameter with invalid trailing text {!r}".format(token))) if value: # Must be a ';' at this point. mime_parameters.append(ValueTerminal(';', 'parameter-separator')) value = value[1:] return mime_parameters def _find_mime_parameters(tokenlist, value): """Do our best to find the parameters in an invalid MIME header """ while value and value[0] != ';': if value[0] in PHRASE_ENDS: tokenlist.append(ValueTerminal(value[0], 'misplaced-special')) value = value[1:] else: token, value = get_phrase(value) tokenlist.append(token) if not value: return tokenlist.append(ValueTerminal(';', 'parameter-separator')) tokenlist.append(parse_mime_parameters(value[1:])) def parse_content_type_header(value): """ maintype "/" subtype ( ";" parameter ) The maintype and substype are tokens. Theoretically they could be checked against the official IANA list + x-token, but we don't do that. """ ctype = ContentType() recover = False if not value: ctype.defects.append(errors.HeaderMissingRequiredValue( "Missing content type specification")) return ctype try: token, value = get_token(value) except errors.HeaderParseError: ctype.defects.append(errors.InvalidHeaderDefect( "Expected content maintype but found {!r}".format(value))) _find_mime_parameters(ctype, value) return ctype ctype.append(token) # XXX: If we really want to follow the formal grammar we should make # mantype and subtype specialized TokenLists here. Probably not worth it. if not value or value[0] != '/': ctype.defects.append(errors.InvalidHeaderDefect( "Invalid content type")) if value: _find_mime_parameters(ctype, value) return ctype ctype.maintype = token.value.strip().lower() ctype.append(ValueTerminal('/', 'content-type-separator')) value = value[1:] try: token, value = get_token(value) except errors.HeaderParseError: ctype.defects.append(errors.InvalidHeaderDefect( "Expected content subtype but found {!r}".format(value))) _find_mime_parameters(ctype, value) return ctype ctype.append(token) ctype.subtype = token.value.strip().lower() if not value: return ctype if value[0] != ';': ctype.defects.append(errors.InvalidHeaderDefect( "Only parameters are valid after content type, but " "found {!r}".format(value))) # The RFC requires that a syntactically invalid content-type be treated # as text/plain. Perhaps we should postel this, but we should probably # only do that if we were checking the subtype value against IANA. del ctype.maintype, ctype.subtype _find_mime_parameters(ctype, value) return ctype ctype.append(ValueTerminal(';', 'parameter-separator')) ctype.append(parse_mime_parameters(value[1:])) return ctype def parse_content_disposition_header(value): """ disposition-type ( ";" parameter ) """ disp_header = ContentDisposition() if not value: disp_header.defects.append(errors.HeaderMissingRequiredValue( "Missing content disposition")) return disp_header try: token, value = get_token(value) except errors.HeaderParseError: disp_header.defects.append(errors.InvalidHeaderDefect( "Expected content disposition but found {!r}".format(value))) _find_mime_parameters(disp_header, value) return disp_header disp_header.append(token) disp_header.content_disposition = token.value.strip().lower() if not value: return disp_header if value[0] != ';': disp_header.defects.append(errors.InvalidHeaderDefect( "Only parameters are valid after content disposition, but " "found {!r}".format(value))) _find_mime_parameters(disp_header, value) return disp_header disp_header.append(ValueTerminal(';', 'parameter-separator')) disp_header.append(parse_mime_parameters(value[1:])) return disp_header def parse_content_transfer_encoding_header(value): """ mechanism """ # We should probably validate the values, since the list is fixed. cte_header = ContentTransferEncoding() if not value: cte_header.defects.append(errors.HeaderMissingRequiredValue( "Missing content transfer encoding")) return cte_header try: token, value = get_token(value) except errors.HeaderParseError: cte_header.defects.append(errors.InvalidHeaderDefect( "Expected content transfer encoding but found {!r}".format(value))) else: cte_header.append(token) cte_header.cte = token.value.strip().lower() if not value: return cte_header while value: cte_header.defects.append(errors.InvalidHeaderDefect( "Extra text after content transfer encoding")) if value[0] in PHRASE_ENDS: cte_header.append(ValueTerminal(value[0], 'misplaced-special')) value = value[1:] else: token, value = get_phrase(value) cte_header.append(token) return cte_header # # Header folding # # Header folding is complex, with lots of rules and corner cases. The # following code does its best to obey the rules and handle the corner # cases, but you can be sure there are few bugs:) # # This folder generally canonicalizes as it goes, preferring the stringified # version of each token. The tokens contain information that supports the # folder, including which tokens can be encoded in which ways. # # Folded text is accumulated in a simple list of strings ('lines'), each # one of which should be less than policy.max_line_length ('maxlen'). # def _steal_trailing_WSP_if_exists(lines): wsp = '' if lines and lines[-1] and lines[-1][-1] in WSP: wsp = lines[-1][-1] lines[-1] = lines[-1][:-1] return wsp def _refold_parse_tree(parse_tree, , policy): """Return string of contents of parse_tree folded according to RFC rules. """ # max_line_length 0/None means no limit, ie: infinitely long. maxlen = policy.max_line_length or sys.maxsize encoding = 'utf-8' if policy.utf8 else 'us-ascii' lines = [''] last_ew = None wrap_as_ew_blocked = 0 want_encoding = False end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') parts = list(parse_tree) while parts: part = parts.pop(0) if part is end_ew_not_allowed: wrap_as_ew_blocked -= 1 continue tstr = str(part) if part.token_type == 'ptext' and set(tstr) & SPECIALS: # Encode if tstr contains special characters. want_encoding = True try: tstr.encode(encoding) charset = encoding except UnicodeEncodeError: if any(isinstance(x, errors.UndecodableBytesDefect) for x in part.all_defects): charset = 'unknown-8bit' else: # If policy.utf8 is false this should really be taken from a # 'charset' property on the policy. charset = 'utf-8' want_encoding = True if part.token_type == 'mime-parameters': # Mime parameter folding (using RFC2231) is extra special. _fold_mime_parameters(part, lines, maxlen, encoding) continue if want_encoding and not wrap_as_ew_blocked: if not part.as_ew_allowed: want_encoding = False last_ew = None if part.syntactic_break: encoded_part = part.fold(policy=policy)[:-len(policy.linesep)] if policy.linesep not in encoded_part: # It fits on a single line if len(encoded_part) > maxlen - len(lines[-1]): # But not on this one, so start a new one. newline = _steal_trailing_WSP_if_exists(lines) # XXX what if encoded_part has no leading FWS? lines.append(newline) lines[-1] += encoded_part continue # Either this is not a major syntactic break, so we don't # want it on a line by itself even if it fits, or it # doesn't fit on a line by itself. Either way, fall through # to unpacking the subparts and wrapping them. if not hasattr(part, 'encode'): # It's not a Terminal, do each piece individually. parts = list(part) + parts else: # It's a terminal, wrap it as an encoded word, possibly # combining it with previously encoded words if allowed. last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, part.ew_combine_allowed, charset) want_encoding = False continue if len(tstr) <= maxlen - len(lines[-1]): lines[-1] += tstr continue # This part is too long to fit. The RFC wants us to break at # "major syntactic breaks", so unless we don't consider this # to be one, check if it will fit on the next line by itself. if (part.syntactic_break and len(tstr) + 1 <= maxlen): newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): lines.append(newline + tstr) last_ew = None continue if not hasattr(part, 'encode'): # It's not a terminal, try folding the subparts. newparts = list(part) if not part.as_ew_allowed: wrap_as_ew_blocked += 1 newparts.append(end_ew_not_allowed) parts = newparts + parts continue if part.as_ew_allowed and not wrap_as_ew_blocked: # It doesn't need CTE encoding, but encode it anyway so we can # wrap it. parts.insert(0, part) want_encoding = True continue # We can't figure out how to wrap, it, so give up. newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): lines.append(newline + tstr) else: # We can't fold it onto the next line either... lines[-1] += tstr return policy.linesep.join(lines) + policy.linesep def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): """Fold string to_encode into lines as encoded word, combining if allowed. Return the new value for last_ew, or None if ew_combine_allowed is False. If there is already an encoded word in the last line of lines (indicated by a non-None value for last_ew) and ew_combine_allowed is true, decode the existing ew, combine it with to_encode, and re-encode. Otherwise, encode to_encode. In either case, split to_encode as necessary so that the encoded segments fit within maxlen. """ if last_ew is not None and ew_combine_allowed: to_encode = str( get_unstructured(lines[-1][last_ew:] + to_encode)) lines[-1] = lines[-1][:last_ew] if to_encode[0] in WSP: # We're joining this to non-encoded text, so don't encode # the leading blank. leading_wsp = to_encode[0] to_encode = to_encode[1:] if (len(lines[-1]) == maxlen): lines.append(_steal_trailing_WSP_if_exists(lines)) lines[-1] += leading_wsp trailing_wsp = '' if to_encode[-1] in WSP: # Likewise for the trailing space. trailing_wsp = to_encode[-1] to_encode = to_encode[:-1] new_last_ew = len(lines[-1]) if last_ew is None else last_ew encode_as = 'utf-8' if charset == 'us-ascii' else charset # The RFC2047 chrome takes up 7 characters plus the length # of the charset name. chrome_len = len(encode_as) + 7 if (chrome_len + 1) >= maxlen: raise errors.HeaderParseError( "max_line_length is too small to fit an encoded word") while to_encode: remaining_space = maxlen - len(lines[-1]) text_space = remaining_space - chrome_len if text_space <= 0: lines.append(' ') continue to_encode_word = to_encode[:text_space] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space while excess > 0: # Since the chunk to encode is guaranteed to fit into less than 100 characters, # shrinking it by one at a time shouldn't take long. to_encode_word = to_encode_word[:-1] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space lines[-1] += encoded_word to_encode = to_encode[len(to_encode_word):] if to_encode: lines.append(' ') new_last_ew = len(lines[-1]) lines[-1] += trailing_wsp return new_last_ew if ew_combine_allowed else None def _fold_mime_parameters(part, lines, maxlen, encoding): """Fold TokenList 'part' into the 'lines' list as mime parameters. Using the decoded list of parameters and values, format them according to the RFC rules, including using RFC2231 encoding if the value cannot be expressed in 'encoding' and/or the parameter+value is too long to fit within 'maxlen'. """ # Special case for RFC2231 encoding: start from decoded values and use # RFC2231 encoding iff needed. # # Note that the 1 and 2s being added to the length calculations are # accounting for the possibly-needed spaces and semicolons we'll be adding. # for name, value in part.params: # XXX What if this ';' puts us over maxlen the first time through the # loop? We should split the header value onto a newline in that case, # but to do that we need to recognize the need earlier or reparse the # header, so I'm going to ignore that bug for now. It'll only put us # one character over. if not lines[-1].rstrip().endswith(';'): lines[-1] += ';' charset = encoding error_handler = 'strict' try: value.encode(encoding) encoding_required = False except UnicodeEncodeError: encoding_required = True if utils._has_surrogates(value): charset = 'unknown-8bit' error_handler = 'surrogateescape' else: charset = 'utf-8' if encoding_required: encoded_value = urllib.parse.quote( value, safe='', errors=error_handler) tstr = "{}={}''{}".format(name, charset, encoded_value) else: tstr = '{}={}'.format(name, quote_string(value)) if len(lines[-1]) + len(tstr) + 1 < maxlen: lines[-1] = lines[-1] + ' ' + tstr continue elif len(tstr) + 2 <= maxlen: lines.append(' ' + tstr) continue # We need multiple sections. We are allowed to mix encoded and # non-encoded sections, but we aren't going to. We'll encode them all. section = 0 extra_chrome = charset + "''" while value: chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome) if maxlen <= chrome_len + 3: # We need room for the leading blank, the trailing semicolon, # and at least one character of the value. If we don't # have that, we'd be stuck, so in that case fall back to # the RFC standard width. maxlen = 78 splitpoint = maxchars = maxlen - chrome_len - 2 while True: partial = value[:splitpoint] encoded_value = urllib.parse.quote( partial, safe='', errors=error_handler) if len(encoded_value) <= maxchars: break splitpoint -= 1 lines.append(" {}{}={}{}".format( name, section, extra_chrome, encoded_value)) extra_chrome = '' section += 1 value = value[splitpoint:] if value: lines[-1] += ';'	99.92 KB	2024-04-17 17:55
📄 _parseaddr.py # Copyright (C) 2002-2007 Python Software Foundation # Contact: email-sig@python.org """Email address parsing code. Lifted directly from rfc822.py. This should eventually be rewritten. """ __all__ = [ 'mktime_tz', 'parsedate', 'parsedate_tz', 'quote', ] import time, calendar SPACE = ' ' EMPTYSTRING = '' COMMASPACE = ', ' # Parse a date field _monthnames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec', 'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december'] _daynames = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] # The timezone table does not include the military time zones defined # in RFC822, other than Z. According to RFC1123, the description in # RFC822 gets the signs wrong, so we can't rely on any such time # zones. RFC1123 recommends that numeric timezone indicators be used # instead of timezone names. _timezones = {'UT':0, 'UTC':0, 'GMT':0, 'Z':0, 'AST': -400, 'ADT': -300, # Atlantic (used in Canada) 'EST': -500, 'EDT': -400, # Eastern 'CST': -600, 'CDT': -500, # Central 'MST': -700, 'MDT': -600, # Mountain 'PST': -800, 'PDT': -700 # Pacific } def parsedate_tz(data): """Convert a date string to a time tuple. Accounts for military timezones. """ res = _parsedate_tz(data) if not res: return if res[9] is None: res[9] = 0 return tuple(res) def _parsedate_tz(data): """Convert date to extended time tuple. The last (additional) element is the time zone offset in seconds, except if the timezone was specified as -0000. In that case the last element is None. This indicates a UTC timestamp that explicitly declaims knowledge of the source timezone, as opposed to a +0000 timestamp that indicates the source timezone really was UTC. """ if not data: return data = data.split() if not data: # This happens for whitespace-only input. return None # The FWS after the comma after the day-of-week is optional, so search and # adjust for this. if data[0].endswith(',') or data[0].lower() in _daynames: # There's a dayname here. Skip it del data[0] else: i = data[0].rfind(',') if i >= 0: data[0] = data[0][i+1:] if len(data) == 3: # RFC 850 date, deprecated stuff = data[0].split('-') if len(stuff) == 3: data = stuff + data[1:] if len(data) == 4: s = data[3] i = s.find('+') if i == -1: i = s.find('-') if i > 0: data[3:] = [s[:i], s[i:]] else: data.append('') # Dummy tz if len(data) < 5: return None data = data[:5] [dd, mm, yy, tm, tz] = data mm = mm.lower() if mm not in _monthnames: dd, mm = mm, dd.lower() if mm not in _monthnames: return None mm = _monthnames.index(mm) + 1 if mm > 12: mm -= 12 if dd[-1] == ',': dd = dd[:-1] i = yy.find(':') if i > 0: yy, tm = tm, yy if yy[-1] == ',': yy = yy[:-1] if not yy[0].isdigit(): yy, tz = tz, yy if tm[-1] == ',': tm = tm[:-1] tm = tm.split(':') if len(tm) == 2: [thh, tmm] = tm tss = '0' elif len(tm) == 3: [thh, tmm, tss] = tm elif len(tm) == 1 and '.' in tm[0]: # Some non-compliant MUAs use '.' to separate time elements. tm = tm[0].split('.') if len(tm) == 2: [thh, tmm] = tm tss = 0 elif len(tm) == 3: [thh, tmm, tss] = tm else: return None try: yy = int(yy) dd = int(dd) thh = int(thh) tmm = int(tmm) tss = int(tss) except ValueError: return None # Check for a yy specified in two-digit format, then convert it to the # appropriate four-digit format, according to the POSIX standard. RFC 822 # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) # mandates a 4-digit yy. For more information, see the documentation for # the time module. if yy < 100: # The year is between 1969 and 1999 (inclusive). if yy > 68: yy += 1900 # The year is between 2000 and 2068 (inclusive). else: yy += 2000 tzoffset = None tz = tz.upper() if tz in _timezones: tzoffset = _timezones[tz] else: try: tzoffset = int(tz) except ValueError: pass if tzoffset==0 and tz.startswith('-'): tzoffset = None # Convert a timezone offset into seconds ; -0500 -> -18000 if tzoffset: if tzoffset < 0: tzsign = -1 tzoffset = -tzoffset else: tzsign = 1 tzoffset = tzsign * ( (tzoffset//100)3600 + (tzoffset % 100)60) # Daylight Saving Time flag is set to -1, since DST is unknown. return [yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset] def parsedate(data): """Convert a time string to a time tuple.""" t = parsedate_tz(data) if isinstance(t, tuple): return t[:9] else: return t def mktime_tz(data): """Turn a 10-tuple as returned by parsedate_tz() into a POSIX timestamp.""" if data[9] is None: # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: t = calendar.timegm(data) return t - data[9] def quote(str): """Prepare string to be used in a quoted string. Turns backslash and double quote characters into quoted pairs. These are the only characters that need to be quoted inside a quoted string. Does not add the surrounding double quotes. """ return str.replace('\\', '\\\\').replace('"', '\\"') class AddrlistClass: """Address parser class by Ben Escoto. To understand what this class does, it helps to have a copy of RFC 2822 in front of you. Note: this class interface is deprecated and may be removed in the future. Use email.utils.AddressList instead. """ def __init__(self, field): """Initialize a new instance. `field' is an unparsed address header field, containing one or more addresses. """ self.specials = '()<>@,:;.\"[]' self.pos = 0 self.LWS = ' \t' self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it # is obsolete syntax. RFC 2822 requires that we recognize obsolete # syntax, so allow dots in phrases. self.phraseends = self.atomends.replace('.', '') self.field = field self.commentlist = [] def gotonext(self): """Skip white space and extract comments.""" wslist = [] while self.pos < len(self.field): if self.field[self.pos] in self.LWS + '\n\r': if self.field[self.pos] not in '\n\r': wslist.append(self.field[self.pos]) self.pos += 1 elif self.field[self.pos] == '(': self.commentlist.append(self.getcomment()) else: break return EMPTYSTRING.join(wslist) def getaddrlist(self): """Parse all addresses. Returns a list containing all of the addresses. """ result = [] while self.pos < len(self.field): ad = self.getaddress() if ad: result += ad else: result.append(('', '')) return result def getaddress(self): """Parse the next address.""" self.commentlist = [] self.gotonext() oldpos = self.pos oldcl = self.commentlist plist = self.getphraselist() self.gotonext() returnlist = [] if self.pos >= len(self.field): # Bad email address technically, no domain. if plist: returnlist = [(SPACE.join(self.commentlist), plist[0])] elif self.field[self.pos] in '.@': # email address is just an addrspec # this isn't very efficient since we start over self.pos = oldpos self.commentlist = oldcl addrspec = self.getaddrspec() returnlist = [(SPACE.join(self.commentlist), addrspec)] elif self.field[self.pos] == ':': # address is a group returnlist = [] fieldlen = len(self.field) self.pos += 1 while self.pos < len(self.field): self.gotonext() if self.pos < fieldlen and self.field[self.pos] == ';': self.pos += 1 break returnlist = returnlist + self.getaddress() elif self.field[self.pos] == '<': # Address is a phrase then a route addr routeaddr = self.getrouteaddr() if self.commentlist: returnlist = [(SPACE.join(plist) + ' (' + ' '.join(self.commentlist) + ')', routeaddr)] else: returnlist = [(SPACE.join(plist), routeaddr)] else: if plist: returnlist = [(SPACE.join(self.commentlist), plist[0])] elif self.field[self.pos] in self.specials: self.pos += 1 self.gotonext() if self.pos < len(self.field) and self.field[self.pos] == ',': self.pos += 1 return returnlist def getrouteaddr(self): """Parse a route address (Return-path value). This method just skips all the route stuff and returns the addrspec. """ if self.field[self.pos] != '<': return expectroute = False self.pos += 1 self.gotonext() adlist = '' while self.pos < len(self.field): if expectroute: self.getdomain() expectroute = False elif self.field[self.pos] == '>': self.pos += 1 break elif self.field[self.pos] == '@': self.pos += 1 expectroute = True elif self.field[self.pos] == ':': self.pos += 1 else: adlist = self.getaddrspec() self.pos += 1 break self.gotonext() return adlist def getaddrspec(self): """Parse an RFC 2822 addr-spec.""" aslist = [] self.gotonext() while self.pos < len(self.field): preserve_ws = True if self.field[self.pos] == '.': if aslist and not aslist[-1].strip(): aslist.pop() aslist.append('.') self.pos += 1 preserve_ws = False elif self.field[self.pos] == '"': aslist.append('"%s"' % quote(self.getquote())) elif self.field[self.pos] in self.atomends: if aslist and not aslist[-1].strip(): aslist.pop() break else: aslist.append(self.getatom()) ws = self.gotonext() if preserve_ws and ws: aslist.append(ws) if self.pos >= len(self.field) or self.field[self.pos] != '@': return EMPTYSTRING.join(aslist) aslist.append('@') self.pos += 1 self.gotonext() domain = self.getdomain() if not domain: # Invalid domain, return an empty address instead of returning a # local part to denote failed parsing. return EMPTYSTRING return EMPTYSTRING.join(aslist) + domain def getdomain(self): """Get the complete domain name from an address.""" sdlist = [] while self.pos < len(self.field): if self.field[self.pos] in self.LWS: self.pos += 1 elif self.field[self.pos] == '(': self.commentlist.append(self.getcomment()) elif self.field[self.pos] == '[': sdlist.append(self.getdomainliteral()) elif self.field[self.pos] == '.': self.pos += 1 sdlist.append('.') elif self.field[self.pos] == '@': # bpo-34155: Don't parse domains with two `@` like # `a@malicious.org@important.com`. return EMPTYSTRING elif self.field[self.pos] in self.atomends: break else: sdlist.append(self.getatom()) return EMPTYSTRING.join(sdlist) def getdelimited(self, beginchar, endchars, allowcomments=True): """Parse a header fragment delimited by special characters. `beginchar' is the start character for the fragment. If self is not looking at an instance of `beginchar' then getdelimited returns the empty string. `endchars' is a sequence of allowable end-delimiting characters. Parsing stops when one of these is encountered. If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed within the parsed fragment. """ if self.field[self.pos] != beginchar: return '' slist = [''] quote = False self.pos += 1 while self.pos < len(self.field): if quote: slist.append(self.field[self.pos]) quote = False elif self.field[self.pos] in endchars: self.pos += 1 break elif allowcomments and self.field[self.pos] == '(': slist.append(self.getcomment()) continue # have already advanced pos from getcomment elif self.field[self.pos] == '\\': quote = True else: slist.append(self.field[self.pos]) self.pos += 1 return EMPTYSTRING.join(slist) def getquote(self): """Get a quote-delimited fragment from self's field.""" return self.getdelimited('"', '"\r', False) def getcomment(self): """Get a parenthesis-delimited fragment from self's field.""" return self.getdelimited('(', ')\r', True) def getdomainliteral(self): """Parse an RFC 2822 domain-literal.""" return '[%s]' % self.getdelimited('[', ']\r', False) def getatom(self, atomends=None): """Parse an RFC 2822 atom. Optional atomends specifies a different set of end token delimiters (the default is to use self.atomends). This is used e.g. in getphraselist() since phrase endings must not include the `.' (which is legal in phrases).""" atomlist = [''] if atomends is None: atomends = self.atomends while self.pos < len(self.field): if self.field[self.pos] in atomends: break else: atomlist.append(self.field[self.pos]) self.pos += 1 return EMPTYSTRING.join(atomlist) def getphraselist(self): """Parse a sequence of RFC 2822 phrases. A phrase is a sequence of words, which are in turn either RFC 2822 atoms or quoted-strings. Phrases are canonicalized by squeezing all runs of continuous whitespace into one space. """ plist = [] while self.pos < len(self.field): if self.field[self.pos] in self.FWS: self.pos += 1 elif self.field[self.pos] == '"': plist.append(self.getquote()) elif self.field[self.pos] == '(': self.commentlist.append(self.getcomment()) elif self.field[self.pos] in self.phraseends: break else: plist.append(self.getatom(self.phraseends)) return plist class AddressList(AddrlistClass): """An AddressList encapsulates a list of parsed RFC 2822 addresses.""" def __init__(self, field): AddrlistClass.__init__(self, field) if field: self.addresslist = self.getaddrlist() else: self.addresslist = [] def __len__(self): return len(self.addresslist) def __add__(self, other): # Set union newaddr = AddressList(None) newaddr.addresslist = self.addresslist[:] for x in other.addresslist: if not x in self.addresslist: newaddr.addresslist.append(x) return newaddr def __iadd__(self, other): # Set union, in-place for x in other.addresslist: if not x in self.addresslist: self.addresslist.append(x) return self def __sub__(self, other): # Set difference newaddr = AddressList(None) for x in self.addresslist: if not x in other.addresslist: newaddr.addresslist.append(x) return newaddr def __isub__(self, other): # Set difference, in-place for x in other.addresslist: if x in self.addresslist: self.addresslist.remove(x) return self def __getitem__(self, index): # Make indexing, slices, and 'in' work return self.addresslist[index]	17.27 KB	2024-04-17 17:55
📄 _policybase.py """Policy framework for the email package. Allows fine grained feature control of how the package parses and emits data. """ import abc from email import header from email import charset as _charset from email.utils import _has_surrogates __all__ = [ 'Policy', 'Compat32', 'compat32', ] class _PolicyBase: """Policy Object basic framework. This class is useless unless subclassed. A subclass should define class attributes with defaults for any values that are to be managed by the Policy object. The constructor will then allow non-default values to be set for these attributes at instance creation time. The instance will be callable, taking these same attributes keyword arguments, and returning a new instance identical to the called instance except for those values changed by the keyword arguments. Instances may be added, yielding new instances with any non-default values from the right hand operand overriding those in the left hand operand. That is, A + B == A(<non-default values of B>) The repr of an instance can be used to reconstruct the object if and only if the repr of the values can be used to reconstruct those values. """ def __init__(self, kw): """Create new Policy, possibly overriding some defaults. See class docstring for a list of overridable attributes. """ for name, value in kw.items(): if hasattr(self, name): super(_PolicyBase,self).__setattr__(name, value) else: raise TypeError( "{!r} is an invalid keyword argument for {}".format( name, self.__class__.__name__)) def __repr__(self): args = [ "{}={!r}".format(name, value) for name, value in self.__dict__.items() ] return "{}({})".format(self.__class__.__name__, ', '.join(args)) def clone(self, kw): """Return a new instance with specified attributes changed. The new instance has the same attribute values as the current object, except for the changes passed in as keyword arguments. """ newpolicy = self.__class__.__new__(self.__class__) for attr, value in self.__dict__.items(): object.__setattr__(newpolicy, attr, value) for attr, value in kw.items(): if not hasattr(self, attr): raise TypeError( "{!r} is an invalid keyword argument for {}".format( attr, self.__class__.__name__)) object.__setattr__(newpolicy, attr, value) return newpolicy def __setattr__(self, name, value): if hasattr(self, name): msg = "{!r} object attribute {!r} is read-only" else: msg = "{!r} object has no attribute {!r}" raise AttributeError(msg.format(self.__class__.__name__, name)) def __add__(self, other): """Non-default values from right operand override those from left. The object returned is a new instance of the subclass. """ return self.clone(**other.__dict__) def _append_doc(doc, added_doc): doc = doc.rsplit('\n', 1)[0] added_doc = added_doc.split('\n', 1)[1] return doc + '\n' + added_doc def _extend_docstrings(cls): if cls.__doc__ and cls.__doc__.startswith('+'): cls.__doc__ = _append_doc(cls.__bases__[0].__doc__, cls.__doc__) for name, attr in cls.__dict__.items(): if attr.__doc__ and attr.__doc__.startswith('+'): for c in (c for base in cls.__bases__ for c in base.mro()): doc = getattr(getattr(c, name), '__doc__') if doc: attr.__doc__ = _append_doc(doc, attr.__doc__) break return cls class Policy(_PolicyBase, metaclass=abc.ABCMeta): r"""Controls for how messages are interpreted and formatted. Most of the classes and many of the methods in the email package accept Policy objects as parameters. A Policy object contains a set of values and functions that control how input is interpreted and how output is rendered. For example, the parameter 'raise_on_defect' controls whether or not an RFC violation results in an error being raised or not, while 'max_line_length' controls the maximum length of output lines when a Message is serialized. Any valid attribute may be overridden when a Policy is created by passing it as a keyword argument to the constructor. Policy objects are immutable, but a new Policy object can be created with only certain values changed by calling the Policy instance with keyword arguments. Policy objects can also be added, producing a new Policy object in which the non-default attributes set in the right hand operand overwrite those specified in the left operand. Settable attributes: raise_on_defect -- If true, then defects should be raised as errors. Default: False. linesep -- string containing the value to use as separation between output lines. Default '\n'. cte_type -- Type of allowed content transfer encodings 7bit -- ASCII only 8bit -- Content-Transfer-Encoding: 8bit is allowed Default: 8bit. Also controls the disposition of (RFC invalid) binary data in headers; see the documentation of the binary_fold method. max_line_length -- maximum length of lines, excluding 'linesep', during serialization. None or 0 means no line wrapping is done. Default is 78. mangle_from_ -- a flag that, when True escapes From_ lines in the body of the message by putting a `>' in front of them. This is used when the message is being serialized by a generator. Default: True. message_factory -- the class to use to create new message objects. If the value is None, the default is Message. """ raise_on_defect = False linesep = '\n' cte_type = '8bit' max_line_length = 78 mangle_from_ = False message_factory = None def handle_defect(self, obj, defect): """Based on policy, either raise defect or call register_defect. handle_defect(obj, defect) defect should be a Defect subclass, but in any case must be an Exception subclass. obj is the object on which the defect should be registered if it is not raised. If the raise_on_defect is True, the defect is raised as an error, otherwise the object and the defect are passed to register_defect. This method is intended to be called by parsers that discover defects. The email package parsers always call it with Defect instances. """ if self.raise_on_defect: raise defect self.register_defect(obj, defect) def register_defect(self, obj, defect): """Record 'defect' on 'obj'. Called by handle_defect if raise_on_defect is False. This method is part of the Policy API so that Policy subclasses can implement custom defect handling. The default implementation calls the append method of the defects attribute of obj. The objects used by the email package by default that get passed to this method will always have a defects attribute with an append method. """ obj.defects.append(defect) def header_max_count(self, name): """Return the maximum allowed number of headers named 'name'. Called when a header is added to a Message object. If the returned value is not 0 or None, and there are already a number of headers with the name 'name' equal to the value returned, a ValueError is raised. Because the default behavior of Message's __setitem__ is to append the value to the list of headers, it is easy to create duplicate headers without realizing it. This method allows certain headers to be limited in the number of instances of that header that may be added to a Message programmatically. (The limit is not observed by the parser, which will faithfully produce as many headers as exist in the message being parsed.) The default implementation returns None for all header names. """ return None @abc.abstractmethod def header_source_parse(self, sourcelines): """Given a list of linesep terminated strings constituting the lines of a single header, return the (name, value) tuple that should be stored in the model. The input lines should retain their terminating linesep characters. The lines passed in by the email package may contain surrogateescaped binary data. """ raise NotImplementedError @abc.abstractmethod def header_store_parse(self, name, value): """Given the header name and the value provided by the application program, return the (name, value) that should be stored in the model. """ raise NotImplementedError @abc.abstractmethod def header_fetch_parse(self, name, value): """Given the header name and the value from the model, return the value to be returned to the application program that is requesting that header. The value passed in by the email package may contain surrogateescaped binary data if the lines were parsed by a BytesParser. The returned value should not contain any surrogateescaped data. """ raise NotImplementedError @abc.abstractmethod def fold(self, name, value): """Given the header name and the value from the model, return a string containing linesep characters that implement the folding of the header according to the policy controls. The value passed in by the email package may contain surrogateescaped binary data if the lines were parsed by a BytesParser. The returned value should not contain any surrogateescaped data. """ raise NotImplementedError @abc.abstractmethod def fold_binary(self, name, value): """Given the header name and the value from the model, return binary data containing linesep characters that implement the folding of the header according to the policy controls. The value passed in by the email package may contain surrogateescaped binary data. """ raise NotImplementedError @_extend_docstrings class Compat32(Policy): """+ This particular policy is the backward compatibility Policy. It replicates the behavior of the email package version 5.1. """ mangle_from_ = True def _sanitize_header(self, name, value): # If the header value contains surrogates, return a Header using # the unknown-8bit charset to encode the bytes as encoded words. if not isinstance(value, str): # Assume it is already a header object return value if _has_surrogates(value): return header.Header(value, charset=_charset.UNKNOWN8BIT, header_name=name) else: return value def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the remainder of the first line, joining all subsequent lines together, and stripping any trailing carriage return or linefeed characters. """ name, value = sourcelines[0].split(':', 1) value = value.lstrip(' \t') + ''.join(sourcelines[1:]) return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): """+ The name and value are returned unmodified. """ return (name, value) def header_fetch_parse(self, name, value): """+ If the value contains binary data, it is converted into a Header object using the unknown-8bit charset. Otherwise it is returned unmodified. """ return self._sanitize_header(name, value) def fold(self, name, value): """+ Headers are folded using the Header folding algorithm, which preserves existing line breaks in the value, and wraps each resulting line to the max_line_length. Non-ASCII binary data are CTE encoded using the unknown-8bit charset. """ return self._fold(name, value, sanitize=True) def fold_binary(self, name, value): """+ Headers are folded using the Header folding algorithm, which preserves existing line breaks in the value, and wraps each resulting line to the max_line_length. If cte_type is 7bit, non-ascii binary data is CTE encoded using the unknown-8bit charset. Otherwise the original source header is used, with its existing line breaks and/or binary data. """ folded = self._fold(name, value, sanitize=self.cte_type=='7bit') return folded.encode('ascii', 'surrogateescape') def _fold(self, name, value, sanitize): parts = [] parts.append('%s: ' % name) if isinstance(value, str): if _has_surrogates(value): if sanitize: h = header.Header(value, charset=_charset.UNKNOWN8BIT, header_name=name) else: # If we have raw 8bit data in a byte string, we have no idea # what the encoding is. There is no safe way to split this # string. If it's ascii-subset, then we could do a normal # ascii split, but if it's multibyte then we could break the # string. There's no way to know so the least harm seems to # be to not split the string and risk it being too long. parts.append(value) h = None else: h = header.Header(value, header_name=name) else: # Assume it is a Header-like object. h = value if h is not None: # The Header class interprets a value of None for maxlinelen as the # default value of 78, as recommended by RFC 2822. maxlinelen = 0 if self.max_line_length is not None: maxlinelen = self.max_line_length parts.append(h.encode(linesep=self.linesep, maxlinelen=maxlinelen)) parts.append(self.linesep) return ''.join(parts) compat32 = Compat32()	14.72 KB	2024-04-17 17:55
📄 architecture.rst :mod:`email` Package Architecture ================================= Overview -------- The email package consists of three major components: Model An object structure that represents an email message, and provides an API for creating, querying, and modifying a message. Parser Takes a sequence of characters or bytes and produces a model of the email message represented by those characters or bytes. Generator Takes a model and turns it into a sequence of characters or bytes. The sequence can either be intended for human consumption (a printable unicode string) or bytes suitable for transmission over the wire. In the latter case all data is properly encoded using the content transfer encodings specified by the relevant RFCs. Conceptually the package is organized around the model. The model provides both "external" APIs intended for use by application programs using the library, and "internal" APIs intended for use by the Parser and Generator components. This division is intentionally a bit fuzzy; the API described by this documentation is all a public, stable API. This allows for an application with special needs to implement its own parser and/or generator. In addition to the three major functional components, there is a third key component to the architecture: Policy An object that specifies various behavioral settings and carries implementations of various behavior-controlling methods. The Policy framework provides a simple and convenient way to control the behavior of the library, making it possible for the library to be used in a very flexible fashion while leveraging the common code required to parse, represent, and generate message-like objects. For example, in addition to the default :rfc:`5322` email message policy, we also have a policy that manages HTTP headers in a fashion compliant with :rfc:`2616`. Individual policy controls, such as the maximum line length produced by the generator, can also be controlled individually to meet specialized application requirements. The Model --------- The message model is implemented by the :class:`~email.message.Message` class. The model divides a message into the two fundamental parts discussed by the RFC: the header section and the body. The `Message` object acts as a pseudo-dictionary of named headers. Its dictionary interface provides convenient access to individual headers by name. However, all headers are kept internally in an ordered list, so that the information about the order of the headers in the original message is preserved. The `Message` object also has a `payload` that holds the body. A `payload` can be one of two things: data, or a list of `Message` objects. The latter is used to represent a multipart MIME message. Lists can be nested arbitrarily deeply in order to represent the message, with all terminal leaves having non-list data payloads. Message Lifecycle ----------------- The general lifecycle of a message is: Creation A `Message` object can be created by a Parser, or it can be instantiated as an empty message by an application. Manipulation The application may examine one or more headers, and/or the payload, and it may modify one or more headers and/or the payload. This may be done on the top level `Message` object, or on any sub-object. Finalization The Model is converted into a unicode or binary stream, or the model is discarded. Header Policy Control During Lifecycle -------------------------------------- One of the major controls exerted by the Policy is the management of headers during the `Message` lifecycle. Most applications don't need to be aware of this. A header enters the model in one of two ways: via a Parser, or by being set to a specific value by an application program after the Model already exists. Similarly, a header exits the model in one of two ways: by being serialized by a Generator, or by being retrieved from a Model by an application program. The Policy object provides hooks for all four of these pathways. The model storage for headers is a list of (name, value) tuples. The Parser identifies headers during parsing, and passes them to the :meth:`~email.policy.Policy.header_source_parse` method of the Policy. The result of that method is the (name, value) tuple to be stored in the model. When an application program supplies a header value (for example, through the `Message` object `__setitem__` interface), the name and the value are passed to the :meth:`~email.policy.Policy.header_store_parse` method of the Policy, which returns the (name, value) tuple to be stored in the model. When an application program retrieves a header (through any of the dict or list interfaces of `Message`), the name and value are passed to the :meth:`~email.policy.Policy.header_fetch_parse` method of the Policy to obtain the value returned to the application. When a Generator requests a header during serialization, the name and value are passed to the :meth:`~email.policy.Policy.fold` method of the Policy, which returns a string containing line breaks in the appropriate places. The :meth:`~email.policy.Policy.cte_type` Policy control determines whether or not Content Transfer Encoding is performed on the data in the header. There is also a :meth:`~email.policy.Policy.binary_fold` method for use by generators that produce binary output, which returns the folded header as binary data, possibly folded at different places than the corresponding string would be. Handling Binary Data -------------------- In an ideal world all message data would conform to the RFCs, meaning that the parser could decode the message into the idealized unicode message that the sender originally wrote. In the real world, the email package must also be able to deal with badly formatted messages, including messages containing non-ASCII characters that either have no indicated character set or are not valid characters in the indicated character set. Since email messages are primarily text data, and operations on message data are primarily text operations (except for binary payloads of course), the model stores all text data as unicode strings. Un-decodable binary inside text data is handled by using the `surrogateescape` error handler of the ASCII codec. As with the binary filenames the error handler was introduced to handle, this allows the email package to "carry" the binary data received during parsing along until the output stage, at which time it is regenerated in its original form. This carried binary data is almost entirely an implementation detail. The one place where it is visible in the API is in the "internal" API. A Parser must do the `surrogateescape` encoding of binary input data, and pass that data to the appropriate Policy method. The "internal" interface used by the Generator to access header values preserves the `surrogateescaped` bytes. All other interfaces convert the binary data either back into bytes or into a safe form (losing information in some cases). Backward Compatibility ---------------------- The :class:`~email.policy.Policy.Compat32` Policy provides backward compatibility with version 5.1 of the email package. It does this via the following implementation of the four+1 Policy methods described above: header_source_parse Splits the first line on the colon to obtain the name, discards any spaces after the colon, and joins the remainder of the line with all of the remaining lines, preserving the linesep characters to obtain the value. Trailing carriage return and/or linefeed characters are stripped from the resulting value string. header_store_parse Returns the name and value exactly as received from the application. header_fetch_parse If the value contains any `surrogateescaped` binary data, return the value as a :class:`~email.header.Header` object, using the character set `unknown-8bit`. Otherwise just returns the value. fold Uses :class:`~email.header.Header`'s folding to fold headers in the same way the email5.1 generator did. binary_fold Same as fold, but encodes to 'ascii'. New Algorithm ------------- header_source_parse Same as legacy behavior. header_store_parse Same as legacy behavior. header_fetch_parse If the value is already a header object, returns it. Otherwise, parses the value using the new parser, and returns the resulting object as the value. `surrogateescaped` bytes get turned into unicode unknown character code points. fold Uses the new header folding algorithm, respecting the policy settings. surrogateescaped bytes are encoded using the ``unknown-8bit`` charset for ``cte_type=7bit`` or ``8bit``. Returns a string. At some point there will also be a ``cte_type=unicode``, and for that policy fold will serialize the idealized unicode message with RFC-like folding, converting any surrogateescaped bytes into the unicode unknown character glyph. binary_fold Uses the new header folding algorithm, respecting the policy settings. surrogateescaped bytes are encoded using the `unknown-8bit` charset for ``cte_type=7bit``, and get turned back into bytes for ``cte_type=8bit``. Returns bytes. At some point there will also be a ``cte_type=unicode``, and for that policy binary_fold will serialize the message according to :rfc:``5335``.	9.34 KB	2023-06-05 20:45
📄 base64mime.py # Copyright (C) 2002-2007 Python Software Foundation # Author: Ben Gertzfield # Contact: email-sig@python.org """Base64 content transfer encoding per RFCs 2045-2047. This module handles the content transfer encoding method defined in RFC 2045 to encode arbitrary 8-bit data using the three 8-bit bytes in four 7-bit characters encoding known as Base64. It is used in the MIME standards for email to attach images, audio, and text using some 8-bit character sets to messages. This module provides an interface to encode and decode both headers and bodies with Base64 encoding. RFC 2045 defines a method for including character set information in an `encoded-word' in a header. This method is commonly used for 8-bit real names in To:, From:, Cc:, etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character conversion necessary for proper internationalized headers; it only does dumb encoding and decoding. To deal with the various line wrapping issues, use the email.header module. """ __all__ = [ 'body_decode', 'body_encode', 'decode', 'decodestring', 'header_encode', 'header_length', ] from base64 import b64encode from binascii import b2a_base64, a2b_base64 CRLF = '\r\n' NL = '\n' EMPTYSTRING = '' # See also Charset.py MISC_LEN = 7 # Helpers def header_length(bytearray): """Return the length of s when it is encoded with base64.""" groups_of_3, leftover = divmod(len(bytearray), 3) # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. n = groups_of_3 * 4 if leftover: n += 4 return n def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with Base64 encoding in a given charset. charset names the character set to use to encode the header. It defaults to iso-8859-1. Base64 encoding is defined in RFC 2045. """ if not header_bytes: return "" if isinstance(header_bytes, str): header_bytes = header_bytes.encode(charset) encoded = b64encode(header_bytes).decode("ascii") return '=?%s?b?%s?=' % (charset, encoded) def body_encode(s, maxlinelen=76, eol=NL): r"""Encode a string with base64. Each line will be wrapped at, at most, maxlinelen characters (defaults to 76 characters). Each line of encoded text will end with eol, which defaults to "\n". Set this to "\r\n" if you will be using the result of this function directly in an email. """ if not s: return s encvec = [] max_unencoded = maxlinelen * 3 // 4 for i in range(0, len(s), max_unencoded): # BAW: should encode() inherit b2a_base64()'s dubious behavior in # adding a newline to the encoded string? enc = b2a_base64(s[i:i + max_unencoded]).decode("ascii") if enc.endswith(NL) and eol != NL: enc = enc[:-1] + eol encvec.append(enc) return EMPTYSTRING.join(encvec) def decode(string): """Decode a raw base64 string, returning a bytes object. This function does not parse a full MIME header value encoded with base64 (like =?iso-8859-1?b?bmloISBuaWgh?=) -- please use the high level email.header class for that functionality. """ if not string: return bytes() elif isinstance(string, str): return a2b_base64(string.encode('raw-unicode-escape')) else: return a2b_base64(string) # For convenience and backwards compatibility w/ standard base64 module body_decode = decode decodestring = decode	3.47 KB	2024-04-17 17:55
📄 charset.py # Copyright (C) 2001-2007 Python Software Foundation # Author: Ben Gertzfield, Barry Warsaw # Contact: email-sig@python.org __all__ = [ 'Charset', 'add_alias', 'add_charset', 'add_codec', ] from functools import partial import email.base64mime import email.quoprimime from email import errors from email.encoders import encode_7or8bit # Flags for types of header encodings QP = 1 # Quoted-Printable BASE64 = 2 # Base64 SHORTEST = 3 # the shorter of QP and base64, but only for headers # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 RFC2047_CHROME_LEN = 7 DEFAULT_CHARSET = 'us-ascii' UNKNOWN8BIT = 'unknown-8bit' EMPTYSTRING = '' # Defaults CHARSETS = { # input header enc body enc output conv 'iso-8859-1': (QP, QP, None), 'iso-8859-2': (QP, QP, None), 'iso-8859-3': (QP, QP, None), 'iso-8859-4': (QP, QP, None), # iso-8859-5 is Cyrillic, and not especially used # iso-8859-6 is Arabic, also not particularly used # iso-8859-7 is Greek, QP will not make it readable # iso-8859-8 is Hebrew, QP will not make it readable 'iso-8859-9': (QP, QP, None), 'iso-8859-10': (QP, QP, None), # iso-8859-11 is Thai, QP will not make it readable 'iso-8859-13': (QP, QP, None), 'iso-8859-14': (QP, QP, None), 'iso-8859-15': (QP, QP, None), 'iso-8859-16': (QP, QP, None), 'windows-1252':(QP, QP, None), 'viscii': (QP, QP, None), 'us-ascii': (None, None, None), 'big5': (BASE64, BASE64, None), 'gb2312': (BASE64, BASE64, None), 'euc-jp': (BASE64, None, 'iso-2022-jp'), 'shift_jis': (BASE64, None, 'iso-2022-jp'), 'iso-2022-jp': (BASE64, None, None), 'koi8-r': (BASE64, BASE64, None), 'utf-8': (SHORTEST, BASE64, 'utf-8'), } # Aliases for other commonly-used names for character sets. Map # them to the real ones used in email. ALIASES = { 'latin_1': 'iso-8859-1', 'latin-1': 'iso-8859-1', 'latin_2': 'iso-8859-2', 'latin-2': 'iso-8859-2', 'latin_3': 'iso-8859-3', 'latin-3': 'iso-8859-3', 'latin_4': 'iso-8859-4', 'latin-4': 'iso-8859-4', 'latin_5': 'iso-8859-9', 'latin-5': 'iso-8859-9', 'latin_6': 'iso-8859-10', 'latin-6': 'iso-8859-10', 'latin_7': 'iso-8859-13', 'latin-7': 'iso-8859-13', 'latin_8': 'iso-8859-14', 'latin-8': 'iso-8859-14', 'latin_9': 'iso-8859-15', 'latin-9': 'iso-8859-15', 'latin_10':'iso-8859-16', 'latin-10':'iso-8859-16', 'cp949': 'ks_c_5601-1987', 'euc_jp': 'euc-jp', 'euc_kr': 'euc-kr', 'ascii': 'us-ascii', } # Map charsets to their Unicode codec strings. CODEC_MAP = { 'gb2312': 'eucgb2312_cn', 'big5': 'big5_tw', # Hack: We don't want any conversion for stuff marked us-ascii, as all # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. # Let that stuff pass through without conversion to/from Unicode. 'us-ascii': None, } # Convenience functions for extending the above mappings def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): """Add character set properties to the global registry. charset is the input character set, and must be the canonical name of a character set. Optional header_enc and body_enc is either Charset.QP for quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for the shortest of qp or base64 encoding, or None for no encoding. SHORTEST is only valid for header_enc. It describes how message headers and message bodies in the input charset are to be encoded. Default is no encoding. Optional output_charset is the character set that the output should be in. Conversions will proceed from input charset, to Unicode, to the output charset when the method Charset.convert() is called. The default is to output in the same character set as the input. Both input_charset and output_charset must have Unicode codec entries in the module's charset-to-codec mapping; use add_codec(charset, codecname) to add codecs the module does not know about. See the codecs module's documentation for more information. """ if body_enc == SHORTEST: raise ValueError('SHORTEST not allowed for body_enc') CHARSETS[charset] = (header_enc, body_enc, output_charset) def add_alias(alias, canonical): """Add a character set alias. alias is the alias name, e.g. latin-1 canonical is the character set's canonical name, e.g. iso-8859-1 """ ALIASES[alias] = canonical def add_codec(charset, codecname): """Add a codec that map characters in the given charset to/from Unicode. charset is the canonical name of a character set. codecname is the name of a Python codec, as appropriate for the second argument to the unicode() built-in, or to the encode() method of a Unicode string. """ CODEC_MAP[charset] = codecname # Convenience function for encoding strings, taking into account # that they might be unknown-8bit (ie: have surrogate-escaped bytes) def _encode(string, codec): if codec == UNKNOWN8BIT: return string.encode('ascii', 'surrogateescape') else: return string.encode(codec) class Charset: """Map character sets to their email properties. This class provides information about the requirements imposed on email for a specific character set. It also provides convenience routines for converting between character sets, given the availability of the applicable codecs. Given a character set, it will do its best to provide information on how to use that character set in an email in an RFC-compliant way. Certain character sets must be encoded with quoted-printable or base64 when used in email headers or bodies. Certain character sets must be converted outright, and are not allowed in email. Instances of this module expose the following information about a character set: input_charset: The initial character set specified. Common aliases are converted to their `official' email names (e.g. latin_1 is converted to iso-8859-1). Defaults to 7-bit us-ascii. header_encoding: If the character set must be encoded before it can be used in an email header, this attribute will be set to Charset.QP (for quoted-printable), Charset.BASE64 (for base64 encoding), or Charset.SHORTEST for the shortest of QP or BASE64 encoding. Otherwise, it will be None. body_encoding: Same as header_encoding, but describes the encoding for the mail message's body, which indeed may be different than the header encoding. Charset.SHORTEST is not allowed for body_encoding. output_charset: Some character sets must be converted before they can be used in email headers or bodies. If the input_charset is one of them, this attribute will contain the name of the charset output will be converted to. Otherwise, it will be None. input_codec: The name of the Python codec used to convert the input_charset to Unicode. If no conversion codec is necessary, this attribute will be None. output_codec: The name of the Python codec used to convert Unicode to the output_charset. If no conversion codec is necessary, this attribute will have the same value as the input_codec. """ def __init__(self, input_charset=DEFAULT_CHARSET): # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to # unicode because its .lower() is locale insensitive. If the argument # is already a unicode, we leave it at that, but ensure that the # charset is ASCII, as the standard (RFC XXX) requires. try: if isinstance(input_charset, str): input_charset.encode('ascii') else: input_charset = str(input_charset, 'ascii') except UnicodeError: raise errors.CharsetError(input_charset) input_charset = input_charset.lower() # Set the input charset after filtering through the aliases self.input_charset = ALIASES.get(input_charset, input_charset) # We can try to guess which encoding and conversion to use by the # charset_map dictionary. Try that first, but let the user override # it. henc, benc, conv = CHARSETS.get(self.input_charset, (SHORTEST, BASE64, None)) if not conv: conv = self.input_charset # Set the attributes, allowing the arguments to override the default. self.header_encoding = henc self.body_encoding = benc self.output_charset = ALIASES.get(conv, conv) # Now set the codecs. If one isn't defined for input_charset, # guess and try a Unicode codec with the same name as input_codec. self.input_codec = CODEC_MAP.get(self.input_charset, self.input_charset) self.output_codec = CODEC_MAP.get(self.output_charset, self.output_charset) def __str__(self): return self.input_charset.lower() __repr__ = __str__ def __eq__(self, other): return str(self) == str(other).lower() def get_body_encoding(self): """Return the content-transfer-encoding used for body encoding. This is either the string `quoted-printable' or `base64' depending on the encoding used, or it is a function in which case you should call the function with a single argument, the Message object being encoded. The function should then set the Content-Transfer-Encoding header itself to whatever is appropriate. Returns "quoted-printable" if self.body_encoding is QP. Returns "base64" if self.body_encoding is BASE64. Returns conversion function otherwise. """ assert self.body_encoding != SHORTEST if self.body_encoding == QP: return 'quoted-printable' elif self.body_encoding == BASE64: return 'base64' else: return encode_7or8bit def get_output_charset(self): """Return the output character set. This is self.output_charset if that is not None, otherwise it is self.input_charset. """ return self.output_charset or self.input_charset def header_encode(self, string): """Header-encode a string by converting it first to bytes. The type of encoding (base64 or quoted-printable) will be based on this charset's `header_encoding`. :param string: A unicode string for the header. It must be possible to encode this string to bytes using the character set's output codec. :return: The encoded string, with RFC 2047 chrome. """ codec = self.output_codec or 'us-ascii' header_bytes = _encode(string, codec) # 7bit/8bit encodings return the string unchanged (modulo conversions) encoder_module = self._get_encoder(header_bytes) if encoder_module is None: return string return encoder_module.header_encode(header_bytes, codec) def header_encode_lines(self, string, maxlengths): """Header-encode a string by converting it first to bytes. This is similar to `header_encode()` except that the string is fit into maximum line lengths as given by the argument. :param string: A unicode string for the header. It must be possible to encode this string to bytes using the character set's output codec. :param maxlengths: Maximum line length iterator. Each element returned from this iterator will provide the next maximum line length. This parameter is used as an argument to built-in next() and should never be exhausted. The maximum line lengths should not count the RFC 2047 chrome. These line lengths are only a hint; the splitter does the best it can. :return: Lines of encoded strings, each with RFC 2047 chrome. """ # See which encoding we should use. codec = self.output_codec or 'us-ascii' header_bytes = _encode(string, codec) encoder_module = self._get_encoder(header_bytes) encoder = partial(encoder_module.header_encode, charset=codec) # Calculate the number of characters that the RFC 2047 chrome will # contribute to each line. charset = self.get_output_charset() extra = len(charset) + RFC2047_CHROME_LEN # Now comes the hard part. We must encode bytes but we can't split on # bytes because some character sets are variable length and each # encoded word must stand on its own. So the problem is you have to # encode to bytes to figure out this word's length, but you must split # on characters. This causes two problems: first, we don't know how # many octets a specific substring of unicode characters will get # encoded to, and second, we don't know how many ASCII characters # those octets will get encoded to. Unless we try it. Which seems # inefficient. In the interest of being correct rather than fast (and # in the hope that there will be few encoded headers in any such # message), brute force it. :( lines = [] current_line = [] maxlen = next(maxlengths) - extra for character in string: current_line.append(character) this_line = EMPTYSTRING.join(current_line) length = encoder_module.header_length(_encode(this_line, charset)) if length > maxlen: # This last character doesn't fit so pop it off. current_line.pop() # Does nothing fit on the first line? if not lines and not current_line: lines.append(None) else: separator = (' ' if lines else '') joined_line = EMPTYSTRING.join(current_line) header_bytes = _encode(joined_line, codec) lines.append(encoder(header_bytes)) current_line = [character] maxlen = next(maxlengths) - extra joined_line = EMPTYSTRING.join(current_line) header_bytes = _encode(joined_line, codec) lines.append(encoder(header_bytes)) return lines def _get_encoder(self, header_bytes): if self.header_encoding == BASE64: return email.base64mime elif self.header_encoding == QP: return email.quoprimime elif self.header_encoding == SHORTEST: len64 = email.base64mime.header_length(header_bytes) lenqp = email.quoprimime.header_length(header_bytes) if len64 < lenqp: return email.base64mime else: return email.quoprimime else: return None def body_encode(self, string): """Body-encode a string by converting it first to bytes. The type of encoding (base64 or quoted-printable) will be based on self.body_encoding. If body_encoding is None, we assume the output charset is a 7bit encoding, so re-encoding the decoded string using the ascii codec produces the correct string version of the content. """ if not string: return string if self.body_encoding is BASE64: if isinstance(string, str): string = string.encode(self.output_charset) return email.base64mime.body_encode(string) elif self.body_encoding is QP: # quopromime.body_encode takes a string, but operates on it as if # it were a list of byte codes. For a (minimal) history on why # this is so, see changeset 0cf700464177. To correctly encode a # character set, then, we must turn it into pseudo bytes via the # latin1 charset, which will encode any byte as a single code point # between 0 and 255, which is what body_encode is expecting. if isinstance(string, str): string = string.encode(self.output_charset) string = string.decode('latin1') return email.quoprimime.body_encode(string) else: if isinstance(string, str): string = string.encode(self.output_charset).decode('ascii') return string	16.75 KB	2024-04-17 17:55
📄 contentmanager.py import binascii import email.charset import email.message import email.errors from email import quoprimime class ContentManager: def __init__(self): self.get_handlers = {} self.set_handlers = {} def add_get_handler(self, key, handler): self.get_handlers[key] = handler def get_content(self, msg, args, kw): content_type = msg.get_content_type() if content_type in self.get_handlers: return self.get_handlers[content_type](msg, args, *kw) maintype = msg.get_content_maintype() if maintype in self.get_handlers: return self.get_handlers[maintype](msg, args, *kw) if '' in self.get_handlers: return self.get_handlers[''](msg, args, *kw) raise KeyError(content_type) def add_set_handler(self, typekey, handler): self.set_handlers[typekey] = handler def set_content(self, msg, obj, args, *kw): if msg.get_content_maintype() == 'multipart': # XXX: is this error a good idea or not? We can remove it later, # but we can't add it later, so do it for now. raise TypeError("set_content not valid on multipart") handler = self._find_set_handler(msg, obj) msg.clear_content() handler(msg, obj, args, *kw) def _find_set_handler(self, msg, obj): full_path_for_error = None for typ in type(obj).__mro__: if typ in self.set_handlers: return self.set_handlers[typ] qname = typ.__qualname__ modname = getattr(typ, '__module__', '') full_path = '.'.join((modname, qname)) if modname else qname if full_path_for_error is None: full_path_for_error = full_path if full_path in self.set_handlers: return self.set_handlers[full_path] if qname in self.set_handlers: return self.set_handlers[qname] name = typ.__name__ if name in self.set_handlers: return self.set_handlers[name] if None in self.set_handlers: return self.set_handlers[None] raise KeyError(full_path_for_error) raw_data_manager = ContentManager() def get_text_content(msg, errors='replace'): content = msg.get_payload(decode=True) charset = msg.get_param('charset', 'ASCII') return content.decode(charset, errors=errors) raw_data_manager.add_get_handler('text', get_text_content) def get_non_text_content(msg): return msg.get_payload(decode=True) for maintype in 'audio image video application'.split(): raw_data_manager.add_get_handler(maintype, get_non_text_content) def get_message_content(msg): return msg.get_payload(0) for subtype in 'rfc822 external-body'.split(): raw_data_manager.add_get_handler('message/'+subtype, get_message_content) def get_and_fixup_unknown_message_content(msg): # If we don't understand a message subtype, we are supposed to treat it as # if it were application/octet-stream, per # tools.ietf.org/html/rfc2046#section-5.2.4. Feedparser doesn't do that, # so do our best to fix things up. Note that it is not* appropriate to # model message/partial content as Message objects, so they are handled # here as well. (How to reassemble them is out of scope for this comment :) return bytes(msg.get_payload(0)) raw_data_manager.add_get_handler('message', get_and_fixup_unknown_message_content) def _prepare_set(msg, maintype, subtype, headers): msg['Content-Type'] = '/'.join((maintype, subtype)) if headers: if not hasattr(headers[0], 'name'): mp = msg.policy headers = [mp.header_factory(mp.header_source_parse([header])) for header in headers] try: for header in headers: if header.defects: raise header.defects[0] msg[header.name] = header except email.errors.HeaderDefect as exc: raise ValueError("Invalid header: {}".format( header.fold(policy=msg.policy))) from exc def _finalize_set(msg, disposition, filename, cid, params): if disposition is None and filename is not None: disposition = 'attachment' if disposition is not None: msg['Content-Disposition'] = disposition if filename is not None: msg.set_param('filename', filename, header='Content-Disposition', replace=True) if cid is not None: msg['Content-ID'] = cid if params is not None: for key, value in params.items(): msg.set_param(key, value) # XXX: This is a cleaned-up version of base64mime.body_encode (including a bug # fix in the calculation of unencoded_bytes_per_line). It would be nice to # drop both this and quoprimime.body_encode in favor of enhanced binascii # routines that accepted a max_line_length parameter. def _encode_base64(data, max_line_length): encoded_lines = [] unencoded_bytes_per_line = max_line_length // 4 3 for i in range(0, len(data), unencoded_bytes_per_line): thisline = data[i:i+unencoded_bytes_per_line] encoded_lines.append(binascii.b2a_base64(thisline).decode('ascii')) return ''.join(encoded_lines) def _encode_text(string, charset, cte, policy): lines = string.encode(charset).splitlines() linesep = policy.linesep.encode('ascii') def embedded_body(lines): return linesep.join(lines) + linesep def normal_body(lines): return b'\n'.join(lines) + b'\n' if cte==None: # Use heuristics to decide on the "best" encoding. try: return '7bit', normal_body(lines).decode('ascii') except UnicodeDecodeError: pass if (policy.cte_type == '8bit' and max(len(x) for x in lines) <= policy.max_line_length): return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') sniff = embedded_body(lines[:10]) sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), policy.max_line_length) sniff_base64 = binascii.b2a_base64(sniff) # This is a little unfair to qp; it includes lineseps, base64 doesn't. if len(sniff_qp) > len(sniff_base64): cte = 'base64' else: cte = 'quoted-printable' if len(lines) <= 10: return cte, sniff_qp if cte == '7bit': data = normal_body(lines).decode('ascii') elif cte == '8bit': data = normal_body(lines).decode('ascii', 'surrogateescape') elif cte == 'quoted-printable': data = quoprimime.body_encode(normal_body(lines).decode('latin-1'), policy.max_line_length) elif cte == 'base64': data = _encode_base64(embedded_body(lines), policy.max_line_length) else: raise ValueError("Unknown content transfer encoding {}".format(cte)) return cte, data def set_text_content(msg, string, subtype="plain", charset='utf-8', cte=None, disposition=None, filename=None, cid=None, params=None, headers=None): _prepare_set(msg, 'text', subtype, headers) cte, payload = _encode_text(string, charset, cte, msg.policy) msg.set_payload(payload) msg.set_param('charset', email.charset.ALIASES.get(charset, charset), replace=True) msg['Content-Transfer-Encoding'] = cte _finalize_set(msg, disposition, filename, cid, params) raw_data_manager.add_set_handler(str, set_text_content) def set_message_content(msg, message, subtype="rfc822", cte=None, disposition=None, filename=None, cid=None, params=None, headers=None): if subtype == 'partial': raise ValueError("message/partial is not supported for Message objects") if subtype == 'rfc822': if cte not in (None, '7bit', '8bit', 'binary'): # http://tools.ietf.org/html/rfc2046#section-5.2.1 mandate. raise ValueError( "message/rfc822 parts do not support cte={}".format(cte)) # 8bit will get coerced on serialization if policy.cte_type='7bit'. We # may end up claiming 8bit when it isn't needed, but the only negative # result of that should be a gateway that needs to coerce to 7bit # having to look through the whole embedded message to discover whether # or not it actually has to do anything. cte = '8bit' if cte is None else cte elif subtype == 'external-body': if cte not in (None, '7bit'): # http://tools.ietf.org/html/rfc2046#section-5.2.3 mandate. raise ValueError( "message/external-body parts do not support cte={}".format(cte)) cte = '7bit' elif cte is None: # http://tools.ietf.org/html/rfc2046#section-5.2.4 says all future # subtypes should be restricted to 7bit, so assume that. cte = '7bit' _prepare_set(msg, 'message', subtype, headers) msg.set_payload([message]) msg['Content-Transfer-Encoding'] = cte _finalize_set(msg, disposition, filename, cid, params) raw_data_manager.add_set_handler(email.message.Message, set_message_content) def set_bytes_content(msg, data, maintype, subtype, cte='base64', disposition=None, filename=None, cid=None, params=None, headers=None): _prepare_set(msg, maintype, subtype, headers) if cte == 'base64': data = _encode_base64(data, max_line_length=msg.policy.max_line_length) elif cte == 'quoted-printable': # XXX: quoprimime.body_encode won't encode newline characters in data, # so we can't use it. This means max_line_length is ignored. Another # bug to fix later. (Note: encoders.quopri is broken on line ends.) data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True) data = data.decode('ascii') elif cte == '7bit': # Make sure it really is only ASCII. The early warning here seems # worth the overhead...if you care write your own content manager :). data.encode('ascii') elif cte in ('8bit', 'binary'): data = data.decode('ascii', 'surrogateescape') msg.set_payload(data) msg['Content-Transfer-Encoding'] = cte _finalize_set(msg, disposition, filename, cid, params) for typ in (bytes, bytearray, memoryview): raw_data_manager.add_set_handler(typ, set_bytes_content)	10.42 KB	2024-04-17 17:55
📄 encoders.py # Copyright (C) 2001-2006 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org """Encodings and related functions.""" __all__ = [ 'encode_7or8bit', 'encode_base64', 'encode_noop', 'encode_quopri', ] from base64 import encodebytes as _bencode from quopri import encodestring as _encodestring def _qencode(s): enc = _encodestring(s, quotetabs=True) # Must encode spaces, which quopri.encodestring() doesn't do return enc.replace(b' ', b'=20') def encode_base64(msg): """Encode the message's payload in Base64. Also, add an appropriate Content-Transfer-Encoding header. """ orig = msg.get_payload(decode=True) encdata = str(_bencode(orig), 'ascii') msg.set_payload(encdata) msg['Content-Transfer-Encoding'] = 'base64' def encode_quopri(msg): """Encode the message's payload in quoted-printable. Also, add an appropriate Content-Transfer-Encoding header. """ orig = msg.get_payload(decode=True) encdata = _qencode(orig) msg.set_payload(encdata) msg['Content-Transfer-Encoding'] = 'quoted-printable' def encode_7or8bit(msg): """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" orig = msg.get_payload(decode=True) if orig is None: # There's no payload. For backwards compatibility we use 7bit msg['Content-Transfer-Encoding'] = '7bit' return # We play a trick to make this go fast. If decoding from ASCII succeeds, # we know the data must be 7bit, otherwise treat it as 8bit. try: orig.decode('ascii') except UnicodeError: msg['Content-Transfer-Encoding'] = '8bit' else: msg['Content-Transfer-Encoding'] = '7bit' def encode_noop(msg): """Do nothing."""	1.74 KB	2024-04-17 17:55
📄 errors.py # Copyright (C) 2001-2006 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org """email package exception classes.""" class MessageError(Exception): """Base class for errors in the email package.""" class MessageParseError(MessageError): """Base class for message parsing errors.""" class HeaderParseError(MessageParseError): """Error while parsing headers.""" class BoundaryError(MessageParseError): """Couldn't find terminating boundary.""" class MultipartConversionError(MessageError, TypeError): """Conversion to a multipart is prohibited.""" class CharsetError(MessageError): """An illegal charset was given.""" # These are parsing defects which the parser was able to work around. class MessageDefect(ValueError): """Base class for a message defect.""" def __init__(self, line=None): if line is not None: super().__init__(line) self.line = line class NoBoundaryInMultipartDefect(MessageDefect): """A message claimed to be a multipart but had no boundary parameter.""" class StartBoundaryNotFoundDefect(MessageDefect): """The claimed start boundary was never found.""" class CloseBoundaryNotFoundDefect(MessageDefect): """A start boundary was found, but not the corresponding close boundary.""" class FirstHeaderLineIsContinuationDefect(MessageDefect): """A message had a continuation line as its first header line.""" class MisplacedEnvelopeHeaderDefect(MessageDefect): """A 'Unix-from' header was found in the middle of a header block.""" class MissingHeaderBodySeparatorDefect(MessageDefect): """Found line with no leading whitespace and no colon before blank line.""" # XXX: backward compatibility, just in case (it was never emitted). MalformedHeaderDefect = MissingHeaderBodySeparatorDefect class MultipartInvariantViolationDefect(MessageDefect): """A message claimed to be a multipart but no subparts were found.""" class InvalidMultipartContentTransferEncodingDefect(MessageDefect): """An invalid content transfer encoding was set on the multipart itself.""" class UndecodableBytesDefect(MessageDefect): """Header contained bytes that could not be decoded""" class InvalidBase64PaddingDefect(MessageDefect): """base64 encoded sequence had an incorrect length""" class InvalidBase64CharactersDefect(MessageDefect): """base64 encoded sequence had characters not in base64 alphabet""" class InvalidBase64LengthDefect(MessageDefect): """base64 encoded sequence had invalid length (1 mod 4)""" # These errors are specific to header parsing. class HeaderDefect(MessageDefect): """Base class for a header defect.""" def __init__(self, args, kw): super().__init__(args, **kw) class InvalidHeaderDefect(HeaderDefect): """Header is not valid, message gives details.""" class HeaderMissingRequiredValue(HeaderDefect): """A header that must have a value had none""" class NonPrintableDefect(HeaderDefect): """ASCII characters outside the ascii-printable range found""" def __init__(self, non_printables): super().__init__(non_printables) self.non_printables = non_printables def __str__(self): return ("the following ASCII non-printables found in header: " "{}".format(self.non_printables)) class ObsoleteHeaderDefect(HeaderDefect): """Header uses syntax declared obsolete by RFC 5322""" class NonASCIILocalPartDefect(HeaderDefect): """local_part contains non-ASCII characters""" # This defect only occurs during unicode parsing, not when # parsing messages decoded from binary.	3.56 KB	2024-04-17 17:55
📄 feedparser.py # Copyright (C) 2004-2006 Python Software Foundation # Authors: Baxter, Wouters and Warsaw # Contact: email-sig@python.org """FeedParser - An email feed parser. The feed parser implements an interface for incrementally parsing an email message, line by line. This has advantages for certain applications, such as those reading email messages off a socket. FeedParser.feed() is the primary interface for pushing new data into the parser. It returns when there's nothing more it can do with the available data. When you have no more data to push into the parser, call .close(). This completes the parsing and returns the root message object. The other advantage of this parser is that it will never raise a parsing exception. Instead, when it finds something unexpected, it adds a 'defect' to the current message. Defects are just instances that live on the message object's .defects attribute. """ __all__ = ['FeedParser', 'BytesFeedParser'] import re from email import errors from email._policybase import compat32 from collections import deque from io import StringIO NLCRE = re.compile(r'\r\n\|\r\|\n') NLCRE_bol = re.compile(r'(\r\n\|\r\|\n)') NLCRE_eol = re.compile(r'(\r\n\|\r\|\n)\Z') NLCRE_crack = re.compile(r'(\r\n\|\r\|\n)') # RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character # except controls, SP, and ":". headerRE = re.compile(r'^(From \|[\041-\071\073-\176]:\|[\t ])') EMPTYSTRING = '' NL = '\n' NeedMoreData = object() class BufferedSubFile(object): """A file-ish object that can have new data loaded into it. You can also push and pop line-matching predicates onto a stack. When the current predicate matches the current line, a false EOF response (i.e. empty string) is returned instead. This lets the parser adhere to a simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): # Text stream of the last partial line pushed into this object. # See issue 22233 for why this is a text stream and not a list. self._partial = StringIO(newline='') # A deque of full, pushed lines self._lines = deque() # The stack of false-EOF checking predicates. self._eofstack = [] # A flag indicating whether the file has been closed or not. self._closed = False def push_eof_matcher(self, pred): self._eofstack.append(pred) def pop_eof_matcher(self): return self._eofstack.pop() def close(self): # Don't forget any trailing partial line. self._partial.seek(0) self.pushlines(self._partial.readlines()) self._partial.seek(0) self._partial.truncate() self._closed = True def readline(self): if not self._lines: if self._closed: return '' return NeedMoreData # Pop the line off the stack and see if it matches the current # false-EOF predicate. line = self._lines.popleft() # RFC 2046, section 5.1.2 requires us to recognize outer level # boundaries at any level of inner nesting. Do this, but be sure it's # in the order of most to least nested. for ateof in reversed(self._eofstack): if ateof(line): # We're at the false EOF. But push the last line back first. self._lines.appendleft(line) return '' return line def unreadline(self, line): # Let the consumer push a line back into the buffer. assert line is not NeedMoreData self._lines.appendleft(line) def push(self, data): """Push some new data into this object.""" self._partial.write(data) if '\n' not in data and '\r' not in data: # No new complete lines, wait for more. return # Crack into lines, preserving the linesep characters. self._partial.seek(0) parts = self._partial.readlines() self._partial.seek(0) self._partial.truncate() # If the last element of the list does not end in a newline, then treat # it as a partial line. We only check for '\n' here because a line # ending with '\r' might be a line that was split in the middle of a # '\r\n' sequence (see bugs 1555570 and 1721862). if not parts[-1].endswith('\n'): self._partial.write(parts.pop()) self.pushlines(parts) def pushlines(self, lines): self._lines.extend(lines) def __iter__(self): return self def __next__(self): line = self.readline() if line == '': raise StopIteration return line class FeedParser: """A feed-style parser of email.""" def __init__(self, _factory=None, , policy=compat32): """_factory is called with no arguments to create a new message obj The policy keyword specifies a policy object that controls a number of aspects of the parser's operation. The default policy maintains backward compatibility. """ self.policy = policy self._old_style_factory = False if _factory is None: if policy.message_factory is None: from email.message import Message self._factory = Message else: self._factory = policy.message_factory else: self._factory = _factory try: _factory(policy=self.policy) except TypeError: # Assume this is an old-style factory self._old_style_factory = True self._input = BufferedSubFile() self._msgstack = [] self._parse = self._parsegen().__next__ self._cur = None self._last = None self._headersonly = False # Non-public interface for supporting Parser's headersonly flag def _set_headersonly(self): self._headersonly = True def feed(self, data): """Push more data into the parser.""" self._input.push(data) self._call_parse() def _call_parse(self): try: self._parse() except StopIteration: pass def close(self): """Parse all remaining data and return the root message object.""" self._input.close() self._call_parse() root = self._pop_message() assert not self._msgstack # Look for final set of defects if root.get_content_maintype() == 'multipart' \ and not root.is_multipart(): defect = errors.MultipartInvariantViolationDefect() self.policy.handle_defect(root, defect) return root def _new_message(self): if self._old_style_factory: msg = self._factory() else: msg = self._factory(policy=self.policy) if self._cur and self._cur.get_content_type() == 'multipart/digest': msg.set_default_type('message/rfc822') if self._msgstack: self._msgstack[-1].attach(msg) self._msgstack.append(msg) self._cur = msg self._last = msg def _pop_message(self): retval = self._msgstack.pop() if self._msgstack: self._cur = self._msgstack[-1] else: self._cur = None return retval def _parsegen(self): # Create a new message and start by parsing headers. self._new_message() headers = [] # Collect the headers, searching for a line that doesn't match the RFC # 2822 header or continuation pattern (including an empty line). for line in self._input: if line is NeedMoreData: yield NeedMoreData continue if not headerRE.match(line): # If we saw the RFC defined header/body separator # (i.e. newline), just throw it away. Otherwise the line is # part of the body so push it back. if not NLCRE.match(line): defect = errors.MissingHeaderBodySeparatorDefect() self.policy.handle_defect(self._cur, defect) self._input.unreadline(line) break headers.append(line) # Done with the headers, so parse them and figure out what we're # supposed to see in the body of the message. self._parse_headers(headers) # Headers-only parsing is a backwards compatibility hack, which was # necessary in the older parser, which could raise errors. All # remaining lines in the input are thrown into the message body. if self._headersonly: lines = [] while True: line = self._input.readline() if line is NeedMoreData: yield NeedMoreData continue if line == '': break lines.append(line) self._cur.set_payload(EMPTYSTRING.join(lines)) return if self._cur.get_content_type() == 'message/delivery-status': # message/delivery-status contains blocks of headers separated by # a blank line. We'll represent each header block as a separate # nested message object, but the processing is a bit different # than standard message/* types because there is no body for the # nested messages. A blank line separates the subparts. while True: self._input.push_eof_matcher(NLCRE.match) for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData continue break msg = self._pop_message() # We need to pop the EOF matcher in order to tell if we're at # the end of the current file, not the end of the last block # of message headers. self._input.pop_eof_matcher() # The input stream must be sitting at the newline or at the # EOF. We want to see if we're at the end of this subpart, so # first consume the blank line, then test the next line to see # if we're at this subpart's EOF. while True: line = self._input.readline() if line is NeedMoreData: yield NeedMoreData continue break while True: line = self._input.readline() if line is NeedMoreData: yield NeedMoreData continue break if line == '': break # Not at EOF so this is a line we're going to need. self._input.unreadline(line) return if self._cur.get_content_maintype() == 'message': # The message claims to be a message/* type, then what follows is # another RFC 2822 message. for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData continue break self._pop_message() return if self._cur.get_content_maintype() == 'multipart': boundary = self._cur.get_boundary() if boundary is None: # The message /claims/ to be a multipart but it has not # defined a boundary. That's a problem which we'll handle by # reading everything until the EOF and marking the message as # defective. defect = errors.NoBoundaryInMultipartDefect() self.policy.handle_defect(self._cur, defect) lines = [] for line in self._input: if line is NeedMoreData: yield NeedMoreData continue lines.append(line) self._cur.set_payload(EMPTYSTRING.join(lines)) return # Make sure a valid content type was specified per RFC 2045:6.4. if (str(self._cur.get('content-transfer-encoding', '8bit')).lower() not in ('7bit', '8bit', 'binary')): defect = errors.InvalidMultipartContentTransferEncodingDefect() self.policy.handle_defect(self._cur, defect) # Create a line match predicate which matches the inter-part # boundary as well as the end-of-multipart boundary. Don't push # this onto the input stream until we've scanned past the # preamble. separator = '--' + boundary boundaryre = re.compile( '(?P<sep>' + re.escape(separator) + r')(?P<end>--)?(?P<ws>[ \t])(?P<linesep>\r\n\|\r\|\n)?$') capturing_preamble = True preamble = [] linesep = False close_boundary_seen = False while True: line = self._input.readline() if line is NeedMoreData: yield NeedMoreData continue if line == '': break mo = boundaryre.match(line) if mo: # If we're looking at the end boundary, we're done with # this multipart. If there was a newline at the end of # the closing boundary, then we need to initialize the # epilogue with the empty string (see below). if mo.group('end'): close_boundary_seen = True linesep = mo.group('linesep') break # We saw an inter-part boundary. Were we in the preamble? if capturing_preamble: if preamble: # According to RFC 2046, the last newline belongs # to the boundary. lastline = preamble[-1] eolmo = NLCRE_eol.search(lastline) if eolmo: preamble[-1] = lastline[:-len(eolmo.group(0))] self._cur.preamble = EMPTYSTRING.join(preamble) capturing_preamble = False self._input.unreadline(line) continue # We saw a boundary separating two parts. Consume any # multiple boundary lines that may be following. Our # interpretation of RFC 2046 BNF grammar does not produce # body parts within such double boundaries. while True: line = self._input.readline() if line is NeedMoreData: yield NeedMoreData continue mo = boundaryre.match(line) if not mo: self._input.unreadline(line) break # Recurse to parse this subpart; the input stream points # at the subpart's first line. self._input.push_eof_matcher(boundaryre.match) for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData continue break # Because of RFC 2046, the newline preceding the boundary # separator actually belongs to the boundary, not the # previous subpart's payload (or epilogue if the previous # part is a multipart). if self._last.get_content_maintype() == 'multipart': epilogue = self._last.epilogue if epilogue == '': self._last.epilogue = None elif epilogue is not None: mo = NLCRE_eol.search(epilogue) if mo: end = len(mo.group(0)) self._last.epilogue = epilogue[:-end] else: payload = self._last._payload if isinstance(payload, str): mo = NLCRE_eol.search(payload) if mo: payload = payload[:-len(mo.group(0))] self._last._payload = payload self._input.pop_eof_matcher() self._pop_message() # Set the multipart up for newline cleansing, which will # happen if we're in a nested multipart. self._last = self._cur else: # I think we must be in the preamble assert capturing_preamble preamble.append(line) # We've seen either the EOF or the end boundary. If we're still # capturing the preamble, we never saw the start boundary. Note # that as a defect and store the captured text as the payload. if capturing_preamble: defect = errors.StartBoundaryNotFoundDefect() self.policy.handle_defect(self._cur, defect) self._cur.set_payload(EMPTYSTRING.join(preamble)) epilogue = [] for line in self._input: if line is NeedMoreData: yield NeedMoreData continue self._cur.epilogue = EMPTYSTRING.join(epilogue) return # If we're not processing the preamble, then we might have seen # EOF without seeing that end boundary...that is also a defect. if not close_boundary_seen: defect = errors.CloseBoundaryNotFoundDefect() self.policy.handle_defect(self._cur, defect) return # Everything from here to the EOF is epilogue. If the end boundary # ended in a newline, we'll need to make sure the epilogue isn't # None if linesep: epilogue = [''] else: epilogue = [] for line in self._input: if line is NeedMoreData: yield NeedMoreData continue epilogue.append(line) # Any CRLF at the front of the epilogue is not technically part of # the epilogue. Also, watch out for an empty string epilogue, # which means a single newline. if epilogue: firstline = epilogue[0] bolmo = NLCRE_bol.match(firstline) if bolmo: epilogue[0] = firstline[len(bolmo.group(0)):] self._cur.epilogue = EMPTYSTRING.join(epilogue) return # Otherwise, it's some non-multipart type, so the entire rest of the # file contents becomes the payload. lines = [] for line in self._input: if line is NeedMoreData: yield NeedMoreData continue lines.append(line) self._cur.set_payload(EMPTYSTRING.join(lines)) def _parse_headers(self, lines): # Passed a list of lines that make up the headers for the current msg lastheader = '' lastvalue = [] for lineno, line in enumerate(lines): # Check for continuation if line[0] in ' \t': if not lastheader: # The first line of the headers was a continuation. This # is illegal, so let's note the defect, store the illegal # line, and ignore it for purposes of headers. defect = errors.FirstHeaderLineIsContinuationDefect(line) self.policy.handle_defect(self._cur, defect) continue lastvalue.append(line) continue if lastheader: self._cur.set_raw(self.policy.header_source_parse(lastvalue)) lastheader, lastvalue = '', [] # Check for envelope header, i.e. unix-from if line.startswith('From '): if lineno == 0: # Strip off the trailing newline mo = NLCRE_eol.search(line) if mo: line = line[:-len(mo.group(0))] self._cur.set_unixfrom(line) continue elif lineno == len(lines) - 1: # Something looking like a unix-from at the end - it's # probably the first line of the body, so push back the # line and stop. self._input.unreadline(line) return else: # Weirdly placed unix-from line. Note this as a defect # and ignore it. defect = errors.MisplacedEnvelopeHeaderDefect(line) self._cur.defects.append(defect) continue # Split the line on the colon separating field name from value. # There will always be a colon, because if there wasn't the part of # the parser that calls us would have started parsing the body. i = line.find(':') # If the colon is on the start of the line the header is clearly # malformed, but we might be able to salvage the rest of the # message. Track the error but keep going. if i == 0: defect = errors.InvalidHeaderDefect("Missing header name.") self._cur.defects.append(defect) continue assert i>0, "_parse_headers fed line with no : and no leading WS" lastheader = line[:i] lastvalue = [line] # Done with all the lines, so handle the last header. if lastheader: self._cur.set_raw(*self.policy.header_source_parse(lastvalue)) class BytesFeedParser(FeedParser): """Like FeedParser, but feed accepts bytes.""" def feed(self, data): super().feed(data.decode('ascii', 'surrogateescape'))	22.25 KB	2024-04-17 17:55
📄 generator.py # Copyright (C) 2001-2010 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org """Classes to generate plain text from a message object tree.""" __all__ = ['Generator', 'DecodedGenerator', 'BytesGenerator'] import re import sys import time import random from copy import deepcopy from io import StringIO, BytesIO from email.utils import _has_surrogates UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. NLCRE = re.compile(r'\r\n\|\r\|\n') fcre = re.compile(r'^From ', re.MULTILINE) class Generator: """Generates output from a Message object tree. This basic generator writes the message to the given file object as plain text. """ # # Public interface # def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, , policy=None): """Create the generator for message flattening. outfp is the output file-like object for writing the message to. It must have a write() method. Optional mangle_from_ is a flag that, when True (the default if policy is not set), escapes From_ lines in the body of the message by putting a `>' in front of them. Optional maxheaderlen specifies the longest length for a non-continued header. When a header line is longer (in characters, with tabs expanded to 8 spaces) than maxheaderlen, the header will split as defined in the Header class. Set maxheaderlen to zero to disable header wrapping. The default is 78, as recommended (but not required) by RFC 2822. The policy keyword specifies a policy object that controls a number of aspects of the generator's operation. If no policy is specified, the policy associated with the Message object passed to the flatten method is used. """ if mangle_from_ is None: mangle_from_ = True if policy is None else policy.mangle_from_ self._fp = outfp self._mangle_from_ = mangle_from_ self.maxheaderlen = maxheaderlen self.policy = policy def write(self, s): # Just delegate to the file object self._fp.write(s) def flatten(self, msg, unixfrom=False, linesep=None): r"""Print the message object tree rooted at msg to the output file specified when the Generator instance was created. unixfrom is a flag that forces the printing of a Unix From_ delimiter before the first object in the message tree. If the original message has no From_ delimiter, a `standard' one is crafted. By default, this is False to inhibit the printing of any From_ delimiter. Note that for subobjects, no From_ line is printed. linesep specifies the characters used to indicate a new line in the output. The default value is determined by the policy specified when the Generator instance was created or, if none was specified, from the policy associated with the msg. """ # We use the _XXX constants for operating on data that comes directly # from the msg, and _encoded_XXX constants for operating on data that # has already been converted (to bytes in the BytesGenerator) and # inserted into a temporary buffer. policy = msg.policy if self.policy is None else self.policy if linesep is not None: policy = policy.clone(linesep=linesep) if self.maxheaderlen is not None: policy = policy.clone(max_line_length=self.maxheaderlen) self._NL = policy.linesep self._encoded_NL = self._encode(self._NL) self._EMPTY = '' self._encoded_EMPTY = self._encode(self._EMPTY) # Because we use clone (below) when we recursively process message # subparts, and because clone uses the computed policy (not None), # submessages will automatically get set to the computed policy when # they are processed by this code. old_gen_policy = self.policy old_msg_policy = msg.policy try: self.policy = policy msg.policy = policy if unixfrom: ufrom = msg.get_unixfrom() if not ufrom: ufrom = 'From nobody ' + time.ctime(time.time()) self.write(ufrom + self._NL) self._write(msg) finally: self.policy = old_gen_policy msg.policy = old_msg_policy def clone(self, fp): """Clone this generator with the exact same options.""" return self.__class__(fp, self._mangle_from_, None, # Use policy setting, which we've adjusted policy=self.policy) # # Protected interface - undocumented ;/ # # Note that we use 'self.write' when what we are writing is coming from # the source, and self._fp.write when what we are writing is coming from a # buffer (because the Bytes subclass has already had a chance to transform # the data in its write method in that case). This is an entirely # pragmatic split determined by experiment; we could be more general by # always using write and having the Bytes subclass write method detect when # it has already transformed the input; but, since this whole thing is a # hack anyway this seems good enough. def _new_buffer(self): # BytesGenerator overrides this to return BytesIO. return StringIO() def _encode(self, s): # BytesGenerator overrides this to encode strings to bytes. return s def _write_lines(self, lines): # We have to transform the line endings. if not lines: return lines = NLCRE.split(lines) for line in lines[:-1]: self.write(line) self.write(self._NL) if lines[-1]: self.write(lines[-1]) # XXX logic tells me this else should be needed, but the tests fail # with it and pass without it. (NLCRE.split ends with a blank element # if and only if there was a trailing newline.) #else: # self.write(self._NL) def _write(self, msg): # We can't write the headers yet because of the following scenario: # say a multipart message includes the boundary string somewhere in # its body. We'd have to calculate the new boundary /before/ we write # the headers so that we can write the correct Content-Type: # parameter. # # The way we do this, so as to make the _handle_() methods simpler, # is to cache any subpart writes into a buffer. The we write the # headers and the buffer contents. That way, subpart handlers can # Do The Right Thing, and can still modify the Content-Type: header if # necessary. oldfp = self._fp try: self._munge_cte = None self._fp = sfp = self._new_buffer() self._dispatch(msg) finally: self._fp = oldfp munge_cte = self._munge_cte del self._munge_cte # If we munged the cte, copy the message again and re-fix the CTE. if munge_cte: msg = deepcopy(msg) msg.replace_header('content-transfer-encoding', munge_cte[0]) msg.replace_header('content-type', munge_cte[1]) # Write the headers. First we see if the message object wants to # handle that itself. If not, we'll do it generically. meth = getattr(msg, '_write_headers', None) if meth is None: self._write_headers(msg) else: meth(self) self._fp.write(sfp.getvalue()) def _dispatch(self, msg): # Get the Content-Type: for the message, then try to dispatch to # self._handle_<maintype>_<subtype>(). If there's no handler for the # full MIME type, then dispatch to self._handle_<maintype>(). If # that's missing too, then dispatch to self._writeBody(). main = msg.get_content_maintype() sub = msg.get_content_subtype() specific = UNDERSCORE.join((main, sub)).replace('-', '_') meth = getattr(self, '_handle_' + specific, None) if meth is None: generic = main.replace('-', '_') meth = getattr(self, '_handle_' + generic, None) if meth is None: meth = self._writeBody meth(msg) # # Default handlers # def _write_headers(self, msg): for h, v in msg.raw_items(): self.write(self.policy.fold(h, v)) # A blank line always separates headers from body self.write(self._NL) # # Handlers for writing types and subtypes # def _handle_text(self, msg): payload = msg.get_payload() if payload is None: return if not isinstance(payload, str): raise TypeError('string payload expected: %s' % type(payload)) if _has_surrogates(msg._payload): charset = msg.get_param('charset') if charset is not None: # XXX: This copy stuff is an ugly hack to avoid modifying the # existing message. msg = deepcopy(msg) del msg['content-transfer-encoding'] msg.set_payload(payload, charset) payload = msg.get_payload() self._munge_cte = (msg['content-transfer-encoding'], msg['content-type']) if self._mangle_from_: payload = fcre.sub('>From ', payload) self._write_lines(payload) # Default body handler _writeBody = _handle_text def _handle_multipart(self, msg): # The trick here is to write out each part separately, merge them all # together, and then make sure that the boundary we've chosen isn't # present in the payload. msgtexts = [] subparts = msg.get_payload() if subparts is None: subparts = [] elif isinstance(subparts, str): # e.g. a non-strict parse of a message with no starting boundary. self.write(subparts) return elif not isinstance(subparts, list): # Scalar payload subparts = [subparts] for part in subparts: s = self._new_buffer() g = self.clone(s) g.flatten(part, unixfrom=False, linesep=self._NL) msgtexts.append(s.getvalue()) # BAW: What about boundaries that are wrapped in double-quotes? boundary = msg.get_boundary() if not boundary: # Create a boundary that doesn't appear in any of the # message texts. alltext = self._encoded_NL.join(msgtexts) boundary = self._make_boundary(alltext) msg.set_boundary(boundary) # If there's a preamble, write it out, with a trailing CRLF if msg.preamble is not None: if self._mangle_from_: preamble = fcre.sub('>From ', msg.preamble) else: preamble = msg.preamble self._write_lines(preamble) self.write(self._NL) # dash-boundary transport-padding CRLF self.write('--' + boundary + self._NL) # body-part if msgtexts: self._fp.write(msgtexts.pop(0)) # encapsulation # --> delimiter transport-padding # --> CRLF body-part for body_part in msgtexts: # delimiter transport-padding CRLF self.write(self._NL + '--' + boundary + self._NL) # body-part self._fp.write(body_part) # close-delimiter transport-padding self.write(self._NL + '--' + boundary + '--' + self._NL) if msg.epilogue is not None: if self._mangle_from_: epilogue = fcre.sub('>From ', msg.epilogue) else: epilogue = msg.epilogue self._write_lines(epilogue) def _handle_multipart_signed(self, msg): # The contents of signed parts has to stay unmodified in order to keep # the signature intact per RFC1847 2.1, so we disable header wrapping. # RDM: This isn't enough to completely preserve the part, but it helps. p = self.policy self.policy = p.clone(max_line_length=0) try: self._handle_multipart(msg) finally: self.policy = p def _handle_message_delivery_status(self, msg): # We can't just write the headers directly to self's file object # because this will leave an extra newline between the last header # block and the boundary. Sigh. blocks = [] for part in msg.get_payload(): s = self._new_buffer() g = self.clone(s) g.flatten(part, unixfrom=False, linesep=self._NL) text = s.getvalue() lines = text.split(self._encoded_NL) # Strip off the unnecessary trailing empty line if lines and lines[-1] == self._encoded_EMPTY: blocks.append(self._encoded_NL.join(lines[:-1])) else: blocks.append(text) # Now join all the blocks with an empty line. This has the lovely # effect of separating each block with an empty line, but not adding # an extra one after the last one. self._fp.write(self._encoded_NL.join(blocks)) def _handle_message(self, msg): s = self._new_buffer() g = self.clone(s) # The payload of a message/rfc822 part should be a multipart sequence # of length 1. The zeroth element of the list should be the Message # object for the subpart. Extract that object, stringify it, and # write it out. # Except, it turns out, when it's a string instead, which happens when # and only when HeaderParser is used on a message of mime type # message/rfc822. Such messages are generated by, for example, # Groupwise when forwarding unadorned messages. (Issue 7970.) So # in that case we just emit the string body. payload = msg._payload if isinstance(payload, list): g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL) payload = s.getvalue() else: payload = self._encode(payload) self._fp.write(payload) # This used to be a module level function; we use a classmethod for this # and _compile_re so we can continue to provide the module level function # for backward compatibility by doing # _make_boundary = Generator._make_boundary # at the end of the module. It is* internal, so we could drop that... @classmethod def _make_boundary(cls, text=None): # Craft a random boundary. If text is given, ensure that the chosen # boundary doesn't appear in the text. token = random.randrange(sys.maxsize) boundary = ('=' * 15) + (_fmt % token) + '==' if text is None: return boundary b = boundary counter = 0 while True: cre = cls._compile_re('^--' + re.escape(b) + '(--)?$', re.MULTILINE) if not cre.search(text): break b = boundary + '.' + str(counter) counter += 1 return b @classmethod def _compile_re(cls, s, flags): return re.compile(s, flags) class BytesGenerator(Generator): """Generates a bytes version of a Message object tree. Functionally identical to the base Generator except that the output is bytes and not string. When surrogates were used in the input to encode bytes, these are decoded back to bytes for output. If the policy has cte_type set to 7bit, then the message is transformed such that the non-ASCII bytes are properly content transfer encoded, using the charset unknown-8bit. The outfp object must accept bytes in its write method. """ def write(self, s): self._fp.write(s.encode('ascii', 'surrogateescape')) def _new_buffer(self): return BytesIO() def _encode(self, s): return s.encode('ascii') def _write_headers(self, msg): # This is almost the same as the string version, except for handling # strings with 8bit bytes. for h, v in msg.raw_items(): self._fp.write(self.policy.fold_binary(h, v)) # A blank line always separates headers from body self.write(self._NL) def _handle_text(self, msg): # If the string has surrogates the original source was bytes, so # just write it back out. if msg._payload is None: return if _has_surrogates(msg._payload) and not self.policy.cte_type=='7bit': if self._mangle_from_: msg._payload = fcre.sub(">From ", msg._payload) self._write_lines(msg._payload) else: super(BytesGenerator,self)._handle_text(msg) # Default body handler _writeBody = _handle_text @classmethod def _compile_re(cls, s, flags): return re.compile(s.encode('ascii'), flags) _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): """Generates a text representation of a message. Like the Generator base class, except that non-text parts are substituted with a format string representing the part. """ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, fmt=None, *, policy=None): """Like Generator.__init__() except that an additional optional argument is allowed. Walks through all subparts of a message. If the subpart is of main type `text', then it prints the decoded payload of the subpart. Otherwise, fmt is a format string that is used instead of the message payload. fmt is expanded with the following keywords (in %(keyword)s format): type : Full MIME type of the non-text part maintype : Main MIME type of the non-text part subtype : Sub-MIME type of the non-text part filename : Filename of the non-text part description: Description associated with the non-text part encoding : Content transfer encoding of the non-text part The default value for fmt is None, meaning [Non-text (%(type)s) part of message omitted, filename %(filename)s] """ Generator.__init__(self, outfp, mangle_from_, maxheaderlen, policy=policy) if fmt is None: self._fmt = _FMT else: self._fmt = fmt def _dispatch(self, msg): for part in msg.walk(): maintype = part.get_content_maintype() if maintype == 'text': print(part.get_payload(decode=False), file=self) elif maintype == 'multipart': # Just skip this pass else: print(self._fmt % { 'type' : part.get_content_type(), 'maintype' : part.get_content_maintype(), 'subtype' : part.get_content_subtype(), 'filename' : part.get_filename('[no filename]'), 'description': part.get('Content-Description', '[no description]'), 'encoding' : part.get('Content-Transfer-Encoding', '[no encoding]'), }, file=self) # Helper used by Generator._make_boundary _width = len(repr(sys.maxsize-1)) _fmt = '%%0%dd' % _width # Backward compatibility _make_boundary = Generator._make_boundary	19.51 KB	2024-04-17 17:55
📄 header.py # Copyright (C) 2002-2007 Python Software Foundation # Author: Ben Gertzfield, Barry Warsaw # Contact: email-sig@python.org """Header encoding and decoding functionality.""" __all__ = [ 'Header', 'decode_header', 'make_header', ] import re import binascii import email.quoprimime import email.base64mime from email.errors import HeaderParseError from email import charset as _charset Charset = _charset.Charset NL = '\n' SPACE = ' ' BSPACE = b' ' SPACE8 = ' ' * 8 EMPTYSTRING = '' MAXLINELEN = 78 FWS = ' \t' USASCII = Charset('us-ascii') UTF8 = Charset('utf-8') # Match encoded-word strings in the form =?charset?q?Hello_World?= ecre = re.compile(r''' =\? # literal =? (?P<charset>[^?]?) # non-greedy up to the next ? is the charset \? # literal ? (?P<encoding>[qQbB]) # either a "q" or a "b", case insensitive \? # literal ? (?P<encoded>.?) # non-greedy up to the next ?= is the encoded string \?= # literal ?= ''', re.VERBOSE \| re.MULTILINE) # Field name regexp, including trailing colon, but not separating whitespace, # according to RFC 2822. Character range is from tilde to exclamation mark. # For use with .match() fcre = re.compile(r'[\041-\176]+:$') # Find a header embedded in a putative header value. Used to check for # header injection attack. _embedded_header = re.compile(r'\n[^ \t]+:') # Helpers _max_append = email.quoprimime._max_append def decode_header(header): """Decode a message header value without converting charset. Returns a list of (string, charset) pairs containing each of the decoded parts of the header. Charset is None for non-encoded parts of the header, otherwise a lower-case string containing the name of the character set specified in the encoded string. header may be a string that may or may not contain RFC2047 encoded words, or it may be a Header object. An email.errors.HeaderParseError may be raised when certain decoding error occurs (e.g. a base64 decoding exception). """ # If it is a Header object, we can just return the encoded chunks. if hasattr(header, '_chunks'): return [(_charset._encode(string, str(charset)), str(charset)) for string, charset in header._chunks] # If no encoding, just return the header with no charset. if not ecre.search(header): return [(header, None)] # First step is to parse all the encoded parts into triplets of the form # (encoded_string, encoding, charset). For unencoded strings, the last # two parts will be None. words = [] for line in header.splitlines(): parts = ecre.split(line) first = True while parts: unencoded = parts.pop(0) if first: unencoded = unencoded.lstrip() first = False if unencoded: words.append((unencoded, None, None)) if parts: charset = parts.pop(0).lower() encoding = parts.pop(0).lower() encoded = parts.pop(0) words.append((encoded, encoding, charset)) # Now loop over words and remove words that consist of whitespace # between two encoded strings. droplist = [] for n, w in enumerate(words): if n>1 and w[1] and words[n-2][1] and words[n-1][0].isspace(): droplist.append(n-1) for d in reversed(droplist): del words[d] # The next step is to decode each encoded word by applying the reverse # base64 or quopri transformation. decoded_words is now a list of the # form (decoded_word, charset). decoded_words = [] for encoded_string, encoding, charset in words: if encoding is None: # This is an unencoded word. decoded_words.append((encoded_string, charset)) elif encoding == 'q': word = email.quoprimime.header_decode(encoded_string) decoded_words.append((word, charset)) elif encoding == 'b': paderr = len(encoded_string) % 4 # Postel's law: add missing padding if paderr: encoded_string += '==='[:4 - paderr] try: word = email.base64mime.decode(encoded_string) except binascii.Error: raise HeaderParseError('Base64 decoding error') else: decoded_words.append((word, charset)) else: raise AssertionError('Unexpected encoding: ' + encoding) # Now convert all words to bytes and collapse consecutive runs of # similarly encoded words. collapsed = [] last_word = last_charset = None for word, charset in decoded_words: if isinstance(word, str): word = bytes(word, 'raw-unicode-escape') if last_word is None: last_word = word last_charset = charset elif charset != last_charset: collapsed.append((last_word, last_charset)) last_word = word last_charset = charset elif last_charset is None: last_word += BSPACE + word else: last_word += word collapsed.append((last_word, last_charset)) return collapsed def make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' '): """Create a Header from a sequence of pairs as returned by decode_header() decode_header() takes a header value string and returns a sequence of pairs of the format (decoded_string, charset) where charset is the string name of the character set. This function takes one of those sequence of pairs and returns a Header instance. Optional maxlinelen, header_name, and continuation_ws are as in the Header constructor. """ h = Header(maxlinelen=maxlinelen, header_name=header_name, continuation_ws=continuation_ws) for s, charset in decoded_seq: # None means us-ascii but we can simply pass it on to h.append() if charset is not None and not isinstance(charset, Charset): charset = Charset(charset) h.append(s, charset) return h class Header: def __init__(self, s=None, charset=None, maxlinelen=None, header_name=None, continuation_ws=' ', errors='strict'): """Create a MIME-compliant header that can contain many character sets. Optional s is the initial header value. If None, the initial header value is not set. You can later append to the header with .append() method calls. s may be a byte string or a Unicode string, but see the .append() documentation for semantics. Optional charset serves two purposes: it has the same meaning as the charset argument to the .append() method. It also sets the default character set for all subsequent .append() calls that omit the charset argument. If charset is not provided in the constructor, the us-ascii charset is used both as s's initial charset and as the default for subsequent .append() calls. The maximum line length can be specified explicitly via maxlinelen. For splitting the first line to a shorter value (to account for the field header which isn't included in s, e.g. `Subject') pass in the name of the field in header_name. The default maxlinelen is 78 as recommended by RFC 2822. continuation_ws must be RFC 2822 compliant folding whitespace (usually either a space or a hard tab) which will be prepended to continuation lines. errors is passed through to the .append() call. """ if charset is None: charset = USASCII elif not isinstance(charset, Charset): charset = Charset(charset) self._charset = charset self._continuation_ws = continuation_ws self._chunks = [] if s is not None: self.append(s, charset, errors) if maxlinelen is None: maxlinelen = MAXLINELEN self._maxlinelen = maxlinelen if header_name is None: self._headerlen = 0 else: # Take the separating colon and space into account. self._headerlen = len(header_name) + 2 def __str__(self): """Return the string value of the header.""" self._normalize() uchunks = [] lastcs = None lastspace = None for string, charset in self._chunks: # We must preserve spaces between encoded and non-encoded word # boundaries, which means for us we need to add a space when we go # from a charset to None/us-ascii, or from None/us-ascii to a # charset. Only do this for the second and subsequent chunks. # Don't add a space if the None/us-ascii string already has # a space (trailing or leading depending on transition) nextcs = charset if nextcs == _charset.UNKNOWN8BIT: original_bytes = string.encode('ascii', 'surrogateescape') string = original_bytes.decode('ascii', 'replace') if uchunks: hasspace = string and self._nonctext(string[0]) if lastcs not in (None, 'us-ascii'): if nextcs in (None, 'us-ascii') and not hasspace: uchunks.append(SPACE) nextcs = None elif nextcs not in (None, 'us-ascii') and not lastspace: uchunks.append(SPACE) lastspace = string and self._nonctext(string[-1]) lastcs = nextcs uchunks.append(string) return EMPTYSTRING.join(uchunks) # Rich comparison operators for equality only. BAW: does it make sense to # have or explicitly disable <, <=, >, >= operators? def __eq__(self, other): # other may be a Header or a string. Both are fine so coerce # ourselves to a unicode (of the unencoded header value), swap the # args and do another comparison. return other == str(self) def append(self, s, charset=None, errors='strict'): """Append a string to the MIME header. Optional charset, if given, should be a Charset instance or the name of a character set (which will be converted to a Charset instance). A value of None (the default) means that the charset given in the constructor is used. s may be a byte string or a Unicode string. If it is a byte string (i.e. isinstance(s, str) is false), then charset is the encoding of that byte string, and a UnicodeError will be raised if the string cannot be decoded with that charset. If s is a Unicode string, then charset is a hint specifying the character set of the characters in the string. In either case, when producing an RFC 2822 compliant header using RFC 2047 rules, the string will be encoded using the output codec of the charset. If the string cannot be encoded to the output codec, a UnicodeError will be raised. Optional `errors' is passed as the errors argument to the decode call if s is a byte string. """ if charset is None: charset = self._charset elif not isinstance(charset, Charset): charset = Charset(charset) if not isinstance(s, str): input_charset = charset.input_codec or 'us-ascii' if input_charset == _charset.UNKNOWN8BIT: s = s.decode('us-ascii', 'surrogateescape') else: s = s.decode(input_charset, errors) # Ensure that the bytes we're storing can be decoded to the output # character set, otherwise an early error is raised. output_charset = charset.output_codec or 'us-ascii' if output_charset != _charset.UNKNOWN8BIT: try: s.encode(output_charset, errors) except UnicodeEncodeError: if output_charset!='us-ascii': raise charset = UTF8 self._chunks.append((s, charset)) def _nonctext(self, s): """True if string s is not a ctext character of RFC822. """ return s.isspace() or s in ('(', ')', '\\') def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'): r"""Encode a message header into an RFC-compliant format. There are many issues involved in converting a given string for use in an email header. Only certain character sets are readable in most email clients, and as header strings can only contain a subset of 7-bit ASCII, care must be taken to properly convert and encode (with Base64 or quoted-printable) header strings. In addition, there is a 75-character length limit on any given encoded header field, so line-wrapping must be performed, even with double-byte character sets. Optional maxlinelen specifies the maximum length of each generated line, exclusive of the linesep string. Individual lines may be longer than maxlinelen if a folding point cannot be found. The first line will be shorter by the length of the header name plus ": " if a header name was specified at Header construction time. The default value for maxlinelen is determined at header construction time. Optional splitchars is a string containing characters which should be given extra weight by the splitting algorithm during normal header wrapping. This is in very rough support of RFC 2822's `higher level syntactic breaks': split points preceded by a splitchar are preferred during line splitting, with the characters preferred in the order in which they appear in the string. Space and tab may be included in the string to indicate whether preference should be given to one over the other as a split point when other split chars do not appear in the line being split. Splitchars does not affect RFC 2047 encoded lines. Optional linesep is a string to be used to separate the lines of the value. The default value is the most useful for typical Python applications, but it can be set to \r\n to produce RFC-compliant line separators when needed. """ self._normalize() if maxlinelen is None: maxlinelen = self._maxlinelen # A maxlinelen of 0 means don't wrap. For all practical purposes, # choosing a huge number here accomplishes that and makes the # _ValueFormatter algorithm much simpler. if maxlinelen == 0: maxlinelen = 1000000 formatter = _ValueFormatter(self._headerlen, maxlinelen, self._continuation_ws, splitchars) lastcs = None hasspace = lastspace = None for string, charset in self._chunks: if hasspace is not None: hasspace = string and self._nonctext(string[0]) if lastcs not in (None, 'us-ascii'): if not hasspace or charset not in (None, 'us-ascii'): formatter.add_transition() elif charset not in (None, 'us-ascii') and not lastspace: formatter.add_transition() lastspace = string and self._nonctext(string[-1]) lastcs = charset hasspace = False lines = string.splitlines() if lines: formatter.feed('', lines[0], charset) else: formatter.feed('', '', charset) for line in lines[1:]: formatter.newline() if charset.header_encoding is not None: formatter.feed(self._continuation_ws, ' ' + line.lstrip(), charset) else: sline = line.lstrip() fws = line[:len(line)-len(sline)] formatter.feed(fws, sline, charset) if len(lines) > 1: formatter.newline() if self._chunks: formatter.add_transition() value = formatter._str(linesep) if _embedded_header.search(value): raise HeaderParseError("header value appears to contain " "an embedded header: {!r}".format(value)) return value def _normalize(self): # Step 1: Normalize the chunks so that all runs of identical charsets # get collapsed into a single unicode string. chunks = [] last_charset = None last_chunk = [] for string, charset in self._chunks: if charset == last_charset: last_chunk.append(string) else: if last_charset is not None: chunks.append((SPACE.join(last_chunk), last_charset)) last_chunk = [string] last_charset = charset if last_chunk: chunks.append((SPACE.join(last_chunk), last_charset)) self._chunks = chunks class _ValueFormatter: def __init__(self, headerlen, maxlen, continuation_ws, splitchars): self._maxlen = maxlen self._continuation_ws = continuation_ws self._continuation_ws_len = len(continuation_ws) self._splitchars = splitchars self._lines = [] self._current_line = _Accumulator(headerlen) def _str(self, linesep): self.newline() return linesep.join(self._lines) def __str__(self): return self._str(NL) def newline(self): end_of_line = self._current_line.pop() if end_of_line != (' ', ''): self._current_line.push(end_of_line) if len(self._current_line) > 0: if self._current_line.is_onlyws() and self._lines: self._lines[-1] += str(self._current_line) else: self._lines.append(str(self._current_line)) self._current_line.reset() def add_transition(self): self._current_line.push(' ', '') def feed(self, fws, string, charset): # If the charset has no header encoding (i.e. it is an ASCII encoding) # then we must split the header at the "highest level syntactic break" # possible. Note that we don't have a lot of smarts about field # syntax; we just try to break on semi-colons, then commas, then # whitespace. Eventually, this should be pluggable. if charset.header_encoding is None: self._ascii_split(fws, string, self._splitchars) return # Otherwise, we're doing either a Base64 or a quoted-printable # encoding which means we don't need to split the line on syntactic # breaks. We can basically just find enough characters to fit on the # current line, minus the RFC 2047 chrome. What makes this trickier # though is that we have to split at octet boundaries, not character # boundaries but it's only safe to split at character boundaries so at # best we can only get close. encoded_lines = charset.header_encode_lines(string, self._maxlengths()) # The first element extends the current line, but if it's None then # nothing more fit on the current line so start a new line. try: first_line = encoded_lines.pop(0) except IndexError: # There are no encoded lines, so we're done. return if first_line is not None: self._append_chunk(fws, first_line) try: last_line = encoded_lines.pop() except IndexError: # There was only one line. return self.newline() self._current_line.push(self._continuation_ws, last_line) # Everything else are full lines in themselves. for line in encoded_lines: self._lines.append(self._continuation_ws + line) def _maxlengths(self): # The first line's length. yield self._maxlen - len(self._current_line) while True: yield self._maxlen - self._continuation_ws_len def _ascii_split(self, fws, string, splitchars): # The RFC 2822 header folding algorithm is simple in principle but # complex in practice. Lines may be folded any place where "folding # white space" appears by inserting a linesep character in front of the # FWS. The complication is that not all spaces or tabs qualify as FWS, # and we are also supposed to prefer to break at "higher level # syntactic breaks". We can't do either of these without intimate # knowledge of the structure of structured headers, which we don't have # here. So the best we can do here is prefer to break at the specified # splitchars, and hope that we don't choose any spaces or tabs that # aren't legal FWS. (This is at least better than the old algorithm, # where we would sometimes introduce* FWS after a splitchar, or the # algorithm before that, where we would turn all white space runs into # single spaces or tabs.) parts = re.split("(["+FWS+"]+)", fws+string) if parts[0]: parts[:0] = [''] else: parts.pop(0) for fws, part in zip([iter(parts)]2): self._append_chunk(fws, part) def _append_chunk(self, fws, string): self._current_line.push(fws, string) if len(self._current_line) > self._maxlen: # Find the best split point, working backward from the end. # There might be none, on a long first line. for ch in self._splitchars: for i in range(self._current_line.part_count()-1, 0, -1): if ch.isspace(): fws = self._current_line[i][0] if fws and fws[0]==ch: break prevpart = self._current_line[i-1][1] if prevpart and prevpart[-1]==ch: break else: continue break else: fws, part = self._current_line.pop() if self._current_line._initial_size > 0: # There will be a header, so leave it on a line by itself. self.newline() if not fws: # We don't use continuation_ws here because the whitespace # after a header should always be a space. fws = ' ' self._current_line.push(fws, part) return remainder = self._current_line.pop_from(i) self._lines.append(str(self._current_line)) self._current_line.reset(remainder) class _Accumulator(list): def __init__(self, initial_size=0): self._initial_size = initial_size super().__init__() def push(self, fws, string): self.append((fws, string)) def pop_from(self, i=0): popped = self[i:] self[i:] = [] return popped def pop(self): if self.part_count()==0: return ('', '') return super().pop() def __len__(self): return sum((len(fws)+len(part) for fws, part in self), self._initial_size) def __str__(self): return EMPTYSTRING.join((EMPTYSTRING.join((fws, part)) for fws, part in self)) def reset(self, startval=None): if startval is None: startval = [] self[:] = startval self._initial_size = 0 def is_onlyws(self): return self._initial_size==0 and (not self or str(self).isspace()) def part_count(self): return super().__len__()	23.54 KB	2024-04-17 17:55
📄 headerregistry.py """Representing and manipulating email headers via custom objects. This module provides an implementation of the HeaderRegistry API. The implementation is designed to flexibly follow RFC5322 rules. Eventually HeaderRegistry will be a public API, but it isn't yet, and will probably change some before that happens. """ from types import MappingProxyType from email import utils from email import errors from email import _header_value_parser as parser class Address: def __init__(self, display_name='', username='', domain='', addr_spec=None): """Create an object representing a full email address. An address can have a 'display_name', a 'username', and a 'domain'. In addition to specifying the username and domain separately, they may be specified together by using the addr_spec keyword instead of the username and domain keywords. If an addr_spec string is specified it must be properly quoted according to RFC 5322 rules; an error will be raised if it is not. An Address object has display_name, username, domain, and addr_spec attributes, all of which are read-only. The addr_spec and the string value of the object are both quoted according to RFC5322 rules, but without any Content Transfer Encoding. """ inputs = ''.join(filter(None, (display_name, username, domain, addr_spec))) if '\r' in inputs or '\n' in inputs: raise ValueError("invalid arguments; address parts cannot contain CR or LF") # This clause with its potential 'raise' may only happen when an # application program creates an Address object using an addr_spec # keyword. The email library code itself must always supply username # and domain. if addr_spec is not None: if username or domain: raise TypeError("addrspec specified when username and/or " "domain also specified") a_s, rest = parser.get_addr_spec(addr_spec) if rest: raise ValueError("Invalid addr_spec; only '{}' " "could be parsed from '{}'".format( a_s, addr_spec)) if a_s.all_defects: raise a_s.all_defects[0] username = a_s.local_part domain = a_s.domain self._display_name = display_name self._username = username self._domain = domain @property def display_name(self): return self._display_name @property def username(self): return self._username @property def domain(self): return self._domain @property def addr_spec(self): """The addr_spec (username@domain) portion of the address, quoted according to RFC 5322 rules, but with no Content Transfer Encoding. """ nameset = set(self.username) if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS): lp = parser.quote_string(self.username) else: lp = self.username if self.domain: return lp + '@' + self.domain if not lp: return '<>' return lp def __repr__(self): return "{}(display_name={!r}, username={!r}, domain={!r})".format( self.__class__.__name__, self.display_name, self.username, self.domain) def __str__(self): nameset = set(self.display_name) if len(nameset) > len(nameset-parser.SPECIALS): disp = parser.quote_string(self.display_name) else: disp = self.display_name if disp: addr_spec = '' if self.addr_spec=='<>' else self.addr_spec return "{} <{}>".format(disp, addr_spec) return self.addr_spec def __eq__(self, other): if type(other) != type(self): return False return (self.display_name == other.display_name and self.username == other.username and self.domain == other.domain) class Group: def __init__(self, display_name=None, addresses=None): """Create an object representing an address group. An address group consists of a display_name followed by colon and a list of addresses (see Address) terminated by a semi-colon. The Group is created by specifying a display_name and a possibly empty list of Address objects. A Group can also be used to represent a single address that is not in a group, which is convenient when manipulating lists that are a combination of Groups and individual Addresses. In this case the display_name should be set to None. In particular, the string representation of a Group whose display_name is None is the same as the Address object, if there is one and only one Address object in the addresses list. """ self._display_name = display_name self._addresses = tuple(addresses) if addresses else tuple() @property def display_name(self): return self._display_name @property def addresses(self): return self._addresses def __repr__(self): return "{}(display_name={!r}, addresses={!r}".format( self.__class__.__name__, self.display_name, self.addresses) def __str__(self): if self.display_name is None and len(self.addresses)==1: return str(self.addresses[0]) disp = self.display_name if disp is not None: nameset = set(disp) if len(nameset) > len(nameset-parser.SPECIALS): disp = parser.quote_string(disp) adrstr = ", ".join(str(x) for x in self.addresses) adrstr = ' ' + adrstr if adrstr else adrstr return "{}:{};".format(disp, adrstr) def __eq__(self, other): if type(other) != type(self): return False return (self.display_name == other.display_name and self.addresses == other.addresses) # Header Classes # class BaseHeader(str): """Base class for message headers. Implements generic behavior and provides tools for subclasses. A subclass must define a classmethod named 'parse' that takes an unfolded value string and a dictionary as its arguments. The dictionary will contain one key, 'defects', initialized to an empty list. After the call the dictionary must contain two additional keys: parse_tree, set to the parse tree obtained from parsing the header, and 'decoded', set to the string value of the idealized representation of the data from the value. (That is, encoded words are decoded, and values that have canonical representations are so represented.) The defects key is intended to collect parsing defects, which the message parser will subsequently dispose of as appropriate. The parser should not, insofar as practical, raise any errors. Defects should be added to the list instead. The standard header parsers register defects for RFC compliance issues, for obsolete RFC syntax, and for unrecoverable parsing errors. The parse method may add additional keys to the dictionary. In this case the subclass must define an 'init' method, which will be passed the dictionary as its keyword arguments. The method should use (usually by setting them as the value of similarly named attributes) and remove all the extra keys added by its parse method, and then use super to call its parent class with the remaining arguments and keywords. The subclass should also make sure that a 'max_count' attribute is defined that is either None or 1. XXX: need to better define this API. """ def __new__(cls, name, value): kwds = {'defects': []} cls.parse(value, kwds) if utils._has_surrogates(kwds['decoded']): kwds['decoded'] = utils._sanitize(kwds['decoded']) self = str.__new__(cls, kwds['decoded']) del kwds['decoded'] self.init(name, *kwds) return self def init(self, name, , parse_tree, defects): self._name = name self._parse_tree = parse_tree self._defects = defects @property def name(self): return self._name @property def defects(self): return tuple(self._defects) def __reduce__(self): return ( _reconstruct_header, ( self.__class__.__name__, self.__class__.__bases__, str(self), ), self.__dict__) @classmethod def _reconstruct(cls, value): return str.__new__(cls, value) def fold(self, , policy): """Fold header according to policy. The parsed representation of the header is folded according to RFC5322 rules, as modified by the policy. If the parse tree contains surrogateescaped bytes, the bytes are CTE encoded using the charset 'unknown-8bit". Any non-ASCII characters in the parse tree are CTE encoded using charset utf-8. XXX: make this a policy setting. The returned value is an ASCII-only string possibly containing linesep characters, and ending with a linesep character. The string includes the header name and the ': ' separator. """ # At some point we need to put fws here if it was in the source. header = parser.Header([ parser.HeaderLabel([ parser.ValueTerminal(self.name, 'header-name'), parser.ValueTerminal(':', 'header-sep')]), ]) if self._parse_tree: header.append( parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')])) header.append(self._parse_tree) return header.fold(policy=policy) def _reconstruct_header(cls_name, bases, value): return type(cls_name, bases, {})._reconstruct(value) class UnstructuredHeader: max_count = None value_parser = staticmethod(parser.get_unstructured) @classmethod def parse(cls, value, kwds): kwds['parse_tree'] = cls.value_parser(value) kwds['decoded'] = str(kwds['parse_tree']) class UniqueUnstructuredHeader(UnstructuredHeader): max_count = 1 class DateHeader: """Header whose value consists of a single timestamp. Provides an additional attribute, datetime, which is either an aware datetime using a timezone, or a naive datetime if the timezone in the input string is -0000. Also accepts a datetime as input. The 'value' attribute is the normalized form of the timestamp, which means it is the output of format_datetime on the datetime. """ max_count = None # This is used only for folding, not for creating 'decoded'. value_parser = staticmethod(parser.get_unstructured) @classmethod def parse(cls, value, kwds): if not value: kwds['defects'].append(errors.HeaderMissingRequiredValue()) kwds['datetime'] = None kwds['decoded'] = '' kwds['parse_tree'] = parser.TokenList() return if isinstance(value, str): value = utils.parsedate_to_datetime(value) kwds['datetime'] = value kwds['decoded'] = utils.format_datetime(kwds['datetime']) kwds['parse_tree'] = cls.value_parser(kwds['decoded']) def init(self, args, *kw): self._datetime = kw.pop('datetime') super().init(args, *kw) @property def datetime(self): return self._datetime class UniqueDateHeader(DateHeader): max_count = 1 class AddressHeader: max_count = None @staticmethod def value_parser(value): address_list, value = parser.get_address_list(value) assert not value, 'this should not happen' return address_list @classmethod def parse(cls, value, kwds): if isinstance(value, str): # We are translating here from the RFC language (address/mailbox) # to our API language (group/address). kwds['parse_tree'] = address_list = cls.value_parser(value) groups = [] for addr in address_list.addresses: groups.append(Group(addr.display_name, [Address(mb.display_name or '', mb.local_part or '', mb.domain or '') for mb in addr.all_mailboxes])) defects = list(address_list.all_defects) else: # Assume it is Address/Group stuff if not hasattr(value, '__iter__'): value = [value] groups = [Group(None, [item]) if not hasattr(item, 'addresses') else item for item in value] defects = [] kwds['groups'] = groups kwds['defects'] = defects kwds['decoded'] = ', '.join([str(item) for item in groups]) if 'parse_tree' not in kwds: kwds['parse_tree'] = cls.value_parser(kwds['decoded']) def init(self, args, *kw): self._groups = tuple(kw.pop('groups')) self._addresses = None super().init(args, *kw) @property def groups(self): return self._groups @property def addresses(self): if self._addresses is None: self._addresses = tuple(address for group in self._groups for address in group.addresses) return self._addresses class UniqueAddressHeader(AddressHeader): max_count = 1 class SingleAddressHeader(AddressHeader): @property def address(self): if len(self.addresses)!=1: raise ValueError(("value of single address header {} is not " "a single address").format(self.name)) return self.addresses[0] class UniqueSingleAddressHeader(SingleAddressHeader): max_count = 1 class MIMEVersionHeader: max_count = 1 value_parser = staticmethod(parser.parse_mime_version) @classmethod def parse(cls, value, kwds): kwds['parse_tree'] = parse_tree = cls.value_parser(value) kwds['decoded'] = str(parse_tree) kwds['defects'].extend(parse_tree.all_defects) kwds['major'] = None if parse_tree.minor is None else parse_tree.major kwds['minor'] = parse_tree.minor if parse_tree.minor is not None: kwds['version'] = '{}.{}'.format(kwds['major'], kwds['minor']) else: kwds['version'] = None def init(self, args, *kw): self._version = kw.pop('version') self._major = kw.pop('major') self._minor = kw.pop('minor') super().init(args, *kw) @property def major(self): return self._major @property def minor(self): return self._minor @property def version(self): return self._version class ParameterizedMIMEHeader: # Mixin that handles the params dict. Must be subclassed and # a property value_parser for the specific header provided. max_count = 1 @classmethod def parse(cls, value, kwds): kwds['parse_tree'] = parse_tree = cls.value_parser(value) kwds['decoded'] = str(parse_tree) kwds['defects'].extend(parse_tree.all_defects) if parse_tree.params is None: kwds['params'] = {} else: # The MIME RFCs specify that parameter ordering is arbitrary. kwds['params'] = {utils._sanitize(name).lower(): utils._sanitize(value) for name, value in parse_tree.params} def init(self, args, *kw): self._params = kw.pop('params') super().init(args, *kw) @property def params(self): return MappingProxyType(self._params) class ContentTypeHeader(ParameterizedMIMEHeader): value_parser = staticmethod(parser.parse_content_type_header) def init(self, args, *kw): super().init(args, *kw) self._maintype = utils._sanitize(self._parse_tree.maintype) self._subtype = utils._sanitize(self._parse_tree.subtype) @property def maintype(self): return self._maintype @property def subtype(self): return self._subtype @property def content_type(self): return self.maintype + '/' + self.subtype class ContentDispositionHeader(ParameterizedMIMEHeader): value_parser = staticmethod(parser.parse_content_disposition_header) def init(self, args, *kw): super().init(args, *kw) cd = self._parse_tree.content_disposition self._content_disposition = cd if cd is None else utils._sanitize(cd) @property def content_disposition(self): return self._content_disposition class ContentTransferEncodingHeader: max_count = 1 value_parser = staticmethod(parser.parse_content_transfer_encoding_header) @classmethod def parse(cls, value, kwds): kwds['parse_tree'] = parse_tree = cls.value_parser(value) kwds['decoded'] = str(parse_tree) kwds['defects'].extend(parse_tree.all_defects) def init(self, args, *kw): super().init(args, **kw) self._cte = utils._sanitize(self._parse_tree.cte) @property def cte(self): return self._cte # The header factory # _default_header_map = { 'subject': UniqueUnstructuredHeader, 'date': UniqueDateHeader, 'resent-date': DateHeader, 'orig-date': UniqueDateHeader, 'sender': UniqueSingleAddressHeader, 'resent-sender': SingleAddressHeader, 'to': UniqueAddressHeader, 'resent-to': AddressHeader, 'cc': UniqueAddressHeader, 'resent-cc': AddressHeader, 'bcc': UniqueAddressHeader, 'resent-bcc': AddressHeader, 'from': UniqueAddressHeader, 'resent-from': AddressHeader, 'reply-to': UniqueAddressHeader, 'mime-version': MIMEVersionHeader, 'content-type': ContentTypeHeader, 'content-disposition': ContentDispositionHeader, 'content-transfer-encoding': ContentTransferEncodingHeader, } class HeaderRegistry: """A header_factory and header registry.""" def __init__(self, base_class=BaseHeader, default_class=UnstructuredHeader, use_default_map=True): """Create a header_factory that works with the Policy API. base_class is the class that will be the last class in the created header class's __bases__ list. default_class is the class that will be used if "name" (see __call__) does not appear in the registry. use_default_map controls whether or not the default mapping of names to specialized classes is copied in to the registry when the factory is created. The default is True. """ self.registry = {} self.base_class = base_class self.default_class = default_class if use_default_map: self.registry.update(_default_header_map) def map_to_type(self, name, cls): """Register cls as the specialized class for handling "name" headers. """ self.registry[name.lower()] = cls def __getitem__(self, name): cls = self.registry.get(name.lower(), self.default_class) return type('_'+cls.__name__, (cls, self.base_class), {}) def __call__(self, name, value): """Create a header instance for header 'name' from 'value'. Creates a header instance by creating a specialized class for parsing and representing the specified header by combining the factory base_class with a specialized class from the registry or the default_class, and passing the name and value to the constructed class's constructor. """ return self[name](name, value)	19.96 KB	2024-04-17 17:55
📄 iterators.py # Copyright (C) 2001-2006 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org """Various types of useful iterators and generators.""" __all__ = [ 'body_line_iterator', 'typed_subpart_iterator', 'walk', # Do not include _structure() since it's part of the debugging API. ] import sys from io import StringIO # This function will become a method of the Message class def walk(self): """Walk over the message tree, yielding each subpart. The walk is performed in depth-first order. This method is a generator. """ yield self if self.is_multipart(): for subpart in self.get_payload(): yield from subpart.walk() # These two functions are imported into the Iterators.py interface module. def body_line_iterator(msg, decode=False): """Iterate over the parts, returning string payloads line-by-line. Optional decode (default False) is passed through to .get_payload(). """ for subpart in msg.walk(): payload = subpart.get_payload(decode=decode) if isinstance(payload, str): yield from StringIO(payload) def typed_subpart_iterator(msg, maintype='text', subtype=None): """Iterate over the subparts with a given MIME type. Use `maintype' as the main MIME type to match against; this defaults to "text". Optional `subtype' is the MIME subtype to match against; if omitted, only the main type is matched. """ for subpart in msg.walk(): if subpart.get_content_maintype() == maintype: if subtype is None or subpart.get_content_subtype() == subtype: yield subpart def _structure(msg, fp=None, level=0, include_default=False): """A handy debugging aid""" if fp is None: fp = sys.stdout tab = ' ' * (level * 4) print(tab + msg.get_content_type(), end='', file=fp) if include_default: print(' [%s]' % msg.get_default_type(), file=fp) else: print(file=fp) if msg.is_multipart(): for subpart in msg.get_payload(): _structure(subpart, fp, level+1, include_default)	2.08 KB	2024-04-17 17:55
📄 message.py # Copyright (C) 2001-2007 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org """Basic message object for the email package object model.""" __all__ = ['Message', 'EmailMessage'] import re import uu import quopri from io import BytesIO, StringIO # Intrapackage imports from email import utils from email import errors from email._policybase import Policy, compat32 from email import charset as _charset from email._encoded_words import decode_b Charset = _charset.Charset SEMISPACE = '; ' # Regular expression that matches `special' characters in parameters, the # existence of which force quoting of the parameter value. tspecials = re.compile(r'[ <>@,;:\\"/\[\]\?=]') def _splitparam(param): # Split header parameters. BAW: this may be too simple. It isn't # strictly RFC 2045 (section 5.1) compliant, but it catches most headers # found in the wild. We may eventually need a full fledged parser. # RDM: we might have a Header here; for now just stringify it. a, sep, b = str(param).partition(';') if not sep: return a.strip(), None return a.strip(), b.strip() def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. This will quote the value if needed or if quote is true. If value is a three tuple (charset, language, value), it will be encoded according to RFC2231 rules. If it contains non-ascii characters it will likewise be encoded according to RFC2231 rules, using the utf-8 charset and a null language. """ if value is not None and len(value) > 0: # A tuple is used for RFC 2231 encoded parameter values where items # are (charset, language, value). charset is a string, not a Charset # instance. RFC 2231 encoded values are never quoted, per RFC. if isinstance(value, tuple): # Encode as per RFC 2231 param += '' value = utils.encode_rfc2231(value[2], value[0], value[1]) return '%s=%s' % (param, value) else: try: value.encode('ascii') except UnicodeEncodeError: param += '' value = utils.encode_rfc2231(value, 'utf-8', '') return '%s=%s' % (param, value) # BAW: Please check this. I think that if quote is set it should # force quoting even if not necessary. if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param def _parseparam(s): # RDM This might be a Header, so for now stringify it. s = ';' + str(s) plist = [] while s[:1] == ';': s = s[1:] end = s.find(';') while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: end = s.find(';', end + 1) if end < 0: end = len(s) f = s[:end] if '=' in f: i = f.index('=') f = f[:i].strip().lower() + '=' + f[i+1:].strip() plist.append(f.strip()) s = s[end:] return plist def _unquotevalue(value): # This is different than utils.collapse_rfc2231_value() because it doesn't # try to convert the value to a unicode. Message.get_param() and # Message.get_params() are both currently defined to return the tuple in # the face of RFC 2231 parameters. if isinstance(value, tuple): return value[0], value[1], utils.unquote(value[2]) else: return utils.unquote(value) class Message: """Basic message object. A message object is defined as something that has a bunch of RFC 2822 headers and a payload. It may optionally have an envelope header (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a multipart or a message/rfc822), then the payload is a list of Message objects, otherwise it is a string. Message objects implement part of the `mapping' interface, which assumes there is exactly one occurrence of the header per message. Some headers do in fact appear multiple times (e.g. Received) and for those headers, you must use the explicit API to set or get all the headers. Not all of the mapping methods are implemented. """ def __init__(self, policy=compat32): self.policy = policy self._headers = [] self._unixfrom = None self._payload = None self._charset = None # Defaults for multipart messages self.preamble = self.epilogue = None self.defects = [] # Default content type self._default_type = 'text/plain' def __str__(self): """Return the entire formatted message as a string. """ return self.as_string() def as_string(self, unixfrom=False, maxheaderlen=0, policy=None): """Return the entire formatted message as a string. Optional 'unixfrom', when true, means include the Unix From_ envelope header. For backward compatibility reasons, if maxheaderlen is not specified it defaults to 0, so you must override it explicitly if you want a different maxheaderlen. 'policy' is passed to the Generator instance used to serialize the mesasge; if it is not specified the policy associated with the message instance is used. If the message object contains binary data that is not encoded according to RFC standards, the non-compliant data will be replaced by unicode "unknown character" code points. """ from email.generator import Generator policy = self.policy if policy is None else policy fp = StringIO() g = Generator(fp, mangle_from_=False, maxheaderlen=maxheaderlen, policy=policy) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() def __bytes__(self): """Return the entire formatted message as a bytes object. """ return self.as_bytes() def as_bytes(self, unixfrom=False, policy=None): """Return the entire formatted message as a bytes object. Optional 'unixfrom', when true, means include the Unix From_ envelope header. 'policy' is passed to the BytesGenerator instance used to serialize the message; if not specified the policy associated with the message instance is used. """ from email.generator import BytesGenerator policy = self.policy if policy is None else policy fp = BytesIO() g = BytesGenerator(fp, mangle_from_=False, policy=policy) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() def is_multipart(self): """Return True if the message consists of multiple parts.""" return isinstance(self._payload, list) # # Unix From_ line # def set_unixfrom(self, unixfrom): self._unixfrom = unixfrom def get_unixfrom(self): return self._unixfrom # # Payload manipulation. # def attach(self, payload): """Add the given payload to the current payload. The current payload will always be a list of objects after this method is called. If you want to set the payload to a scalar object, use set_payload() instead. """ if self._payload is None: self._payload = [payload] else: try: self._payload.append(payload) except AttributeError: raise TypeError("Attach is not valid on a message with a" " non-multipart payload") def get_payload(self, i=None, decode=False): """Return a reference to the payload. The payload will either be a list object or a string. If you mutate the list object, you modify the message's payload in place. Optional i returns that index into the payload. Optional decode is a flag indicating whether the payload should be decoded or not, according to the Content-Transfer-Encoding header (default is False). When True and the message is not a multipart, the payload will be decoded if this header's value is `quoted-printable' or `base64'. If some other encoding is used, or the header is missing, or if the payload has bogus data (i.e. bogus base64 or uuencoded data), the payload is returned as-is. If the message is a multipart and the decode flag is True, then None is returned. """ # Here is the logic table for this code, based on the email5.0.0 code: # i decode is_multipart result # ------ ------ ------------ ------------------------------ # None True True None # i True True None # None False True _payload (a list) # i False True _payload element i (a Message) # i False False error (not a list) # i True False error (not a list) # None False False _payload # None True False _payload decoded (bytes) # Note that Barry planned to factor out the 'decode' case, but that # isn't so easy now that we handle the 8 bit data, which needs to be # converted in both the decode and non-decode path. if self.is_multipart(): if decode: return None if i is None: return self._payload else: return self._payload[i] # For backward compatibility, Use isinstance and this error message # instead of the more logical is_multipart test. if i is not None and not isinstance(self._payload, list): raise TypeError('Expected list, got %s' % type(self._payload)) payload = self._payload # cte might be a Header, so for now stringify it. cte = str(self.get('content-transfer-encoding', '')).lower() # payload may be bytes here. if isinstance(payload, str): if utils._has_surrogates(payload): bpayload = payload.encode('ascii', 'surrogateescape') if not decode: try: payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace') except LookupError: payload = bpayload.decode('ascii', 'replace') elif decode: try: bpayload = payload.encode('ascii') except UnicodeError: # This won't happen for RFC compliant messages (messages # containing only ASCII code points in the unicode input). # If it does happen, turn the string into bytes in a way # guaranteed not to fail. bpayload = payload.encode('raw-unicode-escape') if not decode: return payload if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': # XXX: this is a bit of a hack; decode_b should probably be factored # out somewhere, but I haven't figured out where yet. value, defects = decode_b(b''.join(bpayload.splitlines())) for defect in defects: self.policy.handle_defect(self, defect) return value elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): in_file = BytesIO(bpayload) out_file = BytesIO() try: uu.decode(in_file, out_file, quiet=True) return out_file.getvalue() except uu.Error: # Some decoding problem return bpayload if isinstance(payload, str): return bpayload return payload def set_payload(self, payload, charset=None): """Set the payload to the given value. Optional charset sets the message's default character set. See set_charset() for details. """ if hasattr(payload, 'encode'): if charset is None: self._payload = payload return if not isinstance(charset, Charset): charset = Charset(charset) payload = payload.encode(charset.output_charset) if hasattr(payload, 'decode'): self._payload = payload.decode('ascii', 'surrogateescape') else: self._payload = payload if charset is not None: self.set_charset(charset) def set_charset(self, charset): """Set the charset of the payload to a given character set. charset can be a Charset instance, a string naming a character set, or None. If it is a string it will be converted to a Charset instance. If charset is None, the charset parameter will be removed from the Content-Type field. Anything else will generate a TypeError. The message will be assumed to be of type text/* encoded with charset.input_charset. It will be converted to charset.output_charset and encoded properly, if needed, when generating the plain text representation of the message. MIME headers (MIME-Version, Content-Type, Content-Transfer-Encoding) will be added as needed. """ if charset is None: self.del_param('charset') self._charset = None return if not isinstance(charset, Charset): charset = Charset(charset) self._charset = charset if 'MIME-Version' not in self: self.add_header('MIME-Version', '1.0') if 'Content-Type' not in self: self.add_header('Content-Type', 'text/plain', charset=charset.get_output_charset()) else: self.set_param('charset', charset.get_output_charset()) if charset != charset.get_output_charset(): self._payload = charset.body_encode(self._payload) if 'Content-Transfer-Encoding' not in self: cte = charset.get_body_encoding() try: cte(self) except TypeError: # This 'if' is for backward compatibility, it allows unicode # through even though that won't work correctly if the # message is serialized. payload = self._payload if payload: try: payload = payload.encode('ascii', 'surrogateescape') except UnicodeError: payload = payload.encode(charset.output_charset) self._payload = charset.body_encode(payload) self.add_header('Content-Transfer-Encoding', cte) def get_charset(self): """Return the Charset instance associated with the message's payload. """ return self._charset # # MAPPING INTERFACE (partial) # def __len__(self): """Return the total number of headers, including duplicates.""" return len(self._headers) def __getitem__(self, name): """Get a header value. Return None if the header is missing instead of raising an exception. Note that if the header appeared multiple times, exactly which occurrence gets returned is undefined. Use get_all() to get all the values matching a header field name. """ return self.get(name) def __setitem__(self, name, val): """Set the value of a header. Note: this does not overwrite an existing header with the same field name. Use __delitem__() first to delete any existing headers. """ max_count = self.policy.header_max_count(name) if max_count: lname = name.lower() found = 0 for k, v in self._headers: if k.lower() == lname: found += 1 if found >= max_count: raise ValueError("There may be at most {} {} headers " "in a message".format(max_count, name)) self._headers.append(self.policy.header_store_parse(name, val)) def __delitem__(self, name): """Delete all occurrences of a header, if present. Does not raise an exception if the header is missing. """ name = name.lower() newheaders = [] for k, v in self._headers: if k.lower() != name: newheaders.append((k, v)) self._headers = newheaders def __contains__(self, name): return name.lower() in [k.lower() for k, v in self._headers] def __iter__(self): for field, value in self._headers: yield field def keys(self): """Return a list of all the message's header field names. These will be sorted in the order they appeared in the original message, or were added to the message, and may contain duplicates. Any fields deleted and re-inserted are always appended to the header list. """ return [k for k, v in self._headers] def values(self): """Return a list of all the message's header values. These will be sorted in the order they appeared in the original message, or were added to the message, and may contain duplicates. Any fields deleted and re-inserted are always appended to the header list. """ return [self.policy.header_fetch_parse(k, v) for k, v in self._headers] def items(self): """Get all the message's header fields and values. These will be sorted in the order they appeared in the original message, or were added to the message, and may contain duplicates. Any fields deleted and re-inserted are always appended to the header list. """ return [(k, self.policy.header_fetch_parse(k, v)) for k, v in self._headers] def get(self, name, failobj=None): """Get a header value. Like __getitem__() but return failobj instead of None when the field is missing. """ name = name.lower() for k, v in self._headers: if k.lower() == name: return self.policy.header_fetch_parse(k, v) return failobj # # "Internal" methods (public API, but only intended for use by a parser # or generator, not normal application code. # def set_raw(self, name, value): """Store name and value in the model without modification. This is an "internal" API, intended only for use by a parser. """ self._headers.append((name, value)) def raw_items(self): """Return the (name, value) header pairs without modification. This is an "internal" API, intended only for use by a generator. """ return iter(self._headers.copy()) # # Additional useful stuff # def get_all(self, name, failobj=None): """Return a list of all the values for the named field. These will be sorted in the order they appeared in the original message, and may contain duplicates. Any fields deleted and re-inserted are always appended to the header list. If no such fields exist, failobj is returned (defaults to None). """ values = [] name = name.lower() for k, v in self._headers: if k.lower() == name: values.append(self.policy.header_fetch_parse(k, v)) if not values: return failobj return values def add_header(self, _name, _value, *_params): """Extended header setting. name is the header field to add. keyword arguments can be used to set additional parameters for the header field, with underscores converted to dashes. Normally the parameter will be added as key="value" unless value is None, in which case only the key will be added. If a parameter value contains non-ASCII characters it can be specified as a three-tuple of (charset, language, value), in which case it will be encoded according to RFC2231 rules. Otherwise it will be encoded using the utf-8 charset and a language of ''. Examples: msg.add_header('content-disposition', 'attachment', filename='bud.gif') msg.add_header('content-disposition', 'attachment', filename=('utf-8', '', Fußballer.ppt')) msg.add_header('content-disposition', 'attachment', filename='Fußballer.ppt')) """ parts = [] for k, v in _params.items(): if v is None: parts.append(k.replace('_', '-')) else: parts.append(_formatparam(k.replace('_', '-'), v)) if _value is not None: parts.insert(0, _value) self[_name] = SEMISPACE.join(parts) def replace_header(self, _name, _value): """Replace a header. Replace the first matching header found in the message, retaining header order and case. If no matching header was found, a KeyError is raised. """ _name = _name.lower() for i, (k, v) in zip(range(len(self._headers)), self._headers): if k.lower() == _name: self._headers[i] = self.policy.header_store_parse(k, _value) break else: raise KeyError(_name) # # Use these three methods instead of the three above. # def get_content_type(self): """Return the message's content type. The returned string is coerced to lower case of the form `maintype/subtype'. If there was no Content-Type header in the message, the default type as given by get_default_type() will be returned. Since according to RFC 2045, messages always have a default type this will always return a value. RFC 2045 defines a message's default type to be text/plain unless it appears inside a multipart/digest container, in which case it would be message/rfc822. """ missing = object() value = self.get('content-type', missing) if value is missing: # This should have no parameters return self.get_default_type() ctype = _splitparam(value)[0].lower() # RFC 2045, section 5.2 says if its invalid, use text/plain if ctype.count('/') != 1: return 'text/plain' return ctype def get_content_maintype(self): """Return the message's main content type. This is the `maintype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() return ctype.split('/')[0] def get_content_subtype(self): """Returns the message's sub-content type. This is the `subtype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() return ctype.split('/')[1] def get_default_type(self): """Return the `default' content type. Most messages have a default content type of text/plain, except for messages that are subparts of multipart/digest containers. Such subparts have a default content type of message/rfc822. """ return self._default_type def set_default_type(self, ctype): """Set the `default' content type. ctype should be either "text/plain" or "message/rfc822", although this is not enforced. The default content type is not stored in the Content-Type header. """ self._default_type = ctype def _get_params_preserve(self, failobj, header): # Like get_params() but preserves the quoting of values. BAW: # should this be part of the public interface? missing = object() value = self.get(header, missing) if value is missing: return failobj params = [] for p in _parseparam(value): try: name, val = p.split('=', 1) name = name.strip() val = val.strip() except ValueError: # Must have been a bare attribute name = p.strip() val = '' params.append((name, val)) params = utils.decode_params(params) return params def get_params(self, failobj=None, header='content-type', unquote=True): """Return the message's Content-Type parameters, as a list. The elements of the returned list are 2-tuples of key/value pairs, as split on the `=' sign. The left hand side of the `=' is the key, while the right hand side is the value. If there is no `=' sign in the parameter the value is the empty string. The value is as described in the get_param() method. Optional failobj is the object to return if there is no Content-Type header. Optional header is the header to search instead of Content-Type. If unquote is True, the value is unquoted. """ missing = object() params = self._get_params_preserve(missing, header) if params is missing: return failobj if unquote: return [(k, _unquotevalue(v)) for k, v in params] else: return params def get_param(self, param, failobj=None, header='content-type', unquote=True): """Return the parameter value if found in the Content-Type header. Optional failobj is the object to return if there is no Content-Type header, or the Content-Type header has no such parameter. Optional header is the header to search instead of Content-Type. Parameter keys are always compared case insensitively. The return value can either be a string, or a 3-tuple if the parameter was RFC 2231 encoded. When it's a 3-tuple, the elements of the value are of the form (CHARSET, LANGUAGE, VALUE). Note that both CHARSET and LANGUAGE can be None, in which case you should consider VALUE to be encoded in the us-ascii charset. You can usually ignore LANGUAGE. The parameter value (either the returned string, or the VALUE item in the 3-tuple) is always unquoted, unless unquote is set to False. If your application doesn't care whether the parameter was RFC 2231 encoded, it can turn the return value into a string as follows: rawparam = msg.get_param('foo') param = email.utils.collapse_rfc2231_value(rawparam) """ if header not in self: return failobj for k, v in self._get_params_preserve(failobj, header): if k.lower() == param.lower(): if unquote: return _unquotevalue(v) else: return v return failobj def set_param(self, param, value, header='Content-Type', requote=True, charset=None, language='', replace=False): """Set a parameter in the Content-Type header. If the parameter already exists in the header, its value will be replaced with the new value. If header is Content-Type and has not yet been defined for this message, it will be set to "text/plain" and the new parameter and value will be appended as per RFC 2045. An alternate header can be specified in the header argument, and all parameters will be quoted as necessary unless requote is False. If charset is specified, the parameter will be encoded according to RFC 2231. Optional language specifies the RFC 2231 language, defaulting to the empty string. Both charset and language should be strings. """ if not isinstance(value, tuple) and charset: value = (charset, language, value) if header not in self and header.lower() == 'content-type': ctype = 'text/plain' else: ctype = self.get(header) if not self.get_param(param, header=header): if not ctype: ctype = _formatparam(param, value, requote) else: ctype = SEMISPACE.join( [ctype, _formatparam(param, value, requote)]) else: ctype = '' for old_param, old_value in self.get_params(header=header, unquote=requote): append_param = '' if old_param.lower() == param.lower(): append_param = _formatparam(param, value, requote) else: append_param = _formatparam(old_param, old_value, requote) if not ctype: ctype = append_param else: ctype = SEMISPACE.join([ctype, append_param]) if ctype != self.get(header): if replace: self.replace_header(header, ctype) else: del self[header] self[header] = ctype def del_param(self, param, header='content-type', requote=True): """Remove the given parameter completely from the Content-Type header. The header will be re-written in place without the parameter or its value. All values will be quoted as necessary unless requote is False. Optional header specifies an alternative to the Content-Type header. """ if header not in self: return new_ctype = '' for p, v in self.get_params(header=header, unquote=requote): if p.lower() != param.lower(): if not new_ctype: new_ctype = _formatparam(p, v, requote) else: new_ctype = SEMISPACE.join([new_ctype, _formatparam(p, v, requote)]) if new_ctype != self.get(header): del self[header] self[header] = new_ctype def set_type(self, type, header='Content-Type', requote=True): """Set the main type and subtype for the Content-Type header. type must be a string in the form "maintype/subtype", otherwise a ValueError is raised. This method replaces the Content-Type header, keeping all the parameters in place. If requote is False, this leaves the existing header's quoting as is. Otherwise, the parameters will be quoted (the default). An alternative header can be specified in the header argument. When the Content-Type header is set, we'll always also add a MIME-Version header. """ # BAW: should we be strict? if not type.count('/') == 1: raise ValueError # Set the Content-Type, you get a MIME-Version if header.lower() == 'content-type': del self['mime-version'] self['MIME-Version'] = '1.0' if header not in self: self[header] = type return params = self.get_params(header=header, unquote=requote) del self[header] self[header] = type # Skip the first param; it's the old type. for p, v in params[1:]: self.set_param(p, v, header, requote) def get_filename(self, failobj=None): """Return the filename associated with the payload if present. The filename is extracted from the Content-Disposition header's `filename' parameter, and it is unquoted. If that header is missing the `filename' parameter, this method falls back to looking for the `name' parameter. """ missing = object() filename = self.get_param('filename', missing, 'content-disposition') if filename is missing: filename = self.get_param('name', missing, 'content-type') if filename is missing: return failobj return utils.collapse_rfc2231_value(filename).strip() def get_boundary(self, failobj=None): """Return the boundary associated with the payload if present. The boundary is extracted from the Content-Type header's `boundary' parameter, and it is unquoted. """ missing = object() boundary = self.get_param('boundary', missing) if boundary is missing: return failobj # RFC 2046 says that boundaries may begin but not end in w/s return utils.collapse_rfc2231_value(boundary).rstrip() def set_boundary(self, boundary): """Set the boundary parameter in Content-Type to 'boundary'. This is subtly different than deleting the Content-Type header and adding a new one with a new boundary parameter via add_header(). The main difference is that using the set_boundary() method preserves the order of the Content-Type header in the original message. HeaderParseError is raised if the message has no Content-Type header. """ missing = object() params = self._get_params_preserve(missing, 'content-type') if params is missing: # There was no Content-Type header, and we don't know what type # to set it to, so raise an exception. raise errors.HeaderParseError('No Content-Type header found') newparams = [] foundp = False for pk, pv in params: if pk.lower() == 'boundary': newparams.append(('boundary', '"%s"' % boundary)) foundp = True else: newparams.append((pk, pv)) if not foundp: # The original Content-Type header had no boundary attribute. # Tack one on the end. BAW: should we raise an exception # instead??? newparams.append(('boundary', '"%s"' % boundary)) # Replace the existing Content-Type header with the new value newheaders = [] for h, v in self._headers: if h.lower() == 'content-type': parts = [] for k, v in newparams: if v == '': parts.append(k) else: parts.append('%s=%s' % (k, v)) val = SEMISPACE.join(parts) newheaders.append(self.policy.header_store_parse(h, val)) else: newheaders.append((h, v)) self._headers = newheaders def get_content_charset(self, failobj=None): """Return the charset parameter of the Content-Type header. The returned string is always coerced to lower case. If there is no Content-Type header, or if that header has no charset parameter, failobj is returned. """ missing = object() charset = self.get_param('charset', missing) if charset is missing: return failobj if isinstance(charset, tuple): # RFC 2231 encoded, so decode it, and it better end up as ascii. pcharset = charset[0] or 'us-ascii' try: # LookupError will be raised if the charset isn't known to # Python. UnicodeError will be raised if the encoded text # contains a character not in the charset. as_bytes = charset[2].encode('raw-unicode-escape') charset = str(as_bytes, pcharset) except (LookupError, UnicodeError): charset = charset[2] # charset characters must be in us-ascii range try: charset.encode('us-ascii') except UnicodeError: return failobj # RFC 2046, $4.1.2 says charsets are not case sensitive return charset.lower() def get_charsets(self, failobj=None): """Return a list containing the charset(s) used in this message. The returned list of items describes the Content-Type headers' charset parameter for this message and all the subparts in its payload. Each item will either be a string (the value of the charset parameter in the Content-Type header of that part) or the value of the 'failobj' parameter (defaults to None), if the part does not have a main MIME type of "text", or the charset is not defined. The list will contain one string for each part of the message, plus one for the container message (i.e. self), so that a non-multipart message will still return a list of length 1. """ return [part.get_content_charset(failobj) for part in self.walk()] def get_content_disposition(self): """Return the message's content-disposition if it exists, or None. The return values can be either 'inline', 'attachment' or None according to the rfc2183. """ value = self.get('content-disposition') if value is None: return None c_d = _splitparam(value)[0].lower() return c_d # I.e. def walk(self): ... from email.iterators import walk class MIMEPart(Message): def __init__(self, policy=None): if policy is None: from email.policy import default policy = default Message.__init__(self, policy) def as_string(self, unixfrom=False, maxheaderlen=None, policy=None): """Return the entire formatted message as a string. Optional 'unixfrom', when true, means include the Unix From_ envelope header. maxheaderlen is retained for backward compatibility with the base Message class, but defaults to None, meaning that the policy value for max_line_length controls the header maximum length. 'policy' is passed to the Generator instance used to serialize the mesasge; if it is not specified the policy associated with the message instance is used. """ policy = self.policy if policy is None else policy if maxheaderlen is None: maxheaderlen = policy.max_line_length return super().as_string(maxheaderlen=maxheaderlen, policy=policy) def __str__(self): return self.as_string(policy=self.policy.clone(utf8=True)) def is_attachment(self): c_d = self.get('content-disposition') return False if c_d is None else c_d.content_disposition == 'attachment' def _find_body(self, part, preferencelist): if part.is_attachment(): return maintype, subtype = part.get_content_type().split('/') if maintype == 'text': if subtype in preferencelist: yield (preferencelist.index(subtype), part) return if maintype != 'multipart': return if subtype != 'related': for subpart in part.iter_parts(): yield from self._find_body(subpart, preferencelist) return if 'related' in preferencelist: yield (preferencelist.index('related'), part) candidate = None start = part.get_param('start') if start: for subpart in part.iter_parts(): if subpart['content-id'] == start: candidate = subpart break if candidate is None: subparts = part.get_payload() candidate = subparts[0] if subparts else None if candidate is not None: yield from self._find_body(candidate, preferencelist) def get_body(self, preferencelist=('related', 'html', 'plain')): """Return best candidate mime part for display as 'body' of message. Do a depth first search, starting with self, looking for the first part matching each of the items in preferencelist, and return the part corresponding to the first item that has a match, or None if no items have a match. If 'related' is not included in preferencelist, consider the root part of any multipart/related encountered as a candidate match. Ignore parts with 'Content-Disposition: attachment'. """ best_prio = len(preferencelist) body = None for prio, part in self._find_body(self, preferencelist): if prio < best_prio: best_prio = prio body = part if prio == 0: break return body _body_types = {('text', 'plain'), ('text', 'html'), ('multipart', 'related'), ('multipart', 'alternative')} def iter_attachments(self): """Return an iterator over the non-main parts of a multipart. Skip the first of each occurrence of text/plain, text/html, multipart/related, or multipart/alternative in the multipart (unless they have a 'Content-Disposition: attachment' header) and include all remaining subparts in the returned iterator. When applied to a multipart/related, return all parts except the root part. Return an empty iterator when applied to a multipart/alternative or a non-multipart. """ maintype, subtype = self.get_content_type().split('/') if maintype != 'multipart' or subtype == 'alternative': return payload = self.get_payload() # Certain malformed messages can have content type set to `multipart/` # but still have single part body, in which case payload.copy() can # fail with AttributeError. try: parts = payload.copy() except AttributeError: # payload is not a list, it is most probably a string. return if maintype == 'multipart' and subtype == 'related': # For related, we treat everything but the root as an attachment. # The root may be indicated by 'start'; if there's no start or we # can't find the named start, treat the first subpart as the root. start = self.get_param('start') if start: found = False attachments = [] for part in parts: if part.get('content-id') == start: found = True else: attachments.append(part) if found: yield from attachments return parts.pop(0) yield from parts return # Otherwise we more or less invert the remaining logic in get_body. # This only really works in edge cases (ex: non-text related or # alternatives) if the sending agent sets content-disposition. seen = [] # Only skip the first example of each candidate type. for part in parts: maintype, subtype = part.get_content_type().split('/') if ((maintype, subtype) in self._body_types and not part.is_attachment() and subtype not in seen): seen.append(subtype) continue yield part def iter_parts(self): """Return an iterator over all immediate subparts of a multipart. Return an empty iterator for a non-multipart. """ if self.get_content_maintype() == 'multipart': yield from self.get_payload() def get_content(self, args, content_manager=None, kw): if content_manager is None: content_manager = self.policy.content_manager return content_manager.get_content(self, args, *kw) def set_content(self, args, content_manager=None, *kw): if content_manager is None: content_manager = self.policy.content_manager content_manager.set_content(self, args, *kw) def _make_multipart(self, subtype, disallowed_subtypes, boundary): if self.get_content_maintype() == 'multipart': existing_subtype = self.get_content_subtype() disallowed_subtypes = disallowed_subtypes + (subtype,) if existing_subtype in disallowed_subtypes: raise ValueError("Cannot convert {} to {}".format( existing_subtype, subtype)) keep_headers = [] part_headers = [] for name, value in self._headers: if name.lower().startswith('content-'): part_headers.append((name, value)) else: keep_headers.append((name, value)) if part_headers: # There is existing content, move it to the first subpart. part = type(self)(policy=self.policy) part._headers = part_headers part._payload = self._payload self._payload = [part] else: self._payload = [] self._headers = keep_headers self['Content-Type'] = 'multipart/' + subtype if boundary is not None: self.set_param('boundary', boundary) def make_related(self, boundary=None): self._make_multipart('related', ('alternative', 'mixed'), boundary) def make_alternative(self, boundary=None): self._make_multipart('alternative', ('mixed',), boundary) def make_mixed(self, boundary=None): self._make_multipart('mixed', (), boundary) def _add_multipart(self, _subtype, args, _disp=None, *kw): if (self.get_content_maintype() != 'multipart' or self.get_content_subtype() != _subtype): getattr(self, 'make_' + _subtype)() part = type(self)(policy=self.policy) part.set_content(args, *kw) if _disp and 'content-disposition' not in part: part['Content-Disposition'] = _disp self.attach(part) def add_related(self, args, *kw): self._add_multipart('related', args, _disp='inline', *kw) def add_alternative(self, args, *kw): self._add_multipart('alternative', args, *kw) def add_attachment(self, args, *kw): self._add_multipart('mixed', args, _disp='attachment', *kw) def clear(self): self._headers = [] self._payload = None def clear_content(self): self._headers = [(n, v) for n, v in self._headers if not n.lower().startswith('content-')] self._payload = None class EmailMessage(MIMEPart): def set_content(self, args, *kw): super().set_content(args, **kw) if 'MIME-Version' not in self: self['MIME-Version'] = '1.0'	45.97 KB	2024-04-17 17:55
📁 mime	-	2025-10-13 08:00
📄 parser.py # Copyright (C) 2001-2007 Python Software Foundation # Author: Barry Warsaw, Thomas Wouters, Anthony Baxter # Contact: email-sig@python.org """A parser of RFC 2822 and MIME email messages.""" __all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser', 'FeedParser', 'BytesFeedParser'] from io import StringIO, TextIOWrapper from email.feedparser import FeedParser, BytesFeedParser from email._policybase import compat32 class Parser: def __init__(self, _class=None, , policy=compat32): """Parser of RFC 2822 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. The string must be formatted as a block of RFC 2822 headers and header continuation lines, optionally preceded by a `Unix-from' header. The header block is terminated either by the end of the string or by a blank line. _class is the class to instantiate for new message objects when they must be created. This class must have a constructor that can take zero arguments. Default is Message.Message. The policy keyword specifies a policy object that controls a number of aspects of the parser's operation. The default policy maintains backward compatibility. """ self._class = _class self.policy = policy def parse(self, fp, headersonly=False): """Create a message structure from the data in a file. Reads all the data from the file and returns the root of the message structure. Optional headersonly is a flag specifying whether to stop parsing after reading the headers or not. The default is False, meaning it parses the entire contents of the file. """ feedparser = FeedParser(self._class, policy=self.policy) if headersonly: feedparser._set_headersonly() while True: data = fp.read(8192) if not data: break feedparser.feed(data) return feedparser.close() def parsestr(self, text, headersonly=False): """Create a message structure from a string. Returns the root of the message structure. Optional headersonly is a flag specifying whether to stop parsing after reading the headers or not. The default is False, meaning it parses the entire contents of the file. """ return self.parse(StringIO(text), headersonly=headersonly) class HeaderParser(Parser): def parse(self, fp, headersonly=True): return Parser.parse(self, fp, True) def parsestr(self, text, headersonly=True): return Parser.parsestr(self, text, True) class BytesParser: def __init__(self, args, *kw): """Parser of binary RFC 2822 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. The input must be formatted as a block of RFC 2822 headers and header continuation lines, optionally preceded by a `Unix-from' header. The header block is terminated either by the end of the input or by a blank line. _class is the class to instantiate for new message objects when they must be created. This class must have a constructor that can take zero arguments. Default is Message.Message. """ self.parser = Parser(args, **kw) def parse(self, fp, headersonly=False): """Create a message structure from the data in a binary file. Reads all the data from the file and returns the root of the message structure. Optional headersonly is a flag specifying whether to stop parsing after reading the headers or not. The default is False, meaning it parses the entire contents of the file. """ fp = TextIOWrapper(fp, encoding='ascii', errors='surrogateescape') try: return self.parser.parse(fp, headersonly) finally: fp.detach() def parsebytes(self, text, headersonly=False): """Create a message structure from a byte string. Returns the root of the message structure. Optional headersonly is a flag specifying whether to stop parsing after reading the headers or not. The default is False, meaning it parses the entire contents of the file. """ text = text.decode('ASCII', errors='surrogateescape') return self.parser.parsestr(text, headersonly) class BytesHeaderParser(BytesParser): def parse(self, fp, headersonly=True): return BytesParser.parse(self, fp, headersonly=True) def parsebytes(self, text, headersonly=True): return BytesParser.parsebytes(self, text, headersonly=True)	4.92 KB	2024-04-17 17:55
📄 policy.py """This will be the home for the policy that hooks in the new code that adds all the email6 features. """ import re import sys from email._policybase import Policy, Compat32, compat32, _extend_docstrings from email.utils import _has_surrogates from email.headerregistry import HeaderRegistry as HeaderRegistry from email.contentmanager import raw_data_manager from email.message import EmailMessage __all__ = [ 'Compat32', 'compat32', 'Policy', 'EmailPolicy', 'default', 'strict', 'SMTP', 'HTTP', ] linesep_splitter = re.compile(r'\n\|\r') @_extend_docstrings class EmailPolicy(Policy): """+ PROVISIONAL The API extensions enabled by this policy are currently provisional. Refer to the documentation for details. This policy adds new header parsing and folding algorithms. Instead of simple strings, headers are custom objects with custom attributes depending on the type of the field. The folding algorithm fully implements RFCs 2047 and 5322. In addition to the settable attributes listed above that apply to all Policies, this policy adds the following additional attributes: utf8 -- if False (the default) message headers will be serialized as ASCII, using encoded words to encode any non-ASCII characters in the source strings. If True, the message headers will be serialized using utf8 and will not contain encoded words (see RFC 6532 for more on this serialization format). refold_source -- if the value for a header in the Message object came from the parsing of some source, this attribute indicates whether or not a generator should refold that value when transforming the message back into stream form. The possible values are: none -- all source values use original folding long -- source values that have any line that is longer than max_line_length will be refolded all -- all values are refolded. The default is 'long'. header_factory -- a callable that takes two arguments, 'name' and 'value', where 'name' is a header field name and 'value' is an unfolded header field value, and returns a string-like object that represents that header. A default header_factory is provided that understands some of the RFC5322 header field types. (Currently address fields and date fields have special treatment, while all other fields are treated as unstructured. This list will be completed before the extension is marked stable.) content_manager -- an object with at least two methods: get_content and set_content. When the get_content or set_content method of a Message object is called, it calls the corresponding method of this object, passing it the message object as its first argument, and any arguments or keywords that were passed to it as additional arguments. The default content_manager is :data:`~email.contentmanager.raw_data_manager`. """ message_factory = EmailMessage utf8 = False refold_source = 'long' header_factory = HeaderRegistry() content_manager = raw_data_manager def __init__(self, kw): # Ensure that each new instance gets a unique header factory # (as opposed to clones, which share the factory). if 'header_factory' not in kw: object.__setattr__(self, 'header_factory', HeaderRegistry()) super().__init__(kw) def header_max_count(self, name): """+ The implementation for this class returns the max_count attribute from the specialized header class that would be used to construct a header of type 'name'. """ return self.header_factory[name].max_count # The logic of the next three methods is chosen such that it is possible to # switch a Message object between a Compat32 policy and a policy derived # from this class and have the results stay consistent. This allows a # Message object constructed with this policy to be passed to a library # that only handles Compat32 objects, or to receive such an object and # convert it to use the newer style by just changing its policy. It is # also chosen because it postpones the relatively expensive full rfc5322 # parse until as late as possible when parsing from source, since in many # applications only a few headers will actually be inspected. def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the remainder of the first line, joining all subsequent lines together, and stripping any trailing carriage return or linefeed characters. (This is the same as Compat32). """ name, value = sourcelines[0].split(':', 1) value = value.lstrip(' \t') + ''.join(sourcelines[1:]) return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): """+ The name is returned unchanged. If the input value has a 'name' attribute and it matches the name ignoring case, the value is returned unchanged. Otherwise the name and value are passed to header_factory method, and the resulting custom header object is returned as the value. In this case a ValueError is raised if the input value contains CR or LF characters. """ if hasattr(value, 'name') and value.name.lower() == name.lower(): return (name, value) if isinstance(value, str) and len(value.splitlines())>1: # XXX this error message isn't quite right when we use splitlines # (see issue 22233), but I'm not sure what should happen here. raise ValueError("Header values may not contain linefeed " "or carriage return characters") return (name, self.header_factory(name, value)) def header_fetch_parse(self, name, value): """+ If the value has a 'name' attribute, it is returned to unmodified. Otherwise the name and the value with any linesep characters removed are passed to the header_factory method, and the resulting custom header object is returned. Any surrogateescaped bytes get turned into the unicode unknown-character glyph. """ if hasattr(value, 'name'): return value # We can't use splitlines here because it splits on more than \r and \n. value = ''.join(linesep_splitter.split(value)) return self.header_factory(name, value) def fold(self, name, value): """+ Header folding is controlled by the refold_source policy setting. A value is considered to be a 'source value' if and only if it does not have a 'name' attribute (having a 'name' attribute means it is a header object of some sort). If a source value needs to be refolded according to the policy, it is converted into a custom header object by passing the name and the value with any linesep characters removed to the header_factory method. Folding of a custom header object is done by calling its fold method with the current policy. Source values are split into lines using splitlines. If the value is not to be refolded, the lines are rejoined using the linesep from the policy and returned. The exception is lines containing non-ascii binary data. In that case the value is refolded regardless of the refold_source setting, which causes the binary data to be CTE encoded using the unknown-8bit charset. """ return self._fold(name, value, refold_binary=True) def fold_binary(self, name, value): """+ The same as fold if cte_type is 7bit, except that the returned value is bytes. If cte_type is 8bit, non-ASCII binary data is converted back into bytes. Headers with binary data are not refolded, regardless of the refold_header setting, since there is no way to know whether the binary data consists of single byte characters or multibyte characters. If utf8 is true, headers are encoded to utf8, otherwise to ascii with non-ASCII unicode rendered as encoded words. """ folded = self._fold(name, value, refold_binary=self.cte_type=='7bit') charset = 'utf8' if self.utf8 else 'ascii' return folded.encode(charset, 'surrogateescape') def _fold(self, name, value, refold_binary=False): if hasattr(value, 'name'): return value.fold(policy=self) maxlen = self.max_line_length if self.max_line_length else sys.maxsize lines = value.splitlines() refold = (self.refold_source == 'all' or self.refold_source == 'long' and (lines and len(lines[0])+len(name)+2 > maxlen or any(len(x) > maxlen for x in lines[1:]))) if refold or refold_binary and _has_surrogates(value): return self.header_factory(name, ''.join(lines)).fold(policy=self) return name + ': ' + self.linesep.join(lines) + self.linesep default = EmailPolicy() # Make the default policy use the class default header_factory del default.header_factory strict = default.clone(raise_on_defect=True) SMTP = default.clone(linesep='\r\n') HTTP = default.clone(linesep='\r\n', max_line_length=None) SMTPUTF8 = SMTP.clone(utf8=True)	10.14 KB	2024-04-17 17:55
📄 quoprimime.py # Copyright (C) 2001-2006 Python Software Foundation # Author: Ben Gertzfield # Contact: email-sig@python.org """Quoted-printable content transfer encoding per RFCs 2045-2047. This module handles the content transfer encoding method defined in RFC 2045 to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to safely encode text that is in a character set similar to the 7-bit US ASCII character set, but that includes some 8-bit characters that are normally not allowed in email bodies or headers. Quoted-printable is very space-inefficient for encoding binary files; use the email.base64mime module for that instead. This module provides an interface to encode and decode both headers and bodies with quoted-printable encoding. RFC 2045 defines a method for including character set information in an `encoded-word' in a header. This method is commonly used for 8-bit real names in To:/From:/Cc: etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character conversion necessary for proper internationalized headers; it only does dumb encoding and decoding. To deal with the various line wrapping issues, use the email.header module. """ __all__ = [ 'body_decode', 'body_encode', 'body_length', 'decode', 'decodestring', 'header_decode', 'header_encode', 'header_length', 'quote', 'unquote', ] import re from string import ascii_letters, digits, hexdigits CRLF = '\r\n' NL = '\n' EMPTYSTRING = '' # Build a mapping of octets to the expansion of that octet. Since we're only # going to have 256 of these things, this isn't terribly inefficient # space-wise. Remember that headers and bodies have different sets of safe # characters. Initialize both maps with the full expansion, and then override # the safe bytes with the more compact form. _QUOPRI_MAP = ['=%02X' % c for c in range(256)] _QUOPRI_HEADER_MAP = _QUOPRI_MAP[:] _QUOPRI_BODY_MAP = _QUOPRI_MAP[:] # Safe header bytes which need no encoding. for c in b'-!+/' + ascii_letters.encode('ascii') + digits.encode('ascii'): _QUOPRI_HEADER_MAP[c] = chr(c) # Headers have one other special encoding; spaces become underscores. _QUOPRI_HEADER_MAP[ord(' ')] = '_' # Safe body bytes which need no encoding. for c in (b' !"#$%&\'()+,-./0123456789:;<>' b'?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`' b'abcdefghijklmnopqrstuvwxyz{\|}~\t'): _QUOPRI_BODY_MAP[c] = chr(c) # Helpers def header_check(octet): """Return True if the octet should be escaped with header quopri.""" return chr(octet) != _QUOPRI_HEADER_MAP[octet] def body_check(octet): """Return True if the octet should be escaped with body quopri.""" return chr(octet) != _QUOPRI_BODY_MAP[octet] def header_length(bytearray): """Return a header quoted-printable encoding length. Note that this does not include any RFC 2047 chrome added by `header_encode()`. :param bytearray: An array of bytes (a.k.a. octets). :return: The length in bytes of the byte array when it is encoded with quoted-printable for headers. """ return sum(len(_QUOPRI_HEADER_MAP[octet]) for octet in bytearray) def body_length(bytearray): """Return a body quoted-printable encoding length. :param bytearray: An array of bytes (a.k.a. octets). :return: The length in bytes of the byte array when it is encoded with quoted-printable for bodies. """ return sum(len(_QUOPRI_BODY_MAP[octet]) for octet in bytearray) def _max_append(L, s, maxlen, extra=''): if not isinstance(s, str): s = chr(s) if not L: L.append(s.lstrip()) elif len(L[-1]) + len(s) <= maxlen: L[-1] += extra + s else: L.append(s.lstrip()) def unquote(s): """Turn a string in the form =AB to the ASCII character with value 0xab""" return chr(int(s[1:3], 16)) def quote(c): return _QUOPRI_MAP[ord(c)] def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with quoted-printable (like) encoding. Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but used specifically for email header fields to allow charsets with mostly 7 bit characters (and some 8 bit) to remain more or less readable in non-RFC 2045 aware mail clients. charset names the character set to use in the RFC 2046 header. It defaults to iso-8859-1. """ # Return empty headers as an empty string. if not header_bytes: return '' # Iterate over every byte, encoding if necessary. encoded = header_bytes.decode('latin1').translate(_QUOPRI_HEADER_MAP) # Now add the RFC chrome to each encoded chunk and glue the chunks # together. return '=?%s?q?%s?=' % (charset, encoded) _QUOPRI_BODY_ENCODE_MAP = _QUOPRI_BODY_MAP[:] for c in b'\r\n': _QUOPRI_BODY_ENCODE_MAP[c] = chr(c) def body_encode(body, maxlinelen=76, eol=NL): """Encode with quoted-printable, wrapping at maxlinelen characters. Each line of encoded text will end with eol, which defaults to "\\n". Set this to "\\r\\n" if you will be using the result of this function directly in an email. Each line will be wrapped at, at most, maxlinelen characters before the eol string (maxlinelen defaults to 76 characters, the maximum value permitted by RFC 2045). Long lines will have the 'soft line break' quoted-printable character "=" appended to them, so the decoded text will be identical to the original text. The minimum maxlinelen is 4 to have room for a quoted character ("=XX") followed by a soft line break. Smaller values will generate a ValueError. """ if maxlinelen < 4: raise ValueError("maxlinelen must be at least 4") if not body: return body # quote special characters body = body.translate(_QUOPRI_BODY_ENCODE_MAP) soft_break = '=' + eol # leave space for the '=' at the end of a line maxlinelen1 = maxlinelen - 1 encoded_body = [] append = encoded_body.append for line in body.splitlines(): # break up the line into pieces no longer than maxlinelen - 1 start = 0 laststart = len(line) - 1 - maxlinelen while start <= laststart: stop = start + maxlinelen1 # make sure we don't break up an escape sequence if line[stop - 2] == '=': append(line[start:stop - 1]) start = stop - 2 elif line[stop - 1] == '=': append(line[start:stop]) start = stop - 1 else: append(line[start:stop] + '=') start = stop # handle rest of line, special case if line ends in whitespace if line and line[-1] in ' \t': room = start - laststart if room >= 3: # It's a whitespace character at end-of-line, and we have room # for the three-character quoted encoding. q = quote(line[-1]) elif room == 2: # There's room for the whitespace character and a soft break. q = line[-1] + soft_break else: # There's room only for a soft break. The quoted whitespace # will be the only content on the subsequent line. q = soft_break + quote(line[-1]) append(line[start:-1] + q) else: append(line[start:]) # add back final newline if present if body[-1] in CRLF: append('') return eol.join(encoded_body) # BAW: I'm not sure if the intent was for the signature of this function to be # the same as base64MIME.decode() or not... def decode(encoded, eol=NL): """Decode a quoted-printable string. Lines are separated with eol, which defaults to \\n. """ if not encoded: return encoded # BAW: see comment in encode() above. Again, we're building up the # decoded string with string concatenation, which could be done much more # efficiently. decoded = '' for line in encoded.splitlines(): line = line.rstrip() if not line: decoded += eol continue i = 0 n = len(line) while i < n: c = line[i] if c != '=': decoded += c i += 1 # Otherwise, c == "=". Are we at the end of the line? If so, add # a soft line break. elif i+1 == n: i += 1 continue # Decode if in form =AB elif i+2 < n and line[i+1] in hexdigits and line[i+2] in hexdigits: decoded += unquote(line[i:i+3]) i += 3 # Otherwise, not in form =AB, pass literally else: decoded += c i += 1 if i == n: decoded += eol # Special case if original string did not end with eol if encoded[-1] not in '\r\n' and decoded.endswith(eol): decoded = decoded[:-1] return decoded # For convenience and backwards compatibility w/ standard base64 module body_decode = decode decodestring = decode def _unquote_match(match): """Turn a match in the form =AB to the ASCII character with value 0xab""" s = match.group(0) return unquote(s) # Header decoding is done a bit differently def header_decode(s): """Decode a string encoded with RFC 2045 MIME header `Q' encoding. This function does not parse a full MIME header value encoded with quoted-printable (like =?iso-8859-1?q?Hello_World?=) -- please use the high level email.header class for that functionality. """ s = s.replace('_', ' ') return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s, flags=re.ASCII)	9.63 KB	2024-04-17 17:55
📄 utils.py # Copyright (C) 2001-2010 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org """Miscellaneous utilities.""" __all__ = [ 'collapse_rfc2231_value', 'decode_params', 'decode_rfc2231', 'encode_rfc2231', 'formataddr', 'formatdate', 'format_datetime', 'getaddresses', 'make_msgid', 'mktime_tz', 'parseaddr', 'parsedate', 'parsedate_tz', 'parsedate_to_datetime', 'unquote', ] import os import re import time import random import socket import datetime import urllib.parse from email._parseaddr import quote from email._parseaddr import AddressList as _AddressList from email._parseaddr import mktime_tz from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz # Intrapackage imports from email.charset import Charset COMMASPACE = ', ' EMPTYSTRING = '' UEMPTYSTRING = '' CRLF = '\r\n' TICK = "'" specialsre = re.compile(r'[][\\()<>@,:;".]') escapesre = re.compile(r'[\\"]') def _has_surrogates(s): """Return True if s contains surrogate-escaped binary data.""" # This check is based on the fact that unless there are surrogates, utf8 # (Python's default encoding) can encode any string. This is the fastest # way to check for surrogates, see issue 11454 for timings. try: s.encode() return False except UnicodeEncodeError: return True # How to deal with a string containing bytes before handing it to the # application through the 'normal' interface. def _sanitize(string): # Turn any escaped bytes into unicode 'unknown' char. If the escaped # bytes happen to be utf-8 they will instead get decoded, even if they # were invalid in the charset the source was supposed to be in. This # seems like it is not a bad thing; a defect was still registered. original_bytes = string.encode('utf-8', 'surrogateescape') return original_bytes.decode('utf-8', 'replace') # Helpers def formataddr(pair, charset='utf-8'): """The inverse of parseaddr(), this takes a 2-tuple of the form (realname, email_address) and returns the string value suitable for an RFC 2822 From, To or Cc header. If the first element of pair is false, then the second element is returned unmodified. Optional charset if given is the character set that is used to encode realname in case realname is not ASCII safe. Can be an instance of str or a Charset-like object which has a header_encode method. Default is 'utf-8'. """ name, address = pair # The address MUST (per RFC) be ascii, so raise a UnicodeError if it isn't. address.encode('ascii') if name: try: name.encode('ascii') except UnicodeEncodeError: if isinstance(charset, str): charset = Charset(charset) encoded_name = charset.header_encode(name) return "%s <%s>" % (encoded_name, address) else: quotes = '' if specialsre.search(name): quotes = '"' name = escapesre.sub(r'\\\g<0>', name) return '%s%s%s <%s>' % (quotes, name, quotes, address) return address def getaddresses(fieldvalues): """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" all = COMMASPACE.join(fieldvalues) a = _AddressList(all) return a.addresslist def _format_timetuple_and_zone(timetuple, zone): return '%s, %02d %s %04d %02d:%02d:%02d %s' % ( ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][timetuple[6]], timetuple[2], ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][timetuple[1] - 1], timetuple[0], timetuple[3], timetuple[4], timetuple[5], zone) def formatdate(timeval=None, localtime=False, usegmt=False): """Returns a date string as specified by RFC 2822, e.g.: Fri, 09 Nov 2001 01:08:47 -0000 Optional timeval if given is a floating point time value as accepted by gmtime() and localtime(), otherwise the current time is used. Optional localtime is a flag that when True, interprets timeval, and returns a date relative to the local timezone instead of UTC, properly taking daylight savings time into account. Optional argument usegmt means that the timezone is written out as an ascii string, not numeric one (so "GMT" instead of "+0000"). This is needed for HTTP, and is only used when localtime==False. """ # Note: we cannot use strftime() because that honors the locale and RFC # 2822 requires that day and month names be the English abbreviations. if timeval is None: timeval = time.time() if localtime or usegmt: dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc) else: dt = datetime.datetime.utcfromtimestamp(timeval) if localtime: dt = dt.astimezone() usegmt = False return format_datetime(dt, usegmt) def format_datetime(dt, usegmt=False): """Turn a datetime into a date string as specified in RFC 2822. If usegmt is True, dt must be an aware datetime with an offset of zero. In this case 'GMT' will be rendered instead of the normal +0000 required by RFC2822. This is to support HTTP headers involving date stamps. """ now = dt.timetuple() if usegmt: if dt.tzinfo is None or dt.tzinfo != datetime.timezone.utc: raise ValueError("usegmt option requires a UTC datetime") zone = 'GMT' elif dt.tzinfo is None: zone = '-0000' else: zone = dt.strftime("%z") return _format_timetuple_and_zone(now, zone) def make_msgid(idstring=None, domain=None): """Returns a string suitable for RFC 2822 compliant Message-ID, e.g: <142480216486.20800.16526388040877946887@nightshade.la.mastaler.com> Optional idstring if given is a string used to strengthen the uniqueness of the message id. Optional domain if given provides the portion of the message id after the '@'. It defaults to the locally defined hostname. """ timeval = int(time.time()100) pid = os.getpid() randint = random.getrandbits(64) if idstring is None: idstring = '' else: idstring = '.' + idstring if domain is None: domain = socket.getfqdn() msgid = '<%d.%d.%d%s@%s>' % (timeval, pid, randint, idstring, domain) return msgid def parsedate_to_datetime(data): dtuple, tz = _parsedate_tz(data) if tz is None: return datetime.datetime(dtuple[:6]) return datetime.datetime(dtuple[:6], tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) def parseaddr(addr): """ Parse addr into its constituent realname and email address parts. Return a tuple of realname and email address, unless the parse fails, in which case return a 2-tuple of ('', ''). """ addrs = _AddressList(addr).addresslist if not addrs: return '', '' return addrs[0] # rfc822.unquote() doesn't properly de-backslash-ify in Python pre-2.3. def unquote(str): """Remove quotes from a string.""" if len(str) > 1: if str.startswith('"') and str.endswith('"'): return str[1:-1].replace('\\\\', '\\').replace('\\"', '"') if str.startswith('<') and str.endswith('>'): return str[1:-1] return str # RFC2231-related functions - parameter encoding and decoding def decode_rfc2231(s): """Decode string according to RFC 2231""" parts = s.split(TICK, 2) if len(parts) <= 2: return None, None, s return parts def encode_rfc2231(s, charset=None, language=None): """Encode string according to RFC 2231. If neither charset nor language is given, then s is returned as-is. If charset is given but not language, the string is encoded using the empty string for language. """ s = urllib.parse.quote(s, safe='', encoding=charset or 'ascii') if charset is None and language is None: return s if language is None: language = '' return "%s'%s'%s" % (charset, language, s) rfc2231_continuation = re.compile(r'^(?P<name>\w+)\((?P<num>[0-9]+)\?)?$', re.ASCII) def decode_params(params): """Decode parameters list according to RFC 2231. params is a sequence of 2-tuples containing (param name, string value). """ # Copy params so we don't mess with the original params = params[:] new_params = [] # Map parameter's name to a list of continuations. The values are a # 3-tuple of the continuation number, the string value, and a flag # specifying whether a particular segment is %-encoded. rfc2231_params = {} name, value = params.pop(0) new_params.append((name, value)) while params: name, value = params.pop(0) if name.endswith(''): encoded = True else: encoded = False value = unquote(value) mo = rfc2231_continuation.match(name) if mo: name, num = mo.group('name', 'num') if num is not None: num = int(num) rfc2231_params.setdefault(name, []).append((num, value, encoded)) else: new_params.append((name, '"%s"' % quote(value))) if rfc2231_params: for name, continuations in rfc2231_params.items(): value = [] extended = False # Sort by number continuations.sort() # And now append all values in numerical order, converting # %-encodings for the encoded segments. If any of the # continuation names ends in a , then the entire string, after # decoding segments and concatenating, must have the charset and # language specifiers at the beginning of the string. for num, s, encoded in continuations: if encoded: # Decode as "latin-1", so the characters in s directly # represent the percent-encoded octet values. # collapse_rfc2231_value treats this as an octet sequence. s = urllib.parse.unquote(s, encoding="latin-1") extended = True value.append(s) value = quote(EMPTYSTRING.join(value)) if extended: charset, language, value = decode_rfc2231(value) new_params.append((name, (charset, language, '"%s"' % value))) else: new_params.append((name, '"%s"' % value)) return new_params def collapse_rfc2231_value(value, errors='replace', fallback_charset='us-ascii'): if not isinstance(value, tuple) or len(value) != 3: return unquote(value) # While value comes to us as a unicode string, we need it to be a bytes # object. We do not want bytes() normal utf-8 decoder, we want a straight # interpretation of the string as character bytes. charset, language, text = value if charset is None: # Issue 17369: if charset/lang is None, decode_rfc2231 couldn't parse # the value, so use the fallback_charset. charset = fallback_charset rawbytes = bytes(text, 'raw-unicode-escape') try: return str(rawbytes, charset, errors) except LookupError: # charset is not a known codec. return unquote(text) # # datetime doesn't provide a localtime function yet, so provide one. Code # adapted from the patch in issue 9527. This may not be perfect, but it is # better than not having it. # def localtime(dt=None, isdst=-1): """Return local time as an aware datetime object. If called without arguments, return current time. Otherwise dt argument should be a datetime instance, and it is converted to the local time zone according to the system time zone database. If dt is naive (that is, dt.tzinfo is None), it is assumed to be in local time. In this case, a positive or zero value for isdst causes localtime to presume initially that summer time (for example, Daylight Saving Time) is or is not (respectively) in effect for the specified time. A negative value for isdst causes the localtime() function to attempt to divine whether summer time is in effect for the specified time. """ if dt is None: return datetime.datetime.now(datetime.timezone.utc).astimezone() if dt.tzinfo is not None: return dt.astimezone() # We have a naive datetime. Convert to a (localtime) timetuple and pass to # system mktime together with the isdst hint. System mktime will return # seconds since epoch. tm = dt.timetuple()[:-1] + (isdst,) seconds = time.mktime(tm) localtm = time.localtime(seconds) try: delta = datetime.timedelta(seconds=localtm.tm_gmtoff) tz = datetime.timezone(delta, localtm.tm_zone) except AttributeError: # Compute UTC offset and compare with the value implied by tm_isdst. # If the values match, use the zone name implied by tm_isdst. delta = dt - datetime.datetime(*time.gmtime(seconds)[:6]) dst = time.daylight and localtm.tm_isdst > 0 gmtoff = -(time.altzone if dst else time.timezone) if delta == datetime.timedelta(seconds=gmtoff): tz = datetime.timezone(delta, time.tzname[dst]) else: tz = datetime.timezone(delta) return dt.replace(tzinfo=tz)	13.17 KB	2024-04-17 17:55

Edit File:

Terminal