Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified	Actions
📁 ..	-	2026-03-21 15:45
📄 Grammar.txt # Grammar for 2to3. This grammar supports Python 2.x and 3.x. # NOTE WELL: You should also follow all the steps listed at # https://devguide.python.org/grammar/ # Start symbols for the grammar: # file_input is a module or sequence of commands read from an input file; # single_input is a single interactive statement; # eval_input is the input for the eval() and input() functions. # NB: compound_stmt in single_input is followed by extra NEWLINE! file_input: (NEWLINE \| stmt)* ENDMARKER single_input: NEWLINE \| simple_stmt \| compound_stmt NEWLINE eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef \| funcdef \| async_funcdef) async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' # The following definition for typedarglist is equivalent to this set of rules: # # arguments = argument (',' argument)* # argument = tfpdef ['=' test] # kwargs = '*' tname [','] # args = '' [tname] # kwonly_kwargs = (',' argument)* [',' [kwargs]] # args_kwonly_kwargs = args kwonly_kwargs \| kwargs # poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] # typedargslist_no_posonly = poskeyword_args_kwonly_kwargs \| args_kwonly_kwargs # typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])\|(typedargslist_no_posonly)" # # It needs to be fully expanded to allow our LL(1) parser to work on it. typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [ ',' [((tfpdef ['=' test] ',')* ('' [tname] (',' tname ['=' test]) [',' ['' tname [',']]] \| '' tname [',']) \| tfpdef ['=' test] (',' tfpdef ['=' test])* [','])] ] \| ((tfpdef ['=' test] ',')* ('' [tname] (',' tname ['=' test]) [',' ['' tname [',']]] \| '' tname [',']) \| tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) tname: NAME [':' test] tfpdef: tname \| '(' tfplist ')' tfplist: tfpdef (',' tfpdef)* [','] # The following definition for varargslist is equivalent to this set of rules: # # arguments = argument (',' argument )* # argument = vfpdef ['=' test] # kwargs = '*' vname [','] # args = '' [vname] # kwonly_kwargs = (',' argument )* [',' [kwargs]] # args_kwonly_kwargs = args kwonly_kwargs \| kwargs # poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] # vararglist_no_posonly = poskeyword_args_kwonly_kwargs \| args_kwonly_kwargs # varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] \| (vararglist_no_posonly) # # It needs to be fully expanded to allow our LL(1) parser to work on it. varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ ((vfpdef ['=' test] ',')* ('' [vname] (',' vname ['=' test]) [',' ['' vname [',']]] \| '' vname [',']) \| vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) ]] \| ((vfpdef ['=' test] ',')* ('' [vname] (',' vname ['=' test]) [',' ['' vname [',']]]\| '' vname [',']) \| vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) vname: NAME vfpdef: vname \| '(' vfplist ')' vfplist: vfpdef (',' vfpdef)* [','] stmt: simple_stmt \| compound_stmt simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE small_stmt: (expr_stmt \| print_stmt \| del_stmt \| pass_stmt \| flow_stmt \| import_stmt \| global_stmt \| exec_stmt \| assert_stmt) expr_stmt: testlist_star_expr (annassign \| augassign (yield_expr\|testlist) \| ('=' (yield_expr\|testlist_star_expr))) annassign: ':' test ['=' test] testlist_star_expr: (test\|star_expr) (',' (test\|star_expr)) [','] augassign: ('+=' \| '-=' \| '=' \| '@=' \| '/=' \| '%=' \| '&=' \| '\|=' \| '^=' \| '<<=' \| '>>=' \| '=' \| '//=') # For normal and annotated assignments, additional restrictions enforced by the interpreter print_stmt: 'print' ( [ test (',' test) [','] ] \| '>>' test [ (',' test)+ [','] ] ) del_stmt: 'del' exprlist pass_stmt: 'pass' flow_stmt: break_stmt \| continue_stmt \| return_stmt \| raise_stmt \| yield_stmt break_stmt: 'break' continue_stmt: 'continue' return_stmt: 'return' [testlist_star_expr] yield_stmt: yield_expr raise_stmt: 'raise' [test ['from' test \| ',' test [',' test]]] import_stmt: import_name \| import_from import_name: 'import' dotted_as_names import_from: ('from' ('.'* dotted_name \| '.'+) 'import' ('' \| '(' import_as_names ')' \| import_as_names)) import_as_name: NAME ['as' NAME] dotted_as_name: dotted_name ['as' NAME] import_as_names: import_as_name (',' import_as_name) [','] dotted_as_names: dotted_as_name (',' dotted_as_name)* dotted_name: NAME ('.' NAME)* global_stmt: ('global' \| 'nonlocal') NAME (',' NAME)* exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt \| while_stmt \| for_stmt \| try_stmt \| with_stmt \| funcdef \| classdef \| decorated \| async_stmt async_stmt: ASYNC (funcdef \| with_stmt \| for_stmt) if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] \| 'finally' ':' suite)) with_stmt: 'with' with_item (',' with_item)* ':' suite with_item: test ['as' expr] with_var: 'as' expr # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test [(',' \| 'as') test]] suite: simple_stmt \| NEWLINE INDENT stmt+ DEDENT # Backward compatibility cruft to support: # [ x for x in lambda: True, lambda: False if x() ] # even while also allowing: # lambda x: 5 if x else 2 # (But not a mix of the two) testlist_safe: old_test [(',' old_test)+ [',']] old_test: or_test \| old_lambdef old_lambdef: 'lambda' [varargslist] ':' old_test namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] \| lambdef or_test: and_test ('or' and_test)* and_test: not_test ('and' not_test)* not_test: 'not' not_test \| comparison comparison: expr (comp_op expr)* comp_op: '<'\|'>'\|'=='\|'>='\|'<='\|'<>'\|'!='\|'in'\|'not' 'in'\|'is'\|'is' 'not' star_expr: '' expr expr: xor_expr ('\|' xor_expr) xor_expr: and_expr ('^' and_expr)* and_expr: shift_expr ('&' shift_expr)* shift_expr: arith_expr (('<<'\|'>>') arith_expr)* arith_expr: term (('+'\|'-') term)* term: factor ((''\|'@'\|'/'\|'%'\|'//') factor) factor: ('+'\|'-'\|'~') factor \| power power: [AWAIT] atom trailer* ['*' factor] atom: ('(' [yield_expr\|testlist_gexp] ')' \| '[' [listmaker] ']' \| '{' [dictsetmaker] '}' \| '`' testlist1 '`' \| NAME \| NUMBER \| STRING+ \| '.' '.' '.') listmaker: (namedexpr_test\|star_expr) ( comp_for \| (',' (namedexpr_test\|star_expr)) [','] ) testlist_gexp: (namedexpr_test\|star_expr) ( comp_for \| (',' (namedexpr_test\|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' \| '[' subscriptlist ']' \| '.' NAME subscriptlist: subscript (',' subscript)* [','] subscript: test \| [test] ':' [test] [sliceop] sliceop: ':' [test] exprlist: (expr\|star_expr) (',' (expr\|star_expr))* [','] testlist: test (',' test)* [','] dictsetmaker: ( ((test ':' test \| '' expr) (comp_for \| (',' (test ':' test \| '' expr))* [','])) \| ((test \| star_expr) (comp_for \| (',' (test \| star_expr))* [','])) ) classdef: 'class' NAME ['(' [arglist] ')'] ':' suite arglist: argument (',' argument)* [','] # "test '=' test" is really "keyword '=' test", but we have no such token. # These need to be in a single rule to avoid grammar that is ambiguous # to our LL(1) parser. Even though 'test' includes 'expr' in star_expr, # we explicitly match '' here, too, to give it proper precedence. # Illegal combinations and orderings are blocked in ast.c: # multiple (test comp_for) arguments are blocked; keyword unpackings # that precede iterable unpackings are blocked; etc. argument: ( test [comp_for] \| test ':=' test \| test '=' test \| '*' test \| '' test ) comp_iter: comp_for \| comp_if comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [comp_iter] comp_if: 'if' old_test [comp_iter] testlist1: test (',' test)* # not used in grammar, but may appear in "node" passed from Parser to Compiler encoding_decl: NAME yield_expr: 'yield' [yield_arg] yield_arg: 'from' test \| testlist_star_expr	8.49 KB	2025-10-09 16:16
📄 Grammar3.11.14.final.0.pickle ��;��}�(� symbol2number�}�(� file_input�M��and_expr�M�and_test�M� annassign�M�arglist�M�argument�M� arith_expr�M�assert_stmt�M� async_funcdef�M� async_stmt�M �atom�M � augassign�M� break_stmt�M�classdef�M �comp_for�M�comp_if�M� comp_iter�M�comp_op�M� comparison�M� compound_stmt�M� continue_stmt�M� decorated�M� decorator�M� decorators�M�del_stmt�M�dictsetmaker�M�dotted_as_name�M�dotted_as_names�M�dotted_name�M� encoding_decl�M� eval_input�M� except_clause�M� exec_stmt�M �expr�M!� expr_stmt�M"�exprlist�M#�factor�M$� flow_stmt�M%�for_stmt�M&�funcdef�M'�global_stmt�M(�if_stmt�M)�import_as_name�M�import_as_names�M+�import_from�M,�import_name�M-�import_stmt�M.�lambdef�M/� listmaker�M0�namedexpr_test�M1�not_test�M2�old_lambdef�M3�old_test�M4�or_test�M5� parameters�M6� pass_stmt�M7�power�M8� print_stmt�M9� raise_stmt�M:�return_stmt�M;� shift_expr�M<�simple_stmt�M=�single_input�M>�sliceop�M?� small_stmt�M@� star_expr�MA�stmt�MB� subscript�MC� subscriptlist�MD�suite�ME�term�MF�test�MG�testlist�MH� testlist1�MI� testlist_gexp�MJ� testlist_safe�MK�testlist_star_expr�ML�tfpdef�MM�tfplist�MN�tname�MO�trailer�MP�try_stmt�MQ� typedargslist�MR�varargslist�MS�vfpdef�MT�vfplist�MU�vname�MV� while_stmt�MW� with_item�MX� with_stmt�MY�with_var�MZ�xor_expr�M[� yield_arg�M\� yield_expr�M]� yield_stmt�M^u� number2symbol�}�(M�hMhMhMhMhMhMh Mh MhM hM h MhMhM hMhMhMhMhMhMhMhMhMhMhMhMhMhMhMhMh Mh!Mh"M h#M!h$M"h%M#h&M$h'M%h(M&h)M'hM(h+M)h,Mh-M+h.M,h/M-h0M.h1M/h2M0h3M1h4M2h5M3h6M4h7M5h8M6h9M7h:M8h;M9h<M:h=M;h>M<h?M=h@M>hAM?hBM@hCMAhDMBhEMChFMDhGMEhHMFhIMGhJMHhKMIhLMJhMMKhNMLhOMMhPMNhQMOhRMPhSMQhTMRhUMShVMThWMUhXMVhYMWhZMXh[MYh\MZh]M[h^M\h_M]h`M^hau�states�]�(]�(]�(KK��KK��KK��e]�K�K��ae]�(]�KK��a]�(K+K��K�K��ee]�(]�K,K��a]�(K-K��K�K��ee]�(]�K.K��a]�K/K��a]�(K0K��K�K��e]�K/K��a]�K�K��ae]�(]�K1K��a]�(K2K��K�K��e]�(K1K��K�K��ee]�(]�(KK��K3K��K/K��e]�K/K��a]�(K4K��K0K��K5K��K�K��e]�K�K��ae]�(]�K6K��a]�(KK��KK��K�K��ee]�(]�KK��a]�K/K��a]�(K2K��K�K��e]�K/K��a]�K�K��ae]�(]�K%K��a]�K7K��a]�K�K��ae]�(]�K%K��a]�(K8K��K7K��K9K��e]�K�K��ae]�(]�(KK��KK��K K��KK��K#K��K'K��K(K��K)K��e]�(K:K��K;K��K<K��e]�KK ��a]�(K=K��K>K ��e]�K?K��a]�(K@K��KAK��e]�K�K��a]�(K)K��K�K��e]�K:K��a]�KK��a]�K=K��a]�KK��a]�K@K��ae]�(]�(KBK��KCK��KDK��KEK��KFK��KGK��KHK��KIK��KJK��KKK��KLK��KMK��KNK��e]�K�K��ae]�(]�K K��a]�K�K��ae]�(]�KK��a]�K'K��a]�(KK��K.K��e]�(K:K��KOK��e]�KPK��a]�K.K��a]�K:K��a]�K�K��ae]�(]�(KK��K%K��e]�KQK��a]�KK��a]�KRK��a]�KSK��a]�(KTK��K�K��e]�K�K��ae]�(]�KK��a]�KUK��a]�(KTK��K�K��e]�K�K��ae]�(]�(K5K��KVK��e]�K�K��ae]�(]�(KWK��KXK��KYK��KWK��KZK��K[K��K\K��KRK��K]K��KK��e]�K�K��a]�(KK��K�K��e]�KRK��ae]�(]�K^K��a]�(K_K��K�K��ee]�(]�(K`K��KaK��KbK��K8K��K7K��KcK��KdK��KeK��K9K��e]�K�K��ae]�(]�KK��a]�K�K��ae]�(]�KfK��a]�(KgK��KaK��K7K��e]�K�K��ae]�(]�K K��a]�KhK��a]�(KK��KK��e]�(K:K��KOK��e]�K�K��a]�KK��a]�K:K��ae]�(]�KiK��a]�(KiK��K�K��ee]�(]�KK��a]�KQK��a]�K�K��ae]�(]�(K3K��KjK��K/K��e]�K^K��a]�(K2K��K5K��K�K��e]�(K2K��K.K��K5K��K�K��e]�(K2K��K5K��K�K��e]�(KjK ��K/K ��K�K��e]�K�K��a]�K/K��a]�(K3K ��K/K��K�K��e]�(K2K��K�K ��e]�K^K��a]�K.K ��a]�(K2K��K�K��e]�K/K��ae]�(]�KhK��a]�(KkK��K�K��e]�K'K��a]�K�K��ae]�(]�KlK��a]�(K2K��K�K��ee]�(]�K'K��a]�(KK��K�K��ee]�(]�K'K��a]�K�K��ae]�(]�KmK��a]�(KK��KK��e]�K�K��ae]�(]�KnK��a]�(K/K��K�K��e]�(K2K��KkK��K�K��e]�K/K��a]�K�K��ae]�(]�KK��a]�K^K��a]�(KRK��K�K��e]�K/K��a]�(K2K��K�K��e]�K/K��a]�K�K��ae]�(]�KoK��a]�(KpK��K�K��ee]�(]�KqK��a]�(K0K��KrK��KsK��K�K��e]�(KqK��K<K��e]�K�K��a]�(KmK��K<K��e]�(K0K��K�K��ee]�(]�(K^K��KjK��e]�(K2K��K�K��e]�(K^K��KjK��K�K��ee]�(]�(KK��KK��K$K��KtK��e]�KuK��a]�K�K��ae]�(]�(KvK��KwK��KxK��KyK��KzK��e]�K�K��ae]�(]�KK��a]�KQK��a]�KRK��a]�KmK��a]�K.K��a]�KPK��a]�(K{K��K�K��e]�K.K��a]�KPK ��a]�K�K ��ae]�(]�KK��a]�K'K��a]�K\|K��a]�(K}K��K.K��e]�K/K��a]�KPK��a]�K.K��a]�K�K��ae]�(]�(KK��KK��e]�K'K��a]�(K2K��K�K��ee]�(]�KK��a]�K~K��a]�K.K��a]�KPK��a]�(KK��K{K��K�K��e]�K.K��a]�KPK��a]�K�K��ae]�(]�K'K��a]�(KkK��K�K��e]�K'K��a]�K�K��ae]�(]�K�K��a]�(K2K��K�K��e]�(K�K��K�K��ee]�(]�KK��a]�(KK��KhK��e]�(KK��KK��KhK��e]�KK��a]�(KK��KK��K�K��e]�K�K��a]�K�K��a]�K:K��ae]�(]�KK��a]�K�K��a]�K�K��ae]�(]�(K�K��K�K��e]�K�K��ae]�(]�KK��a]�(K.K��K�K��e]�K/K��a]�K.K��a]�K�K��ae]�(]�(K~K��KjK��e]�(K2K��K5K��K�K��e]�(K~K��KjK��K�K��e]�K�K��a]�(K2K��K�K��ee]�(]�K/K��a]�(K4K��K�K��e]�K/K��a]�K�K��ae]�(]�(KK��K�K��e]�K,K��a]�K�K��ae]�(]�KK��a]�(K.K��K�K��e]�KUK��a]�K.K��a]�K�K��ae]�(]�(K�K��K�K��e]�K�K��ae]�(]�K�K��a]�(K�K��K�K��ee]�(]�KK��a]�(K:K��K�K��e]�K�K��a]�K:K��ae]�(]�KK��a]�K�K��ae]�(]�(K&K��K�K��e]�K�K��a]�(K3K��K�K��K�K��e]�KuK��a]�K�K��ae]�(]�KK��a]�(K�K��K/K��K�K��e]�K/K��a]�(K2K��K�K��e]�(K2K��K�K��e]�(K/K��K�K��e]�K/K��a]�(K2K��K�K��e]�(K/K��K�K��ee]�(]�KK��a]�(K/K��K�K��e]�(K2K��KK��K�K��e]�K/K��a]�K/K��a]�(K2K��K�K��e]�K�K��ae]�(]�KK��a]�(KqK��K�K��e]�K�K��ae]�(]�K�K��a]�(K�K��K�K��K�K��ee]�(]�K�K��a]�(K�K��KK��e]�(KK��K�K��e]�K�K��ae]�(]�(KK��K�K��K�K��e]�K�K��a]�KK��ae]�(]�K.K��a]�(K/K��K�K��e]�K�K��ae]�(]�(K�K��K�K��K�K��K�K��K�K��K�K��K�K��K�K��K�K��e]�K�K��ae]�(]�KK��a]�K^K��a]�K�K��ae]�(]�(K�K��K�K��e]�K�K��ae]�(]�(K.K��K/K��e]�(K�K��K/K��K�K��e]�(K.K��K�K��e]�K�K��a]�(K�K��K�K��ee]�(]�K�K��a]�(K2K��K�K��e]�(K�K��K�K��ee]�(]�(KK��K�K��e]�K�K��a]�K�K��a]�KK��a]�(K�K��KK��ee]�(]�KuK��a]�(K�K��KK��K�K��K�K��K K��K�K��ee]�(]�(K�K��K�K��e]�K�K��a]�(KK��K�K��e]�K�K��a]�K{K��a]�K/K��ae]�(]�K/K��a]�(K2K��K�K��e]�(K/K��K�K��ee]�(]�K/K��a]�(K2K��K�K��ee]�(]�(K~K��KjK��e]�(K2K��K5K��K�K��e]�(K~K��KjK��K�K��e]�K�K��a]�(K2K��K�K��ee]�(]�KUK��a]�(K2K��K�K��e]�KUK��a]�(K2K��K�K��e]�(KUK��K�K��ee]�(]�(KjK��K/K��e]�(K2K��K�K��e]�(KjK��K/K��K�K��ee]�(]�(KK��K�K��e]�K�K��a]�K�K��a]�K:K��ae]�(]�K�K��a]�(K2K��K�K��e]�(K�K��K�K��ee]�(]�K'K��a]�(K.K��K�K��e]�K/K��a]�K�K��ae]�(]�(KK��KK��K K��e]�(K:K��KOK��e]�K'K��a]�K�K��a]�K�K��a]�K:K��a]�K=K��ae]�(]�KK��a]�K.K��a]�KPK��a]�(K�K��K�K��e]�K.K��a]�K.K��a]�KPK��a]�KPK ��a]�K�K��a]�(K{K ��K�K��K�K��K�K ��e]�K.K��a]�KPK��a]�(K�K��K�K��ee]�(]�(KK��K3K��K�K��e]�(K2K��K�K��K�K��e]�K�K��a]�(K2K��K0K��K�K��e]�(K3K��K�K ��K�K��e]�(K2K��K�K��e]�(K2K ��K�K��e]�(KK��K3K��K�K��K�K��K�K��e]�K/K��a]�(K2K��K0K ��K�K ��e]�K�K ��a]�(K2K��K�K��e]�(K2K��K�K��e]�K/K��a]�(KK��K3K��K�K��K�K��e]�(K2K��K�K��K�K��e]�(K2K��K0K��K�K��e]�(K3K��K�K��K�K��e]�(K2K��K�K��e]�K/K��a]�(K2K��K0K��K�K��e]�K/K��ae]�(]�(KK��K3K��K�K��e]�(K2K��K�K��K�K��e]�K�K��a]�(K2K��K0K��K�K��e]�(K3K��K�K ��K�K��e]�(K2K��K�K��e]�(K2K ��K�K��e]�(KK��K3K��K�K��K�K��K�K��e]�K/K��a]�(K2K��K0K ��K�K ��e]�K�K ��a]�(K2K��K�K��e]�(K2K��K�K��e]�K/K��a]�(KK��K3K��K�K��K�K��e]�(K2K��K�K��K�K��e]�(K2K��K0K��K�K��e]�(K3K��K�K��K�K��e]�(K2K��K�K��e]�K/K��a]�(K2K��K0K��K�K��e]�K/K��ae]�(]�(KK��K�K��e]�K�K��a]�K�K��a]�K:K��ae]�(]�K�K��a]�(K2K��K�K��e]�(K�K��K�K��ee]�(]�K'K��a]�K�K��ae]�(]�K K��a]�K~K��a]�K.K��a]�KPK��a]�(K{K��K�K��e]�K.K��a]�KPK��a]�K�K��ae]�(]�K/K��a]�(KkK��K�K��e]�K^K��a]�K�K��ae]�(]�K!K��a]�K�K��a]�(K2K��K.K��e]�KPK��a]�K�K��ae]�(]�KkK��a]�K^K��a]�K�K��ae]�(]�K�K��a]�(K�K��K�K��ee]�(]�(KK��KqK��e]�K/K��a]�K�K��ae]�(]�K"K��a]�(K�K��K�K��e]�K�K��ae]�(]�K<K��a]�K�K��aee�dfas�}�(M�hf}�(KKKKKKKKKKK KK KKKKKK KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKK KK!KK"KK#KK$KK%KK&KKKK'KKKK(KK)Ku��Mhm}�(KKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��Mhs}�(KKKKKKKKK KKKKKK#KK$KK&KK'KK(KK)Ku��Mhy}�K.Ks��Mh�}�(KKKKK3KKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��Mh�}�(KKKKK3KKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��Mh�}�(KKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��Mh�}�KKs��Mh�}�K%Ks��M h�}�K%Ks��M h�}�(KKKKK KKKK#KK'KK(KK)Ku��Mh�}�(KBKKCKKDKKEKKFKKGKKHKKIKKJKKKKKLKKMKKNKu��Mh�}�K Ks��M h�}�KKs��Mj��}�(KKK%Ku��Mj ��}�KKs��Mj��}�(KKKKK%Ku��Mj0��}�(KWKKXKKYKKZKK[KK\KKRKK]KKKu��MjC��}�(KKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��MjI��}�(K KKKKKKKKKKKK KK!KK%Ku��MjV��}�KKs��Mj[��}�K Ks��Mjd��}�K Ks��Mju��}�K Ks��Mj{��}�KKs��Mj��}�(KKKKK3KKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��Mj��}�K'Ks��Mj��}�K'Ks��Mj��}�K'Ks��Mj��}�K'Ks��Mj��}�(KKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��Mj��}�KnKs��M j��}�KKs��M!j��}�(KKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��M"j��}�(KKKKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��M#j ��}�(KKKKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��M$j��}�(KKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��M%j��}�(K KKKKKKKK"Ku��M&j'��}�KKs��M'j=��}�KKs��M(jO��}�(KKKKu��M)jX��}�KKs��Mjk��}�K'Ks��M+ju��}�K'Ks��M,j~��}�KKs��M-j��}�KKs��M.j��}�(KKKKu��M/j��}�KKs��M0j��}�(KKKKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��M1j��}�(KKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��M2j��}�(KKKKKKKKK KKKKKK#KK$KK&KK'KK(KK)Ku��M3j��}�KKs��M4j��}�(KKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��M5j��}�(KKKKKKKKK KKKKKK#KK$KK&KK'KK(KK)Ku��M6j��}�KKs��M7j��}�KKs��M8j��}�(KKKKK KKKK#KK&KK'KK(KK)Ku��M9j��}�KKs��M:j��}�KKs��M;j2��}�KKs��M<j:��}�(KKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��M=jA��}�(KKKKKKKKKKK KKKKKK KKKKKKKKKKKKKKKKKKKKKKKKKKKK"KK#KK$KK&KK'KK(KK)Ku��M>jL��}�(KKKKKKKKKKK KK KKKKKK KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKK KK!KK"KK#KK$KK%KK&KK'KKKK(KK)Ku��M?jU��}�K.Ks��M@j]��}�(KKKKKKKKKKK KKKKKK KKKKKKKKKKKKKKKKKKKKKKKKKKKK"KK#KK$KK&KK'KK(KK)Ku��MAjj��}�KKs��MBjq��}�(KKKKKKKKKKK KK KKKKKK KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKK KK!KK"KK#KK$KK%KK&KK'KK(KK)Ku��MCjw��}�(KKKKKKKKK.KK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MDj��}�(KKKKKKKKK.KK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MEj��}�(KKKKKKKKKKK KKKKKK KKKKKKKKKKKKKKKKKKKKKKKKKKKK"KK#KK$KK&KK'KKKK(KK)Ku��MFj��}�(KKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��MGj��}�(KKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MHj��}�(KKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MIj��}�(KKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MJj��}�(KKKKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MKj��}�(KKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MLj��}�(KKKKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MMj��}�(KKK'Ku��MNj��}�(KKK'Ku��MOj��}�K'Ks��MPj��}�(KKKKK Ku��MQj��}�KKs��MRj>��}�(KKKKK3KK'Ku��MSj��}�(KKKKK3KK'Ku��MTj��}�(KKK'Ku��MUj��}�(KKK'Ku��MVj��}�K'Ks��MWj��}�K Ks��MXj��}�(KKKKKKKKK KKKKKKKK#KK$KK&KK'KK(KK)Ku��MYj��}�K!Ks��MZj��}�KkKs��M[j��}�(KKKKKKKKK KKKK#KK$KK&KK'KK(KK)Ku��M\j!��}�(KKKKKKKKKKK KKKKKKKKKK#KK$KK&KK'KK(KK)Ku��M]j)��}�K"Ks��M^j1��}�K"Ks��u�labels�]�(K��EMPTY��K�N��KN��MBN��KN��KN��KN��KN��KN��K2N��K N��KN��K�assert��K�break��K�class��K�continue��K�def��K�del��K�exec��K�for��K�from��K�global��K�if��K�import��K�lambda��K�nonlocal��K�not��K�pass��K�print��K�raise��K�return��K�try��K�while��K�with��K�yield��KN��K N��K9N��K8N��KN��KN��KN��M<N��KN��M2N��K�and��KN��MGN��KN��MN��KN��K$N��K;N��MN��MFN��M'N��M&N��MYN��KN��MJN��M]N��K N��M0N��MIN��KN��MN��K)N��KN��K/N��K'N��K%N��K&N��K1N��K(N��K-N��K.N��K3N��K,N��K+N��MN��MEN��M#N��K�in��MKN��MN��M4N��MN��KN��KN��KN��KN��KN��KN��K�is��M!N��MN��M N��M N��MN��M)N��MQN��MWN��MN��MN��MN��MN��MAN��K�as��MN��MHN��K�except��M[N��KN��MLN��MN��MN��M8N��M$N��MN��MN��M:N��M;N��M^N��K�else��M6N��K7N��M1N��K�elif��MN��M+N��MN��M,N��M-N��MSN��MN��M3N��M5N��MN��K�or��MRN��M N��MPN��K#N��MN��K"N��M@N��K N��MN��M=N��MN��MN��M N��M"N��M%N��M(N��M.N��M7N��M9N��M?N��MCN��KN��KN��KN��KN��K0N��M/N��MON��MNN��MMN��MDN��K�finally��MN��MTN��MVN��MUN��MXN��MN��K!N��M\N��e�keywords�}�(j��Kj��K j ��Kj��Kj ��Kj��Kj��Kj��Kj��Kj��Kj��Kj��Kj��Kj��Kj!��Kj#��Kj%��Kj'��Kj)��Kj+��Kj-��K j/��K!j1��K"j=��K-jc��KRjo��K]j~��Kkj��Knj��K{j��Kj��K�j��K�u�tokens�}�(K�KKKKKKKKKKKKKK2K K K KKKK#K K$K9K%K8K&KK'KK(KK)KK+KK.KK0KK2K$K3K;K4KK:K K=KK@K)KBKKCK/KDK'KEK%KFK&KGK1KHK(KIK-KJK.KKK3KLK,KMK+KNKKWKKXKKYKKZKK[KK\KKpK7K}K#K�K"K�K K�KK�KK�KK�KK�K0K�K!K�u�symbol2label�}�(�stmt�K� shift_expr�K�not_test�K,�test�K/�argument�K1�comp_for�K5�term�K6�funcdef�K7�for_stmt�K8� with_stmt�K9� testlist_gexp�K;� yield_expr�K<� listmaker�K>� testlist1�K?�dictsetmaker�KA�arglist�KO�suite�KP�exprlist�KQ� testlist_safe�KS� comp_iter�KT�old_test�KU�comp_if�KV�expr�K^�comp_op�K_� async_stmt�K`�classdef�Ka� decorated�Kb�if_stmt�Kc�try_stmt�Kd� while_stmt�Ke� decorators�Kf� async_funcdef�Kg�dotted_name�Kh� decorator�Ki� star_expr�Kj�dotted_as_name�Kl�testlist�Km�xor_expr�Ko�testlist_star_expr�Kq� annassign�Kr� augassign�Ks�power�Kt�factor�Ku� break_stmt�Kv� continue_stmt�Kw� raise_stmt�Kx�return_stmt�Ky� yield_stmt�Kz� parameters�K\|�namedexpr_test�K~�import_as_name�K��import_as_names�K��dotted_as_names�K��import_from�K��import_name�K��varargslist�K�� comparison�K��old_lambdef�K��or_test�K��and_test�K�� typedargslist�K��atom�K��trailer�K�� arith_expr�K�� small_stmt�K�� compound_stmt�K��simple_stmt�K��assert_stmt�K��del_stmt�K�� exec_stmt�K�� expr_stmt�K�� flow_stmt�K��global_stmt�K��import_stmt�K�� pass_stmt�K�� print_stmt�K��sliceop�K�� subscript�K��lambdef�K��tname�K��tfplist�K��tfpdef�K�� subscriptlist�K�� except_clause�K��vfpdef�K��vname�K��vfplist�K�� with_item�K��and_expr�K�� yield_arg�K�u�start�M�u.	14.95 KB	2026-01-07 23:15
📄 PatternGrammar.txt # Copyright 2006 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. # A grammar to describe tree matching patterns. # Not shown here: # - 'TOKEN' stands for any token (leaf node) # - 'any' stands for any node (leaf or interior) # With 'any' we can still specify the sub-structure. # The start symbol is 'Matcher'. Matcher: Alternatives ENDMARKER Alternatives: Alternative ('\|' Alternative)* Alternative: (Unit \| NegatedUnit)+ Unit: [NAME '='] ( STRING [Repeater] \| NAME [Details] [Repeater] \| '(' Alternatives ')' [Repeater] \| '[' Alternatives ']' ) NegatedUnit: 'not' (STRING \| NAME [Details] \| '(' Alternatives ')') Repeater: '*' \| '+' \| '{' NUMBER [',' NUMBER] '}' Details: '<' Alternatives '>'	793 B	2025-10-09 16:16
📄 PatternGrammar3.11.14.final.0.pickle ��}�(� symbol2number�}�(�Matcher�M��Alternative�M�Alternatives�M�Details�M�NegatedUnit�M�Repeater�M�Unit�Mu� number2symbol�}�(M�hMhMhMhMhMhMh u�states�]�(]�(]�KK��a]�KK��a]�K�K��ae]�(]�(KK��K K��e]�(KK��K K��K�K��ee]�(]�K K��a]�(KK��K�K��ee]�(]�KK��a]�KK��a]�K K��a]�K�K��ae]�(]�KK��a]�(KK��KK��KK��e]�KK��a]�(KK��K�K��e]�K�K��a]�KK��ae]�(]�(KK��KK��KK��e]�K�K��a]�KK��a]�(KK��KK��e]�KK��a]�KK��ae]�(]�(KK��KK��KK��KK��e]�KK��a]�KK��a]�(KK��KK��KK��K�K��e]�(KK��K�K��e]�KK��a]�KK��a]�(KK��KK��KK ��KK��e]�K�K��a]�(KK��KK��K�K ��eee�dfas�}�(M�h}�(KKKKKKKKKKu��Mh}�(KKKKKKKKKKu��Mh}�(KKKKKKKKKKu��Mh#}�KKs��Mh,}�KKs��Mh<}�(KKKKKKu��MhL}�(KKKKKKKKu��u�labels�]�(K��EMPTY��MN��K�N��KN��K N��K�not��KN��KN��MN��MN��MN��KN��KN��KN��MN��KN��KN��KN��KN��KN��KN��KN��KN��MN��K N��e�keywords�}�h�Ks�tokens�}�(K�KKKK KKKKKKKKKKK KKKKKKKKKKKKKKKKK Ku�symbol2label�}�(�Alternatives�K�NegatedUnit�K�Unit�K �Alternative�K �Details�K�Repeater�Ku�start�M�u.	1.2 KB	2026-01-07 23:15
📄 __init__.py import warnings warnings.warn( "lib2to3 package is deprecated and may not be able to parse Python 3.10+", DeprecationWarning, stacklevel=2, )	156 B	2025-10-09 16:16
📄 __main__.py import sys from .main import main sys.exit(main("lib2to3.fixes"))	67 B	2025-10-09 16:16
📁 __pycache__	-	2026-03-21 15:45
📄 btm_matcher.py """A bottom-up tree matching algorithm implementation meant to speed up 2to3's matching process. After the tree patterns are reduced to their rarest linear path, a linear Aho-Corasick automaton is created. The linear automaton traverses the linear paths from the leaves to the root of the AST and returns a set of nodes for further matching. This reduces significantly the number of candidate nodes.""" __author__ = "George Boutsioukis <gboutsioukis@gmail.com>" import logging import itertools from collections import defaultdict from . import pytree from .btm_utils import reduce_tree class BMNode(object): """Class for a node of the Aho-Corasick automaton used in matching""" count = itertools.count() def __init__(self): self.transition_table = {} self.fixers = [] self.id = next(BMNode.count) self.content = '' class BottomMatcher(object): """The main matcher class. After instantiating the patterns should be added using the add_fixer method""" def __init__(self): self.match = set() self.root = BMNode() self.nodes = [self.root] self.fixers = [] self.logger = logging.getLogger("RefactoringTool") def add_fixer(self, fixer): """Reduces a fixer's pattern tree to a linear path and adds it to the matcher(a common Aho-Corasick automaton). The fixer is appended on the matching states and called when they are reached""" self.fixers.append(fixer) tree = reduce_tree(fixer.pattern_tree) linear = tree.get_linear_subpattern() match_nodes = self.add(linear, start=self.root) for match_node in match_nodes: match_node.fixers.append(fixer) def add(self, pattern, start): "Recursively adds a linear pattern to the AC automaton" #print("adding pattern", pattern, "to", start) if not pattern: #print("empty pattern") return [start] if isinstance(pattern[0], tuple): #alternatives #print("alternatives") match_nodes = [] for alternative in pattern[0]: #add all alternatives, and add the rest of the pattern #to each end node end_nodes = self.add(alternative, start=start) for end in end_nodes: match_nodes.extend(self.add(pattern[1:], end)) return match_nodes else: #single token #not last if pattern[0] not in start.transition_table: #transition did not exist, create new next_node = BMNode() start.transition_table[pattern[0]] = next_node else: #transition exists already, follow next_node = start.transition_table[pattern[0]] if pattern[1:]: end_nodes = self.add(pattern[1:], start=next_node) else: end_nodes = [next_node] return end_nodes def run(self, leaves): """The main interface with the bottom matcher. The tree is traversed from the bottom using the constructed automaton. Nodes are only checked once as the tree is retraversed. When the automaton fails, we give it one more shot(in case the above tree matches as a whole with the rejected leaf), then we break for the next leaf. There is the special case of multiple arguments(see code comments) where we recheck the nodes Args: The leaves of the AST tree to be matched Returns: A dictionary of node matches with fixers as the keys """ current_ac_node = self.root results = defaultdict(list) for leaf in leaves: current_ast_node = leaf while current_ast_node: current_ast_node.was_checked = True for child in current_ast_node.children: # multiple statements, recheck if isinstance(child, pytree.Leaf) and child.value == ";": current_ast_node.was_checked = False break if current_ast_node.type == 1: #name node_token = current_ast_node.value else: node_token = current_ast_node.type if node_token in current_ac_node.transition_table: #token matches current_ac_node = current_ac_node.transition_table[node_token] for fixer in current_ac_node.fixers: results[fixer].append(current_ast_node) else: #matching failed, reset automaton current_ac_node = self.root if (current_ast_node.parent is not None and current_ast_node.parent.was_checked): #the rest of the tree upwards has been checked, next leaf break #recheck the rejected node once from the root if node_token in current_ac_node.transition_table: #token matches current_ac_node = current_ac_node.transition_table[node_token] for fixer in current_ac_node.fixers: results[fixer].append(current_ast_node) current_ast_node = current_ast_node.parent return results def print_ac(self): "Prints a graphviz diagram of the BM automaton(for debugging)" print("digraph g{") def print_node(node): for subnode_key in node.transition_table.keys(): subnode = node.transition_table[subnode_key] print("%d -> %d [label=%s] //%s" % (node.id, subnode.id, type_repr(subnode_key), str(subnode.fixers))) if subnode_key == 1: print(subnode.content) print_node(subnode) print_node(self.root) print("}") # taken from pytree.py for debugging; only used by print_ac _type_reprs = {} def type_repr(type_num): global _type_reprs if not _type_reprs: from .pygram import python_symbols # printing tokens is possible but not as useful # from .pgen2 import token // token.__dict__.items(): for name, val in python_symbols.__dict__.items(): if type(val) == int: _type_reprs[val] = name return _type_reprs.setdefault(type_num, type_num)	6.47 KB	2025-10-09 16:16
📄 btm_utils.py "Utility functions used by the btm_matcher module" from . import pytree from .pgen2 import grammar, token from .pygram import pattern_symbols, python_symbols syms = pattern_symbols pysyms = python_symbols tokens = grammar.opmap token_labels = token TYPE_ANY = -1 TYPE_ALTERNATIVES = -2 TYPE_GROUP = -3 class MinNode(object): """This class serves as an intermediate representation of the pattern tree during the conversion to sets of leaf-to-root subpatterns""" def __init__(self, type=None, name=None): self.type = type self.name = name self.children = [] self.leaf = False self.parent = None self.alternatives = [] self.group = [] def __repr__(self): return str(self.type) + ' ' + str(self.name) def leaf_to_root(self): """Internal method. Returns a characteristic path of the pattern tree. This method must be run for all leaves until the linear subpatterns are merged into a single""" node = self subp = [] while node: if node.type == TYPE_ALTERNATIVES: node.alternatives.append(subp) if len(node.alternatives) == len(node.children): #last alternative subp = [tuple(node.alternatives)] node.alternatives = [] node = node.parent continue else: node = node.parent subp = None break if node.type == TYPE_GROUP: node.group.append(subp) #probably should check the number of leaves if len(node.group) == len(node.children): subp = get_characteristic_subpattern(node.group) node.group = [] node = node.parent continue else: node = node.parent subp = None break if node.type == token_labels.NAME and node.name: #in case of type=name, use the name instead subp.append(node.name) else: subp.append(node.type) node = node.parent return subp def get_linear_subpattern(self): """Drives the leaf_to_root method. The reason that leaf_to_root must be run multiple times is because we need to reject 'group' matches; for example the alternative form (a \| b c) creates a group [b c] that needs to be matched. Since matching multiple linear patterns overcomes the automaton's capabilities, leaf_to_root merges each group into a single choice based on 'characteristic'ity, i.e. (a\|b c) -> (a\|b) if b more characteristic than c Returns: The most 'characteristic'(as defined by get_characteristic_subpattern) path for the compiled pattern tree. """ for l in self.leaves(): subp = l.leaf_to_root() if subp: return subp def leaves(self): "Generator that returns the leaves of the tree" for child in self.children: yield from child.leaves() if not self.children: yield self def reduce_tree(node, parent=None): """ Internal function. Reduces a compiled pattern tree to an intermediate representation suitable for feeding the automaton. This also trims off any optional pattern elements(like [a], a). """ new_node = None #switch on the node type if node.type == syms.Matcher: #skip node = node.children[0] if node.type == syms.Alternatives : #2 cases if len(node.children) <= 2: #just a single 'Alternative', skip this node new_node = reduce_tree(node.children[0], parent) else: #real alternatives new_node = MinNode(type=TYPE_ALTERNATIVES) #skip odd children('\|' tokens) for child in node.children: if node.children.index(child)%2: continue reduced = reduce_tree(child, new_node) if reduced is not None: new_node.children.append(reduced) elif node.type == syms.Alternative: if len(node.children) > 1: new_node = MinNode(type=TYPE_GROUP) for child in node.children: reduced = reduce_tree(child, new_node) if reduced: new_node.children.append(reduced) if not new_node.children: # delete the group if all of the children were reduced to None new_node = None else: new_node = reduce_tree(node.children[0], parent) elif node.type == syms.Unit: if (isinstance(node.children[0], pytree.Leaf) and node.children[0].value == '('): #skip parentheses return reduce_tree(node.children[1], parent) if ((isinstance(node.children[0], pytree.Leaf) and node.children[0].value == '[') or (len(node.children)>1 and hasattr(node.children[1], "value") and node.children[1].value == '[')): #skip whole unit if its optional return None leaf = True details_node = None alternatives_node = None has_repeater = False repeater_node = None has_variable_name = False for child in node.children: if child.type == syms.Details: leaf = False details_node = child elif child.type == syms.Repeater: has_repeater = True repeater_node = child elif child.type == syms.Alternatives: alternatives_node = child if hasattr(child, 'value') and child.value == '=': # variable name has_variable_name = True #skip variable name if has_variable_name: #skip variable name, '=' name_leaf = node.children[2] if hasattr(name_leaf, 'value') and name_leaf.value == '(': # skip parenthesis name_leaf = node.children[3] else: name_leaf = node.children[0] #set node type if name_leaf.type == token_labels.NAME: #(python) non-name or wildcard if name_leaf.value == 'any': new_node = MinNode(type=TYPE_ANY) else: if hasattr(token_labels, name_leaf.value): new_node = MinNode(type=getattr(token_labels, name_leaf.value)) else: new_node = MinNode(type=getattr(pysyms, name_leaf.value)) elif name_leaf.type == token_labels.STRING: #(python) name or character; remove the apostrophes from #the string value name = name_leaf.value.strip("'") if name in tokens: new_node = MinNode(type=tokens[name]) else: new_node = MinNode(type=token_labels.NAME, name=name) elif name_leaf.type == syms.Alternatives: new_node = reduce_tree(alternatives_node, parent) #handle repeaters if has_repeater: if repeater_node.children[0].value == '': #reduce to None new_node = None elif repeater_node.children[0].value == '+': #reduce to a single occurrence i.e. do nothing pass else: #TODO: handle {min, max} repeaters raise NotImplementedError #add children if details_node and new_node is not None: for child in details_node.children[1:-1]: #skip '<', '>' markers reduced = reduce_tree(child, new_node) if reduced is not None: new_node.children.append(reduced) if new_node: new_node.parent = parent return new_node def get_characteristic_subpattern(subpatterns): """Picks the most characteristic from a list of linear patterns Current order used is: names > common_names > common_chars """ if not isinstance(subpatterns, list): return subpatterns if len(subpatterns)==1: return subpatterns[0] # first pick out the ones containing variable names subpatterns_with_names = [] subpatterns_with_common_names = [] common_names = ['in', 'for', 'if' , 'not', 'None'] subpatterns_with_common_chars = [] common_chars = "[]().,:" for subpattern in subpatterns: if any(rec_test(subpattern, lambda x: type(x) is str)): if any(rec_test(subpattern, lambda x: isinstance(x, str) and x in common_chars)): subpatterns_with_common_chars.append(subpattern) elif any(rec_test(subpattern, lambda x: isinstance(x, str) and x in common_names)): subpatterns_with_common_names.append(subpattern) else: subpatterns_with_names.append(subpattern) if subpatterns_with_names: subpatterns = subpatterns_with_names elif subpatterns_with_common_names: subpatterns = subpatterns_with_common_names elif subpatterns_with_common_chars: subpatterns = subpatterns_with_common_chars # of the remaining subpatterns pick out the longest one return max(subpatterns, key=len) def rec_test(sequence, test_func): """Tests test_func on all items of sequence and items of included sub-iterables""" for x in sequence: if isinstance(x, (list, tuple)): yield from rec_test(x, test_func) else: yield test_func(x)	9.71 KB	2025-10-09 16:16
📄 fixer_base.py # Copyright 2006 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Base class for fixers (optional, but recommended).""" # Python imports import itertools # Local imports from .patcomp import PatternCompiler from . import pygram from .fixer_util import does_tree_import class BaseFix(object): """Optional base class for fixers. The subclass name must be FixFooBar where FooBar is the result of removing underscores and capitalizing the words of the fix name. For example, the class name for a fixer named 'has_key' should be FixHasKey. """ PATTERN = None # Most subclasses should override with a string literal pattern = None # Compiled pattern, set by compile_pattern() pattern_tree = None # Tree representation of the pattern options = None # Options object passed to initializer filename = None # The filename (set by set_filename) numbers = itertools.count(1) # For new_name() used_names = set() # A set of all used NAMEs order = "post" # Does the fixer prefer pre- or post-order traversal explicit = False # Is this ignored by refactor.py -f all? run_order = 5 # Fixers will be sorted by run order before execution # Lower numbers will be run first. _accept_type = None # [Advanced and not public] This tells RefactoringTool # which node type to accept when there's not a pattern. keep_line_order = False # For the bottom matcher: match with the # original line order BM_compatible = False # Compatibility with the bottom matching # module; every fixer should set this # manually # Shortcut for access to Python grammar symbols syms = pygram.python_symbols def __init__(self, options, log): """Initializer. Subclass may override. Args: options: a dict containing the options passed to RefactoringTool that could be used to customize the fixer through the command line. log: a list to append warnings and other messages to. """ self.options = options self.log = log self.compile_pattern() def compile_pattern(self): """Compiles self.PATTERN into self.pattern. Subclass may override if it doesn't want to use self.{pattern,PATTERN} in .match(). """ if self.PATTERN is not None: PC = PatternCompiler() self.pattern, self.pattern_tree = PC.compile_pattern(self.PATTERN, with_tree=True) def set_filename(self, filename): """Set the filename. The main refactoring tool should call this. """ self.filename = filename def match(self, node): """Returns match for a given parse tree node. Should return a true or false object (not necessarily a bool). It may return a non-empty dict of matching sub-nodes as returned by a matching pattern. Subclass may override. """ results = {"node": node} return self.pattern.match(node, results) and results def transform(self, node, results): """Returns the transformation for a given parse tree node. Args: node: the root of the parse tree that matched the fixer. results: a dict mapping symbolic names to part of the match. Returns: None, or a node that is a modified copy of the argument node. The node argument may also be modified in-place to effect the same change. Subclass must override. """ raise NotImplementedError() def new_name(self, template="xxx_todo_changeme"): """Return a string suitable for use as an identifier The new name is guaranteed not to conflict with other identifiers. """ name = template while name in self.used_names: name = template + str(next(self.numbers)) self.used_names.add(name) return name def log_message(self, message): if self.first_log: self.first_log = False self.log.append("### In file %s ###" % self.filename) self.log.append(message) def cannot_convert(self, node, reason=None): """Warn the user that a given chunk of code is not valid Python 3, but that it cannot be converted automatically. First argument is the top-level node for the code in question. Optional second argument is why it can't be converted. """ lineno = node.get_lineno() for_output = node.clone() for_output.prefix = "" msg = "Line %d: could not convert: %s" self.log_message(msg % (lineno, for_output)) if reason: self.log_message(reason) def warning(self, node, reason): """Used for warning the user about possible uncertainty in the translation. First argument is the top-level node for the code in question. Optional second argument is why it can't be converted. """ lineno = node.get_lineno() self.log_message("Line %d: %s" % (lineno, reason)) def start_tree(self, tree, filename): """Some fixers need to maintain tree-wide state. This method is called once, at the start of tree fix-up. tree - the root node of the tree to be processed. filename - the name of the file the tree came from. """ self.used_names = tree.used_names self.set_filename(filename) self.numbers = itertools.count(1) self.first_log = True def finish_tree(self, tree, filename): """Some fixers need to maintain tree-wide state. This method is called once, at the conclusion of tree fix-up. tree - the root node of the tree to be processed. filename - the name of the file the tree came from. """ pass class ConditionalFix(BaseFix): """ Base class for fixers which not execute if an import is found. """ # This is the name of the import which, if found, will cause the test to be skipped skip_on = None def start_tree(self, args): super(ConditionalFix, self).start_tree(args) self._should_skip = None def should_skip(self, node): if self._should_skip is not None: return self._should_skip pkg = self.skip_on.split(".") name = pkg[-1] pkg = ".".join(pkg[:-1]) self._should_skip = does_tree_import(pkg, name, node) return self._should_skip	6.53 KB	2025-10-09 16:16
📄 fixer_util.py """Utility functions, node construction macros, etc.""" # Author: Collin Winter # Local imports from .pgen2 import token from .pytree import Leaf, Node from .pygram import python_symbols as syms from . import patcomp ########################################################### ### Common node-construction "macros" ########################################################### def KeywordArg(keyword, value): return Node(syms.argument, [keyword, Leaf(token.EQUAL, "="), value]) def LParen(): return Leaf(token.LPAR, "(") def RParen(): return Leaf(token.RPAR, ")") def Assign(target, source): """Build an assignment statement""" if not isinstance(target, list): target = [target] if not isinstance(source, list): source.prefix = " " source = [source] return Node(syms.atom, target + [Leaf(token.EQUAL, "=", prefix=" ")] + source) def Name(name, prefix=None): """Return a NAME leaf""" return Leaf(token.NAME, name, prefix=prefix) def Attr(obj, attr): """A node tuple for obj.attr""" return [obj, Node(syms.trailer, [Dot(), attr])] def Comma(): """A comma leaf""" return Leaf(token.COMMA, ",") def Dot(): """A period (.) leaf""" return Leaf(token.DOT, ".") def ArgList(args, lparen=LParen(), rparen=RParen()): """A parenthesised argument list, used by Call()""" node = Node(syms.trailer, [lparen.clone(), rparen.clone()]) if args: node.insert_child(1, Node(syms.arglist, args)) return node def Call(func_name, args=None, prefix=None): """A function call""" node = Node(syms.power, [func_name, ArgList(args)]) if prefix is not None: node.prefix = prefix return node def Newline(): """A newline literal""" return Leaf(token.NEWLINE, "\n") def BlankLine(): """A blank line""" return Leaf(token.NEWLINE, "") def Number(n, prefix=None): return Leaf(token.NUMBER, n, prefix=prefix) def Subscript(index_node): """A numeric or string subscript""" return Node(syms.trailer, [Leaf(token.LBRACE, "["), index_node, Leaf(token.RBRACE, "]")]) def String(string, prefix=None): """A string leaf""" return Leaf(token.STRING, string, prefix=prefix) def ListComp(xp, fp, it, test=None): """A list comprehension of the form [xp for fp in it if test]. If test is None, the "if test" part is omitted. """ xp.prefix = "" fp.prefix = " " it.prefix = " " for_leaf = Leaf(token.NAME, "for") for_leaf.prefix = " " in_leaf = Leaf(token.NAME, "in") in_leaf.prefix = " " inner_args = [for_leaf, fp, in_leaf, it] if test: test.prefix = " " if_leaf = Leaf(token.NAME, "if") if_leaf.prefix = " " inner_args.append(Node(syms.comp_if, [if_leaf, test])) inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)]) return Node(syms.atom, [Leaf(token.LBRACE, "["), inner, Leaf(token.RBRACE, "]")]) def FromImport(package_name, name_leafs): """ Return an import statement in the form: from package import name_leafs""" # XXX: May not handle dotted imports properly (eg, package_name='foo.bar') #assert package_name == '.' or '.' not in package_name, "FromImport has "\ # "not been tested with dotted package names -- use at your own "\ # "peril!" for leaf in name_leafs: # Pull the leaves out of their old tree leaf.remove() children = [Leaf(token.NAME, "from"), Leaf(token.NAME, package_name, prefix=" "), Leaf(token.NAME, "import", prefix=" "), Node(syms.import_as_names, name_leafs)] imp = Node(syms.import_from, children) return imp def ImportAndCall(node, results, names): """Returns an import statement and calls a method of the module: import module module.name()""" obj = results["obj"].clone() if obj.type == syms.arglist: newarglist = obj.clone() else: newarglist = Node(syms.arglist, [obj.clone()]) after = results["after"] if after: after = [n.clone() for n in after] new = Node(syms.power, Attr(Name(names[0]), Name(names[1])) + [Node(syms.trailer, [results["lpar"].clone(), newarglist, results["rpar"].clone()])] + after) new.prefix = node.prefix return new ########################################################### ### Determine whether a node represents a given literal ########################################################### def is_tuple(node): """Does the node represent a tuple literal?""" if isinstance(node, Node) and node.children == [LParen(), RParen()]: return True return (isinstance(node, Node) and len(node.children) == 3 and isinstance(node.children[0], Leaf) and isinstance(node.children[1], Node) and isinstance(node.children[2], Leaf) and node.children[0].value == "(" and node.children[2].value == ")") def is_list(node): """Does the node represent a list literal?""" return (isinstance(node, Node) and len(node.children) > 1 and isinstance(node.children[0], Leaf) and isinstance(node.children[-1], Leaf) and node.children[0].value == "[" and node.children[-1].value == "]") ########################################################### ### Misc ########################################################### def parenthesize(node): return Node(syms.atom, [LParen(), node, RParen()]) consuming_calls = {"sorted", "list", "set", "any", "all", "tuple", "sum", "min", "max", "enumerate"} def attr_chain(obj, attr): """Follow an attribute chain. If you have a chain of objects where a.foo -> b, b.foo-> c, etc, use this to iterate over all objects in the chain. Iteration is terminated by getattr(x, attr) is None. Args: obj: the starting object attr: the name of the chaining attribute Yields: Each successive object in the chain. """ next = getattr(obj, attr) while next: yield next next = getattr(next, attr) p0 = """for_stmt< 'for' any 'in' node=any ':' any* > \| comp_for< 'for' any 'in' node=any any* > """ p1 = """ power< ( 'iter' \| 'list' \| 'tuple' \| 'sorted' \| 'set' \| 'sum' \| 'any' \| 'all' \| 'enumerate' \| (any* trailer< '.' 'join' >) ) trailer< '(' node=any ')' > any* > """ p2 = """ power< ( 'sorted' \| 'enumerate' ) trailer< '(' arglist<node=any any> ')' > any > """ pats_built = False def in_special_context(node): """ Returns true if node is in an environment where all that is required of it is being iterable (ie, it doesn't matter if it returns a list or an iterator). See test_map_nochange in test_fixers.py for some examples and tests. """ global p0, p1, p2, pats_built if not pats_built: p0 = patcomp.compile_pattern(p0) p1 = patcomp.compile_pattern(p1) p2 = patcomp.compile_pattern(p2) pats_built = True patterns = [p0, p1, p2] for pattern, parent in zip(patterns, attr_chain(node, "parent")): results = {} if pattern.match(parent, results) and results["node"] is node: return True return False def is_probably_builtin(node): """ Check that something isn't an attribute or function name etc. """ prev = node.prev_sibling if prev is not None and prev.type == token.DOT: # Attribute lookup. return False parent = node.parent if parent.type in (syms.funcdef, syms.classdef): return False if parent.type == syms.expr_stmt and parent.children[0] is node: # Assignment. return False if parent.type == syms.parameters or \ (parent.type == syms.typedargslist and ( (prev is not None and prev.type == token.COMMA) or parent.children[0] is node )): # The name of an argument. return False return True def find_indentation(node): """Find the indentation of node.""" while node is not None: if node.type == syms.suite and len(node.children) > 2: indent = node.children[1] if indent.type == token.INDENT: return indent.value node = node.parent return "" ########################################################### ### The following functions are to find bindings in a suite ########################################################### def make_suite(node): if node.type == syms.suite: return node node = node.clone() parent, node.parent = node.parent, None suite = Node(syms.suite, [node]) suite.parent = parent return suite def find_root(node): """Find the top level namespace.""" # Scamper up to the top level namespace while node.type != syms.file_input: node = node.parent if not node: raise ValueError("root found before file_input node was found.") return node def does_tree_import(package, name, node): """ Returns true if name is imported from package at the top level of the tree which node belongs to. To cover the case of an import like 'import foo', use None for the package and 'foo' for the name. """ binding = find_binding(name, find_root(node), package) return bool(binding) def is_import(node): """Returns true if the node is an import statement.""" return node.type in (syms.import_name, syms.import_from) def touch_import(package, name, node): """ Works like `does_tree_import` but adds an import statement if it was not imported. """ def is_import_stmt(node): return (node.type == syms.simple_stmt and node.children and is_import(node.children[0])) root = find_root(node) if does_tree_import(package, name, root): return # figure out where to insert the new import. First try to find # the first import and then skip to the last one. insert_pos = offset = 0 for idx, node in enumerate(root.children): if not is_import_stmt(node): continue for offset, node2 in enumerate(root.children[idx:]): if not is_import_stmt(node2): break insert_pos = idx + offset break # if there are no imports where we can insert, find the docstring. # if that also fails, we stick to the beginning of the file if insert_pos == 0: for idx, node in enumerate(root.children): if (node.type == syms.simple_stmt and node.children and node.children[0].type == token.STRING): insert_pos = idx + 1 break if package is None: import_ = Node(syms.import_name, [ Leaf(token.NAME, "import"), Leaf(token.NAME, name, prefix=" ") ]) else: import_ = FromImport(package, [Leaf(token.NAME, name, prefix=" ")]) children = [import_, Newline()] root.insert_child(insert_pos, Node(syms.simple_stmt, children)) _def_syms = {syms.classdef, syms.funcdef} def find_binding(name, node, package=None): """ Returns the node which binds variable name, otherwise None. If optional argument package is supplied, only imports will be returned. See test cases for examples.""" for child in node.children: ret = None if child.type == syms.for_stmt: if _find(name, child.children[1]): return child n = find_binding(name, make_suite(child.children[-1]), package) if n: ret = n elif child.type in (syms.if_stmt, syms.while_stmt): n = find_binding(name, make_suite(child.children[-1]), package) if n: ret = n elif child.type == syms.try_stmt: n = find_binding(name, make_suite(child.children[2]), package) if n: ret = n else: for i, kid in enumerate(child.children[3:]): if kid.type == token.COLON and kid.value == ":": # i+3 is the colon, i+4 is the suite n = find_binding(name, make_suite(child.children[i+4]), package) if n: ret = n elif child.type in _def_syms and child.children[1].value == name: ret = child elif _is_import_binding(child, name, package): ret = child elif child.type == syms.simple_stmt: ret = find_binding(name, child, package) elif child.type == syms.expr_stmt: if _find(name, child.children[0]): ret = child if ret: if not package: return ret if is_import(ret): return ret return None _block_syms = {syms.funcdef, syms.classdef, syms.trailer} def _find(name, node): nodes = [node] while nodes: node = nodes.pop() if node.type > 256 and node.type not in _block_syms: nodes.extend(node.children) elif node.type == token.NAME and node.value == name: return node return None def _is_import_binding(node, name, package=None): """ Will return node if node will import name, or node will import * from package. None is returned otherwise. See test cases for examples. """ if node.type == syms.import_name and not package: imp = node.children[1] if imp.type == syms.dotted_as_names: for child in imp.children: if child.type == syms.dotted_as_name: if child.children[2].value == name: return node elif child.type == token.NAME and child.value == name: return node elif imp.type == syms.dotted_as_name: last = imp.children[-1] if last.type == token.NAME and last.value == name: return node elif imp.type == token.NAME and imp.value == name: return node elif node.type == syms.import_from: # str(...) is used to make life easier here, because # from a.b import parses to ['import', ['a', '.', 'b'], ...] if package and str(node.children[1]).strip() != package: return None n = node.children[3] if package and _find("as", n): # See test_from_import_as for explanation return None elif n.type == syms.import_as_names and _find(name, n): return node elif n.type == syms.import_as_name: child = n.children[2] if child.type == token.NAME and child.value == name: return node elif n.type == token.NAME and n.value == name: return node elif package and n.type == token.STAR: return node return None	14.85 KB	2025-10-09 16:16
📁 fixes	-	2026-03-21 15:45
📄 main.py """ Main program for 2to3. """ from __future__ import with_statement, print_function import sys import os import difflib import logging import shutil import optparse from . import refactor def diff_texts(a, b, filename): """Return a unified diff of two strings.""" a = a.splitlines() b = b.splitlines() return difflib.unified_diff(a, b, filename, filename, "(original)", "(refactored)", lineterm="") class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): """ A refactoring tool that can avoid overwriting its input files. Prints output to stdout. Output files can optionally be written to a different directory and or have an extra file suffix appended to their name for use in situations where you do not want to replace the input files. """ def __init__(self, fixers, options, explicit, nobackups, show_diffs, input_base_dir='', output_dir='', append_suffix=''): """ Args: fixers: A list of fixers to import. options: A dict with RefactoringTool configuration. explicit: A list of fixers to run even if they are explicit. nobackups: If true no backup '.bak' files will be created for those files that are being refactored. show_diffs: Should diffs of the refactoring be printed to stdout? input_base_dir: The base directory for all input files. This class will strip this path prefix off of filenames before substituting it with output_dir. Only meaningful if output_dir is supplied. All files processed by refactor() must start with this path. output_dir: If supplied, all converted files will be written into this directory tree instead of input_base_dir. append_suffix: If supplied, all files output by this tool will have this appended to their filename. Useful for changing .py to .py3 for example by passing append_suffix='3'. """ self.nobackups = nobackups self.show_diffs = show_diffs if input_base_dir and not input_base_dir.endswith(os.sep): input_base_dir += os.sep self._input_base_dir = input_base_dir self._output_dir = output_dir self._append_suffix = append_suffix super(StdoutRefactoringTool, self).__init__(fixers, options, explicit) def log_error(self, msg, args, kwargs): self.errors.append((msg, args, kwargs)) self.logger.error(msg, args, **kwargs) def write_file(self, new_text, filename, old_text, encoding): orig_filename = filename if self._output_dir: if filename.startswith(self._input_base_dir): filename = os.path.join(self._output_dir, filename[len(self._input_base_dir):]) else: raise ValueError('filename %s does not start with the ' 'input_base_dir %s' % ( filename, self._input_base_dir)) if self._append_suffix: filename += self._append_suffix if orig_filename != filename: output_dir = os.path.dirname(filename) if not os.path.isdir(output_dir) and output_dir: os.makedirs(output_dir) self.log_message('Writing converted %s to %s.', orig_filename, filename) if not self.nobackups: # Make backup backup = filename + ".bak" if os.path.lexists(backup): try: os.remove(backup) except OSError: self.log_message("Can't remove backup %s", backup) try: os.rename(filename, backup) except OSError: self.log_message("Can't rename %s to %s", filename, backup) # Actually write the new file write = super(StdoutRefactoringTool, self).write_file write(new_text, filename, old_text, encoding) if not self.nobackups: shutil.copymode(backup, filename) if orig_filename != filename: # Preserve the file mode in the new output directory. shutil.copymode(orig_filename, filename) def print_output(self, old, new, filename, equal): if equal: self.log_message("No changes to %s", filename) else: self.log_message("Refactored %s", filename) if self.show_diffs: diff_lines = diff_texts(old, new, filename) try: if self.output_lock is not None: with self.output_lock: for line in diff_lines: print(line) sys.stdout.flush() else: for line in diff_lines: print(line) except UnicodeEncodeError: warn("couldn't encode %s's diff for your terminal" % (filename,)) return def warn(msg): print("WARNING: %s" % (msg,), file=sys.stderr) def main(fixer_pkg, args=None): """Main program. Args: fixer_pkg: the name of a package where the fixers are located. args: optional; a list of command line arguments. If omitted, sys.argv[1:] is used. Returns a suggested exit status (0, 1, 2). """ # Set up option parser parser = optparse.OptionParser(usage="2to3 [options] file\|dir ...") parser.add_option("-d", "--doctests_only", action="store_true", help="Fix up doctests only") parser.add_option("-f", "--fix", action="append", default=[], help="Each FIX specifies a transformation; default: all") parser.add_option("-j", "--processes", action="store", default=1, type="int", help="Run 2to3 concurrently") parser.add_option("-x", "--nofix", action="append", default=[], help="Prevent a transformation from being run") parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations") parser.add_option("-p", "--print-function", action="store_true", help="Modify the grammar so that print() is a function") parser.add_option("-e", "--exec-function", action="store_true", help="Modify the grammar so that exec() is a function") parser.add_option("-v", "--verbose", action="store_true", help="More verbose logging") parser.add_option("--no-diffs", action="store_true", help="Don't show diffs of the refactoring") parser.add_option("-w", "--write", action="store_true", help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, help="Don't write backups for modified files") parser.add_option("-o", "--output-dir", action="store", type="str", default="", help="Put output files in this directory " "instead of overwriting the input files. Requires -n.") parser.add_option("-W", "--write-unchanged-files", action="store_true", help="Also write files even if no changes were required" " (useful with --output-dir); implies -w.") parser.add_option("--add-suffix", action="store", type="str", default="", help="Append this string to all output filenames." " Requires -n if non-empty. " "ex: --add-suffix='3' will generate .py3 files.") # Parse command line arguments refactor_stdin = False flags = {} options, args = parser.parse_args(args) if options.write_unchanged_files: flags["write_unchanged_files"] = True if not options.write: warn("--write-unchanged-files/-W implies -w.") options.write = True # If we allowed these, the original files would be renamed to backup names # but not replaced. if options.output_dir and not options.nobackups: parser.error("Can't use --output-dir/-o without -n.") if options.add_suffix and not options.nobackups: parser.error("Can't use --add-suffix without -n.") if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") if not options.write and options.nobackups: parser.error("Can't use -n without -w") if options.list_fixes: print("Available transformations for the -f/--fix option:") for fixname in refactor.get_all_fix_names(fixer_pkg): print(fixname) if not args: return 0 if not args: print("At least one file or directory argument required.", file=sys.stderr) print("Use --help to show usage.", file=sys.stderr) return 2 if "-" in args: refactor_stdin = True if options.write: print("Can't write to stdin.", file=sys.stderr) return 2 if options.print_function: flags["print_function"] = True if options.exec_function: flags["exec_function"] = True # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO logging.basicConfig(format='%(name)s: %(message)s', level=level) logger = logging.getLogger('lib2to3.main') # Initialize the refactoring tool avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg)) unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix) explicit = set() if options.fix: all_present = False for fix in options.fix: if fix == "all": all_present = True else: explicit.add(fixer_pkg + ".fix_" + fix) requested = avail_fixes.union(explicit) if all_present else explicit else: requested = avail_fixes.union(explicit) fixer_names = requested.difference(unwanted_fixes) input_base_dir = os.path.commonprefix(args) if (input_base_dir and not input_base_dir.endswith(os.sep) and not os.path.isdir(input_base_dir)): # One or more similar names were passed, their directory is the base. # os.path.commonprefix() is ignorant of path elements, this corrects # for that weird API. input_base_dir = os.path.dirname(input_base_dir) if options.output_dir: input_base_dir = input_base_dir.rstrip(os.sep) logger.info('Output in %r will mirror the input directory %r layout.', options.output_dir, input_base_dir) rt = StdoutRefactoringTool( sorted(fixer_names), flags, sorted(explicit), options.nobackups, not options.no_diffs, input_base_dir=input_base_dir, output_dir=options.output_dir, append_suffix=options.add_suffix) # Refactor all files and directories passed as arguments if not rt.errors: if refactor_stdin: rt.refactor_stdin() else: try: rt.refactor(args, options.write, options.doctests_only, options.processes) except refactor.MultiprocessingUnsupported: assert options.processes > 1 print("Sorry, -j isn't supported on this platform.", file=sys.stderr) return 1 rt.summarize() # Return error status (0 if rt.errors is zero) return int(bool(rt.errors))	11.58 KB	2025-10-09 16:16
📄 patcomp.py # Copyright 2006 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Pattern compiler. The grammar is taken from PatternGrammar.txt. The compiler compiles a pattern to a pytree.Pattern instance. """ __author__ = "Guido van Rossum <guido@python.org>" # Python imports import io # Fairly local imports from .pgen2 import driver, literals, token, tokenize, parse, grammar # Really local imports from . import pytree from . import pygram class PatternSyntaxError(Exception): pass def tokenize_wrapper(input): """Tokenizes a string suppressing significant whitespace.""" skip = {token.NEWLINE, token.INDENT, token.DEDENT} tokens = tokenize.generate_tokens(io.StringIO(input).readline) for quintuple in tokens: type, value, start, end, line_text = quintuple if type not in skip: yield quintuple class PatternCompiler(object): def __init__(self, grammar_file=None): """Initializer. Takes an optional alternative filename for the pattern grammar. """ if grammar_file is None: self.grammar = pygram.pattern_grammar self.syms = pygram.pattern_symbols else: self.grammar = driver.load_grammar(grammar_file) self.syms = pygram.Symbols(self.grammar) self.pygrammar = pygram.python_grammar self.pysyms = pygram.python_symbols self.driver = driver.Driver(self.grammar, convert=pattern_convert) def compile_pattern(self, input, debug=False, with_tree=False): """Compiles a pattern string to a nested pytree.Pattern object.""" tokens = tokenize_wrapper(input) try: root = self.driver.parse_tokens(tokens, debug=debug) except parse.ParseError as e: raise PatternSyntaxError(str(e)) from None if with_tree: return self.compile_node(root), root else: return self.compile_node(root) def compile_node(self, node): """Compiles a node, recursively. This is one big switch on the node type. """ # XXX Optimize certain Wildcard-containing-Wildcard patterns # that can be merged if node.type == self.syms.Matcher: node = node.children[0] # Avoid unneeded recursion if node.type == self.syms.Alternatives: # Skip the odd children since they are just '\|' tokens alts = [self.compile_node(ch) for ch in node.children[::2]] if len(alts) == 1: return alts[0] p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1) return p.optimize() if node.type == self.syms.Alternative: units = [self.compile_node(ch) for ch in node.children] if len(units) == 1: return units[0] p = pytree.WildcardPattern([units], min=1, max=1) return p.optimize() if node.type == self.syms.NegatedUnit: pattern = self.compile_basic(node.children[1:]) p = pytree.NegatedPattern(pattern) return p.optimize() assert node.type == self.syms.Unit name = None nodes = node.children if len(nodes) >= 3 and nodes[1].type == token.EQUAL: name = nodes[0].value nodes = nodes[2:] repeat = None if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater: repeat = nodes[-1] nodes = nodes[:-1] # Now we've reduced it to: STRING \| NAME [Details] \| (...) \| [...] pattern = self.compile_basic(nodes, repeat) if repeat is not None: assert repeat.type == self.syms.Repeater children = repeat.children child = children[0] if child.type == token.STAR: min = 0 max = pytree.HUGE elif child.type == token.PLUS: min = 1 max = pytree.HUGE elif child.type == token.LBRACE: assert children[-1].type == token.RBRACE assert len(children) in (3, 5) min = max = self.get_int(children[1]) if len(children) == 5: max = self.get_int(children[3]) else: assert False if min != 1 or max != 1: pattern = pattern.optimize() pattern = pytree.WildcardPattern([[pattern]], min=min, max=max) if name is not None: pattern.name = name return pattern.optimize() def compile_basic(self, nodes, repeat=None): # Compile STRING \| NAME [Details] \| (...) \| [...] assert len(nodes) >= 1 node = nodes[0] if node.type == token.STRING: value = str(literals.evalString(node.value)) return pytree.LeafPattern(_type_of_literal(value), value) elif node.type == token.NAME: value = node.value if value.isupper(): if value not in TOKEN_MAP: raise PatternSyntaxError("Invalid token: %r" % value) if nodes[1:]: raise PatternSyntaxError("Can't have details for token") return pytree.LeafPattern(TOKEN_MAP[value]) else: if value == "any": type = None elif not value.startswith("_"): type = getattr(self.pysyms, value, None) if type is None: raise PatternSyntaxError("Invalid symbol: %r" % value) if nodes[1:]: # Details present content = [self.compile_node(nodes[1].children[1])] else: content = None return pytree.NodePattern(type, content) elif node.value == "(": return self.compile_node(nodes[1]) elif node.value == "[": assert repeat is None subpattern = self.compile_node(nodes[1]) return pytree.WildcardPattern([[subpattern]], min=0, max=1) assert False, node def get_int(self, node): assert node.type == token.NUMBER return int(node.value) # Map named tokens to the type value for a LeafPattern TOKEN_MAP = {"NAME": token.NAME, "STRING": token.STRING, "NUMBER": token.NUMBER, "TOKEN": None} def _type_of_literal(value): if value[0].isalpha(): return token.NAME elif value in grammar.opmap: return grammar.opmap[value] else: return None def pattern_convert(grammar, raw_node_info): """Converts raw node information to a Node or Leaf instance.""" type, value, context, children = raw_node_info if children or type in grammar.number2symbol: return pytree.Node(type, children, context=context) else: return pytree.Leaf(type, value, context=context) def compile_pattern(pattern): return PatternCompiler().compile_pattern(pattern)	6.89 KB	2025-10-09 16:16
📁 pgen2	-	2026-03-21 15:45
📄 pygram.py # Copyright 2006 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Export the Python grammar and symbols.""" # Python imports import os # Local imports from .pgen2 import token from .pgen2 import driver from . import pytree # The grammar file _GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt") _PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "PatternGrammar.txt") class Symbols(object): def __init__(self, grammar): """Initializer. Creates an attribute for each grammar symbol (nonterminal), whose value is the symbol's type (an int >= 256). """ for name, symbol in grammar.symbol2number.items(): setattr(self, name, symbol) python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE) python_symbols = Symbols(python_grammar) python_grammar_no_print_statement = python_grammar.copy() del python_grammar_no_print_statement.keywords["print"] python_grammar_no_print_and_exec_statement = python_grammar_no_print_statement.copy() del python_grammar_no_print_and_exec_statement.keywords["exec"] pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE) pattern_symbols = Symbols(pattern_grammar)	1.27 KB	2025-10-09 16:16
📄 pytree.py # Copyright 2006 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """ Python parse tree definitions. This is a very concrete parse tree; we need to keep every token and even the comments and whitespace between tokens. There's also a pattern matching implementation here. """ __author__ = "Guido van Rossum <guido@python.org>" import sys from io import StringIO HUGE = 0x7FFFFFFF # maximum repeat count, default max _type_reprs = {} def type_repr(type_num): global _type_reprs if not _type_reprs: from .pygram import python_symbols # printing tokens is possible but not as useful # from .pgen2 import token // token.__dict__.items(): for name, val in python_symbols.__dict__.items(): if type(val) == int: _type_reprs[val] = name return _type_reprs.setdefault(type_num, type_num) class Base(object): """ Abstract base class for Node and Leaf. This provides some default functionality and boilerplate using the template pattern. A node may be a subnode of at most one parent. """ # Default values for instance variables type = None # int: token number (< 256) or symbol number (>= 256) parent = None # Parent node pointer, or None children = () # Tuple of subnodes was_changed = False was_checked = False def __new__(cls, args, kwds): """Constructor that prevents Base from being instantiated.""" assert cls is not Base, "Cannot instantiate Base" return object.__new__(cls) def __eq__(self, other): """ Compare two nodes for equality. This calls the method _eq(). """ if self.__class__ is not other.__class__: return NotImplemented return self._eq(other) __hash__ = None # For Py3 compatibility. def _eq(self, other): """ Compare two nodes for equality. This is called by __eq__ and __ne__. It is only called if the two nodes have the same type. This must be implemented by the concrete subclass. Nodes should be considered equal if they have the same structure, ignoring the prefix string and other context information. """ raise NotImplementedError def clone(self): """ Return a cloned (deep) copy of self. This must be implemented by the concrete subclass. """ raise NotImplementedError def post_order(self): """ Return a post-order iterator for the tree. This must be implemented by the concrete subclass. """ raise NotImplementedError def pre_order(self): """ Return a pre-order iterator for the tree. This must be implemented by the concrete subclass. """ raise NotImplementedError def replace(self, new): """Replace this node with a new one in the parent.""" assert self.parent is not None, str(self) assert new is not None if not isinstance(new, list): new = [new] l_children = [] found = False for ch in self.parent.children: if ch is self: assert not found, (self.parent.children, self, new) if new is not None: l_children.extend(new) found = True else: l_children.append(ch) assert found, (self.children, self, new) self.parent.changed() self.parent.children = l_children for x in new: x.parent = self.parent self.parent = None def get_lineno(self): """Return the line number which generated the invocant node.""" node = self while not isinstance(node, Leaf): if not node.children: return node = node.children[0] return node.lineno def changed(self): if self.parent: self.parent.changed() self.was_changed = True def remove(self): """ Remove the node from the tree. Returns the position of the node in its parent's children before it was removed. """ if self.parent: for i, node in enumerate(self.parent.children): if node is self: self.parent.changed() del self.parent.children[i] self.parent = None return i @property def next_sibling(self): """ The node immediately following the invocant in their parent's children list. If the invocant does not have a next sibling, it is None """ if self.parent is None: return None # Can't use index(); we need to test by identity for i, child in enumerate(self.parent.children): if child is self: try: return self.parent.children[i+1] except IndexError: return None @property def prev_sibling(self): """ The node immediately preceding the invocant in their parent's children list. If the invocant does not have a previous sibling, it is None. """ if self.parent is None: return None # Can't use index(); we need to test by identity for i, child in enumerate(self.parent.children): if child is self: if i == 0: return None return self.parent.children[i-1] def leaves(self): for child in self.children: yield from child.leaves() def depth(self): if self.parent is None: return 0 return 1 + self.parent.depth() def get_suffix(self): """ Return the string immediately following the invocant node. This is effectively equivalent to node.next_sibling.prefix """ next_sib = self.next_sibling if next_sib is None: return "" return next_sib.prefix if sys.version_info < (3, 0): def __str__(self): return str(self).encode("ascii") class Node(Base): """Concrete implementation for interior nodes.""" def __init__(self,type, children, context=None, prefix=None, fixers_applied=None): """ Initializer. Takes a type constant (a symbol number >= 256), a sequence of child nodes, and an optional context keyword argument. As a side effect, the parent pointers of the children are updated. """ assert type >= 256, type self.type = type self.children = list(children) for ch in self.children: assert ch.parent is None, repr(ch) ch.parent = self if prefix is not None: self.prefix = prefix if fixers_applied: self.fixers_applied = fixers_applied[:] else: self.fixers_applied = None def __repr__(self): """Return a canonical string representation.""" return "%s(%s, %r)" % (self.__class__.__name__, type_repr(self.type), self.children) def __unicode__(self): """ Return a pretty string representation. This reproduces the input source exactly. """ return "".join(map(str, self.children)) if sys.version_info > (3, 0): __str__ = __unicode__ def _eq(self, other): """Compare two nodes for equality.""" return (self.type, self.children) == (other.type, other.children) def clone(self): """Return a cloned (deep) copy of self.""" return Node(self.type, [ch.clone() for ch in self.children], fixers_applied=self.fixers_applied) def post_order(self): """Return a post-order iterator for the tree.""" for child in self.children: yield from child.post_order() yield self def pre_order(self): """Return a pre-order iterator for the tree.""" yield self for child in self.children: yield from child.pre_order() @property def prefix(self): """ The whitespace and comments preceding this node in the input. """ if not self.children: return "" return self.children[0].prefix @prefix.setter def prefix(self, prefix): if self.children: self.children[0].prefix = prefix def set_child(self, i, child): """ Equivalent to 'node.children[i] = child'. This method also sets the child's parent attribute appropriately. """ child.parent = self self.children[i].parent = None self.children[i] = child self.changed() def insert_child(self, i, child): """ Equivalent to 'node.children.insert(i, child)'. This method also sets the child's parent attribute appropriately. """ child.parent = self self.children.insert(i, child) self.changed() def append_child(self, child): """ Equivalent to 'node.children.append(child)'. This method also sets the child's parent attribute appropriately. """ child.parent = self self.children.append(child) self.changed() class Leaf(Base): """Concrete implementation for leaf nodes.""" # Default values for instance variables _prefix = "" # Whitespace and comments preceding this token in the input lineno = 0 # Line where this token starts in the input column = 0 # Column where this token tarts in the input def __init__(self, type, value, context=None, prefix=None, fixers_applied=[]): """ Initializer. Takes a type constant (a token number < 256), a string value, and an optional context keyword argument. """ assert 0 <= type < 256, type if context is not None: self._prefix, (self.lineno, self.column) = context self.type = type self.value = value if prefix is not None: self._prefix = prefix self.fixers_applied = fixers_applied[:] def __repr__(self): """Return a canonical string representation.""" return "%s(%r, %r)" % (self.__class__.__name__, self.type, self.value) def __unicode__(self): """ Return a pretty string representation. This reproduces the input source exactly. """ return self.prefix + str(self.value) if sys.version_info > (3, 0): __str__ = __unicode__ def _eq(self, other): """Compare two nodes for equality.""" return (self.type, self.value) == (other.type, other.value) def clone(self): """Return a cloned (deep) copy of self.""" return Leaf(self.type, self.value, (self.prefix, (self.lineno, self.column)), fixers_applied=self.fixers_applied) def leaves(self): yield self def post_order(self): """Return a post-order iterator for the tree.""" yield self def pre_order(self): """Return a pre-order iterator for the tree.""" yield self @property def prefix(self): """ The whitespace and comments preceding this token in the input. """ return self._prefix @prefix.setter def prefix(self, prefix): self.changed() self._prefix = prefix def convert(gr, raw_node): """ Convert raw node information to a Node or Leaf instance. This is passed to the parser driver which calls it whenever a reduction of a grammar rule produces a new complete node, so that the tree is build strictly bottom-up. """ type, value, context, children = raw_node if children or type in gr.number2symbol: # If there's exactly one child, return that child instead of # creating a new node. if len(children) == 1: return children[0] return Node(type, children, context=context) else: return Leaf(type, value, context=context) class BasePattern(object): """ A pattern is a tree matching pattern. It looks for a specific node type (token or symbol), and optionally for a specific content. This is an abstract base class. There are three concrete subclasses: - LeafPattern matches a single leaf node; - NodePattern matches a single node (usually non-leaf); - WildcardPattern matches a sequence of nodes of variable length. """ # Defaults for instance variables type = None # Node type (token if < 256, symbol if >= 256) content = None # Optional content matching pattern name = None # Optional name used to store match in results dict def __new__(cls, args, *kwds): """Constructor that prevents BasePattern from being instantiated.""" assert cls is not BasePattern, "Cannot instantiate BasePattern" return object.__new__(cls) def __repr__(self): args = [type_repr(self.type), self.content, self.name] while args and args[-1] is None: del args[-1] return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args))) def optimize(self): """ A subclass can define this as a hook for optimizations. Returns either self or another node with the same effect. """ return self def match(self, node, results=None): """ Does this pattern exactly match a node? Returns True if it matches, False if not. If results is not None, it must be a dict which will be updated with the nodes matching named subpatterns. Default implementation for non-wildcard patterns. """ if self.type is not None and node.type != self.type: return False if self.content is not None: r = None if results is not None: r = {} if not self._submatch(node, r): return False if r: results.update(r) if results is not None and self.name: results[self.name] = node return True def match_seq(self, nodes, results=None): """ Does this pattern exactly match a sequence of nodes? Default implementation for non-wildcard patterns. """ if len(nodes) != 1: return False return self.match(nodes[0], results) def generate_matches(self, nodes): """ Generator yielding all matches for this pattern. Default implementation for non-wildcard patterns. """ r = {} if nodes and self.match(nodes[0], r): yield 1, r class LeafPattern(BasePattern): def __init__(self, type=None, content=None, name=None): """ Initializer. Takes optional type, content, and name. The type, if given must be a token type (< 256). If not given, this matches any leaf* node; the content may still be required. The content, if given, must be a string. If a name is given, the matching node is stored in the results dict under that key. """ if type is not None: assert 0 <= type < 256, type if content is not None: assert isinstance(content, str), repr(content) self.type = type self.content = content self.name = name def match(self, node, results=None): """Override match() to insist on a leaf node.""" if not isinstance(node, Leaf): return False return BasePattern.match(self, node, results) def _submatch(self, node, results=None): """ Match the pattern's content to the node's children. This assumes the node type matches and self.content is not None. Returns True if it matches, False if not. If results is not None, it must be a dict which will be updated with the nodes matching named subpatterns. When returning False, the results dict may still be updated. """ return self.content == node.value class NodePattern(BasePattern): wildcards = False def __init__(self, type=None, content=None, name=None): """ Initializer. Takes optional type, content, and name. The type, if given, must be a symbol type (>= 256). If the type is None this matches any single node (leaf or not), except if content is not None, in which it only matches non-leaf nodes that also match the content pattern. The content, if not None, must be a sequence of Patterns that must match the node's children exactly. If the content is given, the type must not be None. If a name is given, the matching node is stored in the results dict under that key. """ if type is not None: assert type >= 256, type if content is not None: assert not isinstance(content, str), repr(content) content = list(content) for i, item in enumerate(content): assert isinstance(item, BasePattern), (i, item) if isinstance(item, WildcardPattern): self.wildcards = True self.type = type self.content = content self.name = name def _submatch(self, node, results=None): """ Match the pattern's content to the node's children. This assumes the node type matches and self.content is not None. Returns True if it matches, False if not. If results is not None, it must be a dict which will be updated with the nodes matching named subpatterns. When returning False, the results dict may still be updated. """ if self.wildcards: for c, r in generate_matches(self.content, node.children): if c == len(node.children): if results is not None: results.update(r) return True return False if len(self.content) != len(node.children): return False for subpattern, child in zip(self.content, node.children): if not subpattern.match(child, results): return False return True class WildcardPattern(BasePattern): """ A wildcard pattern can match zero or more nodes. This has all the flexibility needed to implement patterns like: .* .+ .? .{m,n} (a b c \| d e \| f) (...)* (...)+ (...)? (...){m,n} except it always uses non-greedy matching. """ def __init__(self, content=None, min=0, max=HUGE, name=None): """ Initializer. Args: content: optional sequence of subsequences of patterns; if absent, matches one node; if present, each subsequence is an alternative [] min: optional minimum number of times to match, default 0 max: optional maximum number of times to match, default HUGE name: optional name assigned to this match [] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is equivalent to (a b c \| d e \| f g h); if content is None, this is equivalent to '.' in regular expression terms. The min and max parameters work as follows: min=0, max=maxint: .* min=1, max=maxint: .+ min=0, max=1: .? min=1, max=1: . If content is not None, replace the dot with the parenthesized list of alternatives, e.g. (a b c \| d e \| f g h)* """ assert 0 <= min <= max <= HUGE, (min, max) if content is not None: content = tuple(map(tuple, content)) # Protect against alterations # Check sanity of alternatives assert len(content), repr(content) # Can't have zero alternatives for alt in content: assert len(alt), repr(alt) # Can have empty alternatives self.content = content self.min = min self.max = max self.name = name def optimize(self): """Optimize certain stacked wildcard patterns.""" subpattern = None if (self.content is not None and len(self.content) == 1 and len(self.content[0]) == 1): subpattern = self.content[0][0] if self.min == 1 and self.max == 1: if self.content is None: return NodePattern(name=self.name) if subpattern is not None and self.name == subpattern.name: return subpattern.optimize() if (self.min <= 1 and isinstance(subpattern, WildcardPattern) and subpattern.min <= 1 and self.name == subpattern.name): return WildcardPattern(subpattern.content, self.minsubpattern.min, self.maxsubpattern.max, subpattern.name) return self def match(self, node, results=None): """Does this pattern exactly match a node?""" return self.match_seq([node], results) def match_seq(self, nodes, results=None): """Does this pattern exactly match a sequence of nodes?""" for c, r in self.generate_matches(nodes): if c == len(nodes): if results is not None: results.update(r) if self.name: results[self.name] = list(nodes) return True return False def generate_matches(self, nodes): """ Generator yielding matches for a sequence of nodes. Args: nodes: sequence of nodes Yields: (count, results) tuples where: count: the match comprises nodes[:count]; results: dict containing named submatches. """ if self.content is None: # Shortcut for special case (see __init__.__doc__) for count in range(self.min, 1 + min(len(nodes), self.max)): r = {} if self.name: r[self.name] = nodes[:count] yield count, r elif self.name == "bare_name": yield self._bare_name_matches(nodes) else: # The reason for this is that hitting the recursion limit usually # results in some ugly messages about how RuntimeErrors are being # ignored. We only have to do this on CPython, though, because other # implementations don't have this nasty bug in the first place. if hasattr(sys, "getrefcount"): save_stderr = sys.stderr sys.stderr = StringIO() try: for count, r in self._recursive_matches(nodes, 0): if self.name: r[self.name] = nodes[:count] yield count, r except RuntimeError: # Fall back to the iterative pattern matching scheme if the # recursive scheme hits the recursion limit (RecursionError). for count, r in self._iterative_matches(nodes): if self.name: r[self.name] = nodes[:count] yield count, r finally: if hasattr(sys, "getrefcount"): sys.stderr = save_stderr def _iterative_matches(self, nodes): """Helper to iteratively yield the matches.""" nodelen = len(nodes) if 0 >= self.min: yield 0, {} results = [] # generate matches that use just one alt from self.content for alt in self.content: for c, r in generate_matches(alt, nodes): yield c, r results.append((c, r)) # for each match, iterate down the nodes while results: new_results = [] for c0, r0 in results: # stop if the entire set of nodes has been matched if c0 < nodelen and c0 <= self.max: for alt in self.content: for c1, r1 in generate_matches(alt, nodes[c0:]): if c1 > 0: r = {} r.update(r0) r.update(r1) yield c0 + c1, r new_results.append((c0 + c1, r)) results = new_results def _bare_name_matches(self, nodes): """Special optimized matcher for bare_name.""" count = 0 r = {} done = False max = len(nodes) while not done and count < max: done = True for leaf in self.content: if leaf[0].match(nodes[count], r): count += 1 done = False break r[self.name] = nodes[:count] return count, r def _recursive_matches(self, nodes, count): """Helper to recursively yield the matches.""" assert self.content is not None if count >= self.min: yield 0, {} if count < self.max: for alt in self.content: for c0, r0 in generate_matches(alt, nodes): for c1, r1 in self._recursive_matches(nodes[c0:], count+1): r = {} r.update(r0) r.update(r1) yield c0 + c1, r class NegatedPattern(BasePattern): def __init__(self, content=None): """ Initializer. The argument is either a pattern or None. If it is None, this only matches an empty sequence (effectively '$' in regex lingo). If it is not None, this matches whenever the argument pattern doesn't have any matches. """ if content is not None: assert isinstance(content, BasePattern), repr(content) self.content = content def match(self, node): # We never match a node in its entirety return False def match_seq(self, nodes): # We only match an empty sequence of nodes in its entirety return len(nodes) == 0 def generate_matches(self, nodes): if self.content is None: # Return a match if there is an empty sequence if len(nodes) == 0: yield 0, {} else: # Return a match if the argument pattern has no matches for c, r in self.content.generate_matches(nodes): return yield 0, {} def generate_matches(patterns, nodes): """ Generator yielding matches for a sequence of patterns and nodes. Args: patterns: a sequence of patterns nodes: a sequence of nodes Yields: (count, results) tuples where: count: the entire sequence of patterns matches nodes[:count]; results: dict containing named submatches. """ if not patterns: yield 0, {} else: p, rest = patterns[0], patterns[1:] for c0, r0 in p.generate_matches(nodes): if not rest: yield c0, r0 else: for c1, r1 in generate_matches(rest, nodes[c0:]): r = {} r.update(r0) r.update(r1) yield c0 + c1, r	27.32 KB	2025-10-09 16:16
📄 refactor.py # Copyright 2006 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Refactoring framework. Used as a main program, this can refactor any number of files and/or recursively descend down directories. Imported as a module, this provides infrastructure to write your own refactoring tool. """ __author__ = "Guido van Rossum <guido@python.org>" # Python imports import io import os import pkgutil import sys import logging import operator import collections from itertools import chain # Local imports from .pgen2 import driver, tokenize, token from .fixer_util import find_root from . import pytree, pygram from . import btm_matcher as bm def get_all_fix_names(fixer_pkg, remove_prefix=True): """Return a sorted list of all available fix names in the given package.""" pkg = __import__(fixer_pkg, [], [], [""]) fix_names = [] for finder, name, ispkg in pkgutil.iter_modules(pkg.__path__): if name.startswith("fix_"): if remove_prefix: name = name[4:] fix_names.append(name) return fix_names class _EveryNode(Exception): pass def _get_head_types(pat): """ Accepts a pytree Pattern Node and returns a set of the pattern types which will match first. """ if isinstance(pat, (pytree.NodePattern, pytree.LeafPattern)): # NodePatters must either have no type and no content # or a type and content -- so they don't get any farther # Always return leafs if pat.type is None: raise _EveryNode return {pat.type} if isinstance(pat, pytree.NegatedPattern): if pat.content: return _get_head_types(pat.content) raise _EveryNode # Negated Patterns don't have a type if isinstance(pat, pytree.WildcardPattern): # Recurse on each node in content r = set() for p in pat.content: for x in p: r.update(_get_head_types(x)) return r raise Exception("Oh no! I don't understand pattern %s" %(pat)) def _get_headnode_dict(fixer_list): """ Accepts a list of fixers and returns a dictionary of head node type --> fixer list. """ head_nodes = collections.defaultdict(list) every = [] for fixer in fixer_list: if fixer.pattern: try: heads = _get_head_types(fixer.pattern) except _EveryNode: every.append(fixer) else: for node_type in heads: head_nodes[node_type].append(fixer) else: if fixer._accept_type is not None: head_nodes[fixer._accept_type].append(fixer) else: every.append(fixer) for node_type in chain(pygram.python_grammar.symbol2number.values(), pygram.python_grammar.tokens): head_nodes[node_type].extend(every) return dict(head_nodes) def get_fixers_from_package(pkg_name): """ Return the fully qualified names for fixers in the package pkg_name. """ return [pkg_name + "." + fix_name for fix_name in get_all_fix_names(pkg_name, False)] def _identity(obj): return obj def _detect_future_features(source): have_docstring = False gen = tokenize.generate_tokens(io.StringIO(source).readline) def advance(): tok = next(gen) return tok[0], tok[1] ignore = frozenset({token.NEWLINE, tokenize.NL, token.COMMENT}) features = set() try: while True: tp, value = advance() if tp in ignore: continue elif tp == token.STRING: if have_docstring: break have_docstring = True elif tp == token.NAME and value == "from": tp, value = advance() if tp != token.NAME or value != "__future__": break tp, value = advance() if tp != token.NAME or value != "import": break tp, value = advance() if tp == token.OP and value == "(": tp, value = advance() while tp == token.NAME: features.add(value) tp, value = advance() if tp != token.OP or value != ",": break tp, value = advance() else: break except StopIteration: pass return frozenset(features) class FixerError(Exception): """A fixer could not be loaded.""" class RefactoringTool(object): _default_options = {"print_function" : False, "exec_function": False, "write_unchanged_files" : False} CLASS_PREFIX = "Fix" # The prefix for fixer classes FILE_PREFIX = "fix_" # The prefix for modules with a fixer within def __init__(self, fixer_names, options=None, explicit=None): """Initializer. Args: fixer_names: a list of fixers to import options: a dict with configuration. explicit: a list of fixers to run even if they are explicit. """ self.fixers = fixer_names self.explicit = explicit or [] self.options = self._default_options.copy() if options is not None: self.options.update(options) self.grammar = pygram.python_grammar.copy() if self.options['print_function']: del self.grammar.keywords["print"] elif self.options['exec_function']: del self.grammar.keywords["exec"] # When this is True, the refactor() methods will call write_file() for # files processed even if they were not changed during refactoring. If # and only if the refactor method's write parameter was True. self.write_unchanged_files = self.options.get("write_unchanged_files") self.errors = [] self.logger = logging.getLogger("RefactoringTool") self.fixer_log = [] self.wrote = False self.driver = driver.Driver(self.grammar, convert=pytree.convert, logger=self.logger) self.pre_order, self.post_order = self.get_fixers() self.files = [] # List of files that were or should be modified self.BM = bm.BottomMatcher() self.bmi_pre_order = [] # Bottom Matcher incompatible fixers self.bmi_post_order = [] for fixer in chain(self.post_order, self.pre_order): if fixer.BM_compatible: self.BM.add_fixer(fixer) # remove fixers that will be handled by the bottom-up # matcher elif fixer in self.pre_order: self.bmi_pre_order.append(fixer) elif fixer in self.post_order: self.bmi_post_order.append(fixer) self.bmi_pre_order_heads = _get_headnode_dict(self.bmi_pre_order) self.bmi_post_order_heads = _get_headnode_dict(self.bmi_post_order) def get_fixers(self): """Inspects the options to load the requested patterns and handlers. Returns: (pre_order, post_order), where pre_order is the list of fixers that want a pre-order AST traversal, and post_order is the list that want post-order traversal. """ pre_order_fixers = [] post_order_fixers = [] for fix_mod_path in self.fixers: mod = __import__(fix_mod_path, {}, {}, [""]) fix_name = fix_mod_path.rsplit(".", 1)[-1] if fix_name.startswith(self.FILE_PREFIX): fix_name = fix_name[len(self.FILE_PREFIX):] parts = fix_name.split("_") class_name = self.CLASS_PREFIX + "".join([p.title() for p in parts]) try: fix_class = getattr(mod, class_name) except AttributeError: raise FixerError("Can't find %s.%s" % (fix_name, class_name)) from None fixer = fix_class(self.options, self.fixer_log) if fixer.explicit and self.explicit is not True and \ fix_mod_path not in self.explicit: self.log_message("Skipping optional fixer: %s", fix_name) continue self.log_debug("Adding transformation: %s", fix_name) if fixer.order == "pre": pre_order_fixers.append(fixer) elif fixer.order == "post": post_order_fixers.append(fixer) else: raise FixerError("Illegal fixer order: %r" % fixer.order) key_func = operator.attrgetter("run_order") pre_order_fixers.sort(key=key_func) post_order_fixers.sort(key=key_func) return (pre_order_fixers, post_order_fixers) def log_error(self, msg, args, *kwds): """Called when an error occurs.""" raise def log_message(self, msg, args): """Hook to log a message.""" if args: msg = msg % args self.logger.info(msg) def log_debug(self, msg, args): if args: msg = msg % args self.logger.debug(msg) def print_output(self, old_text, new_text, filename, equal): """Called with the old version, new version, and filename of a refactored file.""" pass def refactor(self, items, write=False, doctests_only=False): """Refactor a list of files and directories.""" for dir_or_file in items: if os.path.isdir(dir_or_file): self.refactor_dir(dir_or_file, write, doctests_only) else: self.refactor_file(dir_or_file, write, doctests_only) def refactor_dir(self, dir_name, write=False, doctests_only=False): """Descends down a directory and refactor every Python file found. Python files are assumed to have a .py extension. Files and subdirectories starting with '.' are skipped. """ py_ext = os.extsep + "py" for dirpath, dirnames, filenames in os.walk(dir_name): self.log_debug("Descending into %s", dirpath) dirnames.sort() filenames.sort() for name in filenames: if (not name.startswith(".") and os.path.splitext(name)[1] == py_ext): fullname = os.path.join(dirpath, name) self.refactor_file(fullname, write, doctests_only) # Modify dirnames in-place to remove subdirs with leading dots dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")] def _read_python_source(self, filename): """ Do our best to decode a Python source file correctly. """ try: f = open(filename, "rb") except OSError as err: self.log_error("Can't open %s: %s", filename, err) return None, None try: encoding = tokenize.detect_encoding(f.readline)[0] finally: f.close() with io.open(filename, "r", encoding=encoding, newline='') as f: return f.read(), encoding def refactor_file(self, filename, write=False, doctests_only=False): """Refactors a file.""" input, encoding = self._read_python_source(filename) if input is None: # Reading the file failed. return input += "\n" # Silence certain parse errors if doctests_only: self.log_debug("Refactoring doctests in %s", filename) output = self.refactor_docstring(input, filename) if self.write_unchanged_files or output != input: self.processed_file(output, filename, input, write, encoding) else: self.log_debug("No doctest changes in %s", filename) else: tree = self.refactor_string(input, filename) if self.write_unchanged_files or (tree and tree.was_changed): # The [:-1] is to take off the \n we added earlier self.processed_file(str(tree)[:-1], filename, write=write, encoding=encoding) else: self.log_debug("No changes in %s", filename) def refactor_string(self, data, name): """Refactor a given input string. Args: data: a string holding the code to be refactored. name: a human-readable name for use in error/log messages. Returns: An AST corresponding to the refactored input stream; None if there were errors during the parse. """ features = _detect_future_features(data) if "print_function" in features: self.driver.grammar = pygram.python_grammar_no_print_statement try: tree = self.driver.parse_string(data) except Exception as err: self.log_error("Can't parse %s: %s: %s", name, err.__class__.__name__, err) return finally: self.driver.grammar = self.grammar tree.future_features = features self.log_debug("Refactoring %s", name) self.refactor_tree(tree, name) return tree def refactor_stdin(self, doctests_only=False): input = sys.stdin.read() if doctests_only: self.log_debug("Refactoring doctests in stdin") output = self.refactor_docstring(input, "<stdin>") if self.write_unchanged_files or output != input: self.processed_file(output, "<stdin>", input) else: self.log_debug("No doctest changes in stdin") else: tree = self.refactor_string(input, "<stdin>") if self.write_unchanged_files or (tree and tree.was_changed): self.processed_file(str(tree), "<stdin>", input) else: self.log_debug("No changes in stdin") def refactor_tree(self, tree, name): """Refactors a parse tree (modifying the tree in place). For compatible patterns the bottom matcher module is used. Otherwise the tree is traversed node-to-node for matches. Args: tree: a pytree.Node instance representing the root of the tree to be refactored. name: a human-readable name for this tree. Returns: True if the tree was modified, False otherwise. """ for fixer in chain(self.pre_order, self.post_order): fixer.start_tree(tree, name) #use traditional matching for the incompatible fixers self.traverse_by(self.bmi_pre_order_heads, tree.pre_order()) self.traverse_by(self.bmi_post_order_heads, tree.post_order()) # obtain a set of candidate nodes match_set = self.BM.run(tree.leaves()) while any(match_set.values()): for fixer in self.BM.fixers: if fixer in match_set and match_set[fixer]: #sort by depth; apply fixers from bottom(of the AST) to top match_set[fixer].sort(key=pytree.Base.depth, reverse=True) if fixer.keep_line_order: #some fixers(eg fix_imports) must be applied #with the original file's line order match_set[fixer].sort(key=pytree.Base.get_lineno) for node in list(match_set[fixer]): if node in match_set[fixer]: match_set[fixer].remove(node) try: find_root(node) except ValueError: # this node has been cut off from a # previous transformation ; skip continue if node.fixers_applied and fixer in node.fixers_applied: # do not apply the same fixer again continue results = fixer.match(node) if results: new = fixer.transform(node, results) if new is not None: node.replace(new) #new.fixers_applied.append(fixer) for node in new.post_order(): # do not apply the fixer again to # this or any subnode if not node.fixers_applied: node.fixers_applied = [] node.fixers_applied.append(fixer) # update the original match set for # the added code new_matches = self.BM.run(new.leaves()) for fxr in new_matches: if not fxr in match_set: match_set[fxr]=[] match_set[fxr].extend(new_matches[fxr]) for fixer in chain(self.pre_order, self.post_order): fixer.finish_tree(tree, name) return tree.was_changed def traverse_by(self, fixers, traversal): """Traverse an AST, applying a set of fixers to each node. This is a helper method for refactor_tree(). Args: fixers: a list of fixer instances. traversal: a generator that yields AST nodes. Returns: None """ if not fixers: return for node in traversal: for fixer in fixers[node.type]: results = fixer.match(node) if results: new = fixer.transform(node, results) if new is not None: node.replace(new) node = new def processed_file(self, new_text, filename, old_text=None, write=False, encoding=None): """ Called when a file has been refactored and there may be changes. """ self.files.append(filename) if old_text is None: old_text = self._read_python_source(filename)[0] if old_text is None: return equal = old_text == new_text self.print_output(old_text, new_text, filename, equal) if equal: self.log_debug("No changes to %s", filename) if not self.write_unchanged_files: return if write: self.write_file(new_text, filename, old_text, encoding) else: self.log_debug("Not writing changes to %s", filename) def write_file(self, new_text, filename, old_text, encoding=None): """Writes a string to a file. It first shows a unified diff between the old text and the new text, and then rewrites the file; the latter is only done if the write option is set. """ try: fp = io.open(filename, "w", encoding=encoding, newline='') except OSError as err: self.log_error("Can't create %s: %s", filename, err) return with fp: try: fp.write(new_text) except OSError as err: self.log_error("Can't write %s: %s", filename, err) self.log_debug("Wrote changes to %s", filename) self.wrote = True PS1 = ">>> " PS2 = "... " def refactor_docstring(self, input, filename): """Refactors a docstring, looking for doctests. This returns a modified version of the input string. It looks for doctests, which start with a ">>>" prompt, and may be continued with "..." prompts, as long as the "..." is indented the same as the ">>>". (Unfortunately we can't use the doctest module's parser, since, like most parsers, it is not geared towards preserving the original source.) """ result = [] block = None block_lineno = None indent = None lineno = 0 for line in input.splitlines(keepends=True): lineno += 1 if line.lstrip().startswith(self.PS1): if block is not None: result.extend(self.refactor_doctest(block, block_lineno, indent, filename)) block_lineno = lineno block = [line] i = line.find(self.PS1) indent = line[:i] elif (indent is not None and (line.startswith(indent + self.PS2) or line == indent + self.PS2.rstrip() + "\n")): block.append(line) else: if block is not None: result.extend(self.refactor_doctest(block, block_lineno, indent, filename)) block = None indent = None result.append(line) if block is not None: result.extend(self.refactor_doctest(block, block_lineno, indent, filename)) return "".join(result) def refactor_doctest(self, block, lineno, indent, filename): """Refactors one doctest. A doctest is given as a block of lines, the first of which starts with ">>>" (possibly indented), while the remaining lines start with "..." (identically indented). """ try: tree = self.parse_block(block, lineno, indent) except Exception as err: if self.logger.isEnabledFor(logging.DEBUG): for line in block: self.log_debug("Source: %s", line.rstrip("\n")) self.log_error("Can't parse docstring in %s line %s: %s: %s", filename, lineno, err.__class__.__name__, err) return block if self.refactor_tree(tree, filename): new = str(tree).splitlines(keepends=True) # Undo the adjustment of the line numbers in wrap_toks() below. clipped, new = new[:lineno-1], new[lineno-1:] assert clipped == ["\n"] (lineno-1), clipped if not new[-1].endswith("\n"): new[-1] += "\n" block = [indent + self.PS1 + new.pop(0)] if new: block += [indent + self.PS2 + line for line in new] return block def summarize(self): if self.wrote: were = "were" else: were = "need to be" if not self.files: self.log_message("No files %s modified.", were) else: self.log_message("Files that %s modified:", were) for file in self.files: self.log_message(file) if self.fixer_log: self.log_message("Warnings/messages while refactoring:") for message in self.fixer_log: self.log_message(message) if self.errors: if len(self.errors) == 1: self.log_message("There was 1 error:") else: self.log_message("There were %d errors:", len(self.errors)) for msg, args, kwds in self.errors: self.log_message(msg, args, kwds) def parse_block(self, block, lineno, indent): """Parses a block into a tree. This is necessary to get correct line number / offset information in the parser diagnostics and embedded into the parse tree. """ tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent)) tree.future_features = frozenset() return tree def wrap_toks(self, block, lineno, indent): """Wraps a tokenize stream to systematically modify start/end.""" tokens = tokenize.generate_tokens(self.gen_lines(block, indent).__next__) for type, value, (line0, col0), (line1, col1), line_text in tokens: line0 += lineno - 1 line1 += lineno - 1 # Don't bother updating the columns; this is too complicated # since line_text would also have to be updated and it would # still break for tokens spanning lines. Let the user guess # that the column numbers for doctests are relative to the # end of the prompt string (PS1 or PS2). yield type, value, (line0, col0), (line1, col1), line_text def gen_lines(self, block, indent): """Generates lines as expected by tokenize from a list of lines. This strips the first len(indent + self.PS1) characters off each line. """ prefix1 = indent + self.PS1 prefix2 = indent + self.PS2 prefix = prefix1 for line in block: if line.startswith(prefix): yield line[len(prefix):] elif line == prefix.rstrip() + "\n": yield "\n" else: raise AssertionError("line=%r, prefix=%r" % (line, prefix)) prefix = prefix2 while True: yield "" class MultiprocessingUnsupported(Exception): pass class MultiprocessRefactoringTool(RefactoringTool): def __init__(self, args, *kwargs): super(MultiprocessRefactoringTool, self).__init__(args, *kwargs) self.queue = None self.output_lock = None def refactor(self, items, write=False, doctests_only=False, num_processes=1): if num_processes == 1: return super(MultiprocessRefactoringTool, self).refactor( items, write, doctests_only) try: import multiprocessing except ImportError: raise MultiprocessingUnsupported if self.queue is not None: raise RuntimeError("already doing multiple processes") self.queue = multiprocessing.JoinableQueue() self.output_lock = multiprocessing.Lock() processes = [multiprocessing.Process(target=self._child) for i in range(num_processes)] try: for p in processes: p.start() super(MultiprocessRefactoringTool, self).refactor(items, write, doctests_only) finally: self.queue.join() for i in range(num_processes): self.queue.put(None) for p in processes: if p.is_alive(): p.join() self.queue = None def _child(self): task = self.queue.get() while task is not None: args, kwargs = task try: super(MultiprocessRefactoringTool, self).refactor_file( args, *kwargs) finally: self.queue.task_done() task = self.queue.get() def refactor_file(self, args, *kwargs): if self.queue is not None: self.queue.put((args, kwargs)) else: return super(MultiprocessRefactoringTool, self).refactor_file( args, **kwargs)	26.86 KB	2025-10-09 16:16

Terminal