From 02ed25e7cb6c79dc1bfdbd786007b95571e782f7 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 12 Feb 2024 00:58:42 -0500 Subject: [PATCH] Bugs found in 3.0 decomplation... parsers/parse30.py; fix set comprehension grammar bug uncompyle6/semantics/n_actions.py: evidence of the evils of modifying node data (via node.pop) --- uncompyle6/parsers/parse30.py | 64 +++++++++++++++++++------------ uncompyle6/semantics/n_actions.py | 19 +++------ 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 20405db5a..84721af6f 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2017, 2022-2023 Rocky Bernstein +# Copyright (c) 2016-2017, 2022-2024 Rocky Bernstein """ spark grammar differences over Python 3.1 for Python 3.0. """ @@ -7,8 +7,8 @@ from uncompyle6.parser import PythonParserSingle from uncompyle6.parsers.parse31 import Python31Parser -class Python30Parser(Python31Parser): +class Python30Parser(Python31Parser): def p_30(self, args): """ @@ -78,7 +78,6 @@ def p_30(self, args): set_comp_func ::= set_comp_header LOAD_ARG FOR_ITER store comp_iter JUMP_BACK ending_return - RETURN_VALUE RETURN_LAST list_comp_header ::= BUILD_LIST_0 DUP_TOP STORE_FAST list_comp ::= list_comp_header @@ -118,7 +117,7 @@ def p_30(self, args): # From Python 2.6 - lc_body ::= LOAD_FAST expr LIST_APPEND + lc_body ::= LOAD_FAST expr LIST_APPEND lc_body ::= LOAD_NAME expr LIST_APPEND list_if ::= expr jmp_false_then list_iter list_if_not ::= expr jmp_true list_iter JUMP_BACK come_froms POP_TOP @@ -216,9 +215,9 @@ def p_30(self, args): compare_chained_right ::= expr COMPARE_OP RETURN_END_IF """ - def remove_rules_30(self): - self.remove_rules(""" + self.remove_rules( + """ # The were found using grammar coverage while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP @@ -286,29 +285,30 @@ def remove_rules_30(self): or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM - """) + """ + ) def customize_grammar_rules(self, tokens, customize): super(Python30Parser, self).customize_grammar_rules(tokens, customize) self.remove_rules_30() self.check_reduce["iflaststmtl"] = "AST" - self.check_reduce['ifstmt'] = "AST" + self.check_reduce["ifstmt"] = "AST" self.check_reduce["ifelsestmtc"] = "AST" self.check_reduce["ifelsestmt"] = "AST" # self.check_reduce["and"] = "stmt" return def reduce_is_invalid(self, rule, ast, tokens, first, last): - invalid = super(Python30Parser, - self).reduce_is_invalid(rule, ast, - tokens, first, last) + invalid = super(Python30Parser, self).reduce_is_invalid( + rule, ast, tokens, first, last + ) if invalid: return invalid lhs = rule[0] if ( - lhs in ("iflaststmtl", "ifstmt", - "ifelsestmt", "ifelsestmtc") and ast[0] == "testexpr" + lhs in ("iflaststmtl", "ifstmt", "ifelsestmt", "ifelsestmtc") + and ast[0] == "testexpr" ): testexpr = ast[0] if testexpr[0] == "testfalse": @@ -316,7 +316,10 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last): if lhs == "ifelsestmtc" and ast[2] == "jump_absolute_else": jump_absolute_else = ast[2] come_from = jump_absolute_else[2] - return come_from == "COME_FROM" and come_from.attr < tokens[first].offset + return ( + come_from == "COME_FROM" + and come_from.attr < tokens[first].offset + ) pass elif lhs in ("ifelsestmt", "ifelsestmtc") and ast[2] == "jump_cf_pop": jump_cf_pop = ast[2] @@ -339,11 +342,11 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last): jmp_false = testfalse[1] if last == len(tokens): last -= 1 - while (isinstance(tokens[first].offset, str) and first < last): + while isinstance(tokens[first].offset, str) and first < last: first += 1 if first == last: return True - while (first < last and isinstance(tokens[last].offset, str)): + while first < last and isinstance(tokens[last].offset, str): last -= 1 if rule[0] == "iflaststmtl": return not (jmp_false[0].attr <= tokens[last].offset) @@ -351,8 +354,9 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last): jmp_false_target = jmp_false[0].attr if tokens[first].offset > jmp_false_target: return True - return ( - (jmp_false_target > tokens[last].offset) and tokens[last] != "JUMP_FORWARD") + return (jmp_false_target > tokens[last].offset) and tokens[ + last + ] != "JUMP_FORWARD" pass pass pass @@ -361,33 +365,43 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last): pass + class Python30ParserSingle(Python30Parser, PythonParserSingle): pass -if __name__ == '__main__': + +if __name__ == "__main__": # Check grammar p = Python30Parser() p.remove_rules_30() p.check_grammar() - from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY + from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE + if PYTHON_VERSION_TRIPLE[:2] == (3, 0): lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY) - opcode_set = set(s.opc.opname).union(set( - """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + opcode_set = set(s.opc.opname).union( + set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME LAMBDA_MARKER RETURN_LAST - """.split())) + """.split() + ) + ) ## FIXME: try this remain_tokens = set(tokens) - opcode_set import re - remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) - remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + + remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens]) + remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens]) remain_tokens = set(remain_tokens) - opcode_set print(remain_tokens) import sys + if len(sys.argv) > 1: from spark_parser.spark import rule2str + for rule in sorted(p.rule2name.items()): print(rule2str(rule[0])) diff --git a/uncompyle6/semantics/n_actions.py b/uncompyle6/semantics/n_actions.py index a17683290..2c3a6b698 100644 --- a/uncompyle6/semantics/n_actions.py +++ b/uncompyle6/semantics/n_actions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023 by Rocky Bernstein +# Copyright (c) 2022-2024 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,22 +16,12 @@ Custom Nonterminal action functions. See NonterminalActions docstring. """ -from uncompyle6.semantics.consts import ( - INDENT_PER_LEVEL, - NONE, - PRECEDENCE, - minint, -) - from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6.scanners.tok import Token +from uncompyle6.semantics.consts import INDENT_PER_LEVEL, NONE, PRECEDENCE, minint +from uncompyle6.semantics.helper import find_code_node, flatten_list from uncompyle6.util import better_repr, get_code_name -from uncompyle6.semantics.helper import ( - find_code_node, - flatten_list, -) - class NonterminalActions: """ @@ -826,7 +816,8 @@ def n_list(self, node): p = self.prec self.prec = PRECEDENCE["yield"] - 1 - lastnode = node.pop() + lastnode = node[-1] + node = node[:-1] lastnodetype = lastnode.kind # If this build list is inside a CALL_FUNCTION_VAR,