Skip to content

Commit

Permalink
Updated for Python 3.13
Browse files Browse the repository at this point in the history
  • Loading branch information
@andreas-zeller
andreas-zeller committed Nov 10, 2024
1 parent 9d479cd commit b89f45a
Show file tree
Hide file tree
Showing 2 changed files with 119 additions and 25 deletions.
135 changes: 113 additions & 22 deletions notebooks/PythonFuzzer.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1534,14 +1534,31 @@
" # https://docs.python.org/3/reference/lexical_analysis.html#identifiers\n",
"\n",
" # Function Calls\n",
" '<Call>': [ 'Call(func=<func>, args=<expr_list>, keywords=<keyword_list>)' ],\n",
" '<func>': [ '<expr>' ], # Actually <Expr>, but this is more readable and parses 90%\n",
" '<Call>': [ 'Call(func=<func><args_param><keywords_param>)' ],\n",
" '<args_param>': [ ', args=<expr_list>' ],\n",
" '<keywords_param>': [ ', keywords=<keyword_list>' ],\n",
" '<func>': [ '<expr>' ], # Actually <Expr>, but this is more readable and parses 90%\n",
" '<keyword_list>': [ '[<keywords>?]' ],\n",
" '<keywords>': [ '<keyword>', '<keyword>, <keywords>' ],\n",
" '<keyword>': [ 'keyword(arg=<identifier>, value=<expr>)' ]\n",
"})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4505b2fe",
"metadata": {},
"outputs": [],
"source": [
"# do import this unconditionally\n",
"if sys.version_info >= (3, 13):\n",
" PYTHON_AST_IDS_GRAMMAR: Grammar = extend_grammar(PYTHON_AST_IDS_GRAMMAR, {\n",
" # As of 3.13, args and keywords parameters are optional\n",
" '<Call>': [ 'Call(func=<func><args_param>?<keywords_param>?)' ],\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -1615,7 +1632,8 @@
"metadata": {},
"outputs": [],
"source": [
"call_str = ast.dump(ast.parse('open()').body[0].value) # type: ignore\n",
"call_str = ast.dump(ast.parse('open(\"foo.txt\", \"r\")').body[0].value) # type: ignore\n",
"print(call_str)\n",
"call_solver = ISLaSolver(ast_ids_grammar)\n",
"assert call_solver.check(call_str)"
]
Expand Down Expand Up @@ -1880,7 +1898,10 @@
" ],\n",
"\n",
" '<If>': [\n",
" 'If(test=<expr>, body=<nonempty_stmt_list>, orelse=<stmt_list>)'\n",
" 'If(test=<expr>, body=<nonempty_stmt_list><orelse_param>)'\n",
" ],\n",
" '<orelse_param>': [\n",
" ', orelse=<stmt_list>'\n",
" ],\n",
"\n",
" '<With>': [\n",
Expand Down Expand Up @@ -1909,11 +1930,29 @@
" '<Break>': [ 'Break()' ],\n",
" '<Continue>': [ 'Continue()']\n",
"\n",
" # FIXME: A few more: AsyncFor, AsyncWith, Match, Try, TryStar, With\n",
" # FIXME: A few more: AsyncFor, AsyncWith, Match, Try, TryStar\n",
" # Import, ImportFrom, Global, Nonlocal...\n",
"})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5db1c800",
"metadata": {},
"outputs": [],
"source": [
"# do import this unconditionally\n",
"if sys.version_info >= (3, 13):\n",
" PYTHON_AST_STMTS_GRAMMAR: Grammar = \\\n",
" extend_grammar(PYTHON_AST_STMTS_GRAMMAR, {\n",
" # As of 3.13, orelse is optional\n",
" '<If>': [\n",
" 'If(test=<expr>, body=<nonempty_stmt_list><orelse_param>?)'\n",
" ],\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -1969,6 +2008,7 @@
"source": [
"python_ast_stmts_grammar = convert_ebnf_grammar(PYTHON_AST_STMTS_GRAMMAR)\n",
"with_tree_str = ast.dump(with_tree.body[0]) # get the `With(...)` subtree\n",
"print(with_tree_str)\n",
"with_solver = ISLaSolver(python_ast_stmts_grammar)\n",
"assert with_solver.check(with_tree_str)"
]
Expand Down Expand Up @@ -2031,11 +2071,28 @@
" '<stmt>': PYTHON_AST_STMTS_GRAMMAR['<stmt>'] + [ '<FunctionDef>' ],\n",
"\n",
" '<FunctionDef>': [\n",
" 'FunctionDef(name=<identifier>, args=<arguments>, body=<nonempty_stmt_list>, decorator_list=<expr_list><returns>?<type_comment>?)'\n",
" 'FunctionDef(name=<identifier>, args=<arguments>, body=<nonempty_stmt_list><decorator_list_param><returns>?<type_comment>?)'\n",
" ],\n",
" '<decorator_list_param>': [\n",
" ', decorator_list=<expr_list>'\n",
" ],\n",
"\n",
" '<arguments>': [\n",
" 'arguments(posonlyargs=<arg_list>, args=<arg_list><vararg>?, kwonlyargs=<arg_list>, kw_defaults=<expr_list><kwarg>?, defaults=<expr_list>)'\n",
" 'arguments(<posonlyargs_param>args=<arg_list><vararg>?<kwonlyargs_param><kw_defaults_param><kwarg>?<defaults_param>)'\n",
" ],\n",
" '<posonlyargs_param>': [\n",
" 'posonlyargs=<arg_list>, '\n",
" ],\n",
" '<kwonlyargs_param>': [\n",
" ', kwonlyargs=<arg_list>'\n",
" ],\n",
" '<kw_defaults_param>': [\n",
" ', kw_defaults=<expr_list>'\n",
" ],\n",
" '<defaults_param>': [\n",
" ', defaults=<expr_list>'\n",
" ],\n",
"\n",
"\n",
" '<arg_list>': [ '[<args>?]' ],\n",
" '<args>': [ '<arg>', '<arg>, <arg>' ],\n",
Expand All @@ -2057,16 +2114,6 @@
"In Python 3.12 and later, function definitions also have a `type_param` field:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ac5c79cf",
"metadata": {},
"outputs": [],
"source": [
"import sys"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -2078,7 +2125,7 @@
"if sys.version_info >= (3, 12):\n",
" PYTHON_AST_DEFS_GRAMMAR: Grammar = extend_grammar(PYTHON_AST_DEFS_GRAMMAR, {\n",
" '<FunctionDef>': [\n",
" 'FunctionDef(name=<identifier>, args=<arguments>, body=<nonempty_stmt_list>, decorator_list=<expr_list><returns>?<type_comment>?<type_params>?)'\n",
" 'FunctionDef(name=<identifier>, args=<arguments>, body=<nonempty_stmt_list><decorator_list_param><returns>?<type_comment>?<type_params>?)'\n",
" ],\n",
" '<type_params>': [\n",
" ', type_params=<type_param_list>',\n",
Expand All @@ -2097,6 +2144,33 @@
" })"
]
},
{
"cell_type": "markdown",
"id": "02959d01",
"metadata": {},
"source": [
"In Python 3.13 and later, several `<FunctionDef>` and `<arguments>` attributes are optional:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ee08d09d",
"metadata": {},
"outputs": [],
"source": [
"# do import this unconditionally\n",
"if sys.version_info >= (3, 13):\n",
" PYTHON_AST_DEFS_GRAMMAR: Grammar = extend_grammar(PYTHON_AST_DEFS_GRAMMAR, {\n",
" '<FunctionDef>': [\n",
" 'FunctionDef(name=<identifier>, args=<arguments>, body=<nonempty_stmt_list><decorator_list_param>?<returns>?<type_comment>?<type_params>?)'\n",
" ],\n",
" '<arguments>': [\n",
" 'arguments(<posonlyargs_param>?args=<arg_list><vararg>?<kwonlyargs_param>?<kw_defaults_param>?<kwarg>?<defaults_param>?)'\n",
" ],\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -2155,14 +2229,31 @@
"PYTHON_AST_MODULE_GRAMMAR: Grammar = extend_grammar(PYTHON_AST_DEFS_GRAMMAR, {\n",
" '<start>': [ '<mod>' ],\n",
" '<mod>': [ '<Module>' ],\n",
" '<Module>': [ 'Module(body=<nonempty_stmt_list>, type_ignores=<type_ignore_list>)'],\n",
" '<Module>': [ 'Module(body=<nonempty_stmt_list><type_ignore_param>)'],\n",
"\n",
" '<type_ignore_param>': [ ', type_ignores=<type_ignore_list>' ],\n",
" '<type_ignore_list>': [ '[<type_ignores>?]' ],\n",
" '<type_ignores>': [ '<type_ignore>', '<type_ignore>, <type_ignore>' ],\n",
" '<type_ignore>': [ 'TypeIgnore(lineno=<integer>, tag=<string>)' ],\n",
"})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ef453b00",
"metadata": {},
"outputs": [],
"source": [
"# do import this unconditionally\n",
"if sys.version_info >= (3, 13):\n",
" PYTHON_AST_MODULE_GRAMMAR: Grammar = \\\n",
" extend_grammar(PYTHON_AST_MODULE_GRAMMAR, {\n",
" # As of 3.13, the type_ignore parameter is optional\n",
" '<Module>': [ 'Module(body=<nonempty_stmt_list><type_ignore_param>?)'],\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -2751,7 +2842,7 @@
"outputs": [],
"source": [
"solver = ISLaSolver(python_ast_grammar)\n",
"solver.check(sum_str)"
"assert solver.check(sum_str)"
]
},
{
Expand Down Expand Up @@ -4180,7 +4271,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": "python3.11",
"language": "python",
"name": "python3"
},
Expand All @@ -4194,7 +4285,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.1"
"version": "3.11.10"
}
},
"nbformat": 4,
Expand Down
9 changes: 6 additions & 3 deletions notebooks/ReleaseNotes.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,14 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Version 1.2.2 (released 2024-11-09)\n",
"## Version 1.2.2 (released 2024-11-10)\n",
"\n",
"* Adapted the code to work with Python 3.13:\n",
" - Work around an error in the `showast` module\n",
" - Extended the [chapter on Compiler Testing](PythonFuzzer.ipynb) to work with Python 3.13 and later\n",
" - Added automatic Python 3.13 tests into our workflow\n",
"* Fix: Outputting code coverage using the `Coverage` class would prefix _covered_ code with `#`, rather than _uncovered_ code as should be. This has been fixed.\n",
"* Work around an error in the `showast` module, occurring in notebooks running Python 3.12 and later\n",
"* Lots of additional typos fixed, thanks to Sergey Bronnikov!"
"* Lots of additional typos fixed, [thanks to Sergey Bronnikov](https://github.com/uds-se/fuzzingbook/pull/181)."
]
},
{
Expand Down

0 comments on commit b89f45a

Please sign in to comment.