From 3bd3c798f26e5cabbb7b9e084b2f29dcd7ef51d4 Mon Sep 17 00:00:00 2001 From: julianuspfeuffer Date: Sat, 18 Nov 2023 13:25:58 +0100 Subject: [PATCH 1/2] Allow nogil for most types of methods. This requires a change in conversion providers. +Some other cleanups. --- CONTRIBUTING.md | 19 +- README_DEVELOP | 9 - autowrap/CodeGenerator.py | 45 +- autowrap/ConversionProvider.py | 304 +++++++--- autowrap/DeclResolver.py | 2 +- autowrap/PXDParser.py | 5 +- autowrap/data_files/boost/cstdint.hpp | 552 ++++++++++++++++++ environment.yml | 6 + requirements_dev.txt | 2 - setup.cfg | 2 - setup.py | 2 +- tests/test_code_generator.py | 2 + tests/test_code_generator_minimal.py | 5 +- .../converters/IntHolderConverter.py | 11 +- tests/test_files/gil_testing.hpp | 29 + tests/test_files/gil_testing.pxd | 2 + tests/test_files/inherited.pyx | 7 +- tests/test_files/libcpp_stl_test.pxd | 4 +- 18 files changed, 891 insertions(+), 117 deletions(-) delete mode 100644 README_DEVELOP create mode 100644 autowrap/data_files/boost/cstdint.hpp create mode 100644 environment.yml delete mode 100644 requirements_dev.txt delete mode 100644 setup.cfg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61e03e9a..d7fa6bfb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,19 @@ This document is displayed because you either opened an issue or you want to pro When interacting with other developers, users or anyone else from our community, please adhere to [the OpenMS CODE OF CONDUCT](https://github.com/OpenMS/OpenMS/blob/develop/CODE_OF_CONDUCT.md) +# Setting up dev environment +Install conda/mamba (or use a venv). +Install git. +Install a C++ compiler. + +```bash +git clone https://github.com/OpenMS/autowrap +cd autowrap +mamba create -n autowrap -f environment.yml +python -m pytest +``` + + # Reporting an Issue: You most likely came here to: @@ -18,10 +31,10 @@ If you found a bug, e.g. an autowrap tool crashes during code generation, it is - how you installed autowrap (e.g., from source, pip) - a description on how to reproduce the bug - relevant tool output (e.g., error messages) - - data to repoduce the bug (If possible as a GitHub gist. Other platforms like Dropbox, Google Drive links also work. If you can't share the data publicly please indicate this and we will contact you in private.) + - data to reproduce the bug (If possible as a GitHub gist. Other platforms like Dropbox, Google Drive links also work. If you can't share the data publicly please indicate this and we will contact you in private.) -If you are an official OpenMS team meber: - - label your issue using github labels (e.g. as: question, defect) that indicate the type of issue and which components of autowrap (blue labels) are affected. The severity is usually assigned by OpenMS maintainers and used internally to e.g. indicate if a bug is a blocker for a new release. +If you are an official OpenMS team member: + - label your issue using GitHub labels (e.g. as: question, defect) that indicate the type of issue and which components of autowrap (blue labels) are affected. The severity is usually assigned by OpenMS maintainers and used internally to e.g. indicate if a bug is a blocker for a new release. # Opening a Pull Request diff --git a/README_DEVELOP b/README_DEVELOP deleted file mode 100644 index 0a3927dc..00000000 --- a/README_DEVELOP +++ /dev/null @@ -1,9 +0,0 @@ -Activate the project in "development mode" first: - - $ python setup.py develop - -For testing run - - $ py.test tests - -from the projects directory. diff --git a/autowrap/CodeGenerator.py b/autowrap/CodeGenerator.py index 87111540..a4228746 100644 --- a/autowrap/CodeGenerator.py +++ b/autowrap/CodeGenerator.py @@ -1113,18 +1113,22 @@ def _create_fun_decl_and_input_conversion(self, code, py_name, method, is_free_f call_args = [] checks = [] in_types = [] + decls = [] + decl_calls = [] for arg_num, (t, n) in enumerate(args): # get new ConversionProvider using the converter registry converter = self.cr.get(t) converter.cr = self.cr py_type = converter.matching_python_type(t) py_typing_type = converter.matching_python_type_full(t) - conv_code, call_as, cleanup = converter.input_conversion(t, n, arg_num) + conv_code, call_as, cleanup, (decl, decl_call) = converter.input_conversion(t, n, arg_num) py_signature_parts.append("%s %s " % (py_type, n)) py_typing_signature_parts.append("%s: %s " % (n, py_typing_type)) input_conversion_codes.append(conv_code) cleanups.append(cleanup) call_args.append(call_as) + decls.append(decl) + decl_calls.append(decl_call) in_types.append(t) checks.append((n, converter.type_check_expression(t, n))) @@ -1211,7 +1215,7 @@ def _create_fun_decl_and_input_conversion(self, code, py_name, method, is_free_f for conv_code in input_conversion_codes: code.add(conv_code) - return call_args, cleanups, in_types, stub + return call_args, cleanups, in_types, stub, decls, decl_calls def _create_wrapper_for_attribute(self, attribute): code = Code() @@ -1228,7 +1232,7 @@ def _create_wrapper_for_attribute(self, attribute): converter = self.cr.get(t) py_type = converter.matching_python_type(t) py_typing_type = converter.matching_python_type_full(t) - conv_code, call_as, cleanup = converter.input_conversion(t, name, 0) + conv_code, call_as, cleanup, (decl, decl_call) = converter.input_conversion(t, name, 0) code.add( """ @@ -1323,7 +1327,7 @@ def _create_wrapper_for_attribute(self, attribute): code.add(" return py_result") return code, stubs - def create_wrapper_for_nonoverloaded_method(self, cdcl, py_name, method): + def create_wrapper_for_nonoverloaded_method(self, cdcl, py_name, method: ResolvedMethod): L.info(" create wrapper for %s ('%s')" % (py_name, method)) meth_code = Code() @@ -1332,10 +1336,22 @@ def create_wrapper_for_nonoverloaded_method(self, cdcl, py_name, method): cleanups, in_types, stubs, + decls, + decl_calls ) = self._create_fun_decl_and_input_conversion(meth_code, py_name, method) + init = "" + c_call_args = [] + for decl, decl_call in zip(decls, decl_calls): + init = init + decl + "\n" + c_call_args.append(decl_call) + # call wrapped method and convert result value back to python cpp_name = method.cpp_decl.name + + if method.with_nogil and init.strip("\n"): + call_args = c_call_args + call_args_str = ", ".join(call_args) if method.is_static: cy_call_str = "%s.%s(%s)" % ( @@ -1348,13 +1364,15 @@ def create_wrapper_for_nonoverloaded_method(self, cdcl, py_name, method): res_t = method.result_type out_converter = self.cr.get(res_t) - full_call_stmt = out_converter.call_method(res_t, cy_call_str) + full_call_stmt = out_converter.call_method(res_t, cy_call_str, True, not method.with_nogil) if method.with_nogil: meth_code.add( """ + | $init | with nogil: - """ + | + """, locals() ) indented = Code() else: @@ -1370,6 +1388,10 @@ def create_wrapper_for_nonoverloaded_method(self, cdcl, py_name, method): else: indented.add(full_call_stmt) + if method.with_nogil: + meth_code.add(indented) + indented = meth_code + for cleanup in reversed(cleanups): if not cleanup: continue @@ -1385,9 +1407,6 @@ def create_wrapper_for_nonoverloaded_method(self, cdcl, py_name, method): indented.add(to_py_code) indented.add(" return py_result") - if method.with_nogil: - meth_code.add(indented) - return meth_code, stubs def create_wrapper_for_free_function(self, decl: ResolvedFunction, out_codes: CodeDict) -> None: @@ -1445,6 +1464,8 @@ def _create_wrapper_for_free_function( cleanups, in_types, stubs, + decls, + decl_calls ) = self._create_fun_decl_and_input_conversion(fun_code, name, decl, is_free_fun=True) call_args_str = ", ".join(call_args) @@ -1552,6 +1573,8 @@ def create_wrapper_for_nonoverloaded_constructor(self, class_decl, py_name, cons cleanups, in_types, stubs, + decls, + decl_calls ) = self._create_fun_decl_and_input_conversion(cons_code, py_name, cons_decl) stub_code.extend(stubs) @@ -1663,6 +1686,8 @@ def create_special_getitem_method(self, mdcl): cleanups, (in_type,), stubs, + decls, + decl_calls ) = self._create_fun_decl_and_input_conversion(meth_code, "__getitem__", mdcl) meth_code.add( @@ -1790,7 +1815,7 @@ def create_special_setitem_method(self, mdcl): # CppObject[ idx ] = value # cy_call_str = "deref(self.inst.get())[%s]" % call_arg - code, call_as, cleanup = out_converter.input_conversion(res_t, value_arg, 0) + code, call_as, cleanup, (decl, decl_call) = out_converter.input_conversion(res_t, value_arg, 0) meth_code.add( """ | $cy_call_str = $call_as diff --git a/autowrap/ConversionProvider.py b/autowrap/ConversionProvider.py index df60fefb..681e083e 100644 --- a/autowrap/ConversionProvider.py +++ b/autowrap/ConversionProvider.py @@ -31,6 +31,7 @@ """ from collections import defaultdict +from dataclasses import dataclass from typing import List, Tuple, Optional, Union, AnyStr from autowrap.DeclResolver import ResolvedClass @@ -68,6 +69,16 @@ def mangle(s): class TypeConverterBase(object): + + def __init__(self): + self.enums_to_wrap = None + + def init_as_call_arg(self, cpp_type, argument_var, arg_num): + if cpp_type: + return "cdef %s c_%s = %s" % (cpp_type, argument_var, argument_var) + else: + return "" + def set_enums_to_wrap(self, enums_to_wrap): self.enums_to_wrap = enums_to_wrap @@ -86,7 +97,20 @@ def matches(self, cpp_type: CppType) -> bool: """ raise NotImplementedError() - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: + def call_method_split( + self, res_type: CppType, cy_call_str: str, with_const: bool = True + ) -> Tuple[Union[Code, str], Union[Code, str]]: + return "", "_r = %s" % cy_call_str + + # TODO "with_cdef=False" is currently only used when used "with nogil:" in which context + # a cdef is not allowed. Assignment can still be done untyped but might suffer from a slight + # performance impact. Ideally you would split declaration (to be put before "with nogil:") + # and assignment (inside the nogil context). You might want a different method for this, though. + # see my example of call_method_split. + # Actually I found that in our ConversionProvider classes for more complex types like lists/sets etc. + # we were omitting the cdef completely already (although we needn't to do so [I think, unless replicating + # the type is too complex]). Might give some speed increase to type the _r (result) variable in this case. + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: """ Creates a temporary object which has the type of the current TypeConverter object. @@ -97,16 +121,22 @@ def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = Tr cdef int & _r - is illegal to declare and we have to remove any references from the type. + is illegal to declare, and we have to remove any references from the type. Use with_const = False to return the type non-const """ - cy_res_type = self.converters.cython_type(res_type) # type: CppType + cy_res_type = self.converters.cython_type(res_type) if cy_res_type.is_ref: cy_res_type = cy_res_type.base_type # TODO what about const refs? - return "cdef %s _r = %s" % (cy_res_type, cy_call_str) + if with_cdef: + return "cdef %s _r = %s" % (cy_res_type, cy_call_str) + else: + return "_r = %s" % cy_call_str - return "cdef %s _r = %s" % (cy_res_type.toString(with_const), cy_call_str) + if with_cdef: + return "cdef %s _r = %s" % (cy_res_type.toString(with_const), cy_call_str) + else: + return "_r = %s" % cy_call_str def matching_python_type(self, cpp_type: CppType) -> str: """ @@ -133,7 +163,7 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Union[Code, str], Union[Code, str], Union[Code, str]]: + ) -> Tuple[Union[Code, str], Union[Code, str], Union[Code, str], Tuple[str, str]]: """ Sets up the conversion of input arguments. @@ -141,6 +171,8 @@ def input_conversion( - code : a code object to be added to the beginning of the function - call_as : a piece of code indicating how the argument should be called as going forward - cleanup : a piece of cleanup code to be added to the bottom of the function + - decl : a tuple for split declaration (with cdef) and using the declared variable in the C(++) function call + that produces the result. This is mainly used in the case of using nogil. """ raise NotImplementedError() @@ -181,10 +213,13 @@ class VoidConverter(TypeConverterBase): def get_base_types(self) -> List[str]: return ["void"] + def init_as_call_arg(self, cpp_type, argument_var, arg_num): + return "cdef libcpp_string c_%s = %s" % (argument_var, argument_var) + def matches(self, cpp_type: CppType) -> bool: return not cpp_type.is_ptr - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: return cy_call_str def matching_python_type(self, cpp_type: CppType) -> str: @@ -198,7 +233,7 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: raise NotImplementedError("void has no matching python type") def output_conversion( @@ -240,11 +275,12 @@ def matching_python_type_full(self, cpp_type: CppType) -> str: def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: return "isinstance(%s, int)" % (argument_var,) - def input_conversion(self, cpp_type, argument_var, arg_num) -> Tuple[str, str, str]: + def input_conversion(self, cpp_type, argument_var, arg_num) -> Tuple[str, str, str, Tuple[str, str]]: code = "" call_as = "(<%s>%s)" % (cpp_type, argument_var) + decl = ("cdef %s %s = %s" % (cpp_type, "c_"+argument_var, call_as), "c_"+argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -275,11 +311,12 @@ def matching_python_type_full(self, cpp_type: CppType) -> str: def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: return "isinstance(%s, pybool_t)" % (argument_var,) - def input_conversion(self, cpp_type, argument_var, arg_num) -> Tuple[str, str, str]: + def input_conversion(self, cpp_type, argument_var, arg_num) -> Tuple[str, str, str, Tuple[str, str]]: code = "" call_as = "(<%s>%s)" % (cpp_type, argument_var) + decl = ("cdef %s %s = %s" % (cpp_type, "c_"+argument_var, call_as), "c_"+argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -322,11 +359,12 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: code = "" call_as = "(<%s>%s)" % (cpp_type, argument_var) + decl = ("cdef %s %s = %s" % (cpp_type, "c_"+argument_var, call_as), "c_"+argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -355,11 +393,12 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: code = "" call_as = "(<%s>%s)" % (cpp_type, argument_var) + decl = ("cdef %s %s = %s" % (cpp_type, "c_"+argument_var, call_as), "c_"+argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -385,11 +424,12 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: code = "" call_as = "(<%s>%s)" % (cpp_type, argument_var) + decl = ("cdef %s %s = %s" % (cpp_type, "c_"+argument_var, call_as), "c_"+argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -439,15 +479,17 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: code = "" if not self.enum.scoped: call_as = "(<_%s>%s)" % (cpp_type.base_type, argument_var) else: # for scoped enums we use the python enum class. There you need to use value call_as = "(<_%s>%s.value)" % (cpp_type.base_type, argument_var) + + decl = ("cdef %s %s = %s" % (cpp_type, "c_"+argument_var, call_as), "c_"+argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -477,14 +519,18 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: code = "" call_as = "(((%s)[0]))" % argument_var + decl = ("cdef char %s = %s" % ("c_"+argument_var, call_as), "c_"+argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: - return "cdef char _r = %s" % cy_call_str + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: + if with_cdef: + return "cdef char _r = %s" % cy_call_str + else: + return "_r = %s" % cy_call_str def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -510,16 +556,20 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Code, str, str]: + ) -> Tuple[Code, str, str, Tuple[str, str]]: code = Code().add( "cdef const_char * input_%s = %s" % (argument_var, argument_var) ) call_as = "input_%s" % argument_var + decl = ("", call_as) # declared in the conversion code already cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: - return "cdef const_char * _r = _cast_const_away(%s)" % cy_call_str + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: + if with_cdef: + return "cdef const_char * _r = _cast_const_away(%s)" % cy_call_str + else: + return "_r = _cast_const_away(%s)" % cy_call_str def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -545,14 +595,18 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: code = "" call_as = "(%s)" % argument_var + decl = ("cdef char * %s = %s" % ("c_"+argument_var, call_as), "c_"+argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: - return "cdef char * _r = _cast_const_away(%s)" % cy_call_str + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: + if with_cdef: + return "cdef char * _r = _cast_const_away(%s)" % cy_call_str + else: + return "_r = _cast_const_away(%s)" % cy_call_str def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -564,6 +618,12 @@ class TypeToWrapConverter(TypeConverterBase): def __init__(self, class_: ResolvedClass): self.class_: ResolvedClass = class_ + def init_as_call_arg(self, cpp_type, argument_var, arg_num): + if cpp_type.is_ptr: + return "cdef %s %s = %s.inst.get()" % (cpp_type, argument_var, argument_var) + else: + return "cdef %s %s = deref(%s.inst.get())" % (cpp_type, argument_var, argument_var) + def get_base_types(self) -> List[str]: return [self.class_.name] @@ -582,19 +642,20 @@ def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: code = "" if cpp_type.is_ptr: call_as = "(%s.inst.get())" % (argument_var,) else: call_as = "(deref(%s.inst.get()))" % (argument_var,) + decl = ("", call_as) # Wrapped type can be used directly cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl - def call_method( - self, res_type: CppType, cy_call_str: str, with_const: bool = True - ) -> Union[Code, str]: + def call_method_split( + self, res_type: CppType, cy_call_str: str, with_const: bool = True + ) -> Tuple[Union[Code, str], Union[Code, str]]: t = self.converters.cython_type(res_type) if t.is_ref: @@ -608,18 +669,67 @@ def call_method( # If t is a pointer, we would like to call on the base type t = t.base_type - code = Code().add( + decl = Code().add( """ - |cdef $const $t * __r = ($cy_call_str) + |cdef $const $t * __r + |cdef $t * _r + """, + locals(), + ) + call = Code().add( + """ + |__r = ($cy_call_str) |if __r == NULL: | return None - |cdef $t * _r = new $t(deref(__r)) + |_r = new $t(deref(__r)) """, locals(), ) - return code + return decl, call + + return "cdef %s * _r" % t, "_r = new %s(%s)" % (t, cy_call_str) + + def call_method( + self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True + ) -> Union[Code, str]: + t = self.converters.cython_type(res_type) - return "cdef %s * _r = new %s(%s)" % (t, t, cy_call_str) + if t.is_ref: + # If t is a ref, we would like to call on the base type + t = t.base_type + elif t.is_ptr: + # Special treatment for const raw ptr + const = "" + if t.is_const: + const = "const" + + # If t is a pointer, we would like to call on the base type + t = t.base_type + if with_cdef: + code = Code().add( + """ + |cdef $const $t * __r = ($cy_call_str) + |if __r == NULL: + | return None + |cdef $t * _r = new $t(deref(__r)) + """, + locals(), + ) + else: + code = Code().add( + """ + |__r = ($cy_call_str) + |if __r == NULL: + | return None + |_r = new $t(deref(__r)) + """, + locals(), + ) + return code + if with_cdef: + return "cdef %s * _r = new %s(%s)" % (t, t, cy_call_str) + else: + return "_r = new %s(%s)" % (t, cy_call_str) def output_conversion( self, cpp_type: CppType, input_cpp_var: str, output_py_var: str @@ -690,7 +800,7 @@ def type_check_expression(self, cpp_type, arg_var): def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Code, str, Code]: + ) -> Tuple[Code, str, Code, Tuple[str, str]]: ( t1, t2, @@ -755,10 +865,16 @@ def input_conversion( """, locals(), ) - return code, "%s" % temp_var, cleanup_code + decl = ("", "%s" % temp_var) # declared in code already + return code, "%s" % temp_var, cleanup_code, decl - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: - return "_r = %s" % (cy_call_str) + def call_method_split( + self, res_type: CppType, cy_call_str: str, with_const: bool = True + ) -> Tuple[Union[Code, str], Union[Code, str]]: + return "", "_r = %s" % cy_call_str + + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: + return "_r = %s" % cy_call_str def output_conversion(self, cpp_type: CppType, input_cpp_var: str, output_py_var: str) -> Code: assert not cpp_type.is_ptr @@ -862,7 +978,7 @@ def type_check_expression(self, cpp_type, arg_var): def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Code, str, Union[Code, str]]: + ) -> Tuple[Code, str, Code | str, Tuple[str, str]]: tt_key, tt_value = cpp_type.template_args temp_var = "v%d" % arg_num @@ -964,7 +1080,7 @@ def input_conversion( ) elif tt_value in self.converters: - value_conv_code, value_conv, value_conv_cleanup = self.converters.get( + value_conv_code, value_conv, value_conv_cleanup, _ = self.converters.get( tt_value ).input_conversion(tt_value, "value", 0) else: @@ -975,7 +1091,7 @@ def input_conversion( elif tt_key.base_type in self.converters.names_of_wrapper_classes: key_conv = "deref(<%s *> (<%s> key).inst.get())" % (cy_tt_key, py_tt_key) elif tt_key in self.converters: - key_conv_code, key_conv, key_conv_cleanup = self.converters.get( + key_conv_code, key_conv, key_conv_cleanup, _ = self.converters.get( tt_key ).input_conversion(tt_key, "key", 0) else: @@ -1036,7 +1152,7 @@ def input_conversion( elif ( not cy_tt_value.is_enum and tt_value.base_type in self.converters.names_of_wrapper_classes - and not tt_key.base_type in self.converters.names_of_wrapper_classes + and tt_key.base_type not in self.converters.names_of_wrapper_classes ): cy_tt = tt_value.base_type item = mangle("item_" + argument_var) @@ -1103,9 +1219,15 @@ def input_conversion( else: cleanup_code = "del %s" % temp_var - return code, "deref(%s)" % temp_var, cleanup_code + decl = ("", "deref(%s)" % temp_var) # already declared in conversion code + return code, "deref(%s)" % temp_var, cleanup_code, decl + + def call_method_split( + self, res_type: CppType, cy_call_str: str, with_const: bool = True + ) -> Tuple[Union[Code, str], Union[Code, str]]: + return "", "_r = %s" % cy_call_str - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: return "_r = %s" % cy_call_str def output_conversion(self, cpp_type: CppType, input_cpp_var: str, output_py_var: str) -> Code: @@ -1219,7 +1341,7 @@ def type_check_expression(self, cpp_type, arg_var): def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Code, str, Union[Code, str]]: + ) -> Tuple[Code, str, Code | str, Tuple[str, str]]: (tt,) = cpp_type.template_args temp_var = "v%d" % arg_num inner = self.converters.cython_type(tt) @@ -1251,7 +1373,9 @@ def input_conversion( ) else: cleanup_code = "del %s" % temp_var - return code, "deref(%s)" % temp_var, cleanup_code + + decl = ("", "deref(%s)" % temp_var) + return code, "deref(%s)" % temp_var, cleanup_code, decl elif tt.base_type in self.converters.names_of_wrapper_classes: base_type = tt.base_type @@ -1293,7 +1417,9 @@ def input_conversion( else: cleanup_code = "del %s" % temp_var - return code, "deref(%s)" % temp_var, cleanup_code + + decl = ("", "deref(%s)" % temp_var) + return code, "deref(%s)" % temp_var, cleanup_code, decl else: inner = self.converters.cython_type(tt) # cython cares for conversion of stl containers with std types: @@ -1313,9 +1439,16 @@ def input_conversion( """, locals(), ) - return code, "%s" % temp_var, cleanup_code - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: + decl = ("", temp_var) + return code, temp_var, cleanup_code, decl + + def call_method_split( + self, res_type: CppType, cy_call_str: str, with_const: bool = True + ) -> Tuple[Union[Code, str], Union[Code, str]]: + return "", "_r = %s" % cy_call_str + + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: return "_r = %s" % cy_call_str def output_conversion(self, cpp_type: CppType, input_cpp_var: str, output_py_var: str) -> Code: @@ -1525,7 +1658,6 @@ def _perform_recursion( **kw ): converter = self.cr.get(tt) - py_type = converter.matching_python_type(tt) rec_arg_num = "%s_rec" % arg_num # the current item is the argument var of the recursive call rec_argument_var = item @@ -1534,7 +1666,7 @@ def _perform_recursion( # # Perform the recursive call # - conv_code, call_as, cleanup = converter.input_conversion( + conv_code, call_as, cleanup, _ = converter.input_conversion( tt, rec_argument_var, rec_arg_num, @@ -1649,7 +1781,7 @@ def input_conversion( topmost_code: Optional[Code] = None, bottommost_code: Optional[Code] = None, recursion_cnt: int = 0, - ) -> Tuple[Code, str, Code]: + ) -> Tuple[Code, str, Code | str, Tuple[str, str]]: """Do the input conversion for a std::vector In this case, the template argument is tt (or "inner"). @@ -1726,7 +1858,9 @@ def input_conversion( ) else: cleanup_code = "del %s" % temp_var - return code, "deref(%s)" % temp_var, cleanup_code + + decl = ("", "deref(%s)" % temp_var) + return code, "deref(%s)" % temp_var, cleanup_code, decl elif tt.base_type in self.converters.names_of_wrapper_classes: # Case 2: We wrap a std::vector<> with a base type we need to wrap @@ -1755,7 +1889,8 @@ def input_conversion( else: call_fragment = "deref(%s)" % temp_var - return code, call_fragment, cleanup_code + decl = ("", call_fragment) # already declared in conversion code + return code, call_fragment, cleanup_code, decl elif ( tt.template_args is not None @@ -1802,7 +1937,8 @@ def input_conversion( locals(), ) - return code, "%s" % temp_var, cleanup_code + decl = ("", temp_var) # already declared in conversion code + return code, temp_var, cleanup_code, decl elif inner_contains_classes_to_wrap and tt.base_type != "libcpp_vector": # Only if the template argument which is neither a class-to-wrap nor a std::vector @@ -1857,7 +1993,8 @@ def input_conversion( "Error: For recursion in std::vector to work, we need a ConverterRegistry instance at self.cr" ) - return code, "deref(%s)" % temp_var, cleanup_code + decl = ("", "deref(%s)" % temp_var) + return code, "deref(%s)" % temp_var, cleanup_code, decl else: # Case 5: We wrap a regular type @@ -1879,14 +2016,24 @@ def input_conversion( locals(), ) - return code, "%s" % temp_var, cleanup_code + decl = ("", "%s" % temp_var) + return code, "%s" % temp_var, cleanup_code, decl - def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True) -> str: + def call_method_split( + self, res_type: CppType, cy_call_str: str, with_const: bool = True + ) -> Tuple[Union[Code, str], Union[Code, str]]: t = self.converters.cython_type(res_type) if t.is_ptr: - return "_r = deref(%s)" % (cy_call_str) + return "", "_r = deref(%s)" % cy_call_str - return "_r = %s" % (cy_call_str) + return "", "_r = %s" % cy_call_str + + def call_method(self, res_type: CppType, cy_call_str: str, with_const: bool = True, with_cdef: bool = True) -> str: + t = self.converters.cython_type(res_type) + if t.is_ptr: + return "_r = deref(%s)" % cy_call_str + + return "_r = %s" % cy_call_str def output_conversion(self, cpp_type: CppType, input_cpp_var: str, output_py_var: str) -> Code: (tt,) = cpp_type.template_args @@ -1990,11 +2137,15 @@ def matching_python_type_full(self, cpp_type: CppType) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[str, str, str]: + ) -> Tuple[str, str, str, Tuple[str, str]]: code = "" call_as = "(%s)" % argument_var cleanup = "" - return code, call_as, cleanup + decl = ("cdef libcpp_string %s = %s" % ("c_" + argument_var, call_as), "c_" + argument_var) + return code, call_as, cleanup, decl + + def init_as_call_arg(self, cpp_type, argument_var, arg_num): + return "cdef libcpp_string c_%s = %s" % (argument_var, argument_var) def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: return "isinstance(%s, bytes)" % argument_var @@ -2030,7 +2181,7 @@ def matching_python_type_full(self, cpp_type: CppType) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Code, str, str]: + ) -> Tuple[Code, str, str, Tuple[str, str]]: code = Code() # although python3 does not have "unicode" as a built-in type anymore, # Cython understands it and uses the Py_IsUnicodeCheck @@ -2043,7 +2194,8 @@ def input_conversion( ) call_as = "(%s)" % argument_var cleanup = "" - return code, call_as, cleanup + decl = ("cdef libcpp_string %s = %s" % ("c_" + argument_var, call_as), "c_" + argument_var) + return code, call_as, cleanup, decl def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: return "isinstance(%s, (bytes, str))" % argument_var @@ -2118,7 +2270,8 @@ def input_conversion( """, locals(), ) - return code, call_as, cleanup + decl = ("", call_as) + return code, call_as, cleanup, decl def type_check_expression(self, cpp_type: CppType, argument_var: str) -> str: # We can just use the Python type of the template argument @@ -2158,10 +2311,11 @@ class ConverterRegistry(object): Works with two level lookup: first find converters which support base_type, then call .matches on them to find the finally matching converters - Therefore TypeConverterBase has methods .get_base_types and .matches + Therefore, TypeConverterBase has methods .get_base_types and .matches """ def __init__(self, instance_mapping, names_of_classes_to_wrap, names_of_enums_to_wrap): + self.instance_mapping = dict() self.lookup = defaultdict(list) self.names_of_wrapper_classes = list(instance_mapping.keys()) @@ -2181,7 +2335,7 @@ def process_and_set_type_mapping(self, instance_mapping): (name, CppType("_" + name)) for name in self.names_of_classes_to_wrap + self.names_of_enums_to_wrap ) - self.instance_mapping = dict() + for alias, type_ in instance_mapping.items(): self.instance_mapping[alias] = type_.transformed(map_) diff --git a/autowrap/DeclResolver.py b/autowrap/DeclResolver.py index c3375be4..1e6e8421 100644 --- a/autowrap/DeclResolver.py +++ b/autowrap/DeclResolver.py @@ -229,7 +229,7 @@ class ResolvedMethod(object): def __init__(self, name, is_static, result_type, arguments, decl, instance_map, local_map): self.name: str = name self.is_static: bool = is_static - self.result_type = result_type + self.result_type: Types.CppType = result_type self.arguments = arguments self.cpp_decl = decl self.wrap_ignore: bool = decl.annotations.get("wrap-ignore", False) diff --git a/autowrap/PXDParser.py b/autowrap/PXDParser.py index 469373cc..32f33923 100644 --- a/autowrap/PXDParser.py +++ b/autowrap/PXDParser.py @@ -55,7 +55,7 @@ from collections import defaultdict from .tools import OrderKeepingDictionary -AnnotDict = Dict[str, Union[bool, List[str]]] +AnnotDict = Dict[str, Union[bool, List[str], Code]] """ Methods in this module use Cythons Parser to build an Cython syntax tree @@ -81,6 +81,7 @@ def parse_class_annotations(node, lines: Collection[str]) -> AnnotDict: return _parse_multiline_annotations(lines[start_at_line:]) +# TODO we should really unify the value type of the return value here to either Code or list of str. def _parse_multiline_annotations(lines: Collection[str]) -> AnnotDict: """does the hard work for parse_class_annotations, and is better testable than this. @@ -317,7 +318,7 @@ def _extract_type( class BaseDecl(object): - def __init__(self, name: str, annotations: Dict[str, Union[bool, List[str]]], pxd_path: str): + def __init__(self, name: str, annotations: Dict[str, Union[bool, List[str], Code]], pxd_path: str): self.name: str = name self.annotations: AnnotDict = annotations self.pxd_path: str = pxd_path diff --git a/autowrap/data_files/boost/cstdint.hpp b/autowrap/data_files/boost/cstdint.hpp new file mode 100644 index 00000000..b5fb58a9 --- /dev/null +++ b/autowrap/data_files/boost/cstdint.hpp @@ -0,0 +1,552 @@ +// boost cstdint.hpp header file ------------------------------------------// + +// (C) Copyright Beman Dawes 1999. +// (C) Copyright Jens Mauer 2001 +// (C) Copyright John Maddock 2001 +// Distributed under the Boost +// Software License, Version 1.0. (See accompanying file +// LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +// See http://www.boost.org/libs/integer for documentation. + +// Revision History +// 31 Oct 01 use BOOST_HAS_LONG_LONG to check for "long long" (Jens M.) +// 16 Apr 01 check LONGLONG_MAX when looking for "long long" (Jens Maurer) +// 23 Jan 01 prefer "long" over "int" for int32_t and intmax_t (Jens Maurer) +// 12 Nov 00 Merged (Jens Maurer) +// 23 Sep 00 Added INTXX_C macro support (John Maddock). +// 22 Sep 00 Better 64-bit support (John Maddock) +// 29 Jun 00 Reimplement to avoid including stdint.h within namespace boost +// 8 Aug 99 Initial version (Beman Dawes) + + +#ifndef BOOST_CSTDINT_HPP +#define BOOST_CSTDINT_HPP + +// +// Since we always define the INT#_C macros as per C++0x, +// define __STDC_CONSTANT_MACROS so that does the right +// thing if possible, and so that the user knows that the macros +// are actually defined as per C99. +// +#ifndef __STDC_CONSTANT_MACROS +# define __STDC_CONSTANT_MACROS +#endif + +#include +// +// For the following code we get several warnings along the lines of: +// +// boost/cstdint.hpp:428:35: error: use of C99 long long integer constant +// +// So we declare this a system header to suppress these warnings. +// See also https://github.com/boostorg/config/issues/190 +// +#if defined(__GNUC__) && (__GNUC__ >= 4) +#pragma GCC system_header +#endif + +// +// Note that GLIBC is a bit inconsistent about whether int64_t is defined or not +// depending upon what headers happen to have been included first... +// so we disable use of stdint.h when GLIBC does not define __GLIBC_HAVE_LONG_LONG. +// See https://svn.boost.org/trac/boost/ticket/3548 and http://sources.redhat.com/bugzilla/show_bug.cgi?id=10990 +// +#if defined(BOOST_HAS_STDINT_H) \ + && (!defined(__GLIBC__) \ + || defined(__GLIBC_HAVE_LONG_LONG) \ + || (defined(__GLIBC__) && ((__GLIBC__ > 2) || ((__GLIBC__ == 2) && (__GLIBC_MINOR__ >= 17))))) + +// The following #include is an implementation artifact; not part of interface. +# ifdef __hpux +// HP-UX has a vaguely nice in a non-standard location +# include +# ifdef __STDC_32_MODE__ + // this is triggered with GCC, because it defines __cplusplus < 199707L +# define BOOST_NO_INT64_T +# endif +# elif defined(__FreeBSD__) || defined(__IBMCPP__) || defined(_AIX) +# include +# else +# include + +// There is a bug in Cygwin two _C macros +# if defined(INTMAX_C) && defined(__CYGWIN__) +# undef INTMAX_C +# undef UINTMAX_C +# define INTMAX_C(c) c##LL +# define UINTMAX_C(c) c##ULL +# endif + +# endif + +#if defined(__QNX__) && defined(__EXT_QNX) + +// QNX (Dinkumware stdlib) defines these as non-standard names. +// Reflect to the standard names. + +typedef ::intleast8_t int_least8_t; +typedef ::intfast8_t int_fast8_t; +typedef ::uintleast8_t uint_least8_t; +typedef ::uintfast8_t uint_fast8_t; + +typedef ::intleast16_t int_least16_t; +typedef ::intfast16_t int_fast16_t; +typedef ::uintleast16_t uint_least16_t; +typedef ::uintfast16_t uint_fast16_t; + +typedef ::intleast32_t int_least32_t; +typedef ::intfast32_t int_fast32_t; +typedef ::uintleast32_t uint_least32_t; +typedef ::uintfast32_t uint_fast32_t; + +# ifndef BOOST_NO_INT64_T + +typedef ::intleast64_t int_least64_t; +typedef ::intfast64_t int_fast64_t; +typedef ::uintleast64_t uint_least64_t; +typedef ::uintfast64_t uint_fast64_t; + +# endif + +#endif + +namespace boost +{ + + using ::int8_t; + using ::int_least8_t; + using ::int_fast8_t; + using ::uint8_t; + using ::uint_least8_t; + using ::uint_fast8_t; + + using ::int16_t; + using ::int_least16_t; + using ::int_fast16_t; + using ::uint16_t; + using ::uint_least16_t; + using ::uint_fast16_t; + + using ::int32_t; + using ::int_least32_t; + using ::int_fast32_t; + using ::uint32_t; + using ::uint_least32_t; + using ::uint_fast32_t; + +# ifndef BOOST_NO_INT64_T + + using ::int64_t; + using ::int_least64_t; + using ::int_fast64_t; + using ::uint64_t; + using ::uint_least64_t; + using ::uint_fast64_t; + +# endif + + using ::intmax_t; + using ::uintmax_t; + +} // namespace boost + +#elif defined(__FreeBSD__) && (__FreeBSD__ <= 4) || defined(__osf__) || defined(__VMS) || defined(__SOLARIS9__) || defined(__NetBSD__) +// FreeBSD and Tru64 have an that contains much of what we need. +# include + +namespace boost { + + using ::int8_t; + typedef int8_t int_least8_t; + typedef int8_t int_fast8_t; + using ::uint8_t; + typedef uint8_t uint_least8_t; + typedef uint8_t uint_fast8_t; + + using ::int16_t; + typedef int16_t int_least16_t; + typedef int16_t int_fast16_t; + using ::uint16_t; + typedef uint16_t uint_least16_t; + typedef uint16_t uint_fast16_t; + + using ::int32_t; + typedef int32_t int_least32_t; + typedef int32_t int_fast32_t; + using ::uint32_t; + typedef uint32_t uint_least32_t; + typedef uint32_t uint_fast32_t; + +# ifndef BOOST_NO_INT64_T + + using ::int64_t; + typedef int64_t int_least64_t; + typedef int64_t int_fast64_t; + using ::uint64_t; + typedef uint64_t uint_least64_t; + typedef uint64_t uint_fast64_t; + + typedef int64_t intmax_t; + typedef uint64_t uintmax_t; + +# else + + typedef int32_t intmax_t; + typedef uint32_t uintmax_t; + +# endif + +} // namespace boost + +#else // BOOST_HAS_STDINT_H + +# include // implementation artifact; not part of interface +# include // needed for limits macros + + +namespace boost +{ + +// These are fairly safe guesses for some 16-bit, and most 32-bit and 64-bit +// platforms. For other systems, they will have to be hand tailored. +// +// Because the fast types are assumed to be the same as the undecorated types, +// it may be possible to hand tailor a more efficient implementation. Such +// an optimization may be illusionary; on the Intel x86-family 386 on, for +// example, byte arithmetic and load/stores are as fast as "int" sized ones. + +// 8-bit types ------------------------------------------------------------// + +# if UCHAR_MAX == 0xff + typedef signed char int8_t; + typedef signed char int_least8_t; + typedef signed char int_fast8_t; + typedef unsigned char uint8_t; + typedef unsigned char uint_least8_t; + typedef unsigned char uint_fast8_t; +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif + +// 16-bit types -----------------------------------------------------------// + +# if USHRT_MAX == 0xffff +# if defined(__crayx1) + // The Cray X1 has a 16-bit short, however it is not recommend + // for use in performance critical code. + typedef short int16_t; + typedef short int_least16_t; + typedef int int_fast16_t; + typedef unsigned short uint16_t; + typedef unsigned short uint_least16_t; + typedef unsigned int uint_fast16_t; +# else + typedef short int16_t; + typedef short int_least16_t; + typedef short int_fast16_t; + typedef unsigned short uint16_t; + typedef unsigned short uint_least16_t; + typedef unsigned short uint_fast16_t; +# endif +# elif (USHRT_MAX == 0xffffffff) && defined(__MTA__) + // On MTA / XMT short is 32 bits unless the -short16 compiler flag is specified + // MTA / XMT does support the following non-standard integer types + typedef __short16 int16_t; + typedef __short16 int_least16_t; + typedef __short16 int_fast16_t; + typedef unsigned __short16 uint16_t; + typedef unsigned __short16 uint_least16_t; + typedef unsigned __short16 uint_fast16_t; +# elif (USHRT_MAX == 0xffffffff) && defined(CRAY) + // no 16-bit types on Cray: + typedef short int_least16_t; + typedef short int_fast16_t; + typedef unsigned short uint_least16_t; + typedef unsigned short uint_fast16_t; +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif + +// 32-bit types -----------------------------------------------------------// + +# if UINT_MAX == 0xffffffff + typedef int int32_t; + typedef int int_least32_t; + typedef int int_fast32_t; + typedef unsigned int uint32_t; + typedef unsigned int uint_least32_t; + typedef unsigned int uint_fast32_t; +# elif (USHRT_MAX == 0xffffffff) + typedef short int32_t; + typedef short int_least32_t; + typedef short int_fast32_t; + typedef unsigned short uint32_t; + typedef unsigned short uint_least32_t; + typedef unsigned short uint_fast32_t; +# elif ULONG_MAX == 0xffffffff + typedef long int32_t; + typedef long int_least32_t; + typedef long int_fast32_t; + typedef unsigned long uint32_t; + typedef unsigned long uint_least32_t; + typedef unsigned long uint_fast32_t; +# elif (UINT_MAX == 0xffffffffffffffff) && defined(__MTA__) + // Integers are 64 bits on the MTA / XMT + typedef __int32 int32_t; + typedef __int32 int_least32_t; + typedef __int32 int_fast32_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int32 uint_fast32_t; +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif + +// 64-bit types + intmax_t and uintmax_t ----------------------------------// + +# if defined(BOOST_HAS_LONG_LONG) && \ + !defined(BOOST_MSVC) && !defined(__BORLANDC__) && \ + (!defined(__GLIBCPP__) || defined(_GLIBCPP_USE_LONG_LONG)) && \ + (defined(ULLONG_MAX) || defined(ULONG_LONG_MAX) || defined(ULONGLONG_MAX)) +# if defined(__hpux) + // HP-UX's value of ULONG_LONG_MAX is unusable in preprocessor expressions +# elif (defined(ULLONG_MAX) && ULLONG_MAX == 18446744073709551615ULL) || (defined(ULONG_LONG_MAX) && ULONG_LONG_MAX == 18446744073709551615ULL) || (defined(ULONGLONG_MAX) && ULONGLONG_MAX == 18446744073709551615ULL) + // 2**64 - 1 +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif + + typedef ::boost::long_long_type intmax_t; + typedef ::boost::ulong_long_type uintmax_t; + typedef ::boost::long_long_type int64_t; + typedef ::boost::long_long_type int_least64_t; + typedef ::boost::long_long_type int_fast64_t; + typedef ::boost::ulong_long_type uint64_t; + typedef ::boost::ulong_long_type uint_least64_t; + typedef ::boost::ulong_long_type uint_fast64_t; + +# elif ULONG_MAX != 0xffffffff + +# if ULONG_MAX == 18446744073709551615 // 2**64 - 1 + typedef long intmax_t; + typedef unsigned long uintmax_t; + typedef long int64_t; + typedef long int_least64_t; + typedef long int_fast64_t; + typedef unsigned long uint64_t; + typedef unsigned long uint_least64_t; + typedef unsigned long uint_fast64_t; +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif +# elif defined(__GNUC__) && defined(BOOST_HAS_LONG_LONG) + __extension__ typedef long long intmax_t; + __extension__ typedef unsigned long long uintmax_t; + __extension__ typedef long long int64_t; + __extension__ typedef long long int_least64_t; + __extension__ typedef long long int_fast64_t; + __extension__ typedef unsigned long long uint64_t; + __extension__ typedef unsigned long long uint_least64_t; + __extension__ typedef unsigned long long uint_fast64_t; +# elif defined(BOOST_HAS_MS_INT64) + // + // we have Borland/Intel/Microsoft __int64: + // + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; + typedef __int64 int64_t; + typedef __int64 int_least64_t; + typedef __int64 int_fast64_t; + typedef unsigned __int64 uint64_t; + typedef unsigned __int64 uint_least64_t; + typedef unsigned __int64 uint_fast64_t; +# else // assume no 64-bit integers +# define BOOST_NO_INT64_T + typedef int32_t intmax_t; + typedef uint32_t uintmax_t; +# endif + +} // namespace boost + + +#endif // BOOST_HAS_STDINT_H + +// intptr_t/uintptr_t are defined separately because they are optional and not universally available +#if defined(BOOST_WINDOWS) && !defined(_WIN32_WCE) && !defined(BOOST_HAS_STDINT_H) +// Older MSVC don't have stdint.h and have intptr_t/uintptr_t defined in stddef.h +#include +#endif + +#if (defined(BOOST_WINDOWS) && !defined(_WIN32_WCE)) \ + || (defined(_XOPEN_UNIX) && (_XOPEN_UNIX+0 > 0) && !defined(__UCLIBC__)) \ + || defined(__CYGWIN__) || defined(__VXWORKS__) \ + || defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) \ + || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || (defined(sun) && !defined(BOOST_HAS_STDINT_H)) || defined(INTPTR_MAX) + +namespace boost { + using ::intptr_t; + using ::uintptr_t; +} +#define BOOST_HAS_INTPTR_T + +// Clang pretends to be GCC, so it'll match this condition +#elif defined(__GNUC__) && defined(__INTPTR_TYPE__) && defined(__UINTPTR_TYPE__) + +namespace boost { + typedef __INTPTR_TYPE__ intptr_t; + typedef __UINTPTR_TYPE__ uintptr_t; +} +#define BOOST_HAS_INTPTR_T + +#endif + +#endif // BOOST_CSTDINT_HPP + + +/**************************************************** + +Macro definition section: + +Added 23rd September 2000 (John Maddock). +Modified 11th September 2001 to be excluded when +BOOST_HAS_STDINT_H is defined (John Maddock). +Modified 11th Dec 2009 to always define the +INT#_C macros if they're not already defined (John Maddock). + +******************************************************/ + +#if !defined(BOOST__STDC_CONSTANT_MACROS_DEFINED) && \ + (!defined(INT8_C) || !defined(INT16_C) || !defined(INT32_C) || !defined(INT64_C)) +// +// Undef the macros as a precaution, since we may get here if has failed +// to define them all, see https://svn.boost.org/trac/boost/ticket/12786 +// +#undef INT8_C +#undef INT16_C +#undef INT32_C +#undef INT64_C +#undef INTMAX_C +#undef UINT8_C +#undef UINT16_C +#undef UINT32_C +#undef UINT64_C +#undef UINTMAX_C + +#include +# define BOOST__STDC_CONSTANT_MACROS_DEFINED +# if defined(BOOST_HAS_MS_INT64) +// +// Borland/Intel/Microsoft compilers have width specific suffixes: +// +#ifndef INT8_C +# define INT8_C(value) value##i8 +#endif +#ifndef INT16_C +# define INT16_C(value) value##i16 +#endif +#ifndef INT32_C +# define INT32_C(value) value##i32 +#endif +#ifndef INT64_C +# define INT64_C(value) value##i64 +#endif +# ifdef __BORLANDC__ + // Borland bug: appending ui8 makes the type a signed char +# define UINT8_C(value) static_cast(value##u) +# else +# define UINT8_C(value) value##ui8 +# endif +#ifndef UINT16_C +# define UINT16_C(value) value##ui16 +#endif +#ifndef UINT32_C +# define UINT32_C(value) value##ui32 +#endif +#ifndef UINT64_C +# define UINT64_C(value) value##ui64 +#endif +#ifndef INTMAX_C +# define INTMAX_C(value) value##i64 +# define UINTMAX_C(value) value##ui64 +#endif + +# else +// do it the old fashioned way: + +// 8-bit types ------------------------------------------------------------// + +# if (UCHAR_MAX == 0xff) && !defined(INT8_C) +# define INT8_C(value) static_cast(value) +# define UINT8_C(value) static_cast(value##u) +# endif + +// 16-bit types -----------------------------------------------------------// + +# if (USHRT_MAX == 0xffff) && !defined(INT16_C) +# define INT16_C(value) static_cast(value) +# define UINT16_C(value) static_cast(value##u) +# endif + +// 32-bit types -----------------------------------------------------------// +#ifndef INT32_C +# if (UINT_MAX == 0xffffffff) +# define INT32_C(value) value +# define UINT32_C(value) value##u +# elif ULONG_MAX == 0xffffffff +# define INT32_C(value) value##L +# define UINT32_C(value) value##uL +# endif +#endif + +// 64-bit types + intmax_t and uintmax_t ----------------------------------// +#ifndef INT64_C +# if defined(BOOST_HAS_LONG_LONG) && \ + (defined(ULLONG_MAX) || defined(ULONG_LONG_MAX) || defined(ULONGLONG_MAX) || defined(_ULLONG_MAX) || defined(_LLONG_MAX)) + +# if defined(__hpux) + // HP-UX's value of ULONG_LONG_MAX is unusable in preprocessor expressions +# define INT64_C(value) value##LL +# define UINT64_C(value) value##uLL +# elif (defined(ULLONG_MAX) && ULLONG_MAX == 18446744073709551615ULL) || \ + (defined(ULONG_LONG_MAX) && ULONG_LONG_MAX == 18446744073709551615ULL) || \ + (defined(ULONGLONG_MAX) && ULONGLONG_MAX == 18446744073709551615ULL) || \ + (defined(_ULLONG_MAX) && _ULLONG_MAX == 18446744073709551615ULL) || \ + (defined(_LLONG_MAX) && _LLONG_MAX == 9223372036854775807LL) + +# define INT64_C(value) value##LL +# define UINT64_C(value) value##uLL +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif +# elif ULONG_MAX != 0xffffffff + +# if ULONG_MAX == 18446744073709551615U // 2**64 - 1 +# define INT64_C(value) value##L +# define UINT64_C(value) value##uL +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif +# elif defined(BOOST_HAS_LONG_LONG) + // Usual macros not defined, work things out for ourselves: +# if(~0uLL == 18446744073709551615ULL) +# define INT64_C(value) value##LL +# define UINT64_C(value) value##uLL +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif +# else +# error defaults not correct; you must hand modify boost/cstdint.hpp +# endif + +# ifdef BOOST_NO_INT64_T +# define INTMAX_C(value) INT32_C(value) +# define UINTMAX_C(value) UINT32_C(value) +# else +# define INTMAX_C(value) INT64_C(value) +# define UINTMAX_C(value) UINT64_C(value) +# endif +#endif +# endif // Borland/Microsoft specific width suffixes + +#endif // INT#_C macros. diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..8aa5bbe7 --- /dev/null +++ b/environment.yml @@ -0,0 +1,6 @@ +name: autowrap +channels: + - conda-forge +dependencies: + - cython>=0.19 + - pytest \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 9955decc..00000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest -pytest-cov diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 8dc03ff2..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -#[egg_info] -#tag_build = dev diff --git a/setup.py b/setup.py index 24409fa6..a1a49a74 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import find_packages, setup versionfile = "autowrap/version.py" -try: +try: # py2 execfile(versionfile) except: exec(open(versionfile).read()) diff --git a/tests/test_code_generator.py b/tests/test_code_generator.py index 97efceaa..c2adfe3c 100644 --- a/tests/test_code_generator.py +++ b/tests/test_code_generator.py @@ -279,6 +279,8 @@ def test_gil_unlock(): g = wrapped.GilTesting(b"Jack") g.do_something(b"How are you?") assert g.get_greetings() == b"Hello Jack, How are you?" + g.do_something2(b"How are you2?") + assert g.get_greetings() == b"Hello Jack, How are you2?" def test_automatic_string_conversion(): diff --git a/tests/test_code_generator_minimal.py b/tests/test_code_generator_minimal.py index 01d7cfb3..cd6a8a6a 100644 --- a/tests/test_code_generator_minimal.py +++ b/tests/test_code_generator_minimal.py @@ -68,11 +68,12 @@ def type_check_expression(self, cpp_type, argument_var): def input_conversion(self, cpp_type, argument_var, arg_num): code = "" - # here we inject special behavoir for testing if this converter + # here we inject special behavior for testing if this converter # was called ! call_as = "(1 + %s)" % argument_var + decl = ("cdef int %s = %s" % ("c_" + argument_var, call_as), "c_" + argument_var) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl def output_conversion(self, cpp_type, input_cpp_var, output_py_var): return "%s = %s" % (output_py_var, input_cpp_var) diff --git a/tests/test_files/converters/IntHolderConverter.py b/tests/test_files/converters/IntHolderConverter.py index 292e29d3..0d646874 100644 --- a/tests/test_files/converters/IntHolderConverter.py +++ b/tests/test_files/converters/IntHolderConverter.py @@ -16,19 +16,20 @@ def type_check_expression(self, cpp_type, argument_var): return "isinstance(%s, int)" % (argument_var,) def input_conversion(self, cpp_type, argument_var, arg_num): - # here we inject special behavoir for testing if this converter + # here we inject special behavior for testing if this converter # was called ! ih_name = "ih_" + argument_var code = Code().add( """ - |cdef _Holder[int] $ih_name - |$ih_name.set($argument_var) - """, + |cdef _Holder[int] $ih_name + |$ih_name.set($argument_var) + """, locals(), ) call_as = "(%s)" % ih_name + decl = ("", call_as) cleanup = "" - return code, call_as, cleanup + return code, call_as, cleanup, decl def output_conversion(self, cpp_type, input_cpp_var, output_py_var): return "%s = (%s.get())" % (output_py_var, input_cpp_var) diff --git a/tests/test_files/gil_testing.hpp b/tests/test_files/gil_testing.hpp index 74b23688..800b5bdd 100644 --- a/tests/test_files/gil_testing.hpp +++ b/tests/test_files/gil_testing.hpp @@ -21,6 +21,35 @@ class GilTesting{ has_gil = (tstate && (tstate == PyGILState_GetThisThreadState())); #endif +#else + // For Python 2.0 + PyThreadState * tstate = _PyThreadState_Current; + has_gil = (tstate && (tstate == PyGILState_GetThisThreadState())); +#endif + if (has_gil) { + greetings_ = "Hello "; + greetings_.append(name_); + greetings_.append(", Sorry the GIL is locked, test failed."); + } else { + greetings_ = "Hello "; + greetings_.append(name_); + greetings_.append(", "); + greetings_.append(msg); + } + } + + void do_something2 (const std::string& msg) { + + bool has_gil; +#if PY_MAJOR_VERSION >= 3 +#if PY_MINOR_VERSION >= 4 + // Python >= 3.4 is easy + has_gil = PyGILState_Check(); +#else + PyThreadState * tstate = (PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current); + has_gil = (tstate && (tstate == PyGILState_GetThisThreadState())); +#endif + #else // For Python 2.0 PyThreadState * tstate = _PyThreadState_Current; diff --git a/tests/test_files/gil_testing.pxd b/tests/test_files/gil_testing.pxd index f7595f7f..5ffb465d 100644 --- a/tests/test_files/gil_testing.pxd +++ b/tests/test_files/gil_testing.pxd @@ -1,8 +1,10 @@ # cython: language_level=2 from libc.string cimport const_char +from libcpp.string cimport string as libcpp_string cdef extern from "gil_testing.hpp": cdef cppclass GilTesting: GilTesting(const_char*) void do_something(const_char*) nogil # wrap-with-no-gil const_char* get_greetings() + void do_something2(libcpp_string s) nogil # wrap-with-no-gil diff --git a/tests/test_files/inherited.pyx b/tests/test_files/inherited.pyx index 50ad5224..8a0c2de4 100644 --- a/tests/test_files/inherited.pyx +++ b/tests/test_files/inherited.pyx @@ -1,8 +1,9 @@ -#Generated with autowrap 0.22.10 and Cython (Parser) 0.29.32 +#Generated with autowrap 0.22.12 and Cython (Parser) 3.0.5 #cython: c_string_encoding=ascii #cython: embedsignature=False -from enum import Enum as _PyEnum -from cpython cimport Py_buffer +from enum import Enum as _PyEnum +from cpython cimport Py_buffer +from cpython cimport bool as pybool_t from libcpp.string cimport string as libcpp_string from libcpp.string cimport string as libcpp_utf8_string from libcpp.string cimport string as libcpp_utf8_output_string diff --git a/tests/test_files/libcpp_stl_test.pxd b/tests/test_files/libcpp_stl_test.pxd index 934b0528..c47a664e 100644 --- a/tests/test_files/libcpp_stl_test.pxd +++ b/tests/test_files/libcpp_stl_test.pxd @@ -47,9 +47,9 @@ cdef extern from "libcpp_stl_test.hpp": libcpp_vector[IntWrapper*] process_6_vector(IntWrapper* in_) int process_7_map(libcpp_map[IntWrapper, int] & in_) - libcpp_map[IntWrapper, int] process_8_map(int in_) + libcpp_map[IntWrapper, int] process_8_map(int in_) nogil # wrap-with-no-gil - int process_9_map(libcpp_map[int, IntWrapper] & in_) + int process_9_map(libcpp_map[int, IntWrapper] & in_) nogil # wrap-with-no-gil libcpp_map[int, IntWrapper] process_10_map(int in_) shared_ptr[const IntWrapper] process_11_const() From 8eb1396b201bea3156302901ff3d874359d3a984 Mon Sep 17 00:00:00 2001 From: julianuspfeuffer Date: Tue, 21 Nov 2023 09:47:30 +0100 Subject: [PATCH 2/2] fix some typings --- autowrap/ConversionProvider.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autowrap/ConversionProvider.py b/autowrap/ConversionProvider.py index 681e083e..a9568bf1 100644 --- a/autowrap/ConversionProvider.py +++ b/autowrap/ConversionProvider.py @@ -978,7 +978,7 @@ def type_check_expression(self, cpp_type, arg_var): def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Code, str, Code | str, Tuple[str, str]]: + ) -> Tuple[Code, str, Union[Code, str], Tuple[str, str]]: tt_key, tt_value = cpp_type.template_args temp_var = "v%d" % arg_num @@ -1341,7 +1341,7 @@ def type_check_expression(self, cpp_type, arg_var): def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Code, str, Code | str, Tuple[str, str]]: + ) -> Tuple[Code, str, Union[Code, str], Tuple[str, str]]: (tt,) = cpp_type.template_args temp_var = "v%d" % arg_num inner = self.converters.cython_type(tt) @@ -1781,7 +1781,7 @@ def input_conversion( topmost_code: Optional[Code] = None, bottommost_code: Optional[Code] = None, recursion_cnt: int = 0, - ) -> Tuple[Code, str, Code | str, Tuple[str, str]]: + ) -> Tuple[Code, str, Union[Code, str], Tuple[str, str]]: """Do the input conversion for a std::vector In this case, the template argument is tt (or "inner"). @@ -2250,7 +2250,7 @@ def matching_python_type_full(self, cpp_type: CppType) -> str: def input_conversion( self, cpp_type: CppType, argument_var: str, arg_num: int - ) -> Tuple[Code, str, Union[Code, str]]: + ) -> Tuple[Code, str, Union[Code, str], Tuple[str, str]]: (tt,) = cpp_type.template_args inner = self.converters.cython_type(tt) # Cython expects us to get a C++ type (we cannot just stick var.inst into the function)