Fix #4 and implement opaque wrap

- Fix bug #4
- Add possibility to wrap a class as a PyCapsule of std::any
  - add converter
  - add C2PY_OPAQUE annotation
  - add opaque_match_names optional regex
  - Cf "any" example for usage.

- TODO update the doc.

src/plugins/c2py/codegen/classes.cpp

@@ -18,20 +18,23 @@ static const struct {
 
 // ===================================================================
 
-void codegen_synth_constructor(std::ostream &code, clang::CXXRecordDecl const *cls) {
-
-  logs.cls_details(fmt::format("Synthesize constructor from pydict", clu::get_fully_qualified_name(cls))); //->getQualifiedNameAsString()));
+void codegen_synth_constructor(std::ostream &code, cls_info_t const &cls_info) {
+  auto const *cls = cls_info.ptr;
+  auto cls_name   = clu::get_fully_qualified_name(cls); //cls->getQualifiedNameAsString();
+  logs.cls_details(fmt::format("Synthesize constructor from pydict", cls_name));
   static long counter = 0;
 
   std::vector<std::string> non_default_const_params;
   std::vector<clang::FieldDecl *> simple_fields;
   simple_fields.reserve(100); //NOLINT
 
-  for (clang::FieldDecl *f : cls->fields()) {
-    if (f->getAccess() != clang::AS_public) { // FIXME error is quite late... in codegen ...
+  for (auto *f : cls_info.fields) {
+    // FIXME error is quite late. Move up
+    if (f->getAccess() != clang::AS_public) {
       clu::emit_error(f, "c2py: Synthetizing constructor for pydict. Private fields not supported");
       continue;
     }
+
     // if f is a type, which has no default constructor and no defaut initializer is the class
     // we build it at the construction of the object, using designated initializer (as we skip other fields)
     if (auto *clsf = f->getType()->getAsCXXRecordDecl(); clsf and not clsf->hasDefaultConstructor() and (f->getInClassInitializer() == nullptr)) {
@@ -220,7 +223,7 @@ void codegen_cls(std::ostream &code, str_t const &cls_py_name, cls_info_t const
     // ---- constructor
     if (cls_info.synthetize_init_from_pydict()) {
       // FIXME : check
-      codegen_synth_constructor(MethodDecls, cls_info.ptr);
+      codegen_synth_constructor(MethodDecls, cls_info);
     } else
       codegen::write_dispatch_constructors(MethodDecls, cls_name, cls_info.constructors);
 
@@ -261,7 +264,7 @@ void codegen_cls(std::ostream &code, str_t const &cls_py_name, cls_info_t const
   static long member_counter = 0;
 
   std::stringstream MembersDoc, Members;
-  for (auto f : cls_info.fields) {
+  for (auto *f : cls_info.fields) {
 
     auto name     = str_t{f->getName()};
     auto type     = clu::get_fully_qualified_name(f->getType(), f->getASTContext());

src/plugins/c2py/codegen/doc.cpp

@@ -2,6 +2,7 @@
 #include "clu/doc_string.hpp"
 #include "data.hpp"
 #include "clu/fullqualifiedname.hpp"
+#include <clang/AST/DeclCXX.h>
 #include <fmt/core.h>
 #include <fmt/format.h>
 #include <string>
@@ -25,27 +26,27 @@ std::string pydoc(fnt_info_t const &f) {
   return util::indent_string(util::trim(fs.str()), "   ");
 }
 //---------------------------------------------------------
-std::string doc_of_synthetized_constructor(cls_ptr_t cls);
+std::string doc_of_synthetized_constructor(cls_info_t const &cls_info);
 
 std::string pydoc(cls_info_t const &cls) {
   std::stringstream fs;
   auto doc = clu::doc_string_t{cls.ptr};
   if (not doc.brief.empty()) fs << doc.brief << "\n\n";
   if (not doc.content.empty()) fs << doc.content << "\n\n";
-  if (cls.synthetize_init_from_pydict()) fs << doc_of_synthetized_constructor(cls.ptr);
+  if (cls.synthetize_init_from_pydict()) fs << doc_of_synthetized_constructor(cls);
   return util::indent_string(util::trim(fs.str()), "   ");
 }
 
 // ----------------------------------------------
 
 // vector of [name, c++ type, initializer, doc]
-std::vector<std::vector<std::string>> get_fields_info(const clang::CXXRecordDecl *cls) {
+std::vector<std::vector<std::string>> get_fields_info(cls_info_t const &cls_info) {
   std::vector<std::vector<std::string>> res;
   //res.push_back({"Field name", "C++ type", "Initializer", "Documentation"});
-  clang::ASTContext *ctx = &cls->getASTContext();
+  clang::CXXRecordDecl const *cls = cls_info.ptr;
+  clang::ASTContext *ctx          = &cls->getASTContext();
 
-  for (clang::FieldDecl *f : cls->fields()) {
-    if (f->getAccess() != clang::AS_public) continue;
+  for (auto *f : cls_info.fields) {
     std::vector<std::string> m(4);
     m[0] = f->getNameAsString();
     m[1] = clu::get_fully_qualified_name(f->getType(), *ctx);
@@ -74,15 +75,15 @@ std::vector<std::vector<std::string>> get_fields_info(const clang::CXXRecordDecl
 
 //---------------------------------------------------------
 
-std::string doc_of_synthetized_constructor(cls_ptr_t cls) {
+std::string doc_of_synthetized_constructor(cls_info_t const &cls_info) {
   std::stringstream doc;
   // doc << '\n';
   static std::regex start1(R"RAW(^\s*\/*\s*)RAW");
   static std::regex start2(R"RAW(\n\s*\/*\s*)RAW");
   // FIXME : clean when upgrading
   // gcc 11 does not have the multline implemented ...
   //static std::regex start(R"RAW(^\s*\/*\s*)RAW", std::regex_constants::multiline);
-  for (auto const &f : get_fields_info(cls)) {
+  for (auto const &f : get_fields_info(cls_info)) {
     doc << "* " << f[0] << ": " << f[1];
     if (not f[2].empty()) doc << " = " << f[2];
     doc << "\n" << std::regex_replace(std::regex_replace(f[3], start1, "   "), start2, "\n   ");

src/plugins/c2py/codegen/module.cpp

@@ -128,6 +128,13 @@ str_t codegen_module(module_info_t const &m) {
   std::stringstream ClassesDecls, PyTypeReadyDecls, AddTypeObjectDecls;
   std::stringstream Hdf5C2pyIncluder, Hdf5RegistrationInit, Hdf5Registration;
 
+  // classes wrapped as pycapsule : just use define the converter.
+  for (auto const *cls : m.classes_wrap_opaque) {
+    ClassesDecls << fmt::format(R"RAW(  template <> struct c2py::py_converter<{0}> : c2py::py_converter_as_any<{0}> {{}};)RAW",
+                                clu::get_fully_qualified_name(cls));
+  }
+
+  // classes fully wrapped
   for (auto const &[cls_py_name, cls_info] : m.classes) {
     codegen_cls(ClassesDecls, cls_py_name, cls_info, full_module_name);
 

src/plugins/c2py/data.cpp

@@ -53,3 +53,21 @@ void analyse_dispatch(std::map<str_t, std::vector<fnt_info_t>> &fmap, clang::Var
   } else
     clu::emit_error(decl, "c2py: only c2py::dispatch<...> declaration is authorized here");
 }
+
+// -----------------------------
+bool is_rejected(clang::Decl const *decl, std::optional<std::regex> const &reject_regex, util::logger const *log) {
+  auto *named_decl = llvm::dyn_cast<clang::NamedDecl>(decl);
+  if (!named_decl) return true; // no name -> reject
+  auto name = named_decl->getQualifiedNameAsString();
+  // is annoted explicitely -> reject
+  if (clu::has_annotation(named_decl, "c2py_ignore")) {
+    if (log) (*log)(fmt::format(R"RAW({0} [{1}])RAW", name, "C2PY_IGNORE"));
+    return true;
+  }
+  // matches the regex -> reject
+  if (reject_regex and std::regex_match(name, reject_regex.value())) {
+    if (log) (*log)(fmt::format(R"RAW({0} [{1}])RAW", name, "reject_names"));
+    return true;
+  }
+  return false;
+}

src/plugins/c2py/data.hpp

@@ -3,6 +3,7 @@
 #include <map>
 #include <optional>
 #include "utility/string_tools.hpp"
+#include "utility/logger.hpp"
 //#include "clu/misc.hpp"
 #include "clang/AST/DeclCXX.h"
 
@@ -12,9 +13,9 @@ using fnt_ptr_t = clang::FunctionDecl const *;
 // -----------------------------------------------------------
 
 struct fnt_info_t {
-  fnt_ptr_t ptr                            = nullptr;
-  bool rewrite                             = false; // (as_method() ? false : true); //false; // do we need to rewrite it with a lambda (e.g. friend)
-  clang::CXXRecordDecl const *parent_class = nullptr;
+  fnt_ptr_t ptr          = nullptr;
+  bool rewrite           = false; // (as_method() ? false : true); //false; // do we need to rewrite it with a lambda (e.g. friend)
+  cls_ptr_t parent_class = nullptr;
   //bool no_gil                              = clu::has_annotation(ptr, "c2py_nogil");
   [[nodiscard]] clang::CXXMethodDecl const *as_method() const { return llvm::dyn_cast_or_null<clang::CXXMethodDecl>(ptr); }
 };
@@ -26,6 +27,12 @@ std::vector<fnt_info_t> make_unique(std::vector<fnt_info_t> const &flist);
 // Used by worker and matchers (for methods and funtion resp.)
 void analyse_dispatch(std::map<str_t, std::vector<fnt_info_t>> &fmap, clang::VarDecl const *decl);
 
+/// Should the decl be ignored due to
+/// i) a c2py_ignore annotation
+/// ii) its qualified name matches the regex
+/// If log is present, it logs the rejection
+bool is_rejected(clang::Decl const *decl, std::optional<std::regex> const &reject_regex, util::logger const *log = nullptr);
+
 // -----------------------------------------------------------
 // Serialization method
 enum class Serialization { None, Tuple, H5, Repr };
@@ -70,10 +77,11 @@ struct module_info_t {
 
   // Filters
   std::string match_names, match_files; // string used a regex in the AST Matchers directly
-  std::optional<std::regex> reject_names;
+  std::optional<std::regex> reject_names, opaque_match_names;
   bool get_set_as_properties = false;
 
-  std::map<str_t, std::vector<fnt_info_t>> functions; // vector not unique
-  std::vector<std::pair<str_t, cls_info_t>> classes;  // index of cls_table. Must keep order of insertion to have base first
-  std::vector<clang::EnumDecl const *> enums;         // all enums (including in classes)
+  std::map<str_t, std::vector<fnt_info_t>> functions;            // vector not unique
+  std::vector<std::pair<str_t, cls_info_t>> classes;             // index of cls_table. Must keep order of insertion to have base first
+  std::vector<clang::CXXRecordDecl const *> classes_wrap_opaque; // classes to be wrapped as Pycapsule
+  std::vector<clang::EnumDecl const *> enums;                    // all enums (including in classes)
 };

src/plugins/c2py/matchers.cpp

@@ -7,6 +7,7 @@
 static const struct {
   util::logger rejected = util::logger{&std::cout, "-- ", "\033[1;33mRejecting: \033[0m"};
   util::logger note     = util::logger{&std::cout, "-- ", "\033[1;32mNote:  \033[0m"};
+  util::logger error    = util::logger{&std::cout, "-- ", "\033[1;33mError:  \033[0m"};
 } logs;
 namespace matchers {
 
@@ -40,7 +41,10 @@ namespace matchers {
     extract_literal(d, s);
     try {
       if (not s.empty()) x = std::regex{s};
-    } catch (std::regex_error const &e) { clu::emit_error(d, "c2py: invalid C++ regular expression. Cf std::regex documentation for information."); }
+    } catch (std::regex_error const &e) {
+      logs.error(fmt::format("Regular Expression is invalid: \n {}", e.what()));
+      clu::emit_error(d, "c2py: invalid C++ regular expression. Cf std::regex documentation for information.");
+    }
   }
 
   // Bool
@@ -112,6 +116,7 @@ namespace matchers {
        {"match_names", [](auto *d, auto &M) { extract_literal(d, M.match_names); }},
        {"reject_names", [](auto *d, auto &M) { extract_literal(d, M.reject_names); }},
        {"match_files", [](auto *d, auto &M) { extract_literal(d, M.match_files); }},
+       {"opaque_match_names", [](auto *d, auto &M) { extract_literal(d, M.opaque_match_names); }},
     };
 
     if (auto it = vars.find(decl->getName().str()); it != vars.end())
@@ -186,27 +191,26 @@ namespace matchers {
     // Reject template specialization
     if (llvm::dyn_cast_or_null<clang::ClassTemplateSpecializationDecl>(cls)) return;
 
-    //  Discard if C2PY_ignore annotation is present
-    if (clu::has_annotation(cls, "c2py_ignore")) return;
-
     // Apply the filters
     auto qname = cls->getQualifiedNameAsString();
     auto &M    = worker->module_info;
 
+    // apply c2py_ignore and reject_names
+    if (is_rejected(cls, M.reject_names, &logs.rejected)) return;
+
     // Reject classes defined in c2py_module
     if (qname.starts_with("c2py_module::")) return;
 
-    bool force_wrap = (clu::has_annotation(cls, "c2py_wrap"));
+    if (clu::has_annotation(cls, "c2py_wrap_as_opaque") or (M.opaque_match_names and std::regex_match(qname, M.opaque_match_names.value()))) {
+      // The class is wrapped as pycapsule
+      M.classes_wrap_opaque.push_back(cls);
+    } else {
+      // Insert in the module class list
+      str_t py_name = util::camel_case(cls->getNameAsString());
+      M.classes.emplace_back(py_name, cls_info_t{cls}); //
 
-    if (!force_wrap and M.reject_names and std::regex_match(qname, M.reject_names.value())) {
-      logs.rejected(fmt::format(R"RAW({0} [{1}])RAW", qname, "reject_names"));
-      return;
+      //if (not inserted) clu::emit_error(cls, "Class rejected. Should have another class with the same Python name ??");
     }
-    // Insert in the module class list
-    str_t py_name = util::camel_case(cls->getNameAsString());
-    M.classes.emplace_back(py_name, cls_info_t{cls}); //
-
-    //if (not inserted) clu::emit_error(cls, "Class rejected. Should have another class with the same Python name ??");
   }
 
   // -------------------------------------------------
@@ -236,29 +240,21 @@ namespace matchers {
     // Reject the declaration of the template itself.
     if (f->getDescribedFunctionTemplate()) return;
 
-    // reject method --> FIXME : in matcher ?
+    // reject method
+    // FIXME : in matcher ?
     if (llvm::dyn_cast_or_null<clang::CXXMethodDecl>(f)) return;
 
-    //  Discard if C2PY_ignore annotation is present
-    if (clu::has_annotation(f, "c2py_ignore")) return;
-
     // Discard some special function
     if (f->getNameAsString().starts_with("operator")) return;
 
-    // Apply the filters. regex_filter, regex_exclude, then the _fun version
-    auto fqname = f->getQualifiedNameAsString();
-    auto &M     = worker->module_info;
+    // apply c2py_ignore and the reject_name regex
+    auto &M = worker->module_info;
+    if (is_rejected(f, M.reject_names, &logs.rejected)) return;
 
     // Reject functions defined in c2py_module
+    auto fqname = f->getQualifiedNameAsString();
     if (fqname.starts_with("c2py_module::")) return;
 
-    bool force_wrap = (clu::has_annotation(f, "c2py_wrap"));
-
-    if (!force_wrap and M.reject_names and std::regex_match(fqname, M.reject_names.value())) {
-      logs.rejected(fmt::format(R"RAW({0} [{1}])RAW", fqname, "reject_names"));
-      return;
-    }
-
     // Insert in the module function list. Unicity will be taken care of later by worker.
     str_t py_name = f->getNameAsString();
     M.functions[py_name].push_back(fnt_info_t{f});

src/plugins/c2py/worker.cpp

@@ -1,3 +1,4 @@
+#include <clang/AST/Decl.h>
 #include <regex>
 #include <numeric>
 #include <filesystem>
@@ -59,23 +60,17 @@ void scan_class_elements(cls_info_t &cls_info, module_info_t &m_info, cls_ptr_t
 
   for (clang::Decl *decl : cls->decls()) { // all declarations in the class
     if (decl->getAccess() != clang::AS_public) continue;
+    if (is_rejected(decl, m_info.reject_names, &logs.rejected)) continue;
 
     // -------- method
     if (auto *m = llvm::dyn_cast<clang::CXXMethodDecl>(decl)) {
 
-      if (clu::has_annotation(m, "c2py_ignore")) continue;
       if (llvm::isa<clang::CXXDestructorDecl>(m)) continue;                 // no destructors
       if (m->isMoveAssignmentOperator()) continue;                          // no move assign
       if (auto *ctr = llvm::dyn_cast_or_null<clang::CXXConstructorDecl>(m); //
           ctr and ctr->isCopyOrMoveConstructor())
         continue; // no move or copy constructor
 
-      if (m_info.reject_names and std::regex_match(m->getNameAsString(), m_info.reject_names.value())) {
-        logs.rejected(fmt::format(R"RAW({0} [{1}])RAW", m->getNameAsString(), "reject_names"));
-        //clu::emit_warning(m, "Rejected this method");
-        continue;
-      }
-
       // Operators : keep only [] and ()
       static auto const re = std::regex{"operator(.*)"};
       std::smatch ma;
@@ -251,8 +246,8 @@ void worker_t::separate_properties() {
 // Check convertibility of parameters, return type, and fields
 void worker_t::check_convertibility() {
 
-  // ordred list of all wrapped class pointer
-  std::vector<cls_ptr_t> wcls;
+  // ordred list of all wrapped class pointer (including opaque ones)
+  std::vector<cls_ptr_t> wcls = this->module_info.classes_wrap_opaque; // start with all the classes with pycapsule wrapping
   for (auto const &[n, clsi] : this->module_info.classes) wcls.push_back(clsi.ptr);
   std::sort(wcls.begin(), wcls.end());