Skip to content

Commit

Permalink
.
Browse files Browse the repository at this point in the history
  • Loading branch information
@traversc
traversc committed Nov 11, 2024
1 parent d78be2d commit 1d73eb0
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 38 deletions.
19 changes: 15 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,15 +151,26 @@ using namespace Rcpp;

// [[Rcpp::export]]
SEXP test_qs_serialize(SEXP x) {
uint64_t len = 0;
char * buffer = c_qs_serialize(x, &len, 10, 4); // object, buffer length, compress_level, nthreads
SEXP y = c_qs_deserialize(buffer, len, 4); // buffer, buffer length, nthreads
free(buffer); // must manually free buffer
size_t len = 0;
unsigned char * buffer = c_qs_serialize(x, &len, 10, true, 4); // object, buffer length, compress_level, shuffle, nthreads
SEXP y = c_qs_deserialize(buffer, len, false, 4); // buffer, buffer length, validate_checksum, nthreads
c_qs_free(buffer); // must manually free buffer
return y;
}

// [[Rcpp::export]]
SEXP test_qd_serialize(SEXP x) {
size_t len = 0;
unsigned char * buffer = c_qd_serialize(x, &len, 10, true, 4); // object, buffer length, compress_level, shuffle, nthreads
SEXP y = c_qd_deserialize(buffer, len, false, false, 4); // buffer, buffer length, use_alt_rep, validate_checksum, nthreads
c_qd_free(buffer); // must manually free buffer
return y;
}


/*** R
x <- runif(1e7)
stopifnot(test_qs_serialize(x) == x)
stopifnot(test_qd_serialize(x) == x)
*/
```
24 changes: 16 additions & 8 deletions inst/include/qs2_external.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,28 @@
extern "C" {
#endif

char * c_qs_serialize(SEXP object, uint64_t * len, const int compress_level = 3, const bool shuffle = true, const int nthreads = 1) {
static char *(*fun)(SEXP, uint64_t *, const int, const bool, const int) = (char *(*)(SEXP, uint64_t *, const int, const bool, const int)) R_GetCCallable("qs2", "c_qs_serialize");
unsigned char * c_qs_serialize(SEXP object, size_t * len, const int compress_level = 3, const bool shuffle = true, const int nthreads = 1) {
static unsigned char *(*fun)(SEXP, size_t *, const int, const bool, const int) = (unsigned char *(*)(SEXP, size_t *, const int, const bool, const int)) R_GetCCallable("qs2", "c_qs_serialize");
return fun(object, len, compress_level, shuffle, nthreads);
}
SEXP c_qs_deserialize(const char * buffer, const uint64_t len, const bool validate_checksum = false, const int nthreads = 1) {
static SEXP(*fun)(const char *, const uint64_t, const bool, const int) = (SEXP(*)(const char *, const uint64_t, const bool, const int)) R_GetCCallable("qs2", "c_qs_deserialize");
bool c_qs_free(void *buffer) {
static bool (*fun)(void *) = (bool (*)(void *)) R_GetCCallable("qs2", "c_qs_free");
return fun(buffer);
}
SEXP c_qs_deserialize(const unsigned char * buffer, const size_t len, const bool validate_checksum = false, const int nthreads = 1) {
static SEXP(*fun)(const unsigned char *, const size_t, const bool, const int) = (SEXP(*)(const unsigned char *, const size_t, const bool, const int)) R_GetCCallable("qs2", "c_qs_deserialize");
return fun(buffer, len, validate_checksum, nthreads);
}
char * c_qd_serialize(SEXP object, uint64_t * len, const int compress_level = 3, const bool shuffle = true, const bool warn_unsupported_types = true, const int nthreads = 1) {
static char *(*fun)(SEXP, uint64_t *, const int, const bool, const bool, const int) = (char *(*)(SEXP, uint64_t *, const int, const bool, const bool, const int)) R_GetCCallable("qs2", "c_qd_serialize");
unsigned char * c_qd_serialize(SEXP object, size_t * len, const int compress_level = 3, const bool shuffle = true, const bool warn_unsupported_types = true, const int nthreads = 1) {
static unsigned char *(*fun)(SEXP, size_t *, const int, const bool, const bool, const int) = (unsigned char *(*)(SEXP, size_t *, const int, const bool, const bool, const int)) R_GetCCallable("qs2", "c_qd_serialize");
return fun(object, len, compress_level, shuffle, warn_unsupported_types, nthreads);
}
SEXP c_qd_deserialize(const char * buffer, const uint64_t len, const bool use_alt_rep = false, const bool validate_checksum = false, const int nthreads = 1) {
static SEXP(*fun)(const char *, const uint64_t, const bool, const bool, const int) = (SEXP(*)(const char *, const uint64_t, const bool, const bool, const int)) R_GetCCallable("qs2", "c_qd_deserialize");
bool c_qd_free(void *buffer) {
static bool (*fun)(void *) = (bool (*)(void *)) R_GetCCallable("qs2", "c_qd_free");
return fun(buffer);
}
SEXP c_qd_deserialize(const unsigned char * buffer, const size_t len, const bool use_alt_rep = false, const bool validate_checksum = false, const int nthreads = 1) {
static SEXP(*fun)(const unsigned char *, const size_t, const bool, const bool, const int) = (SEXP(*)(const unsigned char *, const size_t, const bool, const bool, const int)) R_GetCCallable("qs2", "c_qd_deserialize");
return fun(buffer, len, use_alt_rep, validate_checksum, nthreads);
}

Expand Down
50 changes: 36 additions & 14 deletions src/qx_functions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,26 @@
SEXP qs_save(SEXP object, const std::string& file, const int compress_level, const bool shuffle, const int nthreads);
CVectorOut qs_serialize_impl(SEXP object, const int compress_level, const bool shuffle, const int nthreads);
SEXP qs_serialize(SEXP object, const int compress_level, const bool shuffle, const int nthreads);
char* c_qs_serialize(SEXP object, uint64_t* len, const int compress_level, const bool shuffle, const int nthreads);
unsigned char* c_qs_serialize(SEXP object, size_t* len, const int compress_level, const bool shuffle, const int nthreads);
bool c_qs_free(void* ptr);
SEXP qs_read(const std::string& file, const bool validate_checksum, const int nthreads);
SEXP qs_deserialize_impl(CVectorIn& myFile, const bool validate_checksum, const int nthreads);
SEXP qs_deserialize(SEXP input, const bool validate_checksum, const int nthreads);
SEXP c_qs_deserialize(const char* buffer, const uint64_t len, const bool validate_checksum, const int nthreads);
SEXP c_qs_deserialize(const unsigned char* buffer, const size_t len, const bool validate_checksum, const int nthreads);


// qdata format functions
SEXP qd_save(SEXP object, const std::string& file, const int compress_level, const bool shuffle, const bool warn_unsupported_types, const int nthreads);
CVectorOut qd_serialize_impl(SEXP object, const int compress_level, const bool shuffle, const bool warn_unsupported_types, const int nthreads);
SEXP qd_serialize(SEXP object, const int compress_level, const bool shuffle, const bool warn_unsupported_types, const int nthreads);
char* c_qd_serialize(SEXP object, uint64_t* len, const int compress_level, const bool shuffle, const bool warn_unsupported_types, const int nthreads);
unsigned char* c_qd_serialize(SEXP object, size_t* len, const int compress_level, const bool shuffle, const bool warn_unsupported_types, const int nthreads);
bool c_qd_free(void *ptr);
SEXP qd_read(const std::string& file, const bool use_alt_rep, const bool validate_checksum, const int nthreads);
SEXP qd_deserialize_impl(CVectorIn& myFile, const bool use_alt_rep, const bool validate_checksum, const int nthreads);
SEXP qd_deserialize(SEXP input, const bool use_alt_rep, const bool validate_checksum, const int nthreads);
SEXP c_qd_deserialize(const char* buffer, const uint64_t len, const bool use_alt_rep, const bool validate_checksum, const int nthreads);
SEXP c_qd_deserialize(const unsigned char* buffer, const size_t len, const bool use_alt_rep, const bool validate_checksum, const int nthreads);



// qx utility functions
List qx_dump(const std::string& file);
Expand Down Expand Up @@ -175,10 +180,17 @@ SEXP qs_serialize(SEXP object, const int compress_level = 3, const bool shuffle
return output;
}

char* c_qs_serialize(SEXP object, uint64_t* len, const int compress_level = 3, const bool shuffle = true, const int nthreads = 1) {
unsigned char* c_qs_serialize(SEXP object, size_t* len, const int compress_level = 3, const bool shuffle = true, const int nthreads = 1) {
if(len == nullptr) return nullptr;
CVectorOut result = qs_serialize_impl(object, compress_level, shuffle, nthreads);
*len = result.tellp();
return result.release(); // give up ownership of buffer
*len = static_cast<size_t>(result.tellp());
return reinterpret_cast<unsigned char*>(result.release()); // give up ownership of buffer
}

bool c_qs_free(void *buffer) {
if (buffer == nullptr) { return false; }
free(buffer);
return true;
}

// DO_UNWIND_PROTECT macro assigns SEXP output
Expand Down Expand Up @@ -289,8 +301,8 @@ SEXP qs_deserialize(SEXP input, const bool validate_checksum = false, const int
return qs_deserialize_impl(myFile, validate_checksum, nthreads);
}

SEXP c_qs_deserialize(const char* buffer, const uint64_t len, const bool validate_checksum = false, const int nthreads = 1) {
CVectorIn myFile(buffer, len);
SEXP c_qs_deserialize(const unsigned char* buffer, const size_t len, const bool validate_checksum = false, const int nthreads = 1) {
CVectorIn myFile(reinterpret_cast<const char*>(buffer), static_cast<const uint64_t>(len));
return qs_deserialize_impl(myFile, validate_checksum, nthreads);
}

Expand Down Expand Up @@ -382,10 +394,17 @@ SEXP qd_serialize(SEXP object, const int compress_level = 3, const bool shuffle
return output;
}

char* c_qd_serialize(SEXP object, uint64_t* len, const int compress_level = 3, const bool shuffle = true, const bool warn_unsupported_types = true, const int nthreads = 1) {
unsigned char* c_qd_serialize(SEXP object, size_t* len, const int compress_level = 3, const bool shuffle = true, const bool warn_unsupported_types = true, const int nthreads = 1) {
if(len == nullptr) { return nullptr; }
CVectorOut result = qd_serialize_impl(object, compress_level, shuffle, warn_unsupported_types, nthreads);
*len = result.tellp();
return result.release(); // give up ownership of buffer
*len = static_cast<size_t>(result.tellp());
return reinterpret_cast<unsigned char*>(result.release()); // give up ownership of buffer
}

bool c_qd_free(void *buffer) {
if (buffer == nullptr) { return false; }
free(buffer);
return true;
}

#define DO_QD_READ(_STREAM_READER_, _BASE_CLASS_, _DECOMPRESSOR_) \
Expand Down Expand Up @@ -488,11 +507,12 @@ SEXP qd_deserialize(SEXP input, const bool use_alt_rep = false, const bool valid
return qd_deserialize_impl(myFile, use_alt_rep, validate_checksum, nthreads);
}

SEXP c_qd_deserialize(const char* buffer, const uint64_t len, const bool use_alt_rep = false, const bool validate_checksum = false, const int nthreads = 1) {
CVectorIn myFile(buffer, len);
SEXP c_qd_deserialize(const unsigned char* buffer, const size_t len, const bool use_alt_rep = false, const bool validate_checksum = false, const int nthreads = 1) {
CVectorIn myFile(reinterpret_cast<const char*>(buffer), static_cast<const uint64_t>(len));
return qd_deserialize_impl(myFile, use_alt_rep, validate_checksum, nthreads);
}


///////////////////////////////////////////////////////////////////////////////
/* qx utility functions */

Expand Down Expand Up @@ -776,6 +796,8 @@ RawVector c_base91_decode(const std::string& encoded_string) {
void qx_export_functions(DllInfo* dll) {
R_RegisterCCallable("qs2", "c_qs_serialize", (DL_FUNC)&c_qs_serialize);
R_RegisterCCallable("qs2", "c_qs_deserialize", (DL_FUNC)&c_qs_deserialize);
R_RegisterCCallable("qs2", "c_qs_free", (DL_FUNC)&c_qs_free);
R_RegisterCCallable("qs2", "c_qd_serialize", (DL_FUNC)&c_qd_serialize);
R_RegisterCCallable("qs2", "c_qd_deserialize", (DL_FUNC)&c_qd_deserialize);
R_RegisterCCallable("qs2", "c_qd_free", (DL_FUNC)&c_qd_free);
}
27 changes: 19 additions & 8 deletions vignettes/vignette.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -531,17 +531,28 @@ <h1>Usage in C/C++</h1>
<span id="cb8-5"><a href="#cb8-5" aria-hidden="true"></a></span>
<span id="cb8-6"><a href="#cb8-6" aria-hidden="true"></a><span class="co">// [[Rcpp::export]]</span></span>
<span id="cb8-7"><a href="#cb8-7" aria-hidden="true"></a>SEXP test_qs_serialize(SEXP x) {</span>
<span id="cb8-8"><a href="#cb8-8" aria-hidden="true"></a> <span class="dt">uint64_t</span> len = <span class="dv">0</span>;</span>
<span id="cb8-9"><a href="#cb8-9" aria-hidden="true"></a> <span class="dt">char</span> * buffer = c_qs_serialize(x, &amp;len, <span class="dv">10</span>, <span class="dv">4</span>); <span class="co">// object, buffer length, compress_level, nthreads</span></span>
<span id="cb8-10"><a href="#cb8-10" aria-hidden="true"></a> SEXP y = c_qs_deserialize(buffer, len, <span class="dv">4</span>); <span class="co">// buffer, buffer length, nthreads</span></span>
<span id="cb8-11"><a href="#cb8-11" aria-hidden="true"></a> free(buffer); <span class="co">// must manually free buffer</span></span>
<span id="cb8-8"><a href="#cb8-8" aria-hidden="true"></a> <span class="dt">size_t</span> len = <span class="dv">0</span>;</span>
<span id="cb8-9"><a href="#cb8-9" aria-hidden="true"></a> <span class="dt">unsigned</span> <span class="dt">char</span> * buffer = c_qs_serialize(x, &amp;len, <span class="dv">10</span>, <span class="kw">true</span>, <span class="dv">4</span>); <span class="co">// object, buffer length, compress_level, shuffle, nthreads</span></span>
<span id="cb8-10"><a href="#cb8-10" aria-hidden="true"></a> SEXP y = c_qs_deserialize(buffer, len, <span class="kw">false</span>, <span class="dv">4</span>); <span class="co">// buffer, buffer length, validate_checksum, nthreads</span></span>
<span id="cb8-11"><a href="#cb8-11" aria-hidden="true"></a> c_qs_free(buffer); <span class="co">// must manually free buffer</span></span>
<span id="cb8-12"><a href="#cb8-12" aria-hidden="true"></a> <span class="cf">return</span> y;</span>
<span id="cb8-13"><a href="#cb8-13" aria-hidden="true"></a>}</span>
<span id="cb8-14"><a href="#cb8-14" aria-hidden="true"></a></span>
<span id="cb8-15"><a href="#cb8-15" aria-hidden="true"></a><span class="co">/*** R</span></span>
<span id="cb8-16"><a href="#cb8-16" aria-hidden="true"></a><span class="co">x &lt;- runif(1e7)</span></span>
<span id="cb8-17"><a href="#cb8-17" aria-hidden="true"></a><span class="co">stopifnot(test_qs_serialize(x) == x)</span></span>
<span id="cb8-18"><a href="#cb8-18" aria-hidden="true"></a><span class="co">*/</span></span></code></pre></div>
<span id="cb8-15"><a href="#cb8-15" aria-hidden="true"></a><span class="co">// [[Rcpp::export]]</span></span>
<span id="cb8-16"><a href="#cb8-16" aria-hidden="true"></a>SEXP test_qd_serialize(SEXP x) {</span>
<span id="cb8-17"><a href="#cb8-17" aria-hidden="true"></a> <span class="dt">size_t</span> len = <span class="dv">0</span>;</span>
<span id="cb8-18"><a href="#cb8-18" aria-hidden="true"></a> <span class="dt">unsigned</span> <span class="dt">char</span> * buffer = c_qd_serialize(x, &amp;len, <span class="dv">10</span>, <span class="kw">true</span>, <span class="dv">4</span>); <span class="co">// object, buffer length, compress_level, shuffle, nthreads</span></span>
<span id="cb8-19"><a href="#cb8-19" aria-hidden="true"></a> SEXP y = c_qd_deserialize(buffer, len, <span class="kw">false</span>, <span class="kw">false</span>, <span class="dv">4</span>); <span class="co">// buffer, buffer length, use_alt_rep, validate_checksum, nthreads</span></span>
<span id="cb8-20"><a href="#cb8-20" aria-hidden="true"></a> c_qd_free(buffer); <span class="co">// must manually free buffer</span></span>
<span id="cb8-21"><a href="#cb8-21" aria-hidden="true"></a> <span class="cf">return</span> y;</span>
<span id="cb8-22"><a href="#cb8-22" aria-hidden="true"></a>}</span>
<span id="cb8-23"><a href="#cb8-23" aria-hidden="true"></a></span>
<span id="cb8-24"><a href="#cb8-24" aria-hidden="true"></a></span>
<span id="cb8-25"><a href="#cb8-25" aria-hidden="true"></a><span class="co">/*** R</span></span>
<span id="cb8-26"><a href="#cb8-26" aria-hidden="true"></a><span class="co">x &lt;- runif(1e7)</span></span>
<span id="cb8-27"><a href="#cb8-27" aria-hidden="true"></a><span class="co">stopifnot(test_qs_serialize(x) == x)</span></span>
<span id="cb8-28"><a href="#cb8-28" aria-hidden="true"></a><span class="co">stopifnot(test_qd_serialize(x) == x)</span></span>
<span id="cb8-29"><a href="#cb8-29" aria-hidden="true"></a><span class="co">*/</span></span></code></pre></div>
</div>


Expand Down
19 changes: 15 additions & 4 deletions vignettes/vignette.rmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,16 +156,27 @@ using namespace Rcpp;
// [[Rcpp::export]]
SEXP test_qs_serialize(SEXP x) {
uint64_t len = 0;
char * buffer = c_qs_serialize(x, &len, 10, 4); // object, buffer length, compress_level, nthreads
SEXP y = c_qs_deserialize(buffer, len, 4); // buffer, buffer length, nthreads
free(buffer); // must manually free buffer
size_t len = 0;
unsigned char * buffer = c_qs_serialize(x, &len, 10, true, 4); // object, buffer length, compress_level, shuffle, nthreads
SEXP y = c_qs_deserialize(buffer, len, false, 4); // buffer, buffer length, validate_checksum, nthreads
c_qs_free(buffer); // must manually free buffer
return y;
}
// [[Rcpp::export]]
SEXP test_qd_serialize(SEXP x) {
size_t len = 0;
unsigned char * buffer = c_qd_serialize(x, &len, 10, true, 4); // object, buffer length, compress_level, shuffle, nthreads
SEXP y = c_qd_deserialize(buffer, len, false, false, 4); // buffer, buffer length, use_alt_rep, validate_checksum, nthreads
c_qd_free(buffer); // must manually free buffer
return y;
}
/*** R
x <- runif(1e7)
stopifnot(test_qs_serialize(x) == x)
stopifnot(test_qd_serialize(x) == x)
*/
```

0 comments on commit 1d73eb0

Please sign in to comment.