From e0679229575baeb0eb5685e56d87faeae086e8e9 Mon Sep 17 00:00:00 2001 From: Aaron Lun Date: Thu, 16 May 2024 08:26:22 -0700 Subject: [PATCH] Switch to the new caching utilities in tatami_chunked. (#11) This uses the new slab factories, and removes references to TypicalSlabCache. We instead generate a separate subclass for each caching policy, which improves compartmentalization and enables smaller fetch_raw() definitions. We also update the sparsity-checking method names for the UnknownMatrix, and we align with the new policy of avoiding non-type-related templated parameters. This should reduce compile time and binary size without much perf loss. Finally, a minor bugfix for the cache configuration in the tests. --- include/tatami_r/UnknownMatrix.hpp | 278 ++++------- include/tatami_r/dense_extractor.hpp | 433 +++++++++------- include/tatami_r/dense_matrix.hpp | 24 +- include/tatami_r/sparse_extractor.hpp | 684 ++++++++++++++------------ include/tatami_r/sparse_matrix.hpp | 41 +- tests/src/bindings.cpp | 4 +- 6 files changed, 746 insertions(+), 718 deletions(-) diff --git a/include/tatami_r/UnknownMatrix.hpp b/include/tatami_r/UnknownMatrix.hpp index 478dd6c..c83f995 100644 --- a/include/tatami_r/UnknownMatrix.hpp +++ b/include/tatami_r/UnknownMatrix.hpp @@ -3,7 +3,6 @@ #include "Rcpp.h" #include "tatami/tatami.hpp" -#include "tatami_chunked/tatami_chunked.hpp" #include "dense_extractor.hpp" #include "sparse_extractor.hpp" @@ -17,9 +16,9 @@ namespace tatami_r { /** - * @brief Options for R matrix extraction. + * @brief Options for data extraction from an `UnknownMatrix`. */ -struct Options { +struct UnknownMatrixOptions { /** * Size of the cache, in bytes. * If -1, this is determined from `DelayedArray::getAutoBlockSize()`. @@ -53,7 +52,7 @@ class UnknownMatrix : public tatami::Matrix { * @param seed A matrix-like R object. * @param opt Extraction options. */ - UnknownMatrix(Rcpp::RObject seed, const Options& opt) : + UnknownMatrix(Rcpp::RObject seed, const UnknownMatrixOptions& opt) : original_seed(seed), delayed_env(Rcpp::Environment::namespace_env("DelayedArray")), sparse_env(Rcpp::Environment::namespace_env("SparseArray")), @@ -213,7 +212,7 @@ class UnknownMatrix : public tatami::Matrix { * * @param seed A matrix-like R object. */ - UnknownMatrix(Rcpp::RObject seed) : UnknownMatrix(std::move(seed), Options()) {} + UnknownMatrix(Rcpp::RObject seed) : UnknownMatrix(std::move(seed), UnknownMatrixOptions()) {} private: Index_ internal_nrow, internal_ncol; @@ -248,11 +247,11 @@ class UnknownMatrix : public tatami::Matrix { return internal_ncol; } - bool sparse() const { + bool is_sparse() const { return internal_sparse; } - double sparse_proportion() const { + double is_sparse_proportion() const { return static_cast(internal_sparse); } @@ -297,42 +296,44 @@ class UnknownMatrix : public tatami::Matrix { *** Myopic dense *** ********************/ private: - template - std::unique_ptr > populate_dense(bool row, tatami::MaybeOracle ora, const tatami::Options&) const { + template< + bool oracle_, + template class FromDense_, + template class FromSparse_, + typename ... Args_ + > + std::unique_ptr > populate_dense_internal(bool row, Index_ non_target_length, tatami::MaybeOracle ora, Args_&& ... args) const { std::unique_ptr > output; + Index_ max_target_chunk_length = max_primary_chunk_length(row); + tatami_chunked::SlabCacheStats stats(max_target_chunk_length, non_target_length, cache_size_in_bytes, sizeof(CachedValue_), require_minimum_cache); + + const auto& map = chunk_map(row); + const auto& ticks = chunk_ticks(row); + bool solo = (stats.max_slabs_in_cache == 0); + #ifdef TATAMI_R_PARALLELIZE_UNKNOWN // This involves some Rcpp initializations, so we lock it just in case. auto& mexec = executor(); mexec.run([&]() -> void { #endif - if (!internal_sparse) { - output.reset(new UnknownMatrix_internal::DenseFull( - original_seed, - dense_extractor, - std::move(ora), - secondary_dim(row), - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), - cache_size_in_bytes, - require_minimum_cache - )); + if (!internal_sparse) { + if (solo) { + typedef FromDense_ ShortDense; + output.reset(new ShortDense(original_seed, dense_extractor, row, std::move(ora), std::forward(args)..., ticks, map, stats)); + } else { + typedef FromDense_ ShortDense; + output.reset(new ShortDense(original_seed, dense_extractor, row, std::move(ora), std::forward(args)..., ticks, map, stats)); + } } else { - output.reset(new UnknownMatrix_internal::DensifiedSparseFull( - original_seed, - sparse_extractor, - std::move(ora), - secondary_dim(row), - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), - cache_size_in_bytes, - require_minimum_cache - )); + if (solo) { + typedef FromSparse_ ShortSparse; + output.reset(new ShortSparse(original_seed, sparse_extractor, row, std::move(ora), std::forward(args)..., max_target_chunk_length, ticks, map, stats)); + } else { + typedef FromSparse_ ShortSparse; + output.reset(new ShortSparse(original_seed, sparse_extractor, row, std::move(ora), std::forward(args)..., max_target_chunk_length, ticks, map, stats)); + } } #ifdef TATAMI_R_PARALLELIZE_UNKNOWN @@ -343,95 +344,20 @@ class UnknownMatrix : public tatami::Matrix { } template - std::unique_ptr > populate_dense(bool row, tatami::MaybeOracle ora, Index_ block_start, Index_ block_length, const tatami::Options&) const { - std::unique_ptr > output; - -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { -#endif - - if (!internal_sparse) { - output.reset(new UnknownMatrix_internal::DenseBlock( - original_seed, - dense_extractor, - std::move(ora), - block_start, - block_length, - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), - cache_size_in_bytes, - require_minimum_cache - )); - } else { - output.reset(new UnknownMatrix_internal::DensifiedSparseBlock( - original_seed, - sparse_extractor, - std::move(ora), - block_start, - block_length, - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), - cache_size_in_bytes, - require_minimum_cache - )); - } - -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); -#endif + std::unique_ptr > populate_dense(bool row, tatami::MaybeOracle ora, const tatami::Options&) const { + Index_ non_target_dim = secondary_dim(row); + return populate_dense_internal(row, non_target_dim, std::move(ora), non_target_dim); + } - return output; + template + std::unique_ptr > populate_dense(bool row, tatami::MaybeOracle ora, Index_ block_start, Index_ block_length, const tatami::Options&) const { + return populate_dense_internal(row, block_length, std::move(ora), block_start, block_length); } template std::unique_ptr > populate_dense(bool row, tatami::MaybeOracle ora, tatami::VectorPtr indices_ptr, const tatami::Options&) const { - std::unique_ptr > output; - -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { -#endif - - if (!internal_sparse) { - output.reset(new UnknownMatrix_internal::DenseIndexed( - original_seed, - dense_extractor, - std::move(ora), - std::move(indices_ptr), - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), - cache_size_in_bytes, - require_minimum_cache - )); - } else { - output.reset(new UnknownMatrix_internal::DensifiedSparseIndexed( - original_seed, - sparse_extractor, - std::move(ora), - std::move(indices_ptr), - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), - cache_size_in_bytes, - require_minimum_cache - )); - } - -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); -#endif - - return output; + Index_ nidx = indices_ptr->size(); + return populate_dense_internal(row, nidx, std::move(ora), std::move(indices_ptr)); } public: @@ -467,41 +393,34 @@ class UnknownMatrix : public tatami::Matrix { *** Myopic sparse *** *********************/ public: - template - std::unique_ptr > populate_sparse(bool row, tatami::MaybeOracle ora, const tatami::Options& opt) const { + template< + bool oracle_, + template class FromSparse_, + typename ... Args_ + > + std::unique_ptr > populate_sparse_internal( + bool row, + Index_ non_target_length, + tatami::MaybeOracle ora, + const tatami::Options& opt, + Args_&& ... args) + const { std::unique_ptr > output; -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { -#endif - - output.reset(new UnknownMatrix_internal::SparseFull( - original_seed, - sparse_extractor, - std::move(ora), - secondary_dim(row), - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), + Index_ max_target_chunk_length = max_primary_chunk_length(row); + tatami_chunked::SlabCacheStats stats( + max_target_chunk_length, + non_target_length, cache_size_in_bytes, - require_minimum_cache, - opt.sparse_extract_value, - opt.sparse_extract_index - )); - -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); -#endif + (opt.sparse_extract_index ? sizeof(CachedIndex_) : 0) + (opt.sparse_extract_value ? sizeof(CachedValue_) : 0), + require_minimum_cache + ); - return output; - } - - template - std::unique_ptr > populate_sparse(bool row, tatami::MaybeOracle ora, Index_ block_start, Index_ block_length, const tatami::Options& opt) const { - std::unique_ptr > output; + const auto& map = chunk_map(row); + const auto& ticks = chunk_ticks(row); + bool needs_value = opt.sparse_extract_value; + bool needs_index = opt.sparse_extract_index; + bool solo = stats.max_slabs_in_cache == 0; #ifdef TATAMI_R_PARALLELIZE_UNKNOWN // This involves some Rcpp initializations, so we lock it just in case. @@ -509,21 +428,13 @@ class UnknownMatrix : public tatami::Matrix { mexec.run([&]() -> void { #endif - output.reset(new UnknownMatrix_internal::SparseBlock( - original_seed, - sparse_extractor, - std::move(ora), - block_start, - block_length, - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), - cache_size_in_bytes, - require_minimum_cache, - opt.sparse_extract_value, - opt.sparse_extract_index - )); + if (solo) { + typedef FromSparse_ ShortSparse; + output.reset(new ShortSparse(original_seed, sparse_extractor, row, std::move(ora), std::forward(args)..., max_target_chunk_length, ticks, map, stats, needs_value, needs_index)); + } else { + typedef FromSparse_ ShortSparse; + output.reset(new ShortSparse(original_seed, sparse_extractor, row, std::move(ora), std::forward(args)..., max_target_chunk_length, ticks, map, stats, needs_value, needs_index)); + } #ifdef TATAMI_R_PARALLELIZE_UNKNOWN }); @@ -533,35 +444,20 @@ class UnknownMatrix : public tatami::Matrix { } template - std::unique_ptr > populate_sparse(bool row, tatami::MaybeOracle ora, tatami::VectorPtr indices_ptr, const tatami::Options& opt) const { - std::unique_ptr > output; - -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { -#endif - - output.reset(new UnknownMatrix_internal::SparseIndexed( - original_seed, - sparse_extractor, - std::move(ora), - std::move(indices_ptr), - !row, - max_primary_chunk_length(row), - chunk_ticks(row), - chunk_map(row), - cache_size_in_bytes, - require_minimum_cache, - opt.sparse_extract_value, - opt.sparse_extract_index - )); + std::unique_ptr > populate_sparse(bool row, tatami::MaybeOracle ora, const tatami::Options& opt) const { + Index_ non_target_dim = secondary_dim(row); + return populate_sparse_internal(row, non_target_dim, std::move(ora), opt, non_target_dim); + } -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); -#endif + template + std::unique_ptr > populate_sparse(bool row, tatami::MaybeOracle ora, Index_ block_start, Index_ block_length, const tatami::Options& opt) const { + return populate_sparse_internal(row, block_length, std::move(ora), opt, block_start, block_length); + } - return output; + template + std::unique_ptr > populate_sparse(bool row, tatami::MaybeOracle ora, tatami::VectorPtr indices_ptr, const tatami::Options& opt) const { + Index_ nidx = indices_ptr->size(); + return populate_sparse_internal(row, nidx, std::move(ora), opt, std::move(indices_ptr)); } public: diff --git a/include/tatami_r/dense_extractor.hpp b/include/tatami_r/dense_extractor.hpp index 12df5c5..883bc50 100644 --- a/include/tatami_r/dense_extractor.hpp +++ b/include/tatami_r/dense_extractor.hpp @@ -8,278 +8,361 @@ #include #include +#include namespace tatami_r { namespace UnknownMatrix_internal { -template -struct DenseBase : public tatami::DenseExtractor { - DenseBase( +/******************** + *** Core classes *** + ********************/ + +template +struct SoloDenseCore { + SoloDenseCore( const Rcpp::RObject& mat, const Rcpp::Function& dense_extractor, + bool row, tatami::MaybeOracle ora, - Rcpp::IntegerVector secondary_extract, - bool by_column, - Index_ max_primary_chunk_length, - const std::vector& ticks, - const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache) : + Rcpp::IntegerVector non_target_extract, + [[maybe_unused]] const std::vector& ticks, // provided here for compatibility with the other Dense*Core classes. + [[maybe_unused]] const std::vector& map, + [[maybe_unused]] const tatami_chunked::SlabCacheStats& stats) : mat(mat), dense_extractor(dense_extractor), extract_args(2), - by_column(by_column), - chunk_ticks(ticks), - chunk_map(map), - secondary_length(secondary_extract.size()), - cache( - max_primary_chunk_length, - secondary_length, - cache_size_in_bytes / sizeof(CachedValue_), - require_minimum_cache, - std::move(ora) - ) + row(row), + non_target_length(non_target_extract.size()), + oracle(std::move(ora)) { - if (cache.num_slabs_in_cache == 0) { - solo.resize(secondary_length); + extract_args[static_cast(row)] = non_target_extract; + } + +private: + const Rcpp::RObject& mat; + const Rcpp::Function& dense_extractor; + Rcpp::List extract_args; + + bool row; + size_t non_target_length; + + tatami::MaybeOracle oracle; + size_t counter = 0; + +public: + template + void fetch_raw(Index_ i, Value_* buffer) { + if constexpr(oracle_) { + i = oracle->get(counter++); } - if (by_column) { - extract_args[0] = secondary_extract; +#ifdef TATAMI_R_PARALLELIZE_UNKNOWN + // This involves some Rcpp initializations, so we lock it just in case. + auto& mexec = executor(); + mexec.run([&]() -> void { +#endif + + extract_args[static_cast(!row)] = Rcpp::IntegerVector::create(i + 1); + auto obj = dense_extractor(mat, extract_args); + if (row) { + parse_dense_matrix(obj, true, buffer, 0, 0, 1, non_target_length); } else { - extract_args[1] = secondary_extract; + parse_dense_matrix(obj, false, buffer, 0, 0, non_target_length, 1); } + +#ifdef TATAMI_R_PARALLELIZE_UNKNOWN + }); +#endif } +}; - ~DenseBase() = default; +template +struct MyopicDenseCore { + MyopicDenseCore( + const Rcpp::RObject& mat, + const Rcpp::Function& dense_extractor, + bool row, + [[maybe_unused]] tatami::MaybeOracle ora, // provided here for compatibility with the other Dense*Core classes. + Rcpp::IntegerVector non_target_extract, + const std::vector& ticks, + const std::vector& map, + const tatami_chunked::SlabCacheStats& stats) : + mat(mat), + dense_extractor(dense_extractor), + extract_args(2), + row(row), + non_target_length(non_target_extract.size()), + chunk_ticks(ticks), + chunk_map(map), + factory(stats), + cache(stats.max_slabs_in_cache) + { + extract_args[static_cast(row)] = non_target_extract; + } private: - typedef std::vector Slab; - const Rcpp::RObject& mat; const Rcpp::Function& dense_extractor; Rcpp::List extract_args; - bool by_column; + bool row; + size_t non_target_length; + const std::vector& chunk_ticks; const std::vector& chunk_map; - size_t secondary_length; - - tatami_chunked::TypicalSlabCacheWorkspace cache; - Slab solo; - -private: - std::pair fetch_raw(Index_ i) { - if (cache.num_slabs_in_cache == 0) { - if constexpr(oracle_) { - i = cache.cache.next(); - } + tatami_chunked::DenseSlabFactory factory; + typedef typename decltype(factory)::Slab Slab; + tatami_chunked::LruSlabCache cache; +public: + template + void fetch_raw(Index_ i, Value_* buffer) { + auto chosen = chunk_map[i]; + + const auto& slab = cache.find( + chosen, + [&]() -> Slab { + return factory.create(); + }, + [&](Index_ id, Slab& cache) { #ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { + // This involves some Rcpp initializations, so we lock it just in case. + auto& mexec = executor(); + mexec.run([&]() -> void { #endif - extract_args[static_cast(by_column)] = Rcpp::IntegerVector::create(i + 1); - auto obj = dense_extractor(mat, extract_args); - if (by_column) { - parse_dense_matrix(obj, solo, 0, 0, secondary_length, 1); - } else { - parse_dense_matrix(obj, solo, 0, 0, 1, secondary_length); - } + auto chunk_start = chunk_ticks[id]; + size_t chunk_len = chunk_ticks[id + 1] - chunk_start; + Rcpp::IntegerVector primary_extract(chunk_len); + std::iota(primary_extract.begin(), primary_extract.end(), chunk_start + 1); + extract_args[static_cast(!row)] = primary_extract; + + auto obj = dense_extractor(mat, extract_args); + if (row) { + parse_dense_matrix(obj, true, cache.data, 0, 0, chunk_len, non_target_length); + } else { + parse_dense_matrix(obj, false, cache.data, 0, 0, non_target_length, chunk_len); + } #ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); + }); #endif + } + ); + + auto src = slab.data + static_cast(i - chunk_ticks[chosen]) * non_target_length; + std::copy_n(src, non_target_length, buffer); + } +}; + +template +struct OracularDenseCore { + OracularDenseCore( + const Rcpp::RObject& mat, + const Rcpp::Function& dense_extractor, + bool row, + tatami::MaybeOracle ora, + Rcpp::IntegerVector non_target_extract, + const std::vector& ticks, + const std::vector& map, + const tatami_chunked::SlabCacheStats& stats) : + mat(mat), + dense_extractor(dense_extractor), + extract_args(2), + row(row), + non_target_length(non_target_extract.size()), + chunk_ticks(ticks), + chunk_map(map), + factory(stats), + cache(std::move(ora), stats.max_slabs_in_cache) + { + extract_args[static_cast(row)] = non_target_extract; + } - return std::make_pair(&solo, static_cast(0)); +private: + const Rcpp::RObject& mat; + const Rcpp::Function& dense_extractor; + Rcpp::List extract_args; - } else if constexpr(!oracle_) { - auto chosen = chunk_map[i]; + bool row; + size_t non_target_length; - const auto& slab = cache.cache.find( - chosen, - [&]() -> Slab { - return Slab(); - }, - [&](Index_ id, Slab& cache) { -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { -#endif + const std::vector& chunk_ticks; + const std::vector& chunk_map; - auto chunk_start = chunk_ticks[id]; - size_t chunk_len = chunk_ticks[id + 1] - chunk_start; - Rcpp::IntegerVector primary_extract(chunk_len); - std::iota(primary_extract.begin(), primary_extract.end(), chunk_start + 1); - extract_args[static_cast(by_column)] = primary_extract; - auto obj = dense_extractor(mat, extract_args); - if (by_column) { - parse_dense_matrix(obj, cache, 0, 0, secondary_length, chunk_len); - } else { - parse_dense_matrix(obj, cache, 0, 0, chunk_len, secondary_length); - } + tatami_chunked::DenseSlabFactory factory; + typedef typename decltype(factory)::Slab Slab; + tatami_chunked::OracularSlabCache cache; -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); -#endif +public: + template + void fetch_raw(Index_, Value_* buffer) { + auto res = cache.next( + [&](Index_ i) -> std::pair { + auto chosen = chunk_map[i]; + return std::make_pair(chosen, static_cast(i - chunk_ticks[chosen])); + }, + [&]() -> Slab { + return factory.create(); + }, + [&](std::vector >& to_populate) { + // Sorting them so that the indices are in order. + if (!std::is_sorted(to_populate.begin(), to_populate.end(), [&](const std::pair& left, const std::pair right) { return left.first < right.first; })) { + std::sort(to_populate.begin(), to_populate.end(), [&](const std::pair& left, const std::pair right) { return left.first < right.first; }); } - ); - return std::make_pair(&slab, static_cast(i - chunk_ticks[chosen])); - } else { - return cache.cache.next( - [&](Index_ i) -> std::pair { - auto chosen = chunk_map[i]; - return std::make_pair(chosen, static_cast(i - chunk_ticks[chosen])); - }, - [&]() -> Slab { - return Slab(); - }, - [&](std::vector >& to_populate) { - // Sorting them so that the indices are in order. - if (!std::is_sorted(to_populate.begin(), to_populate.end(), [&](const std::pair& left, const std::pair right) { return left.first < right.first; })) { - std::sort(to_populate.begin(), to_populate.end(), [&](const std::pair& left, const std::pair right) { return left.first < right.first; }); - } - - Index_ total_len = 0; - for (const auto& p : to_populate) { - total_len += chunk_ticks[p.first + 1] - chunk_ticks[p.first]; - } + Index_ total_len = 0; + for (const auto& p : to_populate) { + total_len += chunk_ticks[p.first + 1] - chunk_ticks[p.first]; + } #ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { + // This involves some Rcpp initializations, so we lock it just in case. + auto& mexec = executor(); + mexec.run([&]() -> void { #endif - Rcpp::IntegerVector primary_extract(total_len); - Index_ current = 0; - for (const auto& p : to_populate) { - Index_ chunk_start = chunk_ticks[p.first]; - Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_start; - auto start = primary_extract.begin() + current; - std::iota(start, start + chunk_len, chunk_ticks[p.first] + 1); - current += chunk_len; - } + Rcpp::IntegerVector primary_extract(total_len); + Index_ current = 0; + for (const auto& p : to_populate) { + Index_ chunk_start = chunk_ticks[p.first]; + Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_start; + auto start = primary_extract.begin() + current; + std::iota(start, start + chunk_len, chunk_ticks[p.first] + 1); + current += chunk_len; + } + + extract_args[static_cast(!row)] = primary_extract; + auto obj = dense_extractor(mat, extract_args); - extract_args[static_cast(by_column)] = primary_extract; - auto obj = dense_extractor(mat, extract_args); - - current = 0; - for (const auto& p : to_populate) { - auto chunk_start = chunk_ticks[p.first]; - Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_start; - if (by_column) { - parse_dense_matrix(obj, *p.second, 0, current, secondary_length, chunk_len); - } else { - parse_dense_matrix(obj, *p.second, current, 0, chunk_len, secondary_length); - } - current += chunk_len; + current = 0; + for (const auto& p : to_populate) { + auto chunk_start = chunk_ticks[p.first]; + Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_start; + if (row) { + parse_dense_matrix(obj, true, p.second->data, current, 0, chunk_len, non_target_length); + } else { + parse_dense_matrix(obj, false, p.second->data, 0, current, non_target_length, chunk_len); } + current += chunk_len; + } #ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); + }); #endif - } - ); - } - } + } + ); -public: - const Value_* fetch(Index_ i, Value_* buffer) { - auto res = fetch_raw(i); - size_t shift = this->secondary_length * static_cast(res.second); // cast to size_t to avoid overflow. - std::copy_n(res.first->data() + shift, this->secondary_length, buffer); - return buffer; + size_t shift = non_target_length * static_cast(res.second); // cast to size_t to avoid overflow. + std::copy_n(res.first->data + shift, non_target_length, buffer); } }; -template -struct DenseFull : public DenseBase { +template +using DenseCore = typename std::conditional, + typename std::conditional, + MyopicDenseCore + >::type +>::type; + +/************************* + *** Extractor classes *** + *************************/ + +template +struct DenseFull : public tatami::DenseExtractor { DenseFull( const Rcpp::RObject& mat, const Rcpp::Function& dense_extractor, + bool row, tatami::MaybeOracle ora, - Index_ secondary_dim, - bool by_column, - Index_ max_primary_chunk_length, + Index_ non_target_dim, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache) : - DenseBase( + const tatami_chunked::SlabCacheStats& stats) : + core( mat, dense_extractor, + row, std::move(ora), [&]() { - Rcpp::IntegerVector output(secondary_dim); + Rcpp::IntegerVector output(non_target_dim); std::iota(output.begin(), output.end(), 1); return output; }(), - by_column, - max_primary_chunk_length, ticks, map, - cache_size_in_bytes, - require_minimum_cache + stats ) {} + +private: + DenseCore core; + +public: + const Value_* fetch(Index_ i, Value_* buffer) { + core.fetch_raw(i, buffer); + return buffer; + } }; -template -struct DenseBlock : public DenseBase { +template +struct DenseBlock : public tatami::DenseExtractor { DenseBlock( const Rcpp::RObject& mat, const Rcpp::Function& dense_extractor, + bool row, tatami::MaybeOracle ora, Index_ block_start, Index_ block_length, - bool by_column, - Index_ max_primary_chunk_length, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache) : - DenseBase( + const tatami_chunked::SlabCacheStats& stats) : + core( mat, dense_extractor, + row, std::move(ora), [&]() { Rcpp::IntegerVector output(block_length); std::iota(output.begin(), output.end(), block_start + 1); return output; }(), - by_column, - max_primary_chunk_length, ticks, map, - cache_size_in_bytes, - require_minimum_cache + stats ) {} + +private: + DenseCore core; + +public: + const Value_* fetch(Index_ i, Value_* buffer) { + core.fetch_raw(i, buffer); + return buffer; + } }; -template -struct DenseIndexed : public DenseBase { +template +struct DenseIndexed : public tatami::DenseExtractor { DenseIndexed( const Rcpp::RObject& mat, const Rcpp::Function& dense_extractor, + bool row, tatami::MaybeOracle ora, tatami::VectorPtr indices_ptr, - bool by_column, - Index_ max_primary_chunk_length, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache) : - DenseBase( + const tatami_chunked::SlabCacheStats& stats) : + core( mat, dense_extractor, + row, std::move(ora), [&]() { Rcpp::IntegerVector output(indices_ptr->begin(), indices_ptr->end()); @@ -288,14 +371,20 @@ struct DenseIndexed : public DenseBase { } return output; }(), - by_column, - max_primary_chunk_length, ticks, map, - cache_size_in_bytes, - require_minimum_cache + stats ) {} + +private: + DenseCore core; + +public: + const Value_* fetch(Index_ i, Value_* buffer) { + core.fetch_raw(i, buffer); + return buffer; + } }; } diff --git a/include/tatami_r/dense_matrix.hpp b/include/tatami_r/dense_matrix.hpp index b93889e..6f8e82f 100644 --- a/include/tatami_r/dense_matrix.hpp +++ b/include/tatami_r/dense_matrix.hpp @@ -6,37 +6,35 @@ namespace tatami_r { -template -void parse_dense_matrix_internal(const InputObject_& y, std::vector& cache, size_t start_row, size_t start_col, size_t num_rows, size_t num_cols) { - cache.resize(num_rows * num_cols); +template +void parse_dense_matrix_internal(const InputObject_& y, bool row, CachedValue_* cache, size_t start_row, size_t start_col, size_t num_rows, size_t num_cols) { auto input = static_cast(y.begin()) + start_row + start_col * static_cast(y.rows()); - auto output = cache.data(); - if constexpr(transpose_) { + if (row) { // y is a column-major matrix, but transpose() expects a row-major // input, so we just conceptually transpose it. - tatami::transpose(input, num_cols, num_rows, y.rows(), output, num_cols); + tatami::transpose(input, num_cols, num_rows, y.rows(), cache, num_cols); } else { for (size_t c = 0; c < num_cols; ++c) { - std::copy_n(input, num_rows, output); + std::copy_n(input, num_rows, cache); input += y.rows(); - output += num_rows; + cache += num_rows; } } } -template -void parse_dense_matrix(const Rcpp::RObject& seed, std::vector& cache, size_t start_row, size_t start_col, size_t num_rows, size_t num_cols) { +template +void parse_dense_matrix(const Rcpp::RObject& seed, bool row, CachedValue_* cache, size_t start_row, size_t start_col, size_t num_rows, size_t num_cols) { auto stype = seed.sexp_type(); if (stype == REALSXP) { Rcpp::NumericMatrix y(seed); - parse_dense_matrix_internal(y, cache, start_row, start_col, num_rows, num_cols); + parse_dense_matrix_internal(y, row, cache, start_row, start_col, num_rows, num_cols); } else if (stype == INTSXP) { Rcpp::IntegerMatrix y(seed); - parse_dense_matrix_internal(y, cache, start_row, start_col, num_rows, num_cols); + parse_dense_matrix_internal(y, row, cache, start_row, start_col, num_rows, num_cols); } else if (stype == LGLSXP) { Rcpp::LogicalMatrix y(seed); - parse_dense_matrix_internal(y, cache, start_row, start_col, num_rows, num_cols); + parse_dense_matrix_internal(y, row, cache, start_row, start_col, num_rows, num_cols); } else { throw std::runtime_error("unsupported SEXP type (" + std::to_string(stype) + ") from the matrix returned by 'extract_array'"); } diff --git a/include/tatami_r/sparse_extractor.hpp b/include/tatami_r/sparse_extractor.hpp index cac086e..a1f756b 100644 --- a/include/tatami_r/sparse_extractor.hpp +++ b/include/tatami_r/sparse_extractor.hpp @@ -13,320 +13,343 @@ namespace tatami_r { namespace UnknownMatrix_internal { -template -struct SparseBase { - SparseBase( +/******************** + *** Core classes *** + ********************/ + +template +struct SoloSparseCore { + SoloSparseCore( const Rcpp::RObject& mat, const Rcpp::Function& sparse_extractor, + bool row, tatami::MaybeOracle ora, - Rcpp::IntegerVector secondary_extract, - bool by_column, - Index_ max_primary_chunk_length, - const std::vector& ticks, - const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache, + Rcpp::IntegerVector non_target_extract, + [[maybe_unused]] Index_ max_target_chunk_length, // provided here for compatibility with the other Sparse*Core classes. + [[maybe_unused]] const std::vector& ticks, + [[maybe_unused]] const std::vector& map, + [[maybe_unused]] const tatami_chunked::SlabCacheStats& stats, bool needs_value, bool needs_index) : - mat(std::move(mat)), - sparse_extractor(std::move(sparse_extractor)), + mat(mat), + sparse_extractor(sparse_extractor), extract_args(2), - secondary_indices(std::move(secondary_extract)), - by_column(by_column), - chunk_ticks(ticks), - chunk_map(map), - max_primary_chunk_length(max_primary_chunk_length), - secondary_length(secondary_indices.size()), - needs_value(needs_value), - needs_index(needs_index), - cache( - max_primary_chunk_length, - secondary_length, - cache_size_in_bytes / std::max(static_cast(1), static_cast(sizeof(CachedValue_) * needs_value + sizeof(CachedIndex_) * needs_index)), - require_minimum_cache, - std::move(ora) - ) + row(row), + factory(1, non_target_extract.size(), 1, needs_value, needs_index), + solo(factory.create()), + oracle(std::move(ora)) { - if (cache.num_slabs_in_cache == 0) { - solo = Slab(1, secondary_length, needs_value, needs_index); - } - - if (by_column) { - extract_args[0] = secondary_indices; - } else { - extract_args[1] = secondary_indices; - } + extract_args[static_cast(row)] = non_target_extract; } - ~SparseBase() = default; +private: + const Rcpp::RObject& mat; + const Rcpp::Function& sparse_extractor; + Rcpp::List extract_args; + + bool row; + + tatami_chunked::SparseSlabFactory factory; + typedef typename decltype(factory)::Slab Slab; + Slab solo; + + tatami::MaybeOracle oracle; + size_t counter = 0; public: - struct Slab { - Slab() = default; - Slab(size_t max_primary_chunk_length, size_t secondary_length, bool needs_value, bool needs_index) { - if (needs_value) { - value.reserve(max_primary_chunk_length * secondary_length); - } - if (needs_index) { - index.reserve(max_primary_chunk_length * secondary_length); - } - count.reserve(max_primary_chunk_length); + std::pair fetch_raw(Index_ i) { + if constexpr(oracle_) { + i = oracle->get(counter++); } + solo.number[0] = 0; + +#ifdef TATAMI_R_PARALLELIZE_UNKNOWN + // This involves some Rcpp initializations, so we lock it just in case. + auto& mexec = executor(); + mexec.run([&]() -> void { +#endif + + extract_args[static_cast(!row)] = Rcpp::IntegerVector::create(i + 1); + auto obj = sparse_extractor(mat, extract_args); + parse_sparse_matrix(obj, row, solo.values, solo.indices, solo.number); + +#ifdef TATAMI_R_PARALLELIZE_UNKNOWN + }); +#endif - std::vector value; - std::vector index; - std::vector count; - }; + return std::make_pair(&solo, static_cast(0)); + } +}; + +template +struct MyopicSparseCore { + MyopicSparseCore( + const Rcpp::RObject& mat, + const Rcpp::Function& sparse_extractor, + bool row, + [[maybe_unused]] tatami::MaybeOracle ora, // provided here for compatibility with the other Sparse*Core classes. + Rcpp::IntegerVector non_target_extract, + Index_ max_target_chunk_length, + const std::vector& ticks, + const std::vector& map, + const tatami_chunked::SlabCacheStats& stats, + bool needs_value, + bool needs_index) : + mat(mat), + sparse_extractor(sparse_extractor), + extract_args(2), + row(row), + chunk_ticks(ticks), + chunk_map(map), + factory(max_target_chunk_length, non_target_extract.size(), stats, needs_value, needs_index), + cache(stats.max_slabs_in_cache) + { + extract_args[static_cast(row)] = non_target_extract; + } -protected: +private: const Rcpp::RObject& mat; const Rcpp::Function& sparse_extractor; Rcpp::List extract_args; - Rcpp::IntegerVector secondary_indices; - bool by_column; + bool row; + const std::vector& chunk_ticks; const std::vector& chunk_map; - size_t max_primary_chunk_length; - size_t secondary_length; - bool needs_value; - bool needs_index; - - std::vector chunk_value_ptrs; - std::vector chunk_index_ptrs; - std::vector chunk_counts; - - tatami_chunked::TypicalSlabCacheWorkspace cache; - Slab solo; + tatami_chunked::SparseSlabFactory factory; + typedef typename decltype(factory)::Slab Slab; + tatami_chunked::LruSlabCache cache; -protected: +public: std::pair fetch_raw(Index_ i) { - if (cache.num_slabs_in_cache == 0) { - if constexpr(oracle_) { - i = cache.cache.next(); - } - - if (needs_value) { - chunk_value_ptrs.clear(); - chunk_value_ptrs.push_back(solo.value.data()); - } - if (needs_index) { - chunk_index_ptrs.clear(); - chunk_index_ptrs.push_back(solo.index.data()); - } - solo.count[0] = 0; + auto chosen = chunk_map[i]; + + const auto& slab = cache.find( + chosen, + [&]() -> Slab { + return factory.create(); + }, + [&](Index_ id, Slab& cache) { + auto chunk_start = chunk_ticks[id], chunk_end = chunk_ticks[id + 1]; + size_t chunk_len = chunk_end - chunk_start; + std::fill_n(cache.number, chunk_len, 0); #ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { + // This involves some Rcpp initializations, so we lock it just in case. + auto& mexec = executor(); + mexec.run([&]() -> void { #endif - extract_args[static_cast(by_column)] = Rcpp::IntegerVector::create(i + 1); - auto obj = sparse_extractor(mat, extract_args); - - if (by_column) { - parse_sparse_matrix(obj, chunk_value_ptrs, chunk_index_ptrs, solo.count); - } else { - parse_sparse_matrix(obj, chunk_value_ptrs, chunk_index_ptrs, solo.count); - } + Rcpp::IntegerVector primary_extract(chunk_len); + std::iota(primary_extract.begin(), primary_extract.end(), chunk_start + 1); + extract_args[static_cast(!row)] = primary_extract; + auto obj = sparse_extractor(mat, extract_args); + parse_sparse_matrix(obj, row, cache.values, cache.indices, cache.number); #ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); + }); #endif + } + ); + + Index_ offset = i - chunk_ticks[chosen]; + return std::make_pair(&slab, offset); + } +}; + +template +struct OracularSparseCore { + OracularSparseCore( + const Rcpp::RObject& mat, + const Rcpp::Function& sparse_extractor, + bool row, + tatami::MaybeOracle ora, + Rcpp::IntegerVector non_target_extract, + Index_ max_target_chunk_length, + const std::vector& ticks, + const std::vector& map, + const tatami_chunked::SlabCacheStats& stats, + bool needs_value, + bool needs_index) : + mat(mat), + sparse_extractor(sparse_extractor), + extract_args(2), + row(row), + chunk_ticks(ticks), + chunk_map(map), + factory(max_target_chunk_length, non_target_extract.size(), stats, needs_value, needs_index), + cache(std::move(ora), stats.max_slabs_in_cache), + needs_value(needs_value), + needs_index(needs_index) + { + extract_args[static_cast(row)] = non_target_extract; + } - return std::make_pair(&solo, static_cast(0)); +private: + const Rcpp::RObject& mat; + const Rcpp::Function& sparse_extractor; + Rcpp::List extract_args; - } else if constexpr(!oracle_) { - auto chosen = chunk_map[i]; + bool row; - const auto& slab = cache.cache.find( - chosen, - [&]() -> Slab { - return Slab(max_primary_chunk_length, secondary_length, needs_value, needs_index); - }, - [&](Index_ id, Slab& cache) { - auto chunk_start = chunk_ticks[id], chunk_end = chunk_ticks[id + 1]; - size_t chunk_len = chunk_end - chunk_start; + const std::vector& chunk_ticks; + const std::vector& chunk_map; - if (needs_value) { - chunk_value_ptrs.clear(); - auto ptr = cache.value.data(); - for (size_t i = 0; i < chunk_len; ++i, ptr += secondary_length) { - chunk_value_ptrs.push_back(ptr); - } - } - if (needs_index) { - chunk_index_ptrs.clear(); - auto ptr = cache.index.data(); - for (size_t i = 0; i < chunk_len; ++i, ptr += secondary_length) { - chunk_index_ptrs.push_back(ptr); - } - } - cache.count.clear(); - cache.count.resize(chunk_len); + tatami_chunked::SparseSlabFactory factory; + typedef typename decltype(factory)::Slab Slab; + tatami_chunked::OracularSlabCache cache; -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { -#endif + std::vector chunk_value_ptrs; + std::vector chunk_index_ptrs; + std::vector chunk_numbers; - Rcpp::IntegerVector primary_extract(chunk_len); - std::iota(primary_extract.begin(), primary_extract.end(), chunk_start + 1); - extract_args[static_cast(by_column)] = primary_extract; - auto obj = sparse_extractor(mat, extract_args); + bool needs_value; + bool needs_index; - if (by_column) { - parse_sparse_matrix(obj, chunk_value_ptrs, chunk_index_ptrs, cache.count); - } else { - parse_sparse_matrix(obj, chunk_value_ptrs, chunk_index_ptrs, cache.count); - } +public: + std::pair fetch_raw(Index_) { + return cache.next( + [&](Index_ i) -> std::pair { + auto chosen = chunk_map[i]; + return std::make_pair(chosen, static_cast(i - chunk_ticks[chosen])); + }, + [&]() -> Slab { + return factory.create(); + }, + [&](std::vector >& to_populate) { + // Sorting them so that the indices are in order. + if (!std::is_sorted(to_populate.begin(), to_populate.end(), [&](const std::pair& left, const std::pair right) { return left.first < right.first; })) { + std::sort(to_populate.begin(), to_populate.end(), [&](const std::pair& left, const std::pair right) { return left.first < right.first; }); + } -#ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); -#endif + if (needs_value) { + chunk_value_ptrs.clear(); + } + if (needs_index) { + chunk_index_ptrs.clear(); } - ); - return std::make_pair(&slab, static_cast(i - chunk_ticks[chosen])); - - } else { - return cache.cache.next( - [&](Index_ i) -> std::pair { - auto chosen = chunk_map[i]; - return std::make_pair(chosen, static_cast(i - chunk_ticks[chosen])); - }, - [&]() -> Slab { - return Slab(max_primary_chunk_length, secondary_length, needs_value, needs_index); - }, - [&](std::vector >& to_populate) { - // Sorting them so that the indices are in order. - if (!std::is_sorted(to_populate.begin(), to_populate.end(), [&](const std::pair& left, const std::pair right) { return left.first < right.first; })) { - std::sort(to_populate.begin(), to_populate.end(), [&](const std::pair& left, const std::pair right) { return left.first < right.first; }); - } + Index_ total_len = 0; + for (const auto& p : to_populate) { + Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_ticks[p.first]; + total_len += chunk_len; if (needs_value) { - chunk_value_ptrs.clear(); + auto vIt = p.second->values.begin(); + chunk_value_ptrs.insert(chunk_value_ptrs.end(), vIt, vIt + chunk_len); } if (needs_index) { - chunk_index_ptrs.clear(); - } - - Index_ total_len = 0; - for (const auto& p : to_populate) { - Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_ticks[p.first]; - total_len += chunk_len; - if (needs_value) { - auto ptr = p.second->value.data(); - for (Index_ i = 0; i < chunk_len; ++i, ptr += secondary_length) { - chunk_value_ptrs.push_back(ptr); - } - } - if (needs_index) { - auto ptr = p.second->index.data(); - for (Index_ i = 0; i < chunk_len; ++i, ptr += secondary_length) { - chunk_index_ptrs.push_back(ptr); - } - } + auto iIt = p.second->indices.begin(); + chunk_index_ptrs.insert(chunk_index_ptrs.end(), iIt, iIt + chunk_len); } + } - chunk_counts.clear(); - chunk_counts.resize(total_len); + chunk_numbers.clear(); + chunk_numbers.resize(total_len); #ifdef TATAMI_R_PARALLELIZE_UNKNOWN - // This involves some Rcpp initializations, so we lock it just in case. - auto& mexec = executor(); - mexec.run([&]() -> void { + // This involves some Rcpp initializations, so we lock it just in case. + auto& mexec = executor(); + mexec.run([&]() -> void { #endif - Rcpp::IntegerVector primary_extract(total_len); - Index_ current = 0; - for (const auto& p : to_populate) { - Index_ chunk_start = chunk_ticks[p.first]; - Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_start; - auto start = primary_extract.begin() + current; - std::iota(start, start + chunk_len, chunk_start + 1); - current += chunk_len; - } + Rcpp::IntegerVector primary_extract(total_len); + Index_ current = 0; + for (const auto& p : to_populate) { + Index_ chunk_start = chunk_ticks[p.first]; + Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_start; + auto start = primary_extract.begin() + current; + std::iota(start, start + chunk_len, chunk_start + 1); + current += chunk_len; + } - extract_args[static_cast(by_column)] = primary_extract; - auto obj = sparse_extractor(mat, extract_args); - if (by_column) { - parse_sparse_matrix(obj, chunk_value_ptrs, chunk_index_ptrs, chunk_counts); - } else { - parse_sparse_matrix(obj, chunk_value_ptrs, chunk_index_ptrs, chunk_counts); - } + extract_args[static_cast(!row)] = primary_extract; + auto obj = sparse_extractor(mat, extract_args); + parse_sparse_matrix(obj, row, chunk_value_ptrs, chunk_index_ptrs, chunk_numbers.data()); - current = 0; - for (const auto& p : to_populate) { - Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_ticks[p.first]; - p.second->count.resize(chunk_len); - std::copy_n(chunk_counts.begin() + current, chunk_len, p.second->count.begin()); - current += chunk_len; - } + current = 0; + for (const auto& p : to_populate) { + Index_ chunk_len = chunk_ticks[p.first + 1] - chunk_ticks[p.first]; + std::copy_n(chunk_numbers.begin() + current, chunk_len, p.second->number); + current += chunk_len; + } #ifdef TATAMI_R_PARALLELIZE_UNKNOWN - }); + }); #endif - } - ); - } + } + ); } }; +template +using SparseCore = typename std::conditional, + typename std::conditional, + MyopicSparseCore + >::type +>::type; + /****************************** *** Pure sparse extractors *** ******************************/ -template -struct SparseFull : public SparseBase, public tatami::SparseExtractor { +template +struct SparseFull : public tatami::SparseExtractor { SparseFull( const Rcpp::RObject& mat, const Rcpp::Function& sparse_extractor, + bool row, tatami::MaybeOracle ora, - Index_ secondary_dim, - bool by_column, - Index_ max_primary_chunk_length, + Index_ non_target_dim, + Index_ max_target_chunk_length, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache, + const tatami_chunked::SlabCacheStats& stats, bool needs_value, bool needs_index) : - SparseBase( - std::move(mat), - std::move(sparse_extractor), + core( + mat, + sparse_extractor, + row, std::move(ora), [&]() { - Rcpp::IntegerVector output(secondary_dim); + Rcpp::IntegerVector output(non_target_dim); std::iota(output.begin(), output.end(), 1); return output; }(), - by_column, - max_primary_chunk_length, + max_target_chunk_length, ticks, map, - cache_size_in_bytes, - require_minimum_cache, + stats, needs_value, needs_index - ) + ), + non_target_dim(non_target_dim), + needs_value(needs_value), + needs_index(needs_index) {} +private: + SparseCore core; + Index_ non_target_dim; + bool needs_value, needs_index; + public: tatami::SparseRange fetch(Index_ i, Value_* vbuffer, Index_* ibuffer) { - auto res = this->fetch_raw(i); + auto res = core.fetch_raw(i); const auto& slab = *(res.first); Index_ offset = res.second; - tatami::SparseRange output(slab.count[offset]); - if (this->needs_value) { - std::copy_n(slab.value.data() + static_cast(offset) * this->secondary_length, output.number, vbuffer); // cast to size_t to avoid overflow. + tatami::SparseRange output(slab.number[offset]); + if (needs_value) { + std::copy_n(slab.values[offset], non_target_dim, vbuffer); output.value = vbuffer; } - if (this->needs_index) { - std::copy_n(slab.index.data() + static_cast(offset) * this->secondary_length, output.number, ibuffer); + + if (needs_index) { + std::copy_n(slab.indices[offset], non_target_dim, ibuffer); output.index = ibuffer; } @@ -334,56 +357,62 @@ struct SparseFull : public SparseBase -struct SparseBlock : public SparseBase, public tatami::SparseExtractor { +template +struct SparseBlock : public tatami::SparseExtractor { SparseBlock( const Rcpp::RObject& mat, const Rcpp::Function& sparse_extractor, + bool row, tatami::MaybeOracle ora, Index_ block_start, Index_ block_length, - bool by_column, - Index_ max_primary_chunk_length, + Index_ max_target_chunk_length, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache, + const tatami_chunked::SlabCacheStats& stats, bool needs_value, bool needs_index) : - SparseBase( - std::move(mat), - std::move(sparse_extractor), + core( + mat, + sparse_extractor, + row, std::move(ora), [&]() { Rcpp::IntegerVector output(block_length); std::iota(output.begin(), output.end(), block_start + 1); return output; }(), - by_column, - max_primary_chunk_length, + max_target_chunk_length, ticks, map, - cache_size_in_bytes, - require_minimum_cache, + stats, needs_value, needs_index ), - block_start(block_start) + block_start(block_start), + needs_value(needs_value), + needs_index(needs_index) {} +private: + SparseCore core; + Index_ block_start; + bool needs_value, needs_index; + public: tatami::SparseRange fetch(Index_ i, Value_* vbuffer, Index_* ibuffer) { - auto res = this->fetch_raw(i); + auto res = core.fetch_raw(i); const auto& slab = *(res.first); Index_ offset = res.second; - tatami::SparseRange output(slab.count[offset]); - if (this->needs_value) { - std::copy_n(slab.value.data() + static_cast(offset) * this->secondary_length, output.number, vbuffer); // cast to size_t to avoid overflow. + tatami::SparseRange output(slab.number[offset]); + if (needs_value) { + std::copy_n(slab.values[offset], output.number, vbuffer); output.value = vbuffer; } - if (this->needs_index) { - auto iptr = slab.index.data() + static_cast(offset) * this->secondary_length; + + if (needs_index) { + auto iptr = slab.indices[offset]; for (Index_ i = 0; i < output.number; ++i) { ibuffer[i] = static_cast(iptr[i]) + block_start; } @@ -392,29 +421,26 @@ struct SparseBlock : public SparseBase -struct SparseIndexed : public SparseBase, public tatami::SparseExtractor { +template +struct SparseIndexed : public tatami::SparseExtractor { SparseIndexed( const Rcpp::RObject& mat, const Rcpp::Function& sparse_extractor, + bool row, tatami::MaybeOracle ora, tatami::VectorPtr idx_ptr, - bool by_column, - Index_ max_primary_chunk_length, + Index_ max_target_chunk_length, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache, + const tatami_chunked::SlabCacheStats& stats, bool needs_value, bool needs_index) : - SparseBase( - std::move(mat), - std::move(sparse_extractor), + core( + mat, + sparse_extractor, + row, std::move(ora), [&]() { Rcpp::IntegerVector output(idx_ptr->begin(), idx_ptr->end()); @@ -423,31 +449,37 @@ struct SparseIndexed : public SparseBase core; + tatami::VectorPtr indices_ptr; + bool needs_value, needs_index; + public: tatami::SparseRange fetch(Index_ i, Value_* vbuffer, Index_* ibuffer) { - auto res = this->fetch_raw(i); + auto res = core.fetch_raw(i); const auto& slab = *(res.first); Index_ offset = res.second; - tatami::SparseRange output(slab.count[offset]); - if (this->needs_value) { - std::copy_n(slab.value.data() + static_cast(offset) * this->secondary_length, output.number, vbuffer); // cast to size_t to avoid overflow. + tatami::SparseRange output(slab.number[offset]); + if (needs_value) { + std::copy_n(slab.values[offset], output.number, vbuffer); output.value = vbuffer; } - if (this->needs_index) { - auto iptr = slab.index.data() + static_cast(offset) * this->secondary_length; + + if (needs_index) { + auto iptr = slab.indices[offset]; const auto& indices = *indices_ptr; for (Index_ i = 0; i < output.number; ++i) { ibuffer[i] = indices[iptr[i]]; @@ -457,9 +489,6 @@ struct SparseIndexed : public SparseBase indices_ptr; }; /*********************************** @@ -467,115 +496,119 @@ struct SparseIndexed : public SparseBase -const Value_* densify(const Slab_& slab, Index_ offset, size_t secondary_length, Value_* buffer) { - size_t shift = static_cast(offset) * secondary_length; // cast to size_t to avoid overflow. - auto vptr = slab.value.data() + shift; - auto iptr = slab.index.data() + shift; - - std::fill_n(buffer, secondary_length, 0); - for (Index_ i = 0, end = slab.count[offset]; i < end; ++i, ++vptr, ++iptr) { +const Value_* densify(const Slab_& slab, Index_ offset, size_t non_target_length, Value_* buffer) { + auto vptr = slab.values[offset]; + auto iptr = slab.indices[offset]; + std::fill_n(buffer, non_target_length, 0); + for (Index_ i = 0, end = slab.number[offset]; i < end; ++i, ++vptr, ++iptr) { buffer[*iptr] = *vptr; } return buffer; } -template -struct DensifiedSparseFull : public SparseBase, public tatami::DenseExtractor { +template +struct DensifiedSparseFull : public tatami::DenseExtractor { DensifiedSparseFull( const Rcpp::RObject& mat, const Rcpp::Function& sparse_extractor, + bool row, tatami::MaybeOracle ora, - Index_ secondary_dim, - bool by_column, - Index_ max_primary_chunk_length, + Index_ non_target_dim, + Index_ max_target_chunk_length, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache) : - SparseBase( + const tatami_chunked::SlabCacheStats& stats) : + core( mat, sparse_extractor, + row, std::move(ora), [&]() { - Rcpp::IntegerVector output(secondary_dim); + Rcpp::IntegerVector output(non_target_dim); std::iota(output.begin(), output.end(), 1); return output; }(), - by_column, - max_primary_chunk_length, + max_target_chunk_length, ticks, map, - cache_size_in_bytes, - require_minimum_cache, + stats, true, true - ) + ), + non_target_dim(non_target_dim) {} +private: + SparseCore core; + size_t non_target_dim; + public: const Value_* fetch(Index_ i, Value_* buffer) { - auto res = this->fetch_raw(i); - return densify(*(res.first), res.second, this->secondary_length, buffer); + auto res = core.fetch_raw(i); + return densify(*(res.first), res.second, non_target_dim, buffer); } }; -template -struct DensifiedSparseBlock : public SparseBase, public tatami::DenseExtractor { +template +struct DensifiedSparseBlock : public tatami::DenseExtractor { DensifiedSparseBlock( const Rcpp::RObject& mat, const Rcpp::Function& sparse_extractor, + bool row, tatami::MaybeOracle ora, Index_ block_start, Index_ block_length, - bool by_column, - Index_ max_primary_chunk_length, + Index_ max_target_chunk_length, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache) : - SparseBase( + const tatami_chunked::SlabCacheStats& stats) : + core( mat, sparse_extractor, + row, std::move(ora), [&]() { Rcpp::IntegerVector output(block_length); std::iota(output.begin(), output.end(), block_start + 1); return output; }(), - by_column, - max_primary_chunk_length, + max_target_chunk_length, ticks, map, - cache_size_in_bytes, - require_minimum_cache, + stats, true, true - ) + ), + block_length(block_length) {} +private: + SparseCore core; + size_t block_length; + public: const Value_* fetch(Index_ i, Value_* buffer) { - auto res = this->fetch_raw(i); - return densify(*(res.first), res.second, this->secondary_length, buffer); + auto res = core.fetch_raw(i); + return densify(*(res.first), res.second, block_length, buffer); } }; -template -struct DensifiedSparseIndexed : public SparseBase, public tatami::DenseExtractor { +template +struct DensifiedSparseIndexed : public tatami::DenseExtractor { DensifiedSparseIndexed( const Rcpp::RObject& mat, const Rcpp::Function& sparse_extractor, + bool row, tatami::MaybeOracle ora, tatami::VectorPtr idx_ptr, - bool by_column, - Index_ max_primary_chunk_length, + Index_ max_target_chunk_length, const std::vector& ticks, const std::vector& map, - size_t cache_size_in_bytes, - bool require_minimum_cache) : - SparseBase( + const tatami_chunked::SlabCacheStats& stats) : + core( mat, sparse_extractor, + row, std::move(ora), [&]() { Rcpp::IntegerVector output(idx_ptr->begin(), idx_ptr->end()); @@ -584,21 +617,24 @@ struct DensifiedSparseIndexed : public SparseBasesize()) {} +private: + SparseCore core; + size_t num_indices; + public: const Value_* fetch(Index_ i, Value_* buffer) { - auto res = this->fetch_raw(i); - return densify(*(res.first), res.second, this->secondary_length, buffer); + auto res = core.fetch_raw(i); + return densify(*(res.first), res.second, num_indices, buffer); } }; diff --git a/include/tatami_r/sparse_matrix.hpp b/include/tatami_r/sparse_matrix.hpp index 53adf0d..41c62d3 100644 --- a/include/tatami_r/sparse_matrix.hpp +++ b/include/tatami_r/sparse_matrix.hpp @@ -7,12 +7,13 @@ namespace tatami_r { -template +template void parse_sparse_matrix_internal( Rcpp::RObject seed, + bool row, std::vector& value_ptrs, std::vector& index_ptrs, - std::vector& counts) + Index_* counts) { Rcpp::RObject raw_svt = seed.slot("SVT"); if (raw_svt == R_NilValue) { @@ -24,6 +25,9 @@ void parse_sparse_matrix_internal( bool needs_value = !value_ptrs.empty(); bool needs_index = !index_ptrs.empty(); + // Note that non-empty value_ptrs and index_ptrs may be longer than the + // number of rows/columns in the SVT matrix, due to the reuse of slabs. + for (int c = 0; c < NC; ++c) { Rcpp::RObject raw_inner(svt[c]); if (raw_inner == R_NilValue) { @@ -57,17 +61,21 @@ void parse_sparse_matrix_internal( throw std::runtime_error("both vectors of an element of the 'SVT' slot in a " + ctype + " object should have the same length"); } - if constexpr(transpose_) { - for (size_t i = 0; i < nnz; ++i) { - auto ix = curindices[i]; - auto& shift = counts[ix]; - if (needs_value) { - value_ptrs[ix][shift] = curvalues[i]; + if (row) { + if (needs_value) { + for (size_t i = 0; i < nnz; ++i) { + auto ix = curindices[i]; + value_ptrs[ix][counts[ix]] = curvalues[i]; } - if (needs_index) { - index_ptrs[ix][shift] = c; + } + if (needs_index) { + for (size_t i = 0; i < nnz; ++i) { + auto ix = curindices[i]; + index_ptrs[ix][counts[ix]] = c; } - ++shift; + } + for (size_t i = 0; i < nnz; ++i) { + ++(counts[curindices[i]]); } } else { @@ -82,12 +90,13 @@ void parse_sparse_matrix_internal( } } -template +template void parse_sparse_matrix( Rcpp::RObject seed, + bool row, std::vector& value_ptrs, std::vector& index_ptrs, - std::vector& counts) + Index_* counts) { auto ctype = get_class_name(seed); if (ctype != "SVT_SparseMatrix") { @@ -101,11 +110,11 @@ void parse_sparse_matrix( std::string type = Rcpp::as(seed.slot("type")); if (type == "double") { - parse_sparse_matrix_internal(seed, value_ptrs, index_ptrs, counts); + parse_sparse_matrix_internal(seed, row, value_ptrs, index_ptrs, counts); } else if (type == "integer") { - parse_sparse_matrix_internal(seed, value_ptrs, index_ptrs, counts); + parse_sparse_matrix_internal(seed, row, value_ptrs, index_ptrs, counts); } else if (type == "logical") { - parse_sparse_matrix_internal(seed, value_ptrs, index_ptrs, counts); + parse_sparse_matrix_internal(seed, row, value_ptrs, index_ptrs, counts); } else { throw std::runtime_error("unsupported type '" + type + "' for a " + ctype); } diff --git a/tests/src/bindings.cpp b/tests/src/bindings.cpp index ebfa125..fde9278 100644 --- a/tests/src/bindings.cpp +++ b/tests/src/bindings.cpp @@ -18,8 +18,8 @@ typedef Rcpp::XPtr > RatXPtr; //' @export //[[Rcpp::export(rng=false)]] SEXP parse(Rcpp::RObject seed, double cache_size, bool require_min) { - if (cache_size < 0) { - tatami_r::Options opt; + if (cache_size >= 0) { + tatami_r::UnknownMatrixOptions opt; opt.maximum_cache_size = cache_size; opt.require_minimum_cache = require_min; return RatXPtr(new tatami_r::UnknownMatrix(seed, opt));