diff --git a/.gitignore b/.gitignore index 03d7d9dc4e..5313c4b014 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,4 @@ __cache__/ .rendered* installed.json tmp/ +test_7.json.state diff --git a/CMakeLists.txt b/CMakeLists.txt index a8fb090e4a..cc0db00bc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,9 @@ if (MSVC) # /W4 : warnings level 4 (default in visual studio projects and recommended minimum level). # /external:I $ENV{CONDA_PREFIX}: consider the conda env prefix libraries headers as "external" to this project. # /external:W0 : set the warning level to 1 for external libraries headers (severe warnings, default when unspecified) to only see important warnings coming from dependencies headers. - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /DNOMINMAX /EHsc /Zc:__cplusplus /MP /W4 /external:I $ENV{CONDA_PREFIX} /external:W1") + string(REGEX REPLACE "/W[0-9]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /EHsc /Zc:__cplusplus /MP /experimental:external /external:I $ENV{CONDA_PREFIX} /external:W1") + # Force release mode to avoid debug libraries to be linked set(CMAKE_BUILD_TYPE Release) # add_definitions("-DUNICODE -D_UNICODE") else() diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index 97000d593e..6879b4735d 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -430,6 +430,7 @@ macro(libmamba_create_target target_name linkage deps_linkage output_name) find_package(CURL REQUIRED) find_package(LibArchive REQUIRED) find_package(zstd REQUIRED) + find_package(BZip2 REQUIRED) find_package(OpenSSL REQUIRED) find_package(yaml-cpp CONFIG REQUIRED) find_package(reproc++ CONFIG REQUIRED) @@ -444,6 +445,8 @@ macro(libmamba_create_target target_name linkage deps_linkage output_name) zstd::libzstd_shared ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES} + zstd::libzstd_shared + BZip2::BZip2 yaml-cpp reproc++ reproc diff --git a/libmamba/include/mamba/core/context.hpp b/libmamba/include/mamba/core/context.hpp index 65b620b130..c78f226b0c 100644 --- a/libmamba/include/mamba/core/context.hpp +++ b/libmamba/include/mamba/core/context.hpp @@ -236,11 +236,12 @@ namespace mamba bool override_channels_enabled = true; - std::vector pinned_packages = {}; bool use_only_tar_bz2 = false; + bool repodata_use_zst = false; + std::vector repodata_has_zst = { "https://conda.anaconda.org/conda-forge" }; // usernames on anaconda.org can have a underscore, which influences the // first two characters diff --git a/libmamba/include/mamba/core/error_handling.hpp b/libmamba/include/mamba/core/error_handling.hpp index d108d40a1a..ad305520e0 100644 --- a/libmamba/include/mamba/core/error_handling.hpp +++ b/libmamba/include/mamba/core/error_handling.hpp @@ -30,6 +30,7 @@ namespace mamba lockfile_failure, selfupdate_failure, satisfiablitity_error, + user_interrupted, }; class mamba_error : public std::runtime_error diff --git a/libmamba/include/mamba/core/fetch.hpp b/libmamba/include/mamba/core/fetch.hpp index 3255b86364..2c4b55d4b4 100644 --- a/libmamba/include/mamba/core/fetch.hpp +++ b/libmamba/include/mamba/core/fetch.hpp @@ -15,8 +15,8 @@ extern "C" #include #include - -#include "nlohmann/json.hpp" +#include +#include #include "progress_bar.hpp" #include "validate.hpp" @@ -25,6 +25,78 @@ namespace mamba { void init_curl_ssl(); + struct ZstdStream + { + constexpr static size_t BUFFER_SIZE = 256000; + ZstdStream(curl_write_callback write_callback, void* write_callback_data) + : stream(ZSTD_createDCtx()) + , m_write_callback(write_callback) + , m_write_callback_data(write_callback_data) + { + ZSTD_initDStream(stream); + } + + ~ZstdStream() + { + ZSTD_freeDCtx(stream); + } + + size_t write(char* in, size_t size); + + static size_t write_callback(char* ptr, size_t size, size_t nmemb, void* self) + { + return static_cast(self)->write(ptr, size * nmemb); + } + + ZSTD_DCtx* stream; + char buffer[BUFFER_SIZE]; + + // original curl callback + curl_write_callback m_write_callback; + void* m_write_callback_data; + }; + + struct Bzip2Stream + { + constexpr static size_t BUFFER_SIZE = 256000; + + Bzip2Stream(curl_write_callback write_callback, void* write_callback_data) + : m_write_callback(write_callback) + , m_write_callback_data(write_callback_data) + { + stream.bzalloc = nullptr; + stream.bzfree = nullptr; + stream.opaque = nullptr; + + error = BZ2_bzDecompressInit(&stream, 0, false); + if (error != BZ_OK) + { + throw std::runtime_error("BZ2_bzDecompressInit failed"); + } + } + + size_t write(char* in, size_t size); + + static size_t write_callback(char* ptr, size_t size, size_t nmemb, void* self) + { + return static_cast(self)->write(ptr, size * nmemb); + } + + ~Bzip2Stream() + { + BZ2_bzDecompressEnd(&stream); + } + + int error; + bz_stream stream; + char buffer[BUFFER_SIZE]; + + // original curl callback + curl_write_callback m_write_callback; + void* m_write_callback_data; + }; + + class DownloadTarget { public: @@ -45,11 +117,16 @@ namespace mamba static int progress_callback( void*, curl_off_t total_to_download, curl_off_t now_downloaded, curl_off_t, curl_off_t); - void set_mod_etag_headers(const nlohmann::json& mod_etag); + void set_mod_etag_headers(const std::string& mod, const std::string& etag); void set_progress_bar(ProgressProxy progress_proxy); void set_expected_size(std::size_t size); + void set_head_only(bool yes) + { + curl_easy_setopt(m_handle, CURLOPT_NOBODY, yes); + } const std::string& name() const; + const std::string& url() const; std::size_t expected_size() const; void init_curl_target(const std::string& url); @@ -61,9 +138,9 @@ namespace mamba curl_off_t get_speed(); template - inline void set_finalize_callback(bool (C::*cb)(), C* data) + inline void set_finalize_callback(bool (C::*cb)(const DownloadTarget&), C* data) { - m_finalize_callback = std::bind(cb, data); + m_finalize_callback = std::bind(cb, data, std::placeholders::_1); } inline void set_ignore_failure(bool yes) @@ -97,7 +174,9 @@ namespace mamba std::string etag, mod, cache_control; private: - std::function m_finalize_callback; + std::unique_ptr m_zstd_stream; + std::unique_ptr m_bzip2_stream; + std::function m_finalize_callback; std::string m_name, m_filename, m_url; @@ -145,6 +224,7 @@ namespace mamba const int MAMBA_DOWNLOAD_FAILFAST = 1 << 0; const int MAMBA_DOWNLOAD_SORT = 1 << 1; + const int MAMBA_NO_CLEAR_PROGRESS_BARS = 1 << 2; } // namespace mamba #endif // MAMBA_FETCH_HPP diff --git a/libmamba/include/mamba/core/mamba_fs.hpp b/libmamba/include/mamba/core/mamba_fs.hpp index 69d783098b..9f787db332 100644 --- a/libmamba/include/mamba/core/mamba_fs.hpp +++ b/libmamba/include/mamba/core/mamba_fs.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "mamba/core/util_string.hpp" @@ -1170,7 +1171,7 @@ namespace fs } // void last_write_time(const path& p, now _, error_code& ec) noexcept; - inline void last_write_time(const u8path& path, now _, std::error_code& ec) noexcept + inline void last_write_time(const u8path& path, now, std::error_code& ec) noexcept { #if defined(USE_UTIMENSAT) if (utimensat(AT_FDCWD, path.string().c_str(), NULL, 0) == -1) @@ -1380,5 +1381,21 @@ struct std::hash<::fs::u8path> } }; +template <> +struct fmt::formatter<::fs::u8path> +{ + constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) + { + // make sure that range is empty + if (ctx.begin() != ctx.end() && *ctx.begin() != '}') + throw format_error("invalid format"); + return ctx.begin(); + } + template + auto format(const ::fs::u8path& path, FormatContext& ctx) + { + return fmt::format_to(ctx.out(), "'{}'", path.string()); + } +}; #endif diff --git a/libmamba/include/mamba/core/package_handling.hpp b/libmamba/include/mamba/core/package_handling.hpp index f7c987ac00..dc0df1e9df 100644 --- a/libmamba/include/mamba/core/package_handling.hpp +++ b/libmamba/include/mamba/core/package_handling.hpp @@ -17,6 +17,7 @@ namespace mamba { enum compression_algorithm { + none, bzip2, zip, zstd diff --git a/libmamba/include/mamba/core/subdirdata.hpp b/libmamba/include/mamba/core/subdirdata.hpp index df23bb4395..0e4ddd5de0 100644 --- a/libmamba/include/mamba/core/subdirdata.hpp +++ b/libmamba/include/mamba/core/subdirdata.hpp @@ -11,8 +11,6 @@ #include #include -#include "nlohmann/json.hpp" - #include "mamba/core/channel.hpp" #include "mamba/core/context.hpp" #include "mamba/core/error_handling.hpp" @@ -22,15 +20,54 @@ #include "mamba/core/pool.hpp" #include "mamba/core/repo.hpp" #include "mamba/core/util.hpp" - +#include "package_handling.hpp" namespace decompress { - bool raw(const std::string& in, const std::string& out); + bool raw(mamba::compression_algorithm ca, const std::string& in, const std::string& out); } namespace mamba { + struct subdir_metadata + { + struct checked_at + { + bool value; + std::time_t last_checked; + + bool has_expired() const + { + // difference in seconds, check every 14 days + return std::difftime(std::time(nullptr), last_checked) > 60 * 60 * 24 * 14; + } + }; + + static tl::expected from_stream(std::istream& in); + + std::string url; + std::string etag; + std::string mod; + std::string cache_control; +#ifdef _WIN32 + std::chrono::system_clock::time_point stored_mtime; +#else + fs::file_time_type stored_mtime; +#endif + std::size_t stored_file_size; + std::optional has_zst; + std::optional has_bz2; + std::optional has_jlap; + + void store_file_metadata(const fs::u8path& path); + bool check_valid_metadata(const fs::u8path& path); + + void serialize_to_stream(std::ostream& out) const; + void serialize_to_stream_tiny(std::ostream& out) const; + + bool check_zst(const Channel* channel); + }; + /** * Represents a channel subdirectory (i.e. a platform) @@ -55,8 +92,8 @@ namespace mamba MSubdirData& operator=(MSubdirData&&); // TODO return seconds as double - fs::file_time_type::duration check_cache(const fs::u8path& cache_file, - const fs::file_time_type::clock::time_point& ref); + fs::file_time_type::duration check_cache( + const fs::u8path& cache_file, const fs::file_time_type::clock::time_point& ref) const; bool loaded() const; bool forbid_cache(); @@ -65,9 +102,12 @@ namespace mamba expected_t cache_path() const; const std::string& name() const; + std::vector>& check_targets(); DownloadTarget* target(); - bool finalize_transfer(); + bool finalize_check(const DownloadTarget& target); + bool finalize_transfer(const DownloadTarget& target); + void finalize_checks(); expected_t create_repo(MPool& pool); private: @@ -78,11 +118,13 @@ namespace mamba const std::string& repodata_fn = "repodata.json"); bool load(MultiPackageCache& caches); - bool decompress(); - void create_target(nlohmann::json& mod_etag); + void check_repodata_existence(); + void create_target(); std::size_t get_cache_control_max_age(const std::string& val); + void refresh_last_write_time(const fs::u8path& json_file, const fs::u8path& solv_file); std::unique_ptr m_target = nullptr; + std::vector> m_check_targets; bool m_json_cache_valid = false; bool m_solv_cache_valid = false; @@ -92,6 +134,7 @@ namespace mamba fs::u8path m_writable_pkgs_dir; ProgressProxy m_progress_bar; + ProgressProxy m_progress_bar_check; bool m_loaded; bool m_download_complete; @@ -100,7 +143,7 @@ namespace mamba std::string m_json_fn; std::string m_solv_fn; bool m_is_noarch; - nlohmann::json m_mod_etag; + subdir_metadata m_metadata; std::unique_ptr m_temp_file; const Channel* p_channel = nullptr; }; diff --git a/libmamba/include/mamba/core/transaction.hpp b/libmamba/include/mamba/core/transaction.hpp index 4f335d1099..d6d40cc586 100644 --- a/libmamba/include/mamba/core/transaction.hpp +++ b/libmamba/include/mamba/core/transaction.hpp @@ -50,7 +50,7 @@ namespace mamba void write_repodata_record(const fs::u8path& base_path); void add_url(); - bool finalize_callback(); + bool finalize_callback(const DownloadTarget& target); bool finished(); void validate(); bool extract(); diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index 66da5ef09b..21b82524de 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -4,6 +4,7 @@ #include "mamba/core/output.hpp" #include "mamba/core/repo.hpp" #include "mamba/core/subdirdata.hpp" +#include "mamba/core/thread_utils.hpp" namespace mamba @@ -61,15 +62,14 @@ namespace mamba { for (auto& [platform, url] : channel->platform_urls(true)) { - auto sdires = MSubdirData::create(*channel, platform, url, package_caches); + auto sdires + = MSubdirData::create(*channel, platform, url, package_caches, "repodata.json"); if (!sdires.has_value()) { error_list.push_back(std::move(sdires).error()); continue; } auto sdir = std::move(sdires).value(); - - multi_dl.add(sdir.target()); subdirs.push_back(std::move(sdir)); if (ctx.channel_priority == ChannelPriority::kDisabled) { @@ -87,7 +87,34 @@ namespace mamba } } } - // TODO load local channels even when offline + + for (auto& subdir : subdirs) + { + for (auto& check_target : subdir.check_targets()) + { + multi_dl.add(check_target.get()); + } + } + + multi_dl.download(MAMBA_NO_CLEAR_PROGRESS_BARS); + if (is_sig_interrupted()) + { + error_list.push_back( + mamba_error("Interrupted by user", mamba_error_code::user_interrupted)); + return tl::unexpected(mamba_aggregated_error(std::move(error_list))); + } + + for (auto& subdir : subdirs) + { + if (!subdir.check_targets().empty()) + { + // recreate final download target in case HEAD requests succeeded + subdir.finalize_checks(); + } + multi_dl.add(subdir.target()); + } + + // TODO load local channels even when offline if (!ctx.offline) if (!ctx.offline) { try diff --git a/libmamba/src/api/configuration.cpp b/libmamba/src/api/configuration.cpp index 34bd580ee1..0bde67a4de 100644 --- a/libmamba/src/api/configuration.cpp +++ b/libmamba/src/api/configuration.cpp @@ -1131,6 +1131,16 @@ namespace mamba .set_env_var_names() .description("Permit use of the --overide-channels command-line flag")); + insert(Configurable("repodata_use_zst", &ctx.repodata_use_zst) + .group("Repodata") + .set_rc_configurable() + .description("Use zstd encoded repodata when fetching")); + + insert(Configurable("repodata_has_zst", &ctx.repodata_has_zst) + .group("Repodata") + .set_rc_configurable() + .description("Channels that have zstd encoded repodata (saves a HEAD request)")); + // Network insert(Configurable("cacert_path", std::string("")) .group("Network") diff --git a/libmamba/src/core/fetch.cpp b/libmamba/src/core/fetch.cpp index ef0c371992..e088037050 100644 --- a/libmamba/src/core/fetch.cpp +++ b/libmamba/src/core/fetch.cpp @@ -20,6 +20,61 @@ namespace mamba { + size_t ZstdStream::write(char* in, size_t size) + { + ZSTD_inBuffer input = { in, size, 0 }; + ZSTD_outBuffer output = { buffer, BUFFER_SIZE, 0 }; + + while (input.pos < input.size) + { + auto ret = ZSTD_decompressStream(stream, &output, &input); + if (ZSTD_isError(ret)) + { + LOG_ERROR << "ZSTD decompression error: " << ZSTD_getErrorName(ret); + return size + 1; + } + if (output.pos > 0) + { + size_t wcb_res = m_write_callback(buffer, 1, output.pos, m_write_callback_data); + if (wcb_res != output.pos) + { + return size + 1; + } + output.pos = 0; + } + } + return size; + } + + size_t Bzip2Stream::write(char* in, size_t size) + { + bz_stream* stream = static_cast(m_write_callback_data); + stream->next_in = in; + stream->avail_in = size; + + while (stream->avail_in > 0) + { + stream->next_out = buffer; + stream->avail_out = Bzip2Stream::BUFFER_SIZE; + + int ret = BZ2_bzDecompress(stream); + if (ret != BZ_OK && ret != BZ_STREAM_END) + { + LOG_ERROR << "Bzip2 decompression error: " << ret; + return size + 1; + } + + size_t wcb_res = m_write_callback( + buffer, 1, BUFFER_SIZE - stream->avail_out, m_write_callback_data); + if (wcb_res != BUFFER_SIZE - stream->avail_out) + { + return size + 1; + } + } + return size; + } + + void init_curl_ssl() { auto& ctx = Context::instance(); @@ -310,14 +365,37 @@ namespace mamba curl_easy_setopt(m_handle, CURLOPT_HEADERFUNCTION, &DownloadTarget::header_callback); curl_easy_setopt(m_handle, CURLOPT_HEADERDATA, this); - curl_easy_setopt(m_handle, CURLOPT_WRITEFUNCTION, &DownloadTarget::write_callback); - curl_easy_setopt(m_handle, CURLOPT_WRITEDATA, this); + if (ends_with(url, ".json.zst")) + { + m_zstd_stream = std::make_unique(&DownloadTarget::write_callback, this); + if (ends_with(m_filename, ".zst")) + { + m_filename = m_filename.substr(0, m_filename.size() - 4); + } + curl_easy_setopt(m_handle, CURLOPT_WRITEFUNCTION, ZstdStream::write_callback); + curl_easy_setopt(m_handle, CURLOPT_WRITEDATA, m_zstd_stream.get()); + } + else if (ends_with(url, ".json.bz2")) + { + m_bzip2_stream = std::make_unique(&DownloadTarget::write_callback, this); + if (ends_with(m_filename, ".bz2")) + { + m_filename = m_filename.substr(0, m_filename.size() - 4); + } + curl_easy_setopt(m_handle, CURLOPT_WRITEFUNCTION, Bzip2Stream::write_callback); + curl_easy_setopt(m_handle, CURLOPT_WRITEDATA, m_bzip2_stream.get()); + } + else + { + curl_easy_setopt(m_handle, CURLOPT_WRITEFUNCTION, &DownloadTarget::write_callback); + curl_easy_setopt(m_handle, CURLOPT_WRITEDATA, this); + } m_headers = nullptr; if (ends_with(url, ".json")) { - curl_easy_setopt( - m_handle, CURLOPT_ACCEPT_ENCODING, "gzip, deflate, compress, identity"); + // accept all encodings supported by the libcurl build + curl_easy_setopt(m_handle, CURLOPT_ACCEPT_ENCODING, ""); m_headers = curl_slist_append(m_headers, "Content-Type: application/json"); } @@ -395,7 +473,6 @@ namespace mamba } } - size_t DownloadTarget::write_callback(char* ptr, size_t size, size_t nmemb, void* self) { auto* s = reinterpret_cast(self); @@ -522,20 +599,18 @@ namespace mamba return 0; } - void DownloadTarget::set_mod_etag_headers(const nlohmann::json& mod_etag) + void DownloadTarget::set_mod_etag_headers(const std::string& mod, const std::string& etag) { auto to_header = [](const std::string& key, const std::string& value) { return std::string(key + ": " + value); }; - if (mod_etag.find("_etag") != mod_etag.end()) + if (!etag.empty()) { - m_headers = curl_slist_append(m_headers, - to_header("If-None-Match", mod_etag["_etag"]).c_str()); + m_headers = curl_slist_append(m_headers, to_header("If-None-Match", etag).c_str()); } - if (mod_etag.find("_mod") != mod_etag.end()) + if (!mod.empty()) { - m_headers = curl_slist_append(m_headers, - to_header("If-Modified-Since", mod_etag["_mod"]).c_str()); + m_headers = curl_slist_append(m_headers, to_header("If-Modified-Since", mod).c_str()); } } @@ -560,6 +635,11 @@ namespace mamba return m_name; } + const std::string& DownloadTarget::url() const + { + return m_url; + } + std::size_t DownloadTarget::expected_size() const { return m_expected_size; @@ -603,7 +683,7 @@ namespace mamba result = curl_easy_perform(m_handle); set_result(result); - return m_finalize_callback ? m_finalize_callback() : true; + return m_finalize_callback ? m_finalize_callback(*this) : true; } CURL* DownloadTarget::handle() @@ -704,7 +784,7 @@ namespace mamba bool ret = true; if (m_finalize_callback) { - ret = m_finalize_callback(); + ret = m_finalize_callback(*this); } else { @@ -836,6 +916,7 @@ namespace mamba { bool failfast = options & MAMBA_DOWNLOAD_FAILFAST; bool sort = options & MAMBA_DOWNLOAD_SORT; + bool no_clear_progress_bars = options & MAMBA_NO_CLEAR_PROGRESS_BARS; auto& ctx = Context::instance(); @@ -861,7 +942,6 @@ namespace mamba bool pbar_manager_started = pbar_manager.started(); if (!(ctx.no_progress_bars || ctx.json || ctx.quiet || pbar_manager_started)) { - pbar_manager.start(); pbar_manager.watch_print(); } @@ -933,14 +1013,14 @@ namespace mamba if (is_sig_interrupted()) { Console::instance().print("Download interrupted"); - curl_multi_cleanup(m_handle); return false; } if (!(ctx.no_progress_bars || ctx.json || ctx.quiet || pbar_manager_started)) { pbar_manager.terminate(); - pbar_manager.clear_progress_bars(); + if (!no_clear_progress_bars) + pbar_manager.clear_progress_bars(); } return true; diff --git a/libmamba/src/core/repo.cpp b/libmamba/src/core/repo.cpp index 5e096013b8..76adee6882 100644 --- a/libmamba/src/core/repo.cpp +++ b/libmamba/src/core/repo.cpp @@ -312,7 +312,7 @@ namespace mamba m_solv_file.replace_extension("solv"); } - LOG_INFO << "Reading cache files '" << (filename.parent_path() / filename.stem()).string() + LOG_INFO << "Reading cache files '" << (filename.parent_path() / filename).string() << ".*' for repo index '" << m_repo->name << "'"; if (is_solv) @@ -328,7 +328,7 @@ namespace mamba throw std::runtime_error("Could not open repository file " + filename.string()); } - LOG_DEBUG << "Attempt load from solv " << m_solv_file; + LOG_INFO << "Attempt load from solv " << m_solv_file; int ret = repo_add_solv(m_repo, fp, 0); if (ret != 0) @@ -358,6 +358,8 @@ namespace mamba const char* mod = repodata_lookup_str(repodata, SOLVID_META, mod_id); const char* tool_version = repodata_lookup_str(repodata, SOLVID_META, REPOSITORY_TOOLVERSION); + LOG_INFO << "Metadata solv file: " << url << " " << pip_added << " " << etag + << " " << mod << " " << tool_version; bool metadata_valid = !(!url || !etag || !mod || !tool_version || pip_added == failure); @@ -368,15 +370,15 @@ namespace mamba && (std::strcmp(tool_version, mamba_tool_version()) == 0); } - LOG_DEBUG << "Metadata from SOLV are " - << (metadata_valid ? "valid" : "NOT valid"); + LOG_INFO << "Metadata from SOLV are " + << (metadata_valid ? "valid" : "NOT valid"); if (!metadata_valid) { - LOG_DEBUG << "SOLV file was written with a previous version of " - "libsolv or mamba " - << (tool_version != nullptr ? tool_version : "") - << ", updating it now!"; + LOG_INFO << "SOLV file was written with a previous version of " + "libsolv or mamba " + << (tool_version != nullptr ? tool_version : "") + << ", updating it now!"; } else { diff --git a/libmamba/src/core/subdirdata.cpp b/libmamba/src/core/subdirdata.cpp index fdf7f731d2..a7a5eca8f8 100644 --- a/libmamba/src/core/subdirdata.cpp +++ b/libmamba/src/core/subdirdata.cpp @@ -14,68 +14,189 @@ #include "progress_bar_impl.hpp" #include -namespace decompress +namespace mamba { - bool raw(const std::string& in, const std::string& out) + void subdir_metadata::serialize_to_stream(std::ostream& out) const { - int r; - std::ptrdiff_t size; + nlohmann::json j; + j["url"] = url; + j["etag"] = etag; + j["mod"] = mod; + j["cache_control"] = cache_control; + j["file_size"] = stored_file_size; + + auto secs + = std::chrono::duration_cast(stored_mtime.time_since_epoch()); + auto nsecs = std::chrono::duration_cast( + stored_mtime.time_since_epoch() - secs); + + j["file_mtime"]["seconds"] = secs.count(); + j["file_mtime"]["nanoseconds"] = nsecs.count(); + + if (has_zst.has_value()) + { + j["has_zst"]["value"] = has_zst.value().value; + j["has_zst"]["last_checked"] = timestamp(has_zst.value().last_checked); + } + out << j.dump(4); + } - LOG_INFO << "Decompressing from " << in << " to " << out; + void subdir_metadata::serialize_to_stream_tiny(std::ostream& out) const + { + nlohmann::json j; + j["_url"] = url; + j["_etag"] = etag; + j["_mod"] = mod; + j["_cache_control"] = cache_control; + out << j.dump(); + } - struct archive* a = archive_read_new(); - archive_read_support_filter_bzip2(a); - archive_read_support_format_raw(a); - // TODO figure out good value for this - const std::size_t BLOCKSIZE = 16384; - r = archive_read_open_filename(a, in.c_str(), BLOCKSIZE); - if (r != ARCHIVE_OK) + bool subdir_metadata::check_zst(const Channel* channel) + { + if (has_zst.has_value()) { - return false; + if (!has_zst.value().has_expired()) + { + return has_zst.value().value; + } } - struct archive_entry* entry; - std::ofstream out_file = mamba::open_ofstream(out); - char buff[BLOCKSIZE]; - std::size_t buffsize = BLOCKSIZE; - r = archive_read_next_header(a, &entry); - if (r != ARCHIVE_OK) + for (const auto& c : Context::instance().repodata_has_zst) { + if (make_channel(c) == *channel) + { + has_zst + = { true, + std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) }; + return true; + break; + } + } + return false; + } + +#ifdef _WIN32 + std::chrono::system_clock::time_point filetime_to_unix(const fs::file_time_type& filetime) + { + // windows filetime is in 100ns intervals since 1601-01-01 + constexpr static auto epoch_offset = std::chrono::seconds(11644473600ULL); + return std::chrono::system_clock::time_point( + std::chrono::duration_cast( + filetime.time_since_epoch() - epoch_offset)); + } +#endif + + void subdir_metadata::store_file_metadata(const fs::u8path& file) + { +#ifndef _WIN32 + stored_mtime = fs::last_write_time(file); +#else + // convert windows filetime to unix timestamp + stored_mtime = filetime_to_unix(fs::last_write_time(file)); +#endif + stored_file_size = fs::file_size(file); + } + + bool subdir_metadata::check_valid_metadata(const fs::u8path& file) + { + if (stored_file_size != fs::file_size(file)) + { + LOG_INFO << "File size changed, invalidating metadata"; return false; } +#ifndef _WIN32 + bool last_write_time_valid = fs::last_write_time(file) == stored_mtime; +#else + bool last_write_time_valid = filetime_to_unix(fs::last_write_time(file)) == stored_mtime; +#endif + if (!last_write_time_valid) + { + LOG_INFO << "File mtime changed, invalidating metadata"; + } + return last_write_time_valid; + } - while (true) + tl::expected subdir_metadata::from_stream(std::istream& in) + { + nlohmann::json j = nlohmann::json::parse(in); + subdir_metadata m; + try { - size = archive_read_data(a, &buff, buffsize); - if (size < ARCHIVE_OK) + m.url = j["url"].get(); + m.etag = j["etag"].get(); + m.mod = j["mod"].get(); + m.cache_control = j["cache_control"].get(); + m.stored_file_size = j["file_size"].get(); + + using time_type = decltype(m.stored_mtime); + m.stored_mtime = time_type(std::chrono::duration_cast( + std::chrono::seconds(j["file_mtime"]["seconds"].get()) + + std::chrono::nanoseconds(j["file_mtime"]["nanoseconds"].get()))); + + int err_code = 0; + if (j.find("has_zst") != j.end()) { - throw std::runtime_error(std::string("Could not read archive: ") - + archive_error_string(a)); + m.has_zst = { j["has_zst"]["value"].get(), + parse_utc_timestamp(j["has_zst"]["last_checked"].get(), + err_code) }; } - if (size == 0) - { - break; - } - out_file.write(buff, size); } - - archive_read_free(a); - return true; + catch (const std::exception& e) + { + return make_unexpected(fmt::format("Could not load cache state: {}", e.what()), + mamba_error_code::cache_not_loaded); + } + return m; } -} // namespace decompress -namespace mamba -{ + namespace detail { - nlohmann::json read_mod_and_etag(const fs::u8path& file) + tl::expected read_metadata(const fs::u8path& file) { + auto state_file = file; + state_file.replace_extension(".state.json"); + std::error_code ec; + if (fs::exists(state_file, ec)) + { + auto infile = open_ifstream(state_file); + auto m = subdir_metadata::from_stream(infile); + if (!m.has_value()) + { + LOG_WARNING << "Could not parse state file" << m.error().what(); + fs::remove(state_file, ec); + if (ec) + { + LOG_WARNING << "Could not remove state file " << state_file << ": " + << ec.message(); + } + return make_unexpected( + fmt::format("File: {}: {}", state_file, m.error().what()), + mamba_error_code::cache_not_loaded); + } + + if (!m.value().check_valid_metadata(file)) + { + LOG_WARNING << "Cache file " << file << " was modified by another program"; + // TODO clear out json file values? + m.value().etag = ""; + m.value().mod = ""; + m.value().cache_control = ""; + m.value().stored_file_size = 0; + m.value().stored_mtime = decltype(m.value().stored_mtime)::min(); + return make_unexpected( + fmt::format("File: {}: Cache file mtime mismatch", state_file), + mamba_error_code::cache_not_loaded); + } + + return m.value(); + } + // parse json at the beginning of the stream such as // {"_url": "https://conda.anaconda.org/conda-forge/linux-64", // "_etag": "W/\"6092e6a2b6cec6ea5aade4e177c3edda-8\"", // "_mod": "Sat, 04 Apr 2020 03:29:49 GMT", // "_cache_control": "public, max-age=1200" - auto extract_subjson = [](std::ifstream& s) -> std::string { char next; @@ -152,15 +273,22 @@ namespace mamba try { result = nlohmann::json::parse(json); - return result; + subdir_metadata m; + m.url = result.value("_url", ""); + m.etag = result.value("_etag", ""); + m.mod = result.value("_mod", ""); + m.cache_control = result.value("_cache_control", ""); + return m; } - catch (...) + catch (std::exception& e) { LOG_WARNING << "Could not parse mod/etag header"; - return nlohmann::json(); + return make_unexpected(fmt::format("File: {}: Could not parse mod/etag header ({})", + state_file, + e.what()), + mamba_error_code::cache_not_loaded); } } - } expected_t MSubdirData::create(const Channel& channel, @@ -206,12 +334,14 @@ namespace mamba MSubdirData::MSubdirData(MSubdirData&& rhs) : m_target(std::move(rhs.m_target)) + , m_check_targets(std::move(rhs.m_check_targets)) , m_json_cache_valid(rhs.m_json_cache_valid) , m_solv_cache_valid(rhs.m_solv_cache_valid) , m_valid_cache_path(std::move(rhs.m_valid_cache_path)) , m_expired_cache_path(std::move(rhs.m_expired_cache_path)) , m_writable_pkgs_dir(std::move(rhs.m_writable_pkgs_dir)) , m_progress_bar(std::move(rhs.m_progress_bar)) + , m_progress_bar_check(std::move(rhs.m_progress_bar_check)) , m_loaded(rhs.m_loaded) , m_download_complete(rhs.m_download_complete) , m_repodata_url(std::move(rhs.m_repodata_url)) @@ -219,7 +349,7 @@ namespace mamba , m_json_fn(std::move(rhs.m_json_fn)) , m_solv_fn(std::move(rhs.m_solv_fn)) , m_is_noarch(rhs.m_is_noarch) - , m_mod_etag(std::move(rhs.m_mod_etag)) + , m_metadata(std::move(rhs.m_metadata)) , m_temp_file(std::move(rhs.m_temp_file)) , p_channel(rhs.p_channel) { @@ -227,6 +357,10 @@ namespace mamba { m_target->set_finalize_callback(&MSubdirData::finalize_transfer, this); } + for (auto& t : m_check_targets) + { + t->set_finalize_callback(&MSubdirData::finalize_check, this); + } } MSubdirData& MSubdirData::operator=(MSubdirData&& rhs) @@ -239,6 +373,7 @@ namespace mamba swap(m_expired_cache_path, rhs.m_expired_cache_path); swap(m_writable_pkgs_dir, rhs.m_writable_pkgs_dir); swap(m_progress_bar, m_progress_bar); + swap(m_progress_bar_check, m_progress_bar_check); swap(m_loaded, rhs.m_loaded); swap(m_download_complete, rhs.m_download_complete); swap(m_repodata_url, rhs.m_repodata_url); @@ -246,8 +381,9 @@ namespace mamba swap(m_json_fn, rhs.m_json_fn); swap(m_solv_fn, rhs.m_solv_fn); swap(m_is_noarch, rhs.m_is_noarch); - swap(m_mod_etag, rhs.m_mod_etag); + swap(m_metadata, rhs.m_metadata); swap(m_temp_file, rhs.m_temp_file); + swap(m_check_targets, rhs.m_check_targets); swap(p_channel, rhs.p_channel); if (m_target != nullptr) @@ -258,11 +394,21 @@ namespace mamba { rhs.m_target->set_finalize_callback(&MSubdirData::finalize_transfer, &rhs); } + + for (auto& t : m_check_targets) + { + t->set_finalize_callback(&MSubdirData::finalize_check, this); + } + for (auto& t : rhs.m_check_targets) + { + t->set_finalize_callback(&MSubdirData::finalize_check, &rhs); + } + return *this; } fs::file_time_type::duration MSubdirData::check_cache( - const fs::u8path& cache_file, const fs::file_time_type::clock::time_point& ref) + const fs::u8path& cache_file, const fs::file_time_type::clock::time_point& ref) const { try { @@ -287,6 +433,36 @@ namespace mamba return starts_with(m_repodata_url, "file://"); } + void MSubdirData::finalize_checks() + { + create_target(); + } + + bool MSubdirData::finalize_check(const DownloadTarget& target) + { + LOG_INFO << "Checked: " << target.url() << " [" << target.http_status << "]"; + if (m_progress_bar_check) + { + m_progress_bar_check.repr().postfix.set_value("Checked"); + m_progress_bar_check.repr().speed.deactivate(); + m_progress_bar_check.repr().total.deactivate(); + m_progress_bar_check.mark_as_completed(); + } + + if (ends_with(target.url(), ".zst")) + { + this->m_metadata.has_zst = { target.http_status == 200, utc_time_now() }; + } + return true; + } + + std::vector>& MSubdirData::check_targets() + { + // check if zst or (later) jlap are available + return m_check_targets; + } + + bool MSubdirData::load(MultiPackageCache& caches) { auto now = fs::file_time_type::clock::now(); @@ -294,7 +470,6 @@ namespace mamba m_valid_cache_path = ""; m_expired_cache_path = ""; m_loaded = false; - m_mod_etag = nlohmann::json::object(); LOG_INFO << "Searching index cache file for repo '" << m_repodata_url << "'"; @@ -314,11 +489,15 @@ namespace mamba if (cache_age != fs::file_time_type::duration::max() && !forbid_cache()) { - LOG_INFO << "Found cache at '" << json_file.string() << "'"; - m_mod_etag = detail::read_mod_and_etag(json_file); - - if (m_mod_etag.size() != 0) + auto metadata_temp = detail::read_metadata(json_file); + if (!metadata_temp.has_value()) + { + LOG_INFO << "Invalid json cache found, ignoring"; + continue; + } + if (metadata_temp.has_value()) { + m_metadata = std::move(metadata_temp.value()); int max_age = 0; if (Context::instance().local_repodata_ttl > 1) { @@ -327,8 +506,7 @@ namespace mamba else if (Context::instance().local_repodata_ttl == 1) { // TODO error handling if _cache_control key does not exist! - auto el = m_mod_etag.value("_cache_control", std::string("")); - max_age = get_cache_control_max_age(el); + max_age = get_cache_control_max_age(m_metadata.cache_control); } auto cache_age_seconds @@ -351,7 +529,7 @@ namespace mamba // check libsolv cache auto solv_age = check_cache(solv_file, now); if (solv_age != fs::file_time_type::duration::max() - && solv_age.count() <= cache_age.count()) + && solv_age <= cache_age) { // valid libsolv cache found LOG_DEBUG << "Using SOLV cache"; @@ -389,8 +567,36 @@ namespace mamba if (!m_expired_cache_path.empty()) LOG_INFO << "Expired cache (or invalid mod/etag headers) found at '" << m_expired_cache_path.string() << "'"; - if (!Context::instance().offline || forbid_cache()) - create_target(m_mod_etag); + + auto& ctx = Context::instance(); + if (!ctx.offline || forbid_cache()) + { + if (ctx.repodata_use_zst) + { + bool has_value = m_metadata.has_zst.has_value(); + bool is_expired = m_metadata.has_zst.has_value() + && m_metadata.has_zst.value().has_expired(); + bool has_zst = m_metadata.check_zst(p_channel); + if (!has_zst && (is_expired || !has_value)) + { + m_check_targets.push_back(std::make_unique( + m_name + " (check zst)", m_repodata_url + ".zst", "")); + m_check_targets.back()->set_head_only(true); + m_check_targets.back()->set_finalize_callback(&MSubdirData::finalize_check, + this); + m_check_targets.back()->set_ignore_failure(true); + if (!(ctx.no_progress_bars || ctx.quiet || ctx.json)) + { + m_progress_bar_check + = Console::instance().add_progress_bar(m_name + " (check zst)"); + m_check_targets.back()->set_progress_bar(m_progress_bar_check); + m_progress_bar_check.repr().postfix.set_value("Checking"); + } + return true; + } + } + create_target(); + } } return true; } @@ -419,12 +625,43 @@ namespace mamba return m_name; } - bool MSubdirData::finalize_transfer() + void MSubdirData::refresh_last_write_time(const fs::u8path& json_file, + const fs::u8path& solv_file) + { + auto now = fs::file_time_type::clock::now(); + + auto json_age = check_cache(json_file, now); + auto solv_age = check_cache(solv_file, now); + + { + auto lock = LockFile(json_file); + fs::last_write_time(json_file, fs::now()); + } + + if (fs::exists(solv_file) && solv_age.count() <= json_age.count()) + { + auto lock = LockFile(solv_file); + fs::last_write_time(solv_file, fs::now()); + m_solv_cache_valid = true; + } + + if (Context::instance().repodata_use_zst) + { + auto state_file = json_file; + state_file.replace_extension(".state.json"); + auto lock = LockFile(state_file); + m_metadata.store_file_metadata(json_file); + auto outf = open_ofstream(state_file); + m_metadata.serialize_to_stream(outf); + } + } + + bool MSubdirData::finalize_transfer(const DownloadTarget& target) { if (m_target->result != 0 || m_target->http_status >= 400) { LOG_INFO << "Unable to retrieve repodata (response: " << m_target->http_status - << ") for '" << m_repodata_url << "'"; + << ") for '" << m_target->url() << "'"; if (m_progress_bar) { @@ -446,8 +683,8 @@ namespace mamba else { LOG_WARNING << "HTTP response code indicates error, retrying."; - throw std::runtime_error("Unhandled HTTP code: " - + std::to_string(m_target->http_status)); + throw mamba_error("Unhandled HTTP code: " + std::to_string(m_target->http_status), + mamba_error_code::subdirdata_not_loaded); } fs::u8path json_file, solv_file; @@ -459,10 +696,6 @@ namespace mamba json_file = m_expired_cache_path / "cache" / m_json_fn; solv_file = m_expired_cache_path / "cache" / m_solv_fn; - auto now = fs::file_time_type::clock::now(); - auto json_age = check_cache(json_file, now); - auto solv_age = check_cache(solv_file, now); - if (path::is_writable(json_file) && (!fs::exists(solv_file) || path::is_writable(solv_file))) { @@ -474,7 +707,8 @@ namespace mamba if (m_writable_pkgs_dir.empty()) { LOG_ERROR << "Could not find any writable cache directory for repodata file"; - throw std::runtime_error("Non-writable cache error."); + throw mamba_error("Non-writable cache error.", + mamba_error_code::subdirdata_not_loaded); } LOG_DEBUG << "Copying repodata cache files from '" << m_expired_cache_path.string() @@ -500,18 +734,7 @@ namespace mamba m_valid_cache_path = m_writable_pkgs_dir; } - { - LOG_TRACE << "Refreshing '" << json_file.string() << "'"; - auto lock = LockFile(json_file); - fs::last_write_time(json_file, fs::now()); - } - if (fs::exists(solv_file) && solv_age.count() <= json_age.count()) - { - LOG_TRACE << "Refreshing '" << solv_file.string() << "'"; - auto lock = LockFile(solv_file); - fs::last_write_time(solv_file, fs::now()); - m_solv_cache_valid = true; - } + refresh_last_write_time(json_file, solv_file); if (m_progress_bar) { @@ -529,7 +752,7 @@ namespace mamba m_json_cache_valid = true; m_loaded = true; - m_temp_file.reset(nullptr); + m_temp_file.reset(); return true; } else @@ -537,113 +760,117 @@ namespace mamba if (m_writable_pkgs_dir.empty()) { LOG_ERROR << "Could not find any writable cache directory for repodata file"; - throw std::runtime_error("Non-writable cache error."); + throw mamba_error("Non-writable cache error.", + mamba_error_code::subdirdata_not_loaded); } } - LOG_DEBUG << "Finalized transfer of '" << m_repodata_url << "'"; + LOG_DEBUG << "Finalized transfer of '" << m_target->url() << "'"; fs::u8path writable_cache_dir = create_cache_dir(m_writable_pkgs_dir); json_file = writable_cache_dir / m_json_fn; auto lock = LockFile(writable_cache_dir); - m_mod_etag.clear(); - m_mod_etag["_url"] = m_repodata_url; - m_mod_etag["_etag"] = m_target->etag; - m_mod_etag["_mod"] = m_target->mod; - m_mod_etag["_cache_control"] = m_target->cache_control; + auto file_size = fs::file_size(m_temp_file->path()); - LOG_DEBUG << "Opening '" << json_file.string() << "'"; - path::touch(json_file, true); - std::ofstream final_file = open_ofstream(json_file); + m_metadata.url = m_target->url(); + m_metadata.etag = m_target->etag; + m_metadata.mod = m_target->mod; + m_metadata.cache_control = m_target->cache_control; + m_metadata.stored_file_size = file_size; - if (!final_file.is_open()) + if (!Context::instance().repodata_use_zst) { - throw std::runtime_error(fmt::format("Could not open file '{}'", json_file.string())); - } + LOG_DEBUG << "Opening '" << json_file.string() << "'"; + path::touch(json_file, true); + std::ofstream final_file = open_ofstream(json_file); + + if (!final_file.is_open()) + { + throw mamba_error(fmt::format("Could not open file '{}'", json_file.string()), + mamba_error_code::subdirdata_not_loaded); + } - if (ends_with(m_repodata_url, ".bz2")) - { if (m_progress_bar) - m_progress_bar.set_postfix("Decompressing"); - decompress(); + m_progress_bar.set_postfix("Finalizing"); + + std::ifstream temp_file = open_ifstream(m_temp_file->path()); + std::stringstream temp_json; + m_metadata.serialize_to_stream_tiny(temp_json); + + // replace `}` with `,` + temp_json.seekp(-1, temp_json.cur); + temp_json << ','; + final_file << temp_json.str(); + temp_file.seekg(1); + std::copy(std::istreambuf_iterator(temp_file), + std::istreambuf_iterator(), + std::ostreambuf_iterator(final_file)); + + if (!temp_file) + { + std::error_code ec; + fs::remove(json_file, ec); + if (ec) + { + LOG_ERROR << "Could not remove file " << json_file << ": " << ec.message(); + } + throw mamba_error(fmt::format("Could not write out repodata file {}: {}", + json_file, + strerror(errno)), + mamba_error_code::subdirdata_not_loaded); + } + fs::last_write_time(json_file, fs::now()); } - - if (m_progress_bar) - m_progress_bar.set_postfix("Finalizing"); - - std::ifstream temp_file = open_ifstream(m_temp_file->path()); - std::stringstream temp_json; - temp_json << m_mod_etag.dump(); - - // replace `}` with `,` - temp_json.seekp(-1, temp_json.cur); - temp_json << ','; - final_file << temp_json.str(); - temp_file.seekg(1); - std::copy(std::istreambuf_iterator(temp_file), - std::istreambuf_iterator(), - std::ostreambuf_iterator(final_file)); - - if (!temp_file) + else { - fs::remove(json_file); - throw std::runtime_error(fmt::format( - "Could not write out repodata file '{}': {}", json_file.string(), strerror(errno))); + fs::u8path state_file = json_file; + state_file.replace_extension(".state.json"); + fs::rename(m_temp_file->path(), json_file); + fs::last_write_time(json_file, fs::now()); + + m_metadata.store_file_metadata(json_file); + // m_metadata.stored_mtime = fs::last_write_time(json_file); + // LOG_WARNING << "Stored mtime: " << + // m_metadata.stored_mtime.time_since_epoch().count(); + std::ofstream state_file_stream = open_ofstream(state_file); + m_metadata.serialize_to_stream(state_file_stream); } - if (m_progress_bar) { m_progress_bar.repr().postfix.set_value("Downloaded").deactivate(); m_progress_bar.mark_as_completed(); } + m_temp_file.reset(); m_valid_cache_path = m_writable_pkgs_dir; m_json_cache_valid = true; m_loaded = true; - temp_file.close(); - m_temp_file.reset(nullptr); - final_file.close(); - - fs::last_write_time(json_file, fs::now()); - return true; } - bool MSubdirData::decompress() - { - LOG_INFO << "Decompressing metadata"; - auto json_temp_file = std::make_unique(); - bool result - = decompress::raw(m_temp_file->path().string(), json_temp_file->path().string()); - if (!result) - { - LOG_WARNING << "Could not decompress " << m_temp_file->path(); - } - std::swap(json_temp_file, m_temp_file); - return result; - } - - void MSubdirData::create_target(nlohmann::json& mod_etag) + void MSubdirData::create_target() { auto& ctx = Context::instance(); m_temp_file = std::make_unique(); + + bool use_zst = m_metadata.has_zst.has_value() && m_metadata.has_zst.value().value; m_target = std::make_unique( - m_name, m_repodata_url, m_temp_file->path().string()); + m_name, m_repodata_url + (use_zst ? ".zst" : ""), m_temp_file->path().string()); if (!(ctx.no_progress_bars || ctx.quiet || ctx.json)) { m_progress_bar = Console::instance().add_progress_bar(m_name); m_target->set_progress_bar(m_progress_bar); } - // if we get something _other_ than the noarch, we DO NOT throw if the file - // can't be retrieved + // if we get something _other_ than the noarch, we DO NOT throw if the file can't be + // retrieved if (!m_is_noarch) { m_target->set_ignore_failure(true); } m_target->set_finalize_callback(&MSubdirData::finalize_transfer, this); - m_target->set_mod_etag_headers(mod_etag); + m_target->set_mod_etag_headers(m_metadata.mod, m_metadata.etag); } std::size_t MSubdirData::get_cache_control_max_age(const std::string& val) @@ -676,8 +903,8 @@ namespace mamba using return_type = expected_t; RepoMetadata meta{ m_repodata_url, Context::instance().add_pip_as_python_dependency, - m_mod_etag.value("_etag", ""), - m_mod_etag.value("_mod", "") }; + m_metadata.etag, + m_metadata.mod }; auto cache = cache_path(); return cache ? return_type(MRepo::create(pool, m_name, *cache, meta, *p_channel)) diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 457569979b..635e72fc67 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -304,7 +304,7 @@ namespace mamba return result; } - bool PackageDownloadExtractTarget::finalize_callback() + bool PackageDownloadExtractTarget::finalize_callback(const DownloadTarget& target) { if (m_has_progress_bars) { diff --git a/libmamba/src/core/url.cpp b/libmamba/src/core/url.cpp index b56347ed9d..8c1ad222a5 100644 --- a/libmamba/src/core/url.cpp +++ b/libmamba/src/core/url.cpp @@ -155,6 +155,7 @@ namespace mamba } // mimicking conda's behavior by special handling repodata.json + // todo support .zst if (ends_with(u, "/repodata.json")) { u = u.substr(0, u.size() - 13); diff --git a/libmamba/tests/repodata_json_cache/test_7.json b/libmamba/tests/repodata_json_cache/test_7.json new file mode 100644 index 0000000000..a411e66ede --- /dev/null +++ b/libmamba/tests/repodata_json_cache/test_7.json @@ -0,0 +1,5 @@ +{ + "info": { + "subdir": "linux-64" + } +} \ No newline at end of file diff --git a/libmamba/tests/repodata_json_cache/test_7.state.json b/libmamba/tests/repodata_json_cache/test_7.state.json new file mode 100644 index 0000000000..a1f8b38f71 --- /dev/null +++ b/libmamba/tests/repodata_json_cache/test_7.state.json @@ -0,0 +1,15 @@ +{ + "cache_control": "something", + "etag": "something else", + "file_mtime": { + "nanoseconds": 170463081, + "seconds": 1673263108 + }, + "file_size": 44, + "has_zst": { + "last_checked": "2023-01-06T16:33:06Z", + "value": true + }, + "mod": "Fri, 11 Feb 2022 13:52:44 GMT", + "url": "https://conda.anaconda.org/conda-forge/noarch/repodata.json.zst" +} \ No newline at end of file diff --git a/libmamba/tests/test_cpp.cpp b/libmamba/tests/test_cpp.cpp index 24d53bfc08..d8d9844294 100644 --- a/libmamba/tests/test_cpp.cpp +++ b/libmamba/tests/test_cpp.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -9,6 +10,7 @@ #include "mamba/core/link.hpp" #include "mamba/core/match_spec.hpp" #include "mamba/core/output.hpp" +#include "mamba/core/subdirdata.hpp" #include "test_data.hpp" @@ -568,47 +570,99 @@ namespace mamba namespace detail { // read the header that contains json like {"_mod": "...", ...} - nlohmann::json read_mod_and_etag(const fs::u8path& file); + tl::expected read_metadata(const fs::u8path& file); } +#ifdef _WIN32 + std::chrono::system_clock::time_point filetime_to_unix_test(const fs::file_time_type& filetime) + { + // windows filetime is in 100ns intervals since 1601-01-01 + constexpr static auto epoch_offset = std::chrono::seconds(11644473600ULL); + return std::chrono::system_clock::time_point( + std::chrono::duration_cast( + filetime.time_since_epoch() - epoch_offset)); + } +#endif + TEST(subdirdata, parse_mod_etag) { + bool old_value = Context::instance().repodata_use_zst; + Context::instance().repodata_use_zst = true; fs::u8path cache_folder = fs::u8path(test_data_dir / "repodata_json_cache"); - auto j = detail::read_mod_and_etag(cache_folder / "test_1.json"); - EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT"); + auto mq = detail::read_metadata(cache_folder / "test_1.json"); + EXPECT_TRUE(mq.has_value()); + auto j = mq.value(); + EXPECT_EQ(j.mod, "Fri, 11 Feb 2022 13:52:44 GMT"); EXPECT_EQ( - j["_url"], + j.url, "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json"); - j = detail::read_mod_and_etag(cache_folder / "test_2.json"); - EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT"); + j = detail::read_metadata(cache_folder / "test_2.json").value(); + EXPECT_EQ(j.mod, "Fri, 11 Feb 2022 13:52:44 GMT"); EXPECT_EQ( - j["_url"], + j.url, "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json"); - j = detail::read_mod_and_etag(cache_folder / "test_5.json"); - EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT"); + j = detail::read_metadata(cache_folder / "test_5.json").value(); + EXPECT_EQ(j.mod, "Fri, 11 Feb 2022 13:52:44 GMT"); EXPECT_EQ( - j["_url"], + j.url, "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json"); - j = detail::read_mod_and_etag(cache_folder / "test_4.json"); - EXPECT_EQ(j["_cache_control"], "{{}}\",,,\""); - EXPECT_EQ(j["_etag"], "\n\n\"\"randome ecx,,ssd\n,,\""); - EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT"); + j = detail::read_metadata(cache_folder / "test_4.json").value(); + EXPECT_EQ(j.cache_control, "{{}}\",,,\""); + EXPECT_EQ(j.etag, "\n\n\"\"randome ecx,,ssd\n,,\""); + EXPECT_EQ(j.mod, "Fri, 11 Feb 2022 13:52:44 GMT"); EXPECT_EQ( - j["_url"], + j.url, "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json"); - j = detail::read_mod_and_etag(cache_folder / "test_3.json"); - EXPECT_TRUE(j.empty()); + mq = detail::read_metadata(cache_folder / "test_3.json"); + EXPECT_TRUE(mq.has_value() == false); + + j = detail::read_metadata(cache_folder / "test_6.json").value(); + EXPECT_EQ(j.mod, "Thu, 02 Apr 2020 20:21:27 GMT"); + EXPECT_EQ(j.url, "https://conda.anaconda.org/intake/osx-arm64"); + + auto state_file = cache_folder / "test_7.state.json"; + // set file_mtime + + + { +#ifdef _WIN32 + auto file_mtime + = filetime_to_unix_test(fs::last_write_time(cache_folder / "test_7.json")); +#else + auto file_mtime = fs::last_write_time(cache_folder / "test_7.json"); +#endif + + // auto file_size = fs::file_size(state_file); + auto ifs = open_ifstream(state_file, std::ios::in | std::ios::binary); + auto jstate = nlohmann::json::parse(ifs); + ifs.close(); + auto secs + = std::chrono::duration_cast(file_mtime.time_since_epoch()); + auto nsecs = std::chrono::duration_cast( + file_mtime.time_since_epoch() - secs); + + jstate["file_mtime"]["seconds"] = secs.count(); + jstate["file_mtime"]["nanoseconds"] = nsecs.count(); + + auto file_size = fs::file_size(cache_folder / "test_7.json"); + jstate["file_size"] = file_size; + + auto ofs = open_ofstream(state_file); + ofs << jstate.dump(4); + } - j = detail::read_mod_and_etag(cache_folder / "test_6.json"); - EXPECT_EQ(j["_mod"], "Thu, 02 Apr 2020 20:21:27 GMT"); - EXPECT_EQ(j["_url"], "https://conda.anaconda.org/intake/osx-arm64"); + j = detail::read_metadata(cache_folder / "test_7.json").value(); + EXPECT_EQ(j.cache_control, "something"); + EXPECT_EQ(j.etag, "something else"); + EXPECT_EQ(j.mod, "Fri, 11 Feb 2022 13:52:44 GMT"); + EXPECT_EQ(j.url, "https://conda.anaconda.org/conda-forge/noarch/repodata.json.zst"); + EXPECT_EQ(j.has_zst.value().value, true); + EXPECT_EQ(j.has_zst.value().last_checked, parse_utc_timestamp("2023-01-06T16:33:06Z")); - // EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT"); - // EXPECT_EQ(j["_url"], - // "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json"); + Context::instance().repodata_use_zst = old_value; } } // namespace mamba