diff --git a/include/powerloader/context.hpp b/include/powerloader/context.hpp index 5676422a..cbecd6a6 100644 --- a/include/powerloader/context.hpp +++ b/include/powerloader/context.hpp @@ -17,7 +17,7 @@ namespace powerloader namespace fs = std::filesystem; class Context; - struct Mirror; + class Mirror; using mirror_set = std::vector>; // TODO: replace by std::flat_set once available. diff --git a/include/powerloader/mirror.hpp b/include/powerloader/mirror.hpp index b29b4459..638280ee 100644 --- a/include/powerloader/mirror.hpp +++ b/include/powerloader/mirror.hpp @@ -63,10 +63,15 @@ namespace powerloader }; // mirrors should be dict -> urls mapping - struct POWERLOADER_API Mirror + class POWERLOADER_API Mirror { - Mirror(MirrorID id, const Context& ctx, const std::string& url); - Mirror(const Context& ctx, const std::string& url); + public: + Mirror(const MirrorID& id, const Context& ctx, const std::string& url) + : m_id(id) + , m_url(url) + { + } + virtual ~Mirror(); Mirror(const Mirror&) = delete; @@ -156,8 +161,8 @@ namespace powerloader } private: - std::string m_url; const MirrorID m_id; + const std::string m_url; Protocol m_protocol = Protocol::kHTTP; MirrorState m_state = MirrorState::READY; @@ -180,6 +185,36 @@ namespace powerloader std::size_t m_retry_counter = 0; }; + class POWERLOADER_API HTTPMirror : public Mirror + { + public: + HTTPMirror(const Context& ctx, const std::string& url) + : Mirror(HTTPMirror::id(url), ctx, url) + { + } + + static MirrorID id(const std::string& url) + { + return MirrorID{ fmt::format("HTTPMirror[{}]", url) }; + } + + void set_auth(const std::string& user, const std::string& password); + + bool prepare(Target* target) override; + bool prepare(const std::string& path, CURLHandle& handle) override; + + bool needs_preparation(Target* target) const override; + bool authenticate(CURLHandle& handle, const std::string& path) override; + + std::vector get_auth_headers(const std::string& path) const override; + + std::string format_url(Target* target) const override; + + private: + std::string m_auth_user; + std::string m_auth_password; + }; + bool sort_mirrors(std::vector>& mirrors, const std::shared_ptr& mirror, bool success, diff --git a/src/cli/main.cpp b/src/cli/main.cpp index 89acd5cd..6947b381 100644 --- a/src/cli/main.cpp +++ b/src/cli/main.cpp @@ -392,7 +392,7 @@ parse_mirrors(const Context& ctx, const YAML::Node& node) else if (kof == KindOf::kHTTP) { spdlog::info("Adding HTTP mirror: {} -> {}", mirror_name, creds.url.url()); - result.create_unique_mirror(mirror_name, ctx, creds.url.url()); + result.create_unique_mirror(mirror_name, ctx, creds.url.url()); } } } diff --git a/src/download_target.cpp b/src/download_target.cpp index 3c955c70..9aad6561 100644 --- a/src/download_target.cpp +++ b/src/download_target.cpp @@ -46,7 +46,7 @@ namespace powerloader URLHandler uh{ target_url }; if (uh.scheme() == "file") { - ctx.mirror_map.create_unique_mirror("[file]", ctx, "file://"); + ctx.mirror_map.create_unique_mirror("[file]", ctx, "file://"); return std::make_shared(uh.path(), "[file]", destination_path); } @@ -57,7 +57,7 @@ namespace powerloader const fs::path dst = destination_path.empty() ? fs::path{ rsplit(path, "/", 1).back() } : destination_path; - ctx.mirror_map.create_unique_mirror(host, ctx, mirror_url); + ctx.mirror_map.create_unique_mirror(host, ctx, mirror_url); return std::make_shared(path.substr(1, std::string::npos), host, dst); } else diff --git a/src/downloader.cpp b/src/downloader.cpp index 4911c18f..1a181543 100644 --- a/src/downloader.cpp +++ b/src/downloader.cpp @@ -571,7 +571,6 @@ namespace powerloader bool retry = false; spdlog::error("Error during transfer"); - // Call mirrorfailure callback // LrMirrorFailureCb mf_cb = target->target().mirrorfailurecb; // if (mf_cb) diff --git a/src/mirror.cpp b/src/mirror.cpp index 9981be37..02013586 100644 --- a/src/mirror.cpp +++ b/src/mirror.cpp @@ -7,24 +7,6 @@ namespace powerloader { - Mirror::Mirror(MirrorID id, const Context& ctx, const std::string& url) - : m_url(url) - , m_id(id) - { - if (url.back() == '/' && url != "file://") - m_url = m_url.substr(0, m_url.size() - 1); - - if (ctx.max_downloads_per_mirror > 0) - { - m_stats.allowed_parallel_connections = ctx.max_downloads_per_mirror; - } - } - - Mirror::Mirror(const Context& ctx, const std::string& url) - : Mirror(Mirror::id(url), ctx, url) - { - } - Mirror::~Mirror() = default; void Mirror::change_max_ranges(int new_value) @@ -202,4 +184,49 @@ namespace powerloader return true; } + + bool HTTPMirror::prepare(Target* target) + { + return true; + } + + bool HTTPMirror::prepare(const std::string& path, CURLHandle& handle) + { + return true; + } + + bool HTTPMirror::needs_preparation(Target* target) const + { + return false; + } + + bool HTTPMirror::authenticate(CURLHandle& handle, const std::string& path) + { + if (!m_auth_password.empty()) + { + spdlog::warn( + "Setting HTTP authentication for {} to {}:{}", path, m_auth_user, m_auth_password); + handle.setopt(CURLOPT_USERNAME, m_auth_user.c_str()); + handle.setopt(CURLOPT_PASSWORD, m_auth_password.c_str()); + } + return true; + } + + void HTTPMirror::set_auth(const std::string& user, const std::string& password) + { + m_auth_user = user; + m_auth_password = password; + } + + + std::vector HTTPMirror::get_auth_headers(const std::string& path) const + { + return {}; + } + + std::string HTTPMirror::format_url(Target* target) const + { + return join_url(url(), target->target().path()); + } + } diff --git a/src/python/main.cpp b/src/python/main.cpp index aee61157..0ec0d82f 100644 --- a/src/python/main.cpp +++ b/src/python/main.cpp @@ -24,13 +24,14 @@ PYBIND11_MODULE(pypowerloader, m) py::class_>(m, "DownloadTarget") .def(py::init()) - .def_property_readonly("complete_url", &DownloadTarget::complete_url) .def_property("progress_callback", &DownloadTarget::progress_callback, &DownloadTarget::set_progress_callback); + py::class_(m, "MirrorID").def(py::init()); + py::class_>(m, "Mirror") - .def(py::init()); + .def(py::init()); py::class_(m, "MirrorMap") .def(py::init<>()) diff --git a/src/target.cpp b/src/target.cpp index cb6f24e1..73303742 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -38,6 +38,7 @@ namespace powerloader void Target::reset() { + spdlog::warn("Resetting target {}", m_target->destination_path().string()); if (m_target->outfile() && !zck_running()) { std::error_code ec; @@ -108,15 +109,29 @@ namespace powerloader bool Target::truncate_transfer_file() { + spdlog::warn("Truncating transfer file ... "); std::ptrdiff_t offset = 0; std::error_code ec; - if (!m_target->outfile() || !m_target->outfile()->open()) + spdlog::warn("NOT TRUNCATING BECAUSE FILE IS {} or {}", + (bool) m_target->outfile(), + m_target->outfile()->open()); + if (!m_target->outfile()) + { return true; + } + + if (!m_target->outfile()->open()) + { + fs::remove(m_target->outfile()->path()); + return true; + } if (m_original_offset >= 0) offset = m_original_offset; + spdlog::warn("Truncating transfer file offset {}", offset); + m_target->outfile()->truncate(offset, ec); if (ec) { diff --git a/test.py b/test.py index 2811fd4b..fcccce34 100644 --- a/test.py +++ b/test.py @@ -10,7 +10,6 @@ baseurl = "https://conda.anaconda.org/conda-forge" filename = "python3.9_test" downTarg = pypowerloader.DownloadTarget(path, baseurl, filename) -print("complete url: " + downTarg.complete_url) def progress(total, done):