33#include <zypp-curl/parser/MetaLinkParser>
36#include <zypp-curl/auth/CurlAuthData>
43#undef CURLVERSION_AT_LEAST
44#define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O)
187 void adddnsfd( std::vector<GPollFD> &waitFds );
188 void dnsevent(
const std::vector<GPollFD> &waitFds );
219 size_t writefunction (
char *ptr, std::optional<off_t> offset,
size_t bytes )
override;
221 bool beginRange ( off_t range, std::string &cancelReason )
override;
222 bool finishedRange ( off_t range,
bool validated, std::string &cancelReason )
override;
235 Url baseurl, CURLM *multi, FILE *fp,
244 void run(std::vector<Url> &urllist);
266 std::list< std::unique_ptr<multifetchworker> >
_workers;
319#if _POSIX_C_SOURCE >= 199309L
321 if ( clock_gettime( CLOCK_MONOTONIC, &ts) )
323 return ts.tv_sec + ts.tv_nsec / 1000000000.;
326 if (gettimeofday(&tv, NULL))
328 return tv.tv_sec + tv.tv_usec / 1000000.;
336 return bytes ? 0 : 1;
358 const auto &blk =
_blocks[*currRange];
359 off_t seekTo = blk.start + blk.bytesWritten;
364 return bytes ? 0 : 1;
376 const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
379 cancelReason =
"Cancelled because stripe block is already finalized";
381 WAR <<
"#" <<
_workerno <<
": trying to start a range ("<<stripeRangeOff<<
"["<<
_blocks[workerRangeOff].start <<
" : "<<
_blocks[workerRangeOff].len<<
"]) that was already finalized, cancelling. Stealing was: " <<
_request->
_stealing << endl;
392 const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
396 cancelReason =
"Block failed to validate";
423 if (l > 9 && !strncasecmp(p,
"Location:", 9)) {
424 std::string line(p + 9, l - 9);
425 if (line[l - 10] ==
'\r')
426 line.erase(l - 10, 1);
427 XXX <<
"#" <<
_workerno <<
": redirecting to" << line << endl;
433 XXX <<
"#" <<
_workerno <<
": filesize mismatch" << endl;
445, _maxspeed( request._maxspeed )
446, _request ( &request )
452 XXX <<
"reused worker from pool" << endl;
472 curl_easy_cleanup(
_curl);
478 curl_easy_setopt(
_curl, CURLOPT_PRIVATE,
this);
491 if (use_auth.empty())
492 use_auth =
"digest,basic";
494 if( auth != CURLAUTH_NONE)
496 XXX <<
"#" <<
_workerno <<
": Enabling HTTP authentication methods: " << use_auth
497 <<
" (CURLOPT_HTTPAUTH=" << auth <<
")" << std::endl;
498 curl_easy_setopt(
_curl, CURLOPT_HTTPAUTH, auth);
513#if CURLVERSION_AT_LEAST(7,15,5)
514 curl_easy_setopt(
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)0);
516 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
517 curl_easy_setopt(
_curl, CURLOPT_WRITEFUNCTION, (
void *)0);
518 curl_easy_setopt(
_curl, CURLOPT_WRITEDATA, (
void *)0);
519 curl_easy_setopt(
_curl, CURLOPT_HEADERFUNCTION, (
void *)0);
520 curl_easy_setopt(
_curl, CURLOPT_HEADERDATA, (
void *)0);
524 curl_easy_cleanup(
_curl);
546 const char *s = getenv(name.c_str());
547 return s && *s ? true :
false;
563 if (inet_pton(AF_INET, host.c_str(), addrbuf) == 1)
565 if (inet_pton(AF_INET6, host.c_str(), addrbuf) == 1)
576 if (schemeproxy !=
"http_proxy")
578 std::transform(schemeproxy.begin(), schemeproxy.end(), schemeproxy.begin(), ::toupper);
583 XXX <<
"checking DNS lookup of " << host << endl;
592 if (
_pid == pid_t(-1))
605 struct addrinfo *ai =
nullptr, aihints;
606 memset(&aihints, 0,
sizeof(aihints));
607 aihints.ai_family = PF_UNSPEC;
608 int tstsock = socket(PF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
610 aihints.ai_family = PF_INET;
613 aihints.ai_socktype = SOCK_STREAM;
614 aihints.ai_flags = AI_CANONNAME;
617 alarm(connecttimeout);
618 signal(SIGALRM, SIG_DFL);
619 if (getaddrinfo(host.c_str(), NULL, &aihints, &ai))
637 .events = G_IO_IN | G_IO_HUP | G_IO_ERR,
645 bool hasEvent = std::any_of( waitFds.begin (), waitFds.end(),[
this](
const GPollFD &waitfd ){
646 return ( waitfd.fd == _dnspipe && waitfd.revents != 0 );
659 if (!WIFEXITED(status))
666 int exitcode = WEXITSTATUS(status);
667 XXX <<
"#" <<
_workerno <<
": DNS lookup returned " << exitcode << endl;
685 auto &blk =
_blocks[workerRangeIdx];
701 size_t cnt = l >
sizeof(buf) ?
sizeof(buf) : l;
711 blk._digest = std::move(newDig);
726 std::optional<zypp::Digest> digest;
727 std::optional<size_t> relDigLen;
728 std::optional<size_t> blkSumPad;
733 relDigLen = sum.size( );
739 return MultiByteHandler::Range::make(
745 std::move(relDigLen),
746 std::move(blkSumPad) );
753 XXX <<
"start stealing!" << endl;
767 if (worker->
_pass == -1)
834 XXX <<
"#" <<
_workerno <<
": going to sleep for " << sl * 1000 <<
" ms" << endl;
891 for ( uint i = 0; i < stripeDesc.blocks.size(); i++ ) {
911 DBG <<
"#" <<
_workerno <<
"Done adding blocks to download, going to download: " <<
_blocks.size() <<
" nr of block with " <<
_datasize <<
" nr of bytes" << std::endl;
922 bool hadRangeFail =
_multiByteHandler->lastError() == MultiByteHandler::Code::RangeFail;
928 if ( hadRangeFail ) {
935 curl_easy_reset(
_curl );
973 :
internal::CurlPollHelper::CurlPoll{ multi }
975 , _filename(
std::move(filename))
976 , _baseurl(
std::move(baseurl))
979 , _blklist(
std::move(blklist))
980 , _filesize(filesize)
982 , _timeout(context->_settings.timeout())
983 , _connect_timeout(context->_settings.connectTimeout())
984 , _maxspeed(context->_settings.maxDownloadSpeed())
985 , _maxworkers(context->_settings.maxConcurrentConnections())
1012 currStripeSize += blk.
size;
1027 std::vector<Url>::iterator urliter = urllist.begin();
1033 if (mcode != CURLM_OK)
1039 std::vector<GPollFD> waitFds;
1044 XXX <<
"finished!" << endl;
1051 _workers.push_back(std::make_unique<multifetchworker>(workerno++, *
this, *urliter));
1068 WAR <<
"No more active workers!" << endl;
1070 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1080 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1081 (*workeriter)->adddnsfd( waitFds );
1090 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter) {
1104 timeoutMs = sl * 1000;
1108 timeoutMs = std::min<long>( timeoutMs, _curlHelper.
timeout_ms.value() );
1110 dnsFdCount = waitFds.size();
1111 waitFds.insert( waitFds.end(), _curlHelper.
socks.begin(), _curlHelper.
socks.end() );
1114 if ( !waitFds.empty() || timeoutMs != -1) {
1119 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1124 (*workeriter)->
dnsevent( waitFds );
1133 if (mcode != CURLM_OK)
1137 if (mcode != CURLM_OK)
1162 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1171 XXX <<
"#" << worker->
_workerno <<
": sleep done, wake up" << endl;
1179 CURLMsg *msg =
nullptr;
1181 while ((msg = curl_multi_info_read(
_multi, &nqueue)) != 0)
1183 if (msg->msg != CURLMSG_DONE)
1185 CURL *easy = msg->easy_handle;
1186 CURLcode cc = msg->data.result;
1189 if (curl_easy_getinfo(easy, CURLINFO_PRIVATE, &worker) != CURLE_OK)
1200 curl_multi_remove_handle(
_multi, easy);
1202 const auto &setWorkerBroken = [&](
const std::string &
str = {} ){
1204 if ( !
str.empty () )
1215 WAR <<
"#" << worker->
_workerno <<
": has no multibyte handler, this is a bug" << endl;
1216 setWorkerBroken(
"Multibyte handler error");
1225 WAR <<
"#" << worker->
_workerno <<
": still has work to do or can recover from a error, continuing the job!" << endl;
1236 if ( cc != CURLE_OK ) {
1242 WAR <<
"#" << worker->
_workerno <<
": failed, but was set to discard, reusing for new requests" << endl;
1261 bool done = std::all_of( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), [](
const Stripe::RState s ) { return s == Stripe::FINALIZED; } );
1264 std::for_each( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), [](
Stripe::RState &s ) {
1265 if ( s != Stripe::FINALIZED)
1266 s = Stripe::PENDING;
1277 int maxworkerno = 0;
1279 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1294 double ratio = worker->
_avgspeed / maxavg;
1297 ratio = ratio * ratio;
1300 XXX <<
"#" << worker->
_workerno <<
": too slow ("<< ratio <<
", " << worker->
_avgspeed <<
", #" << maxworkerno <<
": " << maxavg <<
"), going to sleep for " << ratio * 1000 <<
" ms" << endl;
1321#if CURLVERSION_AT_LEAST(7,15,5)
1322 curl_easy_setopt(worker->
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)(avg));
1354 WAR <<
"overall result" << endl;
1355 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1375 MIL <<
"MediaMultiCurl::MediaMultiCurl(" << url_r <<
", " << attach_point_hint_r <<
")" << endl;
1389 curl_multi_cleanup(
_multi);
1392 std::map<std::string, CURL *>::iterator it;
1395 CURL *easy = it->second;
1398 curl_easy_cleanup(easy);
1414 for (; sl; sl = sl->next)
1432 if ( curl_easy_getinfo(
_curl, CURLINFO_PRIVATE, &fp ) != CURLE_OK || !fp )
1434 if ( ftell( fp ) == 0 )
1439 long httpReturnCode = 0;
1440 if (curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0)
1444 bool ismetalink =
false;
1445 if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1447 std::string ct = std::string(ptr);
1448 if (ct.find(
"application/x-zsync") == 0 || ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
1451 if (!ismetalink && dlnow < 256)
1460 DBG <<
"looks_like_meta_file: " << ismetalink << endl;
1477 if( assert_dir( dest.
dirname() ) )
1479 DBG <<
"assert_dir " << dest.
dirname() <<
" failed" << endl;
1489 ERR <<
"out of memory for temp file name" << endl;
1493 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1496 ERR <<
"mkstemp failed for file '" << destNew <<
"'" << endl;
1501 file = ::fdopen( tmp_fd,
"we" );
1504 ERR <<
"fopen failed for file '" << destNew <<
"'" << endl;
1507 tmp_fd.resetDispose();
1510 DBG <<
"dest: " << dest << endl;
1511 DBG <<
"temp: " << destNew << endl;
1516 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1517 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, (
long)
PathInfo(target).mtime());
1521 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1522 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1528 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (*file) );
1535 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1536 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1538 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
1541 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1542 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1544 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
1545 long httpReturnCode = 0;
1546 CURLcode infoRet = curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode);
1547 if (infoRet == CURLE_OK)
1550 if ( httpReturnCode == 304
1553 DBG <<
"not modified: " <<
PathInfo(dest) << endl;
1559 WAR <<
"Could not get the response code." << endl;
1565 if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1567 std::string ct = std::string(ptr);
1568 if (ct.find(
"application/x-zsync") == 0 )
1570 else if (ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
1584 bool userabort =
false;
1590 std::vector<Url> urls;
1593 parser.
parse( destNew );
1610 XXX <<
"With no blocks" << std::endl;
1617 XXX <<
"No filesize in metalink file and no expected filesize, aborting multicurl." << std::endl;
1622 Disabling
this workaround
for now, since we now
do zip ranges into bigger requests
1633 file = fopen((*destNew).c_str(),
"w+e");
1638 XXX <<
"reusing blocks from file " << target << endl;
1644 XXX <<
"reusing blocks from file " << failedFile << endl;
1652 XXX <<
"reusing blocks from file " << df << endl;
1662 userabort = ex.
errstr() ==
"User abort";
1673 WAR<<
"Failed to multifetch file " << ex <<
" falling back to single Curl download!" << std::endl;
1674 if (
PathInfo(destNew).size() >= 63336)
1676 ::unlink(failedFile.
asString().c_str());
1683 file = fopen((*destNew).c_str(),
"w+e");
1695 ERR <<
"Failed to chmod file " << destNew << endl;
1702 ERR <<
"Fclose failed for file '" << destNew <<
"'" << endl;
1706 if ( rename( destNew, dest ) != 0 )
1708 ERR <<
"Rename failed" << endl;
1711 destNew.resetDispose();
1719 if (filesize == off_t(-1) && blklist.haveFilesize())
1720 filesize = blklist.getFilesize();
1721 if (!blklist.haveBlocks() && filesize != 0) {
1722 if ( filesize == -1 ) {
1727 MIL <<
"Generate blocklist, since there was none in the metalink file." << std::endl;
1732 while ( currOff < filesize ) {
1734 auto blksize = filesize - currOff ;
1735 if ( blksize > prefSize )
1738 blklist.addBlock( currOff, blksize );
1742 XXX <<
"Generated blocklist: " << std::endl << blklist << std::endl <<
" End blocklist " << std::endl;
1745 if (filesize == 0 || !blklist.numBlocks()) {
1754 _multi = curl_multi_init();
1760 std::vector<Url> myurllist;
1761 for (std::vector<Url>::iterator urliter = urllist->begin(); urliter != urllist->end(); ++urliter)
1765 std::string scheme = urliter->getScheme();
1766 if (scheme ==
"http" || scheme ==
"https" || scheme ==
"ftp" || scheme ==
"tftp")
1776 if (!myurllist.size())
1777 myurllist.push_back(baseurl);
1786 if (fseeko(fp, off_t(0), SEEK_SET))
1792 while ((l = fread(buf, 1,
sizeof(buf), fp)) > 0)
1800 return _dnsok.find(host) ==
_dnsok.end() ? false :
true;
1822 curl_easy_cleanup(oldeasy);
void resetDispose()
Set no dispose function.
Store and operate with byte count.
static const Unit MB
1000^2 Byte
static const Unit K
1024 Byte
std::string asString(unsigned field_width_r=0, unsigned unit_width_r=1) const
Auto selected Unit and precision.
Compute Message Digests (MD5, SHA1 etc)
bool update(const char *bytes, size_t len)
feed data into digest computation algorithm
Digest clone() const
Returns a clone of the current Digest and returns it.
Base class for Exception.
std::string getScheme() const
Returns the scheme name of the URL.
std::string asString() const
Returns a default string representation of the Url object.
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Pathname repoCachePath() const
Path where the caches are kept (/var/cache/zypp)
static ZConfig & instance()
Singleton ctor.
Wrapper class for stat/lstat.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Pathname dirname() const
Return all but the last component od this path.
const std::string & asString() const
String representation.
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
The CurlMultiPartHandler class.
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
String related utilities and Regular expression matching.
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
int unlink(const Pathname &path)
Like 'unlink'.
std::string numstring(char n, int w=0)
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
auto eintrSafeCall(Fun &&function, Args &&... args)
CURLMcode handleSocketActions(const std::vector< GPollFD > &actionsFds, int first=0)
std::vector< GPollFD > socks
std::optional< long > timeout_ms
AutoDispose<int> calling ::close
AutoDispose<FILE*> calling ::fclose
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.