26#include <zypp/base/Logger.h>
28#include <zypp-curl/parser/MetaLinkParser>
29#include <zypp/ManagedFile.h>
31#include <zypp-curl/auth/CurlAuthData>
36#undef CURLVERSION_AT_LEAST
37#define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O)
46class multifetchrequest;
64 void adddnsfd( std::vector<curl_waitfd> &waitFds );
65 void dnsevent(
const std::vector<curl_waitfd> &waitFds );
90 static size_t _writefunction(
void *ptr,
size_t size,
size_t nmemb,
void *stream);
93 static size_t _headerfunction(
void *ptr,
size_t size,
size_t nmemb,
void *stream);
106#define WORKER_STARTING 0
107#define WORKER_LOOKUP 1
108#define WORKER_FETCH 2
109#define WORKER_DISCARD 3
111#define WORKER_SLEEP 5
112#define WORKER_BROKEN 6
121 void run(std::vector<Url> &urllist);
179 if (gettimeofday(&tv, NULL))
181 return tv.tv_sec + tv.tv_usec / 1000000.;
205 (void)curl_easy_getinfo(
_curl, CURLINFO_EFFECTIVE_URL, &effurl);
206 if (effurl && !strncasecmp(effurl,
"http", 4))
209 (void)curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &statuscode);
210 if (statuscode != 206)
257 if (l > 9 && !strncasecmp(p,
"Location:", 9))
259 std::string line(p + 9, l - 9);
260 if (line[l - 10] ==
'\r')
261 line.erase(l - 10, 1);
262 XXX <<
"#" <<
_workerno <<
": redirecting to" << line << endl;
265 if (l <= 14 || l >= 128 || strncasecmp(p,
"Content-Range:", 14) != 0)
269 while (l && (*p ==
' ' || *p ==
'\t'))
271 if (l < 6 || strncasecmp(p,
"bytes", 5))
278 unsigned long long start, off, filesize;
279 if (sscanf(buf,
"%llu-%llu/%llu", &start, &off, &filesize) != 3)
283 WAR <<
"#" <<
_workerno <<
": setting request filesize to " << filesize << endl;
290 XXX <<
"#" <<
_workerno <<
": filesize mismatch" << endl;
292 strncpy(
_curlError,
"filesize mismatch", CURL_ERROR_SIZE);
329 XXX <<
"reused worker from pool" << endl;
333 strncpy(
_curlError,
"curl_easy_init failed", CURL_ERROR_SIZE);
342 curl_easy_cleanup(
_curl);
345 strncpy(
_curlError,
"curl_easy_setopt failed", CURL_ERROR_SIZE);
348 curl_easy_setopt(
_curl, CURLOPT_PRIVATE,
this);
351 curl_easy_setopt(
_curl, CURLOPT_WRITEDATA,
this);
355 curl_easy_setopt(
_curl, CURLOPT_HEADERDATA,
this);
369 if (use_auth.empty())
370 use_auth =
"digest,basic";
372 if( auth != CURLAUTH_NONE)
374 XXX <<
"#" <<
_workerno <<
": Enabling HTTP authentication methods: " << use_auth
375 <<
" (CURLOPT_HTTPAUTH=" << auth <<
")" << std::endl;
376 curl_easy_setopt(
_curl, CURLOPT_HTTPAUTH, auth);
391#if CURLVERSION_AT_LEAST(7,15,5)
392 curl_easy_setopt(
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)0);
394 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
395 curl_easy_setopt(
_curl, CURLOPT_WRITEFUNCTION, (
void *)0);
396 curl_easy_setopt(
_curl, CURLOPT_WRITEDATA, (
void *)0);
397 curl_easy_setopt(
_curl, CURLOPT_HEADERFUNCTION, (
void *)0);
398 curl_easy_setopt(
_curl, CURLOPT_HEADERDATA, (
void *)0);
402 curl_easy_cleanup(
_curl);
409 while (waitpid(
_pid, &status, 0) == -1)
426 const char *s = getenv(name.c_str());
427 return s && *s ? true :
false;
443 if (inet_pton(AF_INET, host.c_str(), addrbuf) == 1)
445 if (inet_pton(AF_INET6, host.c_str(), addrbuf) == 1)
456 if (schemeproxy !=
"http_proxy")
458 std::transform(schemeproxy.begin(), schemeproxy.end(), schemeproxy.begin(), ::toupper);
463 XXX <<
"checking DNS lookup of " << host << endl;
468 strncpy(
_curlError,
"DNS pipe creation failed", CURL_ERROR_SIZE);
472 if (
_pid == pid_t(-1))
478 strncpy(
_curlError,
"DNS checker fork failed", CURL_ERROR_SIZE);
485 struct addrinfo *ai, aihints;
486 memset(&aihints, 0,
sizeof(aihints));
487 aihints.ai_family = PF_UNSPEC;
488 int tstsock = socket(PF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
490 aihints.ai_family = PF_INET;
493 aihints.ai_socktype = SOCK_STREAM;
494 aihints.ai_flags = AI_CANONNAME;
497 alarm(connecttimeout);
498 signal(SIGALRM, SIG_DFL);
499 if (getaddrinfo(host.c_str(), NULL, &aihints, &ai))
517 .events = CURL_WAIT_POLLIN,
526 bool hasEvent = std::any_of( waitFds.begin (), waitFds.end(),[
this](
const curl_waitfd &waitfd ){
527 return ( waitfd.fd == _dnspipe && waitfd.revents != 0 );
533 while (waitpid(
_pid, &status, 0) == -1)
544 if (!WIFEXITED(status))
547 strncpy(
_curlError,
"DNS lookup failed", CURL_ERROR_SIZE);
551 int exitcode = WEXITSTATUS(status);
552 XXX <<
"#" <<
_workerno <<
": DNS lookup returned " << exitcode << endl;
556 strncpy(
_curlError,
"DNS lookup failed", CURL_ERROR_SIZE);
586 size_t cnt = l >
sizeof(buf) ?
sizeof(buf) : l;
601 XXX <<
"start stealing!" << endl;
605 std::list<multifetchworker *>::iterator workeriter =
_request->
_workers.begin();
612 if (worker->
_pass == -1)
671 XXX <<
"#" <<
_workerno <<
": going to sleep for " << sl * 1000 <<
" ms" << endl;
692 std::list<multifetchworker *>::iterator workeriter =
_request->
_workers.begin();
768 sprintf(rangebuf,
"%llu-", (
unsigned long long)
_blkstart);
776 strncpy(
_curlError,
"curl_easy_setopt range failed", CURL_ERROR_SIZE);
783 strncpy(
_curlError,
"curl_multi_add_handle failed", CURL_ERROR_SIZE);
834 for (
size_t blkno = 0; blkno < blklist->
numBlocks(); blkno++)
840 else if (filesize != off_t(-1))
846 for (std::list<multifetchworker *>::iterator workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
859 std::vector<Url>::iterator urliter = urllist.begin();
863 std::vector<curl_waitfd> extraWaitFds;
867 XXX <<
"finished!" << endl;
891 WAR <<
"No more active workers!" << endl;
893 for (std::list<multifetchworker *>::iterator workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
903 for (std::list<multifetchworker *>::iterator workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
904 (*workeriter)->adddnsfd( extraWaitFds );
911 for (std::list<multifetchworker *>::iterator workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter) {
925 timeoutMs = sl * 1000;
929#if CURLVERSION_AT_LEAST(7,66,0)
930 CURLMcode mcode = curl_multi_poll(
_multi, extraWaitFds.data(), extraWaitFds.size(), timeoutMs, &r );
932 CURLMcode mcode = curl_multi_wait(
_multi, extraWaitFds.data(), extraWaitFds.size(), timeoutMs, &r );
934 if ( mcode != CURLM_OK ) {
937 if (r == -1 && errno != EINTR)
940 for (std::list<multifetchworker *>::iterator workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
945 (*workeriter)->
dnsevent( extraWaitFds );
956 mcode = curl_multi_perform(
_multi, &tasks);
957 if (mcode == CURLM_CALL_MULTI_PERFORM)
959 if (mcode != CURLM_OK)
980 for (std::list<multifetchworker *>::iterator workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
989 XXX <<
"#" << worker->
_workerno <<
": sleep done, wake up" << endl;
999 while ((msg = curl_multi_info_read(
_multi, &nqueue)) != 0)
1001 if (msg->msg != CURLMSG_DONE)
1003 CURL *easy = msg->easy_handle;
1004 CURLcode cc = msg->data.result;
1006 if (curl_easy_getinfo(easy, CURLINFO_PRIVATE, &worker) != CURLE_OK)
1016 curl_multi_remove_handle(
_multi, easy);
1017 if (cc == CURLE_HTTP_RETURNED_ERROR)
1019 long statuscode = 0;
1020 (void)curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &statuscode);
1021 XXX <<
"HTTP status " << statuscode << endl;
1022 if (statuscode == 416 && !
_blklist)
1028 XXX <<
"#" << worker->
_workerno <<
": retrying with no end range" << endl;
1048 WAR <<
"#" << worker->
_workerno <<
": checksum error, disable worker" << endl;
1050 strncpy(worker->
_curlError,
"checksum error", CURL_ERROR_SIZE);
1064 XXX <<
"#" << worker->
_workerno <<
": recheck checksum error, refetch block" << endl;
1077 int maxworkerno = 0;
1079 for (std::list<multifetchworker *>::iterator workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1094 double ratio = worker->
_avgspeed / maxavg;
1097 ratio = ratio * ratio;
1100 XXX <<
"#" << worker->
_workerno <<
": too slow ("<< ratio <<
", " << worker->
_avgspeed <<
", #" << maxworkerno <<
": " << maxavg <<
"), going to sleep for " << ratio * 1000 <<
" ms" << endl;
1121#if CURLVERSION_AT_LEAST(7,15,5)
1122 curl_easy_setopt(worker->
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)(avg));
1164 WAR <<
"overall result" << endl;
1165 for (std::list<multifetchworker *>::iterator workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1175 if ( filesize == 0 )
return 2 * 1024 * 1024;
1176 else if ( filesize < 2*256*1024 )
return filesize;
1177 else if ( filesize < 8*1024*1024 )
return 256*1024;
1178 else if ( filesize < 256*1024*1024 )
return 1024*1024;
1188 MIL <<
"MediaMultiCurl::MediaMultiCurl(" << url_r <<
", " << attach_point_hint_r <<
")" << endl;
1202 curl_multi_cleanup(
_multi);
1205 std::map<std::string, CURL *>::iterator it;
1208 CURL *easy = it->second;
1211 curl_easy_cleanup(easy);
1227 for (; sl; sl = sl->next)
1236 while ((l = pread(fd, buf,
sizeof(buf) - 1, (off_t)0)) == -1 && errno == EINTR)
1242 while (*p ==
' ' || *p ==
'\t' || *p ==
'\r' || *p ==
'\n')
1244 if (!strncasecmp(p,
"<?xml", 5))
1246 while (*p && *p !=
'>')
1250 while (*p ==
' ' || *p ==
'\t' || *p ==
'\r' || *p ==
'\n')
1253 bool ret = !strncasecmp(p,
"<metalink", 9) ? true :
false;
1260 if ((fd = open(file.
asString().c_str(), O_RDONLY|O_CLOEXEC)) == -1)
1264 DBG <<
"looks_like_metalink(" << file <<
"): " << ret << endl;
1280 if ( curl_easy_getinfo(
_curl, CURLINFO_PRIVATE, &fp ) != CURLE_OK || !fp )
1282 if ( ftell( fp ) == 0 )
1287 long httpReturnCode = 0;
1288 if (curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0)
1292 bool ismetalink =
false;
1293 if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1295 std::string ct = std::string(ptr);
1296 if (ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
1299 if (!ismetalink && dlnow < 256)
1308 DBG <<
"looks_like_metalink_fd: " << ismetalink << endl;
1325 if( assert_dir( dest.
dirname() ) )
1327 DBG <<
"assert_dir " << dest.
dirname() <<
" failed" << endl;
1337 ERR <<
"out of memory for temp file name" << endl;
1341 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1344 ERR <<
"mkstemp failed for file '" << destNew <<
"'" << endl;
1349 file = ::fdopen( tmp_fd,
"we" );
1352 ERR <<
"fopen failed for file '" << destNew <<
"'" << endl;
1355 tmp_fd.resetDispose();
1358 DBG <<
"dest: " << dest << endl;
1359 DBG <<
"temp: " << destNew << endl;
1364 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1365 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, (
long)
PathInfo(target).mtime());
1369 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1370 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1376 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (*file) );
1383 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1384 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1386 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
1389 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1390 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1392 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
1393 long httpReturnCode = 0;
1394 CURLcode infoRet = curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode);
1395 if (infoRet == CURLE_OK)
1398 if ( httpReturnCode == 304
1401 DBG <<
"not modified: " <<
PathInfo(dest) << endl;
1407 WAR <<
"Could not get the response code." << endl;
1410 bool ismetalink =
false;
1413 if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1415 std::string ct = std::string(ptr);
1416 if (ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
1431 bool userabort =
false;
1445 XXX <<
"No filesize in metalink file and no expected filesize, aborting multicurl." << std::endl;
1449 std::vector<Url> urls = mlp.
getUrls();
1459 file = fopen((*destNew).c_str(),
"w+e");
1464 XXX <<
"reusing blocks from file " << target << endl;
1470 XXX <<
"reusing blocks from file " << failedFile << endl;
1478 XXX <<
"reusing blocks from file " << df << endl;
1488 userabort = ex.
errstr() ==
"User abort";
1499 if (
PathInfo(destNew).size() >= 63336)
1501 ::unlink(failedFile.
asString().c_str());
1508 file = fopen((*destNew).c_str(),
"w+e");
1520 ERR <<
"Failed to chmod file " << destNew << endl;
1527 ERR <<
"Fclose failed for file '" << destNew <<
"'" << endl;
1531 if ( rename( destNew, dest ) != 0 )
1533 ERR <<
"Rename failed" << endl;
1536 destNew.resetDispose();
1544 if (blklist && filesize == off_t(-1) && blklist->
haveFilesize())
1546 if (blklist && !blklist->
haveBlocks() && filesize != 0)
1548 if (blklist && (filesize == 0 || !blklist->
numBlocks()))
1557 _multi = curl_multi_init();
1571 std::vector<Url> myurllist;
1572 for (std::vector<Url>::iterator urliter = urllist->begin(); urliter != urllist->end(); ++urliter)
1576 std::string scheme = urliter->getScheme();
1577 if (scheme ==
"http" || scheme ==
"https" || scheme ==
"ftp" || scheme ==
"tftp")
1587 if (!myurllist.size())
1588 myurllist.push_back(baseurl);
1597 if (fseeko(fp, off_t(0), SEEK_SET))
1603 while ((l = fread(buf, 1,
sizeof(buf), fp)) > 0)
1611 return _dnsok.find(host) ==
_dnsok.end() ? false :
true;
1633 curl_easy_cleanup(oldeasy);
std::optional< KeyManagerCtx > _context
void resetDispose()
Set no dispose function.
Store and operate with byte count.
static const Unit MB
1000^2 Byte
std::string asString(unsigned field_width_r=0, unsigned unit_width_r=1) const
Auto selected Unit and precision.
Compute Message Digests (MD5, SHA1 etc)
bool update(const char *bytes, size_t len)
feed data into digest computation algorithm
Base class for Exception.
std::string getScheme() const
Returns the scheme name of the URL.
std::string asString() const
Returns a default string representation of the Url object.
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Pathname repoCachePath() const
Path where the caches are kept (/var/cache/zypp)
static ZConfig & instance()
Singleton ctor.
Wrapper class for stat/lstat.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Pathname dirname() const
Return all but the last component od this path.
const std::string & asString() const
String representation.
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
int unlink(const Pathname &path)
Like 'unlink'.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
std::string numstring(char n, int w=0)
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
AutoDispose<int> calling ::close
AutoDispose<FILE*> calling ::fclose
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.