libzypp 17.31.28
MediaMultiCurl.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
13#include <ctype.h>
14#include <sys/types.h>
15#include <signal.h>
16#include <sys/wait.h>
17#include <netdb.h>
18#include <arpa/inet.h>
19#include <glib.h>
20
21#include <vector>
22#include <iostream>
23#include <algorithm>
24
25
26#include <zypp/ZConfig.h>
27#include <zypp/base/Logger.h>
29#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
30#include <zypp-curl/parser/MetaLinkParser>
32#include <zypp/ManagedFile.h>
34#include <zypp-curl/auth/CurlAuthData>
37
38using std::endl;
39using namespace zypp::base;
40
41#undef CURLVERSION_AT_LEAST
42#define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O)
43
44namespace zypp {
45 namespace media {
46
48
49
113
114struct Stripe {
115
116 enum RState {
117 PENDING, //< Pending Range
118 FETCH, //< Fetch is running!
119 COMPETING, //< Competing workers, needs checksum recheck
120 FINALIZED, //< Done, don't write to it anymore
121 REFETCH //< This block needs a refetch
122 };
123
124 std::vector<off_t> blocks; //< required block numbers from blocklist
125 std::vector<RState> blockStates; //< current state of each block in blocks
126};
127
137
138// Hack: we derive from MediaCurl just to get the storage space for
139// settings, url, curlerrors and the like
141 friend class multifetchrequest;
142
143public:
144 multifetchworker(int no, multifetchrequest &request, const Url &url);
146
151 void nextjob();
152
157 void runjob();
158
164 bool continueJob();
165
170 bool recheckChecksum( off_t blockIdx );
171
175 void disableCompetition();
176
180 void checkdns();
181 void adddnsfd( std::vector<GPollFD> &waitFds );
182 void dnsevent(const std::vector<GPollFD> &waitFds );
183
184 const int _workerno;
185
187 bool _competing = false;
188
189 std::vector<MultiByteHandler::Range> _blocks;
190 std::vector<off_t> _rangeToStripeBlock;
191
193 std::unique_ptr<MultiByteHandler> _multiByteHandler;
194
195 off_t _stripe = 0; //< The current stripe we are downloading
196 size_t _datasize = 0; //< The nr of bytes we need to download overall
197
198 double _starttime = 0; //< When was the current job started
199 size_t _datareceived = 0; //< Data downloaded in the current job only
200 off_t _received = 0; //< Overall data"MultiByteHandler::prepare failed" fetched by this worker
201
202 double _avgspeed = 0;
203 double _maxspeed = 0;
204
205 double _sleepuntil = 0;
206
207private:
208 void run();
209 void stealjob();
210 bool setupHandle();
211 MultiByteHandler::Range rangeFromBlock( off_t blockNo ) const;
212
213 size_t writefunction ( char *ptr, std::optional<off_t> offset, size_t bytes ) override;
214 size_t headerfunction ( char *ptr, size_t bytes ) override;
215 bool beginRange ( off_t range, std::string &cancelReason ) override;
216 bool finishedRange ( off_t range, bool validated, std::string &cancelReason ) override;
217
219 int _pass = 0;
220 std::string _urlbuf;
221
222 pid_t _pid = 0;
223 int _dnspipe = -1;
224};
225
227public:
228 multifetchrequest(const MediaMultiCurl *context, const Pathname &filename, const Url &baseurl, CURLM *multi, FILE *fp, callback::SendReport<DownloadProgressReport> *report, MediaBlockList &&blklist, off_t filesize);
230
231 void run(std::vector<Url> &urllist);
232 static ByteCount makeBlksize( uint maxConns, size_t filesize );
233
235 return _blklist;
236 }
237
238protected:
239 friend class multifetchworker;
240
244
245 FILE *_fp = nullptr;
248
249 std::vector<Stripe> _requiredStripes; // all the data we need
250
251 off_t _filesize = 0; //< size of the file we want to download
252
253 CURLM *_multi = nullptr;
254
255 std::list< std::unique_ptr<multifetchworker> > _workers;
256 bool _stealing = false;
257 bool _havenewjob = false;
258
259 zypp::ByteCount _defaultBlksize = 0; //< The blocksize to use if the metalink file does not specify one
260 off_t _stripeNo = 0; //< next stripe to download
261
262 size_t _activeworkers = 0;
263 size_t _lookupworkers = 0;
264 size_t _sleepworkers = 0;
265 double _minsleepuntil = 0;
266 bool _finished = false;
267
268 off_t _totalsize = 0; //< nr of bytes we need to download ( e.g. filesize - ( bytes reused from deltafile ) )
269 off_t _fetchedsize = 0;
271
272 double _starttime = 0;
273 double _lastprogress = 0;
274
277 double _periodavg = 0;
278
279public:
280 double _timeout = 0;
282 double _maxspeed = 0;
283 int _maxworkers = 0;
284};
285
286constexpr auto MIN_REQ_MIRRS = 4;
287constexpr auto MAXURLS = 10;
288
290
291static double
293{
294#if _POSIX_C_SOURCE >= 199309L
295 struct timespec ts;
296 if ( clock_gettime( CLOCK_MONOTONIC, &ts) )
297 return 0;
298 return ts.tv_sec + ts.tv_nsec / 1000000000.;
299#else
300 struct timeval tv;
301 if (gettimeofday(&tv, NULL))
302 return 0;
303 return tv.tv_sec + tv.tv_usec / 1000000.;
304#endif
305}
306
307size_t
308multifetchworker::writefunction(char *ptr, std::optional<off_t> offset, size_t bytes)
309{
311 return bytes ? 0 : 1;
312
313 double now = currentTime();
314
315 // update stats of overall data
316 _datareceived += bytes;
317 _received += bytes;
318 _request->_lastprogress = now;
319
320 const auto &currRange = _multiByteHandler->currentRange();
321 if (!currRange)
322 return 0; // we always write to a range
323
324 auto &stripeDesc = _request->_requiredStripes[_stripe];
325 if ( !_request->_fp || stripeDesc.blockStates[ _rangeToStripeBlock[*currRange] ] == Stripe::FINALIZED ) {
326 // someone else finished our block first!
327 // we stop here and fetch new jobs if there are still some
329 _competing = false;
330 return 0;
331 }
332
333 const auto &blk = _blocks[*currRange];
334 off_t seekTo = blk.start + blk.bytesWritten;
335
336 if ( ftell( _request->_fp ) != seekTo ) {
337 // if we can't seek the file there is no purpose in trying again
338 if (fseeko(_request->_fp, seekTo, SEEK_SET))
339 return bytes ? 0 : 1;
340 }
341
342 size_t cnt = fwrite(ptr, 1, bytes, _request->_fp);
343 _request->_fetchedsize += cnt;
344 return cnt;
345}
346
347bool multifetchworker::beginRange ( off_t workerRangeOff, std::string &cancelReason )
348{
349 auto &stripeDesc = _request->_requiredStripes[_stripe];
350 auto stripeRangeOff = _rangeToStripeBlock[workerRangeOff];
351 const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
352
353 if ( currRangeState == Stripe::FINALIZED ){
354 cancelReason = "Cancelled because stripe block is already finalized";
356 WAR << "#" << _workerno << ": trying to start a range ("<<stripeRangeOff<<"["<< _blocks[workerRangeOff].start <<" : "<<_blocks[workerRangeOff].len<<"]) that was already finalized, cancelling. Stealing was: " << _request->_stealing << endl;
357 return false;
358 }
359 stripeDesc.blockStates[stripeRangeOff] = currRangeState == Stripe::PENDING ? Stripe::FETCH : Stripe::COMPETING;
360 return true;
361}
362
363bool multifetchworker::finishedRange ( off_t workerRangeOff, bool validated, std::string &cancelReason )
364{
365 auto &stripeDesc = _request->_requiredStripes[_stripe];
366 auto stripeRangeOff = _rangeToStripeBlock[workerRangeOff];
367 const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
368
369 if ( !validated ) {
370 // fail, worker will go into WORKER_BROKEN
371 cancelReason = "Block failed to validate";
372 return false;
373 }
374
375 if ( currRangeState == Stripe::FETCH ) {
376 // only us who wrote here, block is finalized
377 stripeDesc.blockStates[stripeRangeOff] = Stripe::FINALIZED;
378 _request->_fetchedgoodsize += _blocks[workerRangeOff].len;
379 } else {
380 // others wrote here, we need to check the full checksum
381 if ( recheckChecksum ( workerRangeOff ) ) {
382 stripeDesc.blockStates[stripeRangeOff] = Stripe::FINALIZED;
383 _request->_fetchedgoodsize += _blocks[workerRangeOff].len;
384 } else {
385 // someone messed that block up, set it to refetch but continue since our
386 // data is valid
387 WAR << "#" << _workerno << ": Broken data in COMPETING block, requesting refetch. Stealing is: " << _request->_stealing << endl;
388 stripeDesc.blockStates[stripeRangeOff] = Stripe::REFETCH;
389 }
390 }
391 return true;
392}
393
394size_t
395multifetchworker::headerfunction( char *p, size_t bytes )
396{
397 size_t l = bytes;
398 if (l > 9 && !strncasecmp(p, "Location:", 9)) {
399 std::string line(p + 9, l - 9);
400 if (line[l - 10] == '\r')
401 line.erase(l - 10, 1);
402 XXX << "#" << _workerno << ": redirecting to" << line << endl;
403 return bytes;
404 }
405
406 const auto &repSize = _multiByteHandler->reportedFileSize ();
407 if ( repSize && *repSize != _request->_filesize ) {
408 XXX << "#" << _workerno << ": filesize mismatch" << endl;
410 strncpy(_curlError, "filesize mismatch", CURL_ERROR_SIZE);
411 return 0;
412 }
413
414 return bytes;
415}
416
418: MediaCurl(url, Pathname())
419, _workerno( no )
420, _maxspeed( request._maxspeed )
421, _request ( &request )
422{
423 Url curlUrl( clearQueryString(url) );
424 _urlbuf = curlUrl.asString();
426 if (_curl)
427 XXX << "reused worker from pool" << endl;
428 if (!_curl && !(_curl = curl_easy_init()))
429 {
431 strncpy(_curlError, "curl_easy_init failed", CURL_ERROR_SIZE);
432 return;
433 }
434
435 if ( url.getScheme() == "http" || url.getScheme() == "https" )
437
438 setupHandle();
439 checkdns();
440}
441
443{
444 try {
445 setupEasy();
446 } catch (Exception &ex) {
447 curl_easy_cleanup(_curl);
448 _curl = 0;
450 strncpy(_curlError, "curl_easy_setopt failed", CURL_ERROR_SIZE);
451 return false;
452 }
453 curl_easy_setopt(_curl, CURLOPT_PRIVATE, this);
454 curl_easy_setopt(_curl, CURLOPT_URL, _urlbuf.c_str());
455
456 // if this is the same host copy authorization
457 // (the host check is also what curl does when doing a redirect)
458 // (note also that unauthorized exceptions are thrown with the request host)
459 if ( _url.getHost() == _request->_context->_url.getHost()) {
463 if ( _settings.userPassword().size() ) {
464 curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
465 std::string use_auth = _settings.authType();
466 if (use_auth.empty())
467 use_auth = "digest,basic"; // our default
468 long auth = CurlAuthData::auth_type_str2long(use_auth);
469 if( auth != CURLAUTH_NONE)
470 {
471 XXX << "#" << _workerno << ": Enabling HTTP authentication methods: " << use_auth
472 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
473 curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, auth);
474 }
475 }
476 }
477 return true;
478}
479
481{
482 if (_curl)
483 {
485 curl_multi_remove_handle(_request->_multi, _curl);
487 {
488#if CURLVERSION_AT_LEAST(7,15,5)
489 curl_easy_setopt(_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)0);
490#endif
491 curl_easy_setopt(_curl, CURLOPT_PRIVATE, (void *)0);
492 curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, (void *)0);
493 curl_easy_setopt(_curl, CURLOPT_WRITEDATA, (void *)0);
494 curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, (void *)0);
495 curl_easy_setopt(_curl, CURLOPT_HEADERDATA, (void *)0);
497 }
498 else
499 curl_easy_cleanup(_curl);
500 _curl = 0;
501 }
502 if (_pid)
503 {
504 kill(_pid, SIGKILL);
505 int status;
506 while (waitpid(_pid, &status, 0) == -1)
507 if (errno != EINTR)
508 break;
509 _pid = 0;
510 }
511 if (_dnspipe != -1)
512 {
513 close(_dnspipe);
514 _dnspipe = -1;
515 }
516 // the destructor in MediaCurl doesn't call disconnect() if
517 // the media is not attached, so we do it here manually
519}
520
521static inline bool env_isset(std::string name)
522{
523 const char *s = getenv(name.c_str());
524 return s && *s ? true : false;
525}
526
527void
529{
530 std::string host = _url.getHost();
531
532 if (host.empty())
533 return;
534
535 if (_request->_context->isDNSok(host))
536 return;
537
538 // no need to do dns checking for numeric hosts
539 char addrbuf[128];
540 if (inet_pton(AF_INET, host.c_str(), addrbuf) == 1)
541 return;
542 if (inet_pton(AF_INET6, host.c_str(), addrbuf) == 1)
543 return;
544
545 // no need to do dns checking if we use a proxy
546 if (!_settings.proxy().empty())
547 return;
548 if (env_isset("all_proxy") || env_isset("ALL_PROXY"))
549 return;
550 std::string schemeproxy = _url.getScheme() + "_proxy";
551 if (env_isset(schemeproxy))
552 return;
553 if (schemeproxy != "http_proxy")
554 {
555 std::transform(schemeproxy.begin(), schemeproxy.end(), schemeproxy.begin(), ::toupper);
556 if (env_isset(schemeproxy))
557 return;
558 }
559
560 XXX << "checking DNS lookup of " << host << endl;
561 int pipefds[2];
562 if (pipe(pipefds))
563 {
565 strncpy(_curlError, "DNS pipe creation failed", CURL_ERROR_SIZE);
566 return;
567 }
568 _pid = fork();
569 if (_pid == pid_t(-1))
570 {
571 close(pipefds[0]);
572 close(pipefds[1]);
573 _pid = 0;
575 strncpy(_curlError, "DNS checker fork failed", CURL_ERROR_SIZE);
576 return;
577 }
578 else if (_pid == 0)
579 {
580 close(pipefds[0]);
581 // XXX: close all other file descriptors
582 struct addrinfo *ai, aihints;
583 memset(&aihints, 0, sizeof(aihints));
584 aihints.ai_family = PF_UNSPEC;
585 int tstsock = socket(PF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
586 if (tstsock == -1)
587 aihints.ai_family = PF_INET;
588 else
589 close(tstsock);
590 aihints.ai_socktype = SOCK_STREAM;
591 aihints.ai_flags = AI_CANONNAME;
592 unsigned int connecttimeout = _request->_connect_timeout;
593 if (connecttimeout)
594 alarm(connecttimeout);
595 signal(SIGALRM, SIG_DFL);
596 if (getaddrinfo(host.c_str(), NULL, &aihints, &ai))
597 _exit(1);
598 _exit(0);
599 }
600 close(pipefds[1]);
601 _dnspipe = pipefds[0];
603}
604
605void
606multifetchworker::adddnsfd(std::vector<GPollFD> &waitFds)
607{
608 if (_state != WORKER_LOOKUP)
609 return;
610
611 waitFds.push_back (
612 GPollFD {
613 .fd = _dnspipe,
614 .events = G_IO_IN | G_IO_HUP | G_IO_ERR,
615 .revents = 0
616 });
617}
618
619void
620multifetchworker::dnsevent( const std::vector<GPollFD> &waitFds )
621{
622 bool hasEvent = std::any_of( waitFds.begin (), waitFds.end(),[this]( const GPollFD &waitfd ){
623 return ( waitfd.fd == _dnspipe && waitfd.revents != 0 );
624 });
625
626 if (_state != WORKER_LOOKUP || !hasEvent)
627 return;
628 int status;
629 while (waitpid(_pid, &status, 0) == -1)
630 {
631 if (errno != EINTR)
632 return;
633 }
634 _pid = 0;
635 if (_dnspipe != -1)
636 {
637 close(_dnspipe);
638 _dnspipe = -1;
639 }
640 if (!WIFEXITED(status))
641 {
643 strncpy(_curlError, "DNS lookup failed", CURL_ERROR_SIZE);
645 return;
646 }
647 int exitcode = WEXITSTATUS(status);
648 XXX << "#" << _workerno << ": DNS lookup returned " << exitcode << endl;
649 if (exitcode != 0)
650 {
652 strncpy(_curlError, "DNS lookup failed", CURL_ERROR_SIZE);
654 return;
655 }
657 nextjob();
658}
659
660bool multifetchworker::recheckChecksum( off_t workerRangeIdx )
661{
662 // XXX << "recheckChecksum block " << _blkno << endl;
663 if (!_request->_fp || !_datasize || !_blocks.size() )
664 return true;
665
666 auto &blk = _blocks[workerRangeIdx];
667 if ( !blk._digest )
668 return true;
669
670 const auto currOf = ftell( _request->_fp );
671 if ( currOf == -1 )
672 return false;
673
674 if (fseeko(_request->_fp, blk.start, SEEK_SET))
675 return false;
676
677 zypp::Digest newDig = blk._digest->clone();
678
679 char buf[4096];
680 size_t l = blk.len;
681 while (l) {
682 size_t cnt = l > sizeof(buf) ? sizeof(buf) : l;
683 if (fread(buf, cnt, 1, _request->_fp) != 1)
684 return false;
685 newDig.update(buf, cnt);
686 l -= cnt;
687 }
688
689 if (fseeko(_request->_fp, currOf, SEEK_SET))
690 return false;
691
692 blk._digest = std::move(newDig);
693 if (!_multiByteHandler->validateRange(blk)) {
694 WAR << "#" << _workerno << " Stripe: " << _stripe << ": Stripe-Block: " << _rangeToStripeBlock[workerRangeIdx] << " failed to validate" << endl;
695 return false;
696 }
697
698 return true;
699}
700
705{
706 UByteArray sum;
707 std::optional<zypp::Digest> digest;
708 std::optional<size_t> relDigLen;
709 std::optional<size_t> blkSumPad;
710
711 const auto &blk = _request->_blklist.getBlock( blkNo );
712 if ( _request->_blklist.haveChecksum ( blkNo ) ) {
713 sum = _request->_blklist.getChecksum( blkNo );
714 relDigLen = sum.size( );
715 blkSumPad = _request->_blklist.checksumPad( );
716 digest = zypp::Digest();
717 digest->create( _request->_blklist.getChecksumType() );
718 }
719
720 return MultiByteHandler::Range::make(
721 blk.off,
722 blk.size,
723 std::move(digest),
724 std::move(sum),
725 {}, // empty user data
726 std::move(relDigLen),
727 std::move(blkSumPad) );
728}
729
731{
732 if (!_request->_stealing)
733 {
734 XXX << "start stealing!" << endl;
735 _request->_stealing = true;
736 }
737
738 multifetchworker *best = 0; // best choice for the worker we want to compete with
739 double now = 0;
740
741 // look through all currently running workers to find the best candidate we
742 // could steal from
743 for (auto workeriter = _request->_workers.begin(); workeriter != _request->_workers.end(); ++workeriter)
744 {
745 multifetchworker *worker = workeriter->get();
746 if (worker == this)
747 continue;
748 if (worker->_pass == -1)
749 continue; // do not steal!
750 if (worker->_state == WORKER_DISCARD || worker->_state == WORKER_DONE || worker->_state == WORKER_SLEEP || !worker->_datasize)
751 continue; // do not steal finished jobs
752 if (!worker->_avgspeed && worker->_datareceived)
753 {
754 // calculate avg speed for the worker if that was not done yet
755 if (!now)
756 now = currentTime();
757 if (now > worker->_starttime)
758 worker->_avgspeed = worker->_datareceived / (now - worker->_starttime);
759 }
760 // only consider worker who still have work
761 if ( worker->_datasize - worker->_datareceived <= 0 )
762 continue;
763 if (!best || best->_pass > worker->_pass)
764 {
765 best = worker;
766 continue;
767 }
768 if (best->_pass < worker->_pass)
769 continue;
770 // if it is the same stripe, our current best choice is competing with the worker we are looking at
771 // we need to check if we are faster than the fastest one competing for this stripe, so we want the best.
772 // Otherwise the worst.
773 if (worker->_stripe == best->_stripe)
774 {
775 if ((worker->_datasize - worker->_datareceived) * best->_avgspeed < (best->_datasize - best->_datareceived) * worker->_avgspeed)
776 best = worker;
777 }
778 else
779 {
780 if ((worker->_datasize - worker->_datareceived) * best->_avgspeed > (best->_datasize - best->_datareceived) * worker->_avgspeed)
781 best = worker;
782 }
783 }
784 if (!best)
785 {
788 _request->_finished = true;
789 return;
790 }
791 // do not sleep twice
792 if (_state != WORKER_SLEEP)
793 {
794 if (!_avgspeed && _datareceived)
795 {
796 if (!now)
797 now = currentTime();
798 if (now > _starttime)
800 }
801
802 // lets see if we should sleep a bit
803 XXX << "me #" << _workerno << ": " << _avgspeed << ", size " << best->_datasize << endl;
804 XXX << "best #" << best->_workerno << ": " << best->_avgspeed << ", size " << (best->_datasize - best->_datareceived) << endl;
805
806 // check if we could download the full data from best faster than best could download its remaining data
807 if ( _avgspeed && best->_avgspeed // we and best have average speed information
808 && _avgspeed <= best->_avgspeed ) // and we are not faster than best
809 {
810 if (!now)
811 now = currentTime();
812 double sl = (best->_datasize - best->_datareceived) / best->_avgspeed * 2;
813 if (sl > 1)
814 sl = 1;
815 XXX << "#" << _workerno << ": going to sleep for " << sl * 1000 << " ms" << endl;
816 _sleepuntil = now + sl;
819 return;
820 }
821 }
822
823 _competing = true;
824 best->_competing = true;
825 _stripe = best->_stripe;
826
827 best->_pass++;
828 _pass = best->_pass;
829
830 runjob();
831}
832
833void
835{
836 for ( auto workeriter = _request->_workers.begin(); workeriter != _request->_workers.end(); ++workeriter)
837 {
838 multifetchworker *worker = workeriter->get();
839 if (worker == this)
840 continue;
841 if (worker->_stripe == _stripe)
842 {
843 if (worker->_state == WORKER_FETCH)
844 worker->_state = WORKER_DISCARD;
845 worker->_pass = -1; /* do not steal this one, we already have it */
846 }
847 }
848}
849
851{
852 _datasize = 0;
853 _blocks.clear();
854
855 // claim next stripe for us, or steal if there nothing left to claim
857 stealjob();
858 return;
859 }
860
862 runjob();
863}
864
866{
867 _datasize = 0;
868 _blocks.clear ();
869 _rangeToStripeBlock.clear ();
870
871 auto &stripeDesc = _request->_requiredStripes[_stripe];
872 for ( uint i = 0; i < stripeDesc.blocks.size(); i++ ) {
873 // ignore verified and finalized ranges
874 if( stripeDesc.blockStates[i] == Stripe::FINALIZED ) {
875 continue;
876 } else {
877 _blocks.push_back( rangeFromBlock(stripeDesc.blocks[i]) );
878 _rangeToStripeBlock.push_back( i );
879 _datasize += _blocks.back().len;
880 }
881 }
882
883 if ( _datasize == 0 ) {
884 // no blocks left in this stripe
887 if ( !_request->_activeworkers )
888 _request->_finished = true;
889 return;
890 }
891
892 DBG << "#" << _workerno << "Done adding blocks to download, going to download: " << _blocks.size() << " nr of block with " << _datasize << " nr of bytes" << std::endl;
893
894 _multiByteHandler.reset( nullptr );
895 _multiByteHandler = std::make_unique<MultiByteHandler>(_protocolMode, _curl, _blocks, *this );
897 _datareceived = 0;
898 run();
899}
900
902{
903 bool hadRangeFail = _multiByteHandler->lastError() == MultiByteHandler::Code::RangeFail;
904 if ( !_multiByteHandler->prepareToContinue() ) {
905 strncpy( _curlError, _multiByteHandler->lastErrorMessage().c_str(), CURL_ERROR_SIZE );
906 return false;
907 }
908
909 if ( hadRangeFail ) {
910 // we reset the handle to default values. We do this to not run into
911 // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
912 // we cancel a connection because of a range error to request a smaller batch.
913 // The error will still happen but much less frequently than without resetting the handle.
914 //
915 // Note: Even creating a new handle will NOT fix the issue
916 curl_easy_reset( _curl );
917 if ( !setupHandle())
918 return false;
919 }
920
921 run();
922 return true;
923}
924
925void
927{
929 return; // just in case...
930
931 if ( !_multiByteHandler->prepare() ) {
934 strncpy( _curlError, _multiByteHandler->lastErrorMessage ().c_str() , CURL_ERROR_SIZE );
935 return;
936 }
937
938 if (curl_multi_add_handle(_request->_multi, _curl) != CURLM_OK) {
941 strncpy( _curlError, "curl_multi_add_handle failed", CURL_ERROR_SIZE );
942 return;
943 }
944
945 _request->_havenewjob = true;
947}
948
949
951
952
953multifetchrequest::multifetchrequest(const MediaMultiCurl *context, const Pathname &filename, const Url &baseurl, CURLM *multi, FILE *fp, callback::SendReport<DownloadProgressReport> *report, MediaBlockList &&blklist, off_t filesize)
954 : _context(context)
955 , _filename(filename)
956 , _baseurl(baseurl)
957 , _fp(fp)
958 , _report(report)
959 , _blklist(std::move(blklist))
960 , _filesize(filesize)
961 , _multi(multi)
962 , _starttime(currentTime())
963 , _timeout(context->_settings.timeout())
964 , _connect_timeout(context->_settings.connectTimeout())
965 , _maxspeed(context->_settings.maxDownloadSpeed())
966 , _maxworkers(context->_settings.maxConcurrentConnections())
967 {
969 if (_maxworkers > MAXURLS)
971 if (_maxworkers <= 0)
972 _maxworkers = 1;
973
974 // calculate the total size of our download
975 for (size_t blkno = 0; blkno < _blklist.numBlocks(); blkno++)
977
978 // equally distribute the data we want to download over all workers
980
981 // lets build stripe informations
982 zypp::ByteCount currStripeSize = 0;
983 for (size_t blkno = 0; blkno < _blklist.numBlocks(); blkno++) {
984
985 const MediaBlock &blk = _blklist.getBlock(blkno);
986 if ( _requiredStripes.empty() || currStripeSize >= _defaultBlksize ) {
987 _requiredStripes.push_back( Stripe{} );
988 currStripeSize = 0;
989 }
990
991 _requiredStripes.back().blocks.push_back(blkno);
992 _requiredStripes.back().blockStates.push_back(Stripe::PENDING);
993 currStripeSize += blk.size;
994 }
995
996 MIL << "Downloading " << _blklist.numBlocks() << " blocks via " << _requiredStripes.size() << " stripes on " << _maxworkers << " connections." << endl;
997}
998
1000{
1001 _workers.clear();
1002}
1003
1004void
1005multifetchrequest::run(std::vector<Url> &urllist)
1006{
1007 int workerno = 0;
1008 std::vector<Url>::iterator urliter = urllist.begin();
1009
1010 struct CurlMuliSockHelper {
1011 CurlMuliSockHelper( multifetchrequest &p ) : _parent(p) {
1012 curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETFUNCTION, socketcb );
1013 curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETDATA, this );
1014 curl_multi_setopt( _parent._multi, CURLMOPT_TIMERFUNCTION, timercb );
1015 curl_multi_setopt( _parent._multi, CURLMOPT_TIMERDATA, this );
1016 }
1017
1018 ~CurlMuliSockHelper() {
1019 curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETFUNCTION, nullptr );
1020 curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETDATA, nullptr );
1021 curl_multi_setopt( _parent._multi, CURLMOPT_TIMERFUNCTION, nullptr );
1022 curl_multi_setopt( _parent._multi, CURLMOPT_TIMERDATA, nullptr );
1023 }
1024
1025 static int socketcb (CURL * easy, curl_socket_t s, int what, CurlMuliSockHelper *userp, void *sockp ) {
1026 auto it = std::find_if( userp->socks.begin(), userp->socks.end(), [&]( const GPollFD &fd){ return fd.fd == s; });
1027 gushort events = 0;
1028 if ( what == CURL_POLL_REMOVE ) {
1029 if ( it == userp->socks.end() ) {
1030 WAR << "Ignoring unknown socket in static_socketcb" << std::endl;
1031 return 0;
1032 }
1033 userp->socks.erase(it);
1034 return 0;
1035 } else if ( what == CURL_POLL_IN ) {
1036 events = G_IO_IN | G_IO_HUP | G_IO_ERR;
1037 } else if ( what == CURL_POLL_OUT ) {
1038 events = G_IO_OUT | G_IO_ERR;
1039 } else if ( what == CURL_POLL_INOUT ) {
1040 events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR;
1041 }
1042
1043 if ( it != userp->socks.end() ) {
1044 it->events = events;
1045 it->revents = 0;
1046 } else {
1047 userp->socks.push_back(
1048 GPollFD{
1049 .fd = s,
1050 .events = events,
1051 .revents = 0
1052 }
1053 );
1054 }
1055
1056 return 0;
1057 }
1058
1059 //called by curl to setup a timer
1060 static int timercb( CURLM *, long timeout_ms, CurlMuliSockHelper *thatPtr ) {
1061 if ( !thatPtr )
1062 return 0;
1063 if ( timeout_ms == -1 )
1064 thatPtr->timeout_ms.reset(); // curl wants to delete its timer
1065 else
1066 thatPtr->timeout_ms = timeout_ms; // maximum time curl wants us to sleep
1067 return 0;
1068 }
1069
1070 multifetchrequest &_parent;
1071 std::vector<GPollFD> socks;
1072 std::optional<long> timeout_ms = 0; //if set curl wants a timeout
1073 } _curlHelper(*this) ;
1074
1075 // kickstart curl
1076 int handles = 0;
1077 CURLMcode mcode = curl_multi_socket_action( _multi, CURL_SOCKET_TIMEOUT, 0, &handles );
1078 if (mcode != CURLM_OK)
1079 ZYPP_THROW(MediaCurlException(_baseurl, "curl_multi_socket_action", "unknown error"));
1080
1081 for (;;)
1082 {
1083 // list of all fds we want to poll
1084 std::vector<GPollFD> waitFds;
1085 int dnsFdCount = 0;
1086
1087 if (_finished)
1088 {
1089 XXX << "finished!" << endl;
1090 break;
1091 }
1092
1093 if ((int)_activeworkers < _maxworkers && urliter != urllist.end() && _workers.size() < MAXURLS)
1094 {
1095 // spawn another worker!
1096 _workers.push_back(std::make_unique<multifetchworker>(workerno++, *this, *urliter));
1097 auto &worker = _workers.back();
1098 if (worker->_state != WORKER_BROKEN)
1099 {
1101 if (worker->_state != WORKER_LOOKUP)
1102 {
1103 worker->nextjob();
1104 }
1105 else
1107 }
1108 ++urliter;
1109 continue;
1110 }
1111 if (!_activeworkers)
1112 {
1113 WAR << "No more active workers!" << endl;
1114 // show the first worker error we find
1115 for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1116 {
1117 if ((*workeriter)->_state != WORKER_BROKEN)
1118 continue;
1119 ZYPP_THROW(MediaCurlException(_baseurl, "Server error", (*workeriter)->_curlError));
1120 }
1121 break;
1122 }
1123
1124 if (_lookupworkers)
1125 for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1126 (*workeriter)->adddnsfd( waitFds );
1127
1128 // if we added a new job we have to call multi_perform once
1129 // to make it show up in the fd set. do not sleep in this case.
1130 int timeoutMs = _havenewjob ? 0 : 200;
1131 if ( _sleepworkers && !_havenewjob ) {
1132 if (_minsleepuntil == 0) {
1133 for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter) {
1134 multifetchworker *worker = workeriter->get();
1135 if (worker->_state != WORKER_SLEEP)
1136 continue;
1137 if (!_minsleepuntil || _minsleepuntil > worker->_sleepuntil)
1138 _minsleepuntil = worker->_sleepuntil;
1139 }
1140 }
1141 double sl = _minsleepuntil - currentTime();
1142 if (sl < 0) {
1143 sl = 0;
1144 _minsleepuntil = 0;
1145 }
1146 if (sl < .2)
1147 timeoutMs = sl * 1000;
1148 }
1149
1150 if ( _curlHelper.timeout_ms.has_value() )
1151 timeoutMs = std::min<long>( timeoutMs, _curlHelper.timeout_ms.value() );
1152
1153 dnsFdCount = waitFds.size(); // remember how many dns fd's we have
1154 waitFds.insert( waitFds.end(), _curlHelper.socks.begin(), _curlHelper.socks.end() ); // add the curl fd's to the poll data
1155
1156 int r = zyppng::eintrSafeCall( g_poll, waitFds.data(), waitFds.size(), timeoutMs );
1157 if ( r == -1 )
1158 ZYPP_THROW(MediaCurlException(_baseurl, "g_poll() failed", "unknown error"));
1159 if ( r != 0 && _lookupworkers ) {
1160 for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1161 {
1162 multifetchworker *worker = workeriter->get();
1163 if (worker->_state != WORKER_LOOKUP)
1164 continue;
1165 (*workeriter)->dnsevent( waitFds );
1166 if (worker->_state != WORKER_LOOKUP)
1168 }
1169 }
1170 _havenewjob = false;
1171
1172 // run curl
1173 if ( r == 0 ) {
1174 int handles = 0;
1175 CURLMcode mcode = curl_multi_socket_action( _multi, CURL_SOCKET_TIMEOUT, 0, &handles );
1176 if (mcode != CURLM_OK)
1177 ZYPP_THROW(MediaCurlException(_baseurl, "curl_multi_socket_action", "unknown error"));
1178 } else {
1179 for ( int sock = dnsFdCount; sock < waitFds.size(); sock++ ) {
1180 const auto &waitFd = waitFds[sock];
1181 if ( waitFd.revents == 0 )
1182 continue;
1183
1184 int ev = 0;
1185 if ( (waitFd.revents & G_IO_HUP) == G_IO_HUP
1186 || (waitFd.revents & G_IO_IN) == G_IO_IN ) {
1187 ev = CURL_CSELECT_IN;
1188 }
1189 if ( (waitFd.revents & G_IO_OUT) == G_IO_OUT ) {
1190 ev |= CURL_CSELECT_OUT;
1191 }
1192 if ( (waitFd.revents & G_IO_ERR) == G_IO_ERR ) {
1193 ev |= CURL_CSELECT_ERR;
1194 }
1195
1196 int runn = 0;
1197 CURLMcode mcode = curl_multi_socket_action( _multi, waitFd.fd, ev, &runn );
1198 if (mcode != CURLM_OK)
1199 ZYPP_THROW(MediaCurlException(_baseurl, "curl_multi_socket_action", "unknown error"));
1200 }
1201 }
1202
1203 double now = currentTime();
1204
1205 // update periodavg
1206 if (now > _lastperiodstart + .5)
1207 {
1208 if (!_periodavg)
1210 else
1213 _lastperiodstart = now;
1214 }
1215
1216 // wake up sleepers
1217 if (_sleepworkers)
1218 {
1219 for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1220 {
1221 multifetchworker *worker = workeriter->get();
1222 if (worker->_state != WORKER_SLEEP)
1223 continue;
1224 if (worker->_sleepuntil > now)
1225 continue;
1226 if (_minsleepuntil == worker->_sleepuntil)
1227 _minsleepuntil = 0;
1228 XXX << "#" << worker->_workerno << ": sleep done, wake up" << endl;
1229 _sleepworkers--;
1230 // nextjob changes the state
1231 worker->nextjob();
1232 }
1233 }
1234
1235 // collect all curl results, (re)schedule jobs
1236 CURLMsg *msg;
1237 int nqueue;
1238 while ((msg = curl_multi_info_read(_multi, &nqueue)) != 0)
1239 {
1240 if (msg->msg != CURLMSG_DONE)
1241 continue;
1242 CURL *easy = msg->easy_handle;
1243 CURLcode cc = msg->data.result;
1244 multifetchworker *worker;
1245
1246 if (curl_easy_getinfo(easy, CURLINFO_PRIVATE, &worker) != CURLE_OK)
1247 ZYPP_THROW(MediaCurlException(_baseurl, "curl_easy_getinfo", "unknown error"));
1248
1249 if (worker->_datareceived && now > worker->_starttime) {
1250 if (worker->_avgspeed)
1251 worker->_avgspeed = (worker->_avgspeed + worker->_datareceived / (now - worker->_starttime)) / 2;
1252 else
1253 worker->_avgspeed = worker->_datareceived / (now - worker->_starttime);
1254 }
1255
1256 XXX << "#" << worker->_workerno << " done code " << cc << " speed " << worker->_avgspeed << endl;
1257 curl_multi_remove_handle(_multi, easy);
1258
1259 const auto &setWorkerBroken = [&]( const std::string &str = {} ){
1260 worker->_state = WORKER_BROKEN;
1261 if ( !str.empty () )
1262 strncpy(worker->_curlError, str.c_str(), CURL_ERROR_SIZE);
1264
1265 if (!_activeworkers && !(urliter != urllist.end() && _workers.size() < MAXURLS)) {
1266 // end of workers reached! goodbye!
1267 worker->evaluateCurlCode(Pathname(), cc, false);
1268 }
1269 };
1270
1271 if ( !worker->_multiByteHandler ) {
1272 WAR << "#" << worker->_workerno << ": has no multibyte handler, this is a bug" << endl;
1273 setWorkerBroken("Multibyte handler error");
1274 continue;
1275 }
1276
1277 // tell the worker to finalize the current block
1278 worker->_multiByteHandler->finalize();
1279
1280 if ( worker->_multiByteHandler->hasMoreWork() && ( cc == CURLE_OK || worker->_multiByteHandler->canRecover() ) ) {
1281
1282 WAR << "#" << worker->_workerno << ": still has work to do or can recover from a error, continuing the job!" << endl;
1283 // the current job is not done, or we failed and need to try more, enqueue and start again
1284 if ( !worker->continueJob() ) {
1285 WAR << "#" << worker->_workerno << ": failed to continue (" << worker->_multiByteHandler->lastErrorMessage() << ")" << endl;
1286 setWorkerBroken( worker->_multiByteHandler->lastErrorMessage() );
1287 }
1288 continue;
1289 }
1290
1291 // --- from here on worker has no more ranges in its current job, or had a error it can't recover from ---
1292
1293 if ( cc != CURLE_OK ) {
1294 if ( worker->_state != WORKER_DISCARD ) {
1295 // something went wrong and we can not recover, broken worker!
1296 setWorkerBroken();
1297 continue;
1298 } else {
1299 WAR << "#" << worker->_workerno << ": failed, but was set to discard, reusing for new requests" << endl;
1300 }
1301 } else {
1302
1303 // we got what we asked for, maybe. Lets see if all ranges have been marked as finalized
1304 if( !worker->_multiByteHandler->verifyData() ) {
1305 WAR << "#" << worker->_workerno << ": error: " << worker->_multiByteHandler->lastErrorMessage() << ", disable worker" << endl;
1306 setWorkerBroken();
1307 continue;
1308 }
1309
1310 // from here on we know THIS worker only got data that verified
1311 // now lets see if the stripe was finished too
1312 // stripe blocks can now be only in FINALIZED or ERROR states
1313 if (worker->_state == WORKER_FETCH ) {
1314 if ( worker->_competing ) {
1315 worker->disableCompetition ();
1316 }
1317 auto &wrkerStripe = _requiredStripes[worker->_stripe];
1318 bool done = std::all_of( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), []( const Stripe::RState s ) { return s == Stripe::FINALIZED; } );
1319 if ( !done ) {
1320 // all ranges that are not finalized are in a bogus state, refetch them
1321 std::for_each( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), []( Stripe::RState &s ) {
1322 if ( s != Stripe::FINALIZED)
1323 s = Stripe::PENDING;
1324 });
1325
1326 _finished = false; //reset finished flag
1327 worker->runjob();
1328 continue;
1329 }
1330 }
1331
1332 // make bad workers ( bad as in really slow ) sleep a little
1333 double maxavg = 0;
1334 int maxworkerno = 0;
1335 int numbetter = 0;
1336 for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1337 {
1338 multifetchworker *oworker = workeriter->get();
1339 if (oworker->_state == WORKER_BROKEN)
1340 continue;
1341 if (oworker->_avgspeed > maxavg)
1342 {
1343 maxavg = oworker->_avgspeed;
1344 maxworkerno = oworker->_workerno;
1345 }
1346 if (oworker->_avgspeed > worker->_avgspeed)
1347 numbetter++;
1348 }
1349 if (maxavg && !_stealing)
1350 {
1351 double ratio = worker->_avgspeed / maxavg;
1352 ratio = 1 - ratio;
1353 if (numbetter < 3) // don't sleep that much if we're in the top two
1354 ratio = ratio * ratio;
1355 if (ratio > .01)
1356 {
1357 XXX << "#" << worker->_workerno << ": too slow ("<< ratio << ", " << worker->_avgspeed << ", #" << maxworkerno << ": " << maxavg << "), going to sleep for " << ratio * 1000 << " ms" << endl;
1358 worker->_sleepuntil = now + ratio;
1359 worker->_state = WORKER_SLEEP;
1360 _sleepworkers++;
1361 continue;
1362 }
1363 }
1364
1365 // do rate control (if requested)
1366 // should use periodavg, but that's not what libcurl does
1367 if (_maxspeed && now > _starttime)
1368 {
1369 double avg = _fetchedsize / (now - _starttime);
1370 avg = worker->_maxspeed * _maxspeed / avg;
1371 if (avg < _maxspeed / _maxworkers)
1372 avg = _maxspeed / _maxworkers;
1373 if (avg > _maxspeed)
1374 avg = _maxspeed;
1375 if (avg < 1024)
1376 avg = 1024;
1377 worker->_maxspeed = avg;
1378#if CURLVERSION_AT_LEAST(7,15,5)
1379 curl_easy_setopt(worker->_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)(avg));
1380#endif
1381 }
1382
1383 worker->nextjob();
1384 }
1385
1386 if ( _filesize > 0 && _fetchedgoodsize > _filesize ) {
1388 }
1389 }
1390
1391 // send report
1392 if (_report)
1393 {
1394 int percent = _totalsize ? (100 * (_fetchedgoodsize + _fetchedsize)) / (_totalsize + _fetchedsize) : 0;
1395
1396 double avg = 0;
1397 if (now > _starttime)
1398 avg = _fetchedsize / (now - _starttime);
1399 if (!(*(_report))->progress(percent, _baseurl, avg, _lastperiodstart == _starttime ? avg : _periodavg))
1400 ZYPP_THROW(MediaCurlException(_baseurl, "User abort", "cancelled"));
1401 }
1402
1403 if (_timeout && now - _lastprogress > _timeout)
1404 break;
1405 }
1406
1407 if (!_finished)
1409
1410 // print some download stats
1411 WAR << "overall result" << endl;
1412 for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1413 {
1414 multifetchworker *worker = workeriter->get();
1415 WAR << "#" << worker->_workerno << ": state: " << worker->_state << " received: " << worker->_received << " url: " << worker->_url << endl;
1416 }
1417}
1418
1419inline zypp::ByteCount multifetchrequest::makeBlksize ( uint maxConns, size_t filesize )
1420{
1421 return std::max<zypp::ByteCount>( filesize / std::min( std::max<int>( 1, maxConns ) , MAXURLS ), zypp::ByteCount(4, zypp::ByteCount::K) );
1422}
1423
1425
1426
1427MediaMultiCurl::MediaMultiCurl(const Url &url_r, const Pathname & attach_point_hint_r)
1428 : MediaCurl(url_r, attach_point_hint_r)
1429{
1430 MIL << "MediaMultiCurl::MediaMultiCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
1431 _multi = 0;
1433}
1434
1436{
1438 {
1439 curl_slist_free_all(_customHeadersMetalink);
1441 }
1442 if (_multi)
1443 {
1444 curl_multi_cleanup(_multi);
1445 _multi = 0;
1446 }
1447 std::map<std::string, CURL *>::iterator it;
1448 for (it = _easypool.begin(); it != _easypool.end(); it++)
1449 {
1450 CURL *easy = it->second;
1451 if (easy)
1452 {
1453 curl_easy_cleanup(easy);
1454 it->second = NULL;
1455 }
1456 }
1457}
1458
1460{
1462
1464 {
1465 curl_slist_free_all(_customHeadersMetalink);
1467 }
1468 struct curl_slist *sl = _customHeaders;
1469 for (; sl; sl = sl->next)
1470 _customHeadersMetalink = curl_slist_append(_customHeadersMetalink, sl->data);
1471 //, application/x-zsync
1472 _customHeadersMetalink = curl_slist_append(_customHeadersMetalink, "Accept: */*, application/x-zsync, application/metalink+xml, application/metalink4+xml");
1473}
1474
1475// here we try to suppress all progress coming from a metalink download
1476// bsc#1021291: Nevertheless send alive trigger (without stats), so UIs
1477// are able to abort a hanging metalink download via callback response.
1478int MediaMultiCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
1479{
1481 if (!_curl)
1482 return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1483
1484 // bsc#408814: Don't report any sizes before we don't have data on disk. Data reported
1485 // due to redirection etc. are not interesting, but may disturb filesize checks.
1486 FILE *fp = 0;
1487 if ( curl_easy_getinfo( _curl, CURLINFO_PRIVATE, &fp ) != CURLE_OK || !fp )
1488 return MediaCurl::aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1489 if ( ftell( fp ) == 0 )
1490 return MediaCurl::aliveCallback( clientp, dltotal, 0.0, ultotal, ulnow );
1491
1492 // (no longer needed due to the filesize check above?)
1493 // work around curl bug that gives us old data
1494 long httpReturnCode = 0;
1495 if (curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0)
1496 return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1497
1498 char *ptr = NULL;
1499 bool ismetalink = false;
1500 if (curl_easy_getinfo(_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1501 {
1502 std::string ct = std::string(ptr);
1503 if (ct.find("application/x-zsync") == 0 || ct.find("application/metalink+xml") == 0 || ct.find("application/metalink4+xml") == 0)
1504 ismetalink = true;
1505 }
1506 if (!ismetalink && dlnow < 256)
1507 {
1508 // can't tell yet, ...
1509 return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1510 }
1511 if (!ismetalink)
1512 {
1513 fflush(fp);
1514 ismetalink = looks_like_meta_file(fp) != MetaDataType::None;
1515 DBG << "looks_like_meta_file: " << ismetalink << endl;
1516 }
1517 if (ismetalink)
1518 {
1519 // this is a metalink file change the expected filesize
1521 // we're downloading the metalink file. Just trigger aliveCallbacks
1522 curl_easy_setopt(_curl, CURLOPT_XFERINFOFUNCTION, &MediaCurl::aliveCallback);
1523 return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1524 }
1525 curl_easy_setopt(_curl, CURLOPT_XFERINFOFUNCTION, &MediaCurl::progressCallback);
1526 return MediaCurl::progressCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1527}
1528
1529void MediaMultiCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1530{
1531 Pathname dest = target.absolutename();
1532 if( assert_dir( dest.dirname() ) )
1533 {
1534 DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1535 ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1536 }
1537
1538 ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1539 AutoFILE file;
1540 {
1541 AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1542 if( ! buf )
1543 {
1544 ERR << "out of memory for temp file name" << endl;
1545 ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1546 }
1547
1548 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1549 if( tmp_fd == -1 )
1550 {
1551 ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1553 }
1554 destNew = ManagedFile( (*buf), filesystem::unlink );
1555
1556 file = ::fdopen( tmp_fd, "we" );
1557 if ( ! file )
1558 {
1559 ERR << "fopen failed for file '" << destNew << "'" << endl;
1561 }
1562 tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1563 }
1564
1565 DBG << "dest: " << dest << endl;
1566 DBG << "temp: " << destNew << endl;
1567
1568 // set IFMODSINCE time condition (no download if not modified)
1569 if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1570 {
1571 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1572 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1573 }
1574 else
1575 {
1576 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1577 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1578 }
1579 // change header to include Accept: metalink
1580 curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeadersMetalink);
1581 // change to our own progress funcion
1582 curl_easy_setopt(_curl, CURLOPT_XFERINFOFUNCTION, &progressCallback);
1583 curl_easy_setopt(_curl, CURLOPT_PRIVATE, (*file) ); // important to pass the FILE* explicitly (passing through varargs)
1584 try
1585 {
1586 MediaCurl::doGetFileCopyFile( srcFile, dest, file, report, options );
1587 }
1588 catch (Exception &ex)
1589 {
1590 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1591 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1592 curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeaders);
1593 curl_easy_setopt(_curl, CURLOPT_PRIVATE, (void *)0);
1594 ZYPP_RETHROW(ex);
1595 }
1596 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1597 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1598 curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeaders);
1599 curl_easy_setopt(_curl, CURLOPT_PRIVATE, (void *)0);
1600 long httpReturnCode = 0;
1601 CURLcode infoRet = curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode);
1602 if (infoRet == CURLE_OK)
1603 {
1604 DBG << "HTTP response: " + str::numstring(httpReturnCode) << endl;
1605 if ( httpReturnCode == 304
1606 || ( httpReturnCode == 213 && _url.getScheme() == "ftp" ) ) // not modified
1607 {
1608 DBG << "not modified: " << PathInfo(dest) << endl;
1609 return;
1610 }
1611 }
1612 else
1613 {
1614 WAR << "Could not get the response code." << endl;
1615 }
1616
1617 MetaDataType ismetalink = MetaDataType::None;
1618
1619 char *ptr = NULL;
1620 if (curl_easy_getinfo(_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1621 {
1622 std::string ct = std::string(ptr);
1623 if (ct.find("application/x-zsync") == 0 )
1624 ismetalink = MetaDataType::Zsync;
1625 else if (ct.find("application/metalink+xml") == 0 || ct.find("application/metalink4+xml") == 0)
1626 ismetalink = MetaDataType::MetaLink;
1627 }
1628
1629 if ( ismetalink == MetaDataType::None )
1630 {
1631 // some proxies do not store the content type, so also look at the file to find
1632 // out if we received a metalink (bnc#649925)
1633 fflush(file);
1634 ismetalink = looks_like_meta_file(destNew);
1635 }
1636
1637 if ( ismetalink != MetaDataType::None )
1638 {
1639 bool userabort = false;
1640 Pathname failedFile = ZConfig::instance().repoCachePath() / "MultiCurl.failed";
1641 file = nullptr; // explicitly close destNew before the parser reads it.
1642 try
1643 {
1644 MediaBlockList bl;
1645 std::vector<Url> urls;
1646 if ( ismetalink == MetaDataType::Zsync ) {
1647 ZsyncParser parser;
1648 parser.parse( destNew );
1649 bl = parser.getBlockList();
1650 urls = parser.getUrls();
1651
1652 XXX << getFileUrl(srcFile.filename()) << " returned zsync meta data." << std::endl;
1653 } else {
1654 MetaLinkParser mlp;
1655 mlp.parse(destNew);
1656 bl = mlp.getBlockList();
1657 urls = mlp.getUrls();
1658
1659 XXX << getFileUrl(srcFile.filename()) << " returned metalink meta data." << std::endl;
1660 }
1661
1662 if ( bl.numBlocks() )
1663 XXX << "With " << bl.numBlocks() << " nr of blocks and a blocksize of " << bl.getBlock(0).size << std::endl;
1664 else
1665 XXX << "With no blocks" << std::endl;
1666
1667 /*
1668 * gihub issue libzipp:#277 Multicurl backend breaks with MirrorCache and Metalink with unknown filesize.
1669 * Fall back to a normal download if we have no knowledge about the filesize we want to download.
1670 */
1671 if ( !bl.haveFilesize() && ! srcFile.downloadSize() ) {
1672 XXX << "No filesize in metalink file and no expected filesize, aborting multicurl." << std::endl;
1673 ZYPP_THROW( MediaException("Multicurl requires filesize but none was provided.") );
1674 }
1675
1676#if 0
1677 Disabling this workaround for now, since we now do zip ranges into bigger requests
1678 /*
1679 * bsc#1191609 In certain locations we do not receive a suitable number of metalink mirrors, and might even
1680 * download chunks serially from one and the same server. In those cases we need to fall back to a normal download.
1681 */
1682 if ( urls.size() < MIN_REQ_MIRRS ) {
1683 ZYPP_THROW( MediaException("Multicurl enabled but not enough mirrors provided") );
1684 }
1685#endif
1686
1687 // XXX << bl << endl;
1688 file = fopen((*destNew).c_str(), "w+e");
1689 if (!file)
1691 if (PathInfo(target).isExist())
1692 {
1693 XXX << "reusing blocks from file " << target << endl;
1694 bl.reuseBlocks(file, target.asString());
1695 XXX << bl << endl;
1696 }
1697 if (bl.haveChecksum(1) && PathInfo(failedFile).isExist())
1698 {
1699 XXX << "reusing blocks from file " << failedFile << endl;
1700 bl.reuseBlocks(file, failedFile.asString());
1701 XXX << bl << endl;
1702 filesystem::unlink(failedFile);
1703 }
1704 Pathname df = srcFile.deltafile();
1705 if (!df.empty())
1706 {
1707 XXX << "reusing blocks from file " << df << endl;
1708 bl.reuseBlocks(file, df.asString());
1709 XXX << bl << endl;
1710 }
1711 try
1712 {
1713 multifetch(srcFile.filename(), file, &urls, &report, std::move(bl), srcFile.downloadSize());
1714 }
1715 catch (MediaCurlException &ex)
1716 {
1717 userabort = ex.errstr() == "User abort";
1718 ZYPP_RETHROW(ex);
1719 }
1720 }
1721 catch (MediaFileSizeExceededException &ex) {
1722 ZYPP_RETHROW(ex);
1723 }
1724 catch (Exception &ex)
1725 {
1726 // something went wrong. fall back to normal download
1727 file = nullptr; // explicitly close destNew before moving it
1728 WAR<< "Failed to multifetch file " << ex << " falling back to single Curl download!" << std::endl;
1729 if (PathInfo(destNew).size() >= 63336)
1730 {
1731 ::unlink(failedFile.asString().c_str());
1732 filesystem::hardlinkCopy(destNew, failedFile);
1733 }
1734 if (userabort)
1735 {
1736 ZYPP_RETHROW(ex);
1737 }
1738 file = fopen((*destNew).c_str(), "w+e");
1739 if (!file)
1741
1742 // use the default progressCallback
1743 curl_easy_setopt(_curl, CURLOPT_XFERINFOFUNCTION, &MediaCurl::progressCallback);
1744 MediaCurl::doGetFileCopyFile(srcFile, dest, file, report, options | OPTION_NO_REPORT_START);
1745 }
1746 }
1747
1748 if (::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 )))
1749 {
1750 ERR << "Failed to chmod file " << destNew << endl;
1751 }
1752
1753 file.resetDispose(); // we're going to close it manually here
1754 if (::fclose(file))
1755 {
1756 filesystem::unlink(destNew);
1757 ERR << "Fclose failed for file '" << destNew << "'" << endl;
1759 }
1760
1761 if ( rename( destNew, dest ) != 0 )
1762 {
1763 ERR << "Rename failed" << endl;
1765 }
1766 destNew.resetDispose(); // no more need to unlink it
1767
1768 DBG << "done: " << PathInfo(dest) << endl;
1769}
1770
1771void MediaMultiCurl::multifetch(const Pathname & filename, FILE *fp, std::vector<Url> *urllist, MediaBlockList &&blklist, callback::SendReport<DownloadProgressReport> *report, off_t filesize) const
1772{
1773 Url baseurl(getFileUrl(filename));
1774 if (filesize == off_t(-1) && blklist.haveFilesize())
1775 filesize = blklist.getFilesize();
1776 if (!blklist.haveBlocks() && filesize != 0) {
1777 if ( filesize == -1 ) {
1778 ZYPP_THROW(MediaException("No filesize and no blocklist, falling back to normal download."));
1779 }
1780
1781 // build a blocklist on demand just so that we have ranges
1782 MIL << "Generate blocklist, since there was none in the metalink file." << std::endl;
1783
1784 off_t currOff = 0;
1785 const auto prefSize = multifetchrequest::makeBlksize( _settings.maxConcurrentConnections(), filesize );
1786
1787 while ( currOff < filesize ) {
1788
1789 auto blksize = filesize - currOff ;
1790 if ( blksize > prefSize )
1791 blksize = prefSize;
1792
1793 blklist.addBlock( currOff, blksize );
1794 currOff += blksize;
1795 }
1796
1797 XXX << "Generated blocklist: " << std::endl << blklist << std::endl << " End blocklist " << std::endl;
1798
1799 }
1800 if (filesize == 0 || !blklist.numBlocks()) {
1801 checkFileDigest(baseurl, fp, blklist);
1802 return;
1803 }
1804 if (filesize == 0)
1805 return;
1806
1807 if (!_multi)
1808 {
1809 _multi = curl_multi_init();
1810 if (!_multi)
1812 }
1813
1814 multifetchrequest req(this, filename, baseurl, _multi, fp, report, std::move(blklist), filesize);
1815 std::vector<Url> myurllist;
1816 for (std::vector<Url>::iterator urliter = urllist->begin(); urliter != urllist->end(); ++urliter)
1817 {
1818 try
1819 {
1820 std::string scheme = urliter->getScheme();
1821 if (scheme == "http" || scheme == "https" || scheme == "ftp" || scheme == "tftp")
1822 {
1823 checkProtocol(*urliter);
1824 myurllist.push_back(internal::propagateQueryParams(*urliter, _url));
1825 }
1826 }
1827 catch (...)
1828 {
1829 }
1830 }
1831 if (!myurllist.size())
1832 myurllist.push_back(baseurl);
1833 req.run(myurllist);
1834 checkFileDigest(baseurl, fp, req.blockList() );
1835}
1836
1837void MediaMultiCurl::checkFileDigest(Url &url, FILE *fp, MediaBlockList &blklist) const
1838{
1839 if ( !blklist.haveFileChecksum() )
1840 return;
1841 if (fseeko(fp, off_t(0), SEEK_SET))
1842 ZYPP_THROW(MediaCurlException(url, "fseeko", "seek error"));
1843 Digest dig;
1844 blklist.createFileDigest(dig);
1845 char buf[4096];
1846 size_t l;
1847 while ((l = fread(buf, 1, sizeof(buf), fp)) > 0)
1848 dig.update(buf, l);
1849 if (!blklist.verifyFileDigest(dig))
1850 ZYPP_THROW(MediaCurlException(url, "file verification failed", "checksum error"));
1851}
1852
1853bool MediaMultiCurl::isDNSok(const std::string &host) const
1854{
1855 return _dnsok.find(host) == _dnsok.end() ? false : true;
1856}
1857
1858void MediaMultiCurl::setDNSok(const std::string &host) const
1859{
1860 _dnsok.insert(host);
1861}
1862
1863CURL *MediaMultiCurl::fromEasyPool(const std::string &host) const
1864{
1865 if (_easypool.find(host) == _easypool.end())
1866 return 0;
1867 CURL *ret = _easypool[host];
1868 _easypool.erase(host);
1869 return ret;
1870}
1871
1872void MediaMultiCurl::toEasyPool(const std::string &host, CURL *easy) const
1873{
1874 CURL *oldeasy = _easypool[host];
1875 _easypool[host] = easy;
1876 if (oldeasy)
1877 curl_easy_cleanup(oldeasy);
1878}
1879
1880 } // namespace media
1881} // namespace zypp
std::optional< KeyManagerCtx > _context
Definition: KeyRing.cc:157
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
Store and operate with byte count.
Definition: ByteCount.h:31
static const Unit MB
1000^2 Byte
Definition: ByteCount.h:60
static const Unit K
1024 Byte
Definition: ByteCount.h:45
std::string asString(unsigned field_width_r=0, unsigned unit_width_r=1) const
Auto selected Unit and precision.
Definition: ByteCount.h:133
Compute Message Digests (MD5, SHA1 etc)
Definition: Digest.h:38
bool update(const char *bytes, size_t len)
feed data into digest computation algorithm
Definition: Digest.cc:309
Digest clone() const
Returns a clone of the current Digest and returns it.
Definition: Digest.cc:228
Base class for Exception.
Definition: Exception.h:146
Describes a resource file located on a medium.
const ByteCount & downloadSize() const
The size of the resource on the server.
const Pathname & filename() const
The path to the resource on the medium.
const Pathname & deltafile() const
The existing deltafile that can be used to reduce download size ( zchunk or metalink )
Url manipulation class.
Definition: Url.h:92
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:588
Pathname repoCachePath() const
Path where the caches are kept (/var/cache/zypp)
Definition: ZConfig.cc:1039
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
Wrapper class for stat/lstat.
Definition: PathInfo.h:221
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
const std::string & asString() const
String representation.
Definition: Pathname.h:91
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
Definition: Pathname.h:139
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
bool haveChecksum(size_t blkno) const
void reuseBlocks(FILE *wfp, std::string filename)
const MediaBlock & getBlock(size_t blkno) const
return the offset/size of a block with number blkno
UByteArray getChecksum(size_t blkno) const
std::string getChecksumType() const
bool createFileDigest(Digest &digest) const
size_t numBlocks() const
return the number of blocks in the blocklist
bool verifyFileDigest(Digest &digest) const
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:32
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:406
@ OPTION_NO_IFMODSINCE
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
@ OPTION_NO_REPORT_START
do not send a start ProgressReport
Definition: MediaCurl.h:45
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1404
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1361
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:369
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:167
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1198
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:381
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:833
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1377
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1347
virtual void disconnectFrom() override
Definition: MediaCurl.cc:705
curl_slist * _customHeaders
Definition: MediaCurl.h:168
Just inherits Exception to separate media exceptions.
Url url() const
Url used.
Definition: MediaHandler.h:503
const Url _url
Url to handle.
Definition: MediaHandler.h:113
virtual void setupEasy() override
initializes the curl easy handle with the data from the url
std::map< std::string, CURL * > _easypool
curl_slist * _customHeadersMetalink
void checkFileDigest(Url &url, FILE *fp, MediaBlockList &blklist) const
void setDNSok(const std::string &host) const
MediaMultiCurl(const Url &url_r, const Pathname &attach_point_hint_r)
std::set< std::string > _dnsok
bool isDNSok(const std::string &host) const
void multifetch(const Pathname &filename, FILE *fp, std::vector< Url > *urllist, MediaBlockList &&blklist, callback::SendReport< DownloadProgressReport > *report=0, off_t filesize=off_t(-1)) const
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
CURL * fromEasyPool(const std::string &host) const
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const override
void toEasyPool(const std::string &host, CURL *easy) const
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
void parse(const Pathname &filename)
parse a file consisting of metalink xml data
MediaBlockList getBlockList() const
return the block list from the parsed metalink data
std::vector< Url > getUrls() const
return the download urls from the parsed metalink data
const std::string & password() const
auth password
const std::string & authType() const
get the allowed authentication types
void setUsername(const std::string &val_r)
sets the auth username
std::string userPassword() const
returns the user and password as a user:pass string
const std::string & proxy() const
proxy host
long maxConcurrentConnections() const
Maximum number of concurrent connections for a single transfer.
void setPassword(const std::string &val_r)
sets the auth password
const std::string & username() const
auth username
void setAuthType(const std::string &val_r)
set the allowed authentication types
void parse(const Pathname &filename)
parse a file consisting of zlink data
Definition: zsyncparser.cc:77
MediaBlockList getBlockList()
return the block list from the parsed metalink data
Definition: zsyncparser.cc:197
std::vector< Url > getUrls()
return the download urls from the parsed metalink data
Definition: zsyncparser.cc:187
std::vector< Stripe > _requiredStripes
callback::SendReport< DownloadProgressReport > * _report
void run(std::vector< Url > &urllist)
multifetchrequest(const MediaMultiCurl *context, const Pathname &filename, const Url &baseurl, CURLM *multi, FILE *fp, callback::SendReport< DownloadProgressReport > *report, MediaBlockList &&blklist, off_t filesize)
std::list< std::unique_ptr< multifetchworker > > _workers
static ByteCount makeBlksize(uint maxConns, size_t filesize)
const MediaMultiCurl * _context
bool beginRange(off_t range, std::string &cancelReason) override
multifetchworker(int no, multifetchrequest &request, const Url &url)
multifetchrequest * _request
size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes) override
size_t headerfunction(char *ptr, size_t bytes) override
MultiByteHandler::Range rangeFromBlock(off_t blockNo) const
std::vector< MultiByteHandler::Range > _blocks
void adddnsfd(std::vector< GPollFD > &waitFds)
MultiByteHandler::ProtocolMode _protocolMode
std::unique_ptr< MultiByteHandler > _multiByteHandler
bool recheckChecksum(off_t blockIdx)
void dnsevent(const std::vector< GPollFD > &waitFds)
std::vector< off_t > _rangeToStripeBlock
bool finishedRange(off_t range, bool validated, std::string &cancelReason) override
MultiFetchWorkerState _state
The CurlMultiPartHandler class.
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
Definition: curlhelper.cc:394
Definition: Arch.h:364
String related utilities and Regular expression matching.
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:789
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:700
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:883
constexpr auto MAXURLS
static bool env_isset(std::string name)
static double currentTime()
constexpr auto MIN_REQ_MIRRS
MetaDataType looks_like_meta_file(const Pathname &file)
std::string numstring(char n, int w=0)
Definition: String.h:289
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
AutoDispose<int> calling ::close
Definition: AutoDispose.h:302
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:313
a single block from the blocklist, consisting of an offset and a size
std::vector< off_t > blocks
std::vector< RState > blockStates
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
#define DBG
Definition: Logger.h:95
#define MIL
Definition: Logger.h:96
#define ERR
Definition: Logger.h:98
#define WAR
Definition: Logger.h:97
#define XXX
Definition: Logger.h:94