libzypp  17.31.31
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp/base/Logger.h>
18 #include <zypp/ExternalProgram.h>
19 #include <zypp/base/String.h>
20 #include <zypp/base/Gettext.h>
21 #include <zypp-core/parser/Sysconfig>
22 #include <zypp/base/Gettext.h>
23 
24 #include <zypp/media/MediaCurl.h>
25 #include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
27 #include <zypp-curl/ProxyInfo>
28 #include <zypp-curl/auth/CurlAuthData>
29 #include <zypp-media/auth/CredentialManager>
30 #include <zypp-curl/CurlConfig>
32 #include <zypp/Target.h>
33 #include <zypp/ZYppFactory.h>
34 #include <zypp/ZConfig.h>
35 #include <zypp/zypp_detail/ZYppImpl.h> // for zypp_poll
36 
37 #include <cstdlib>
38 #include <sys/types.h>
39 #include <sys/stat.h>
40 #include <sys/mount.h>
41 #include <dirent.h>
42 #include <unistd.h>
43 #include <glib.h>
44 
45 using std::endl;
46 
47 namespace internal {
48  using namespace zypp;
52  struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport>
53  {
54  using TimePoint = std::chrono::steady_clock::time_point;
55 
56  OptionalDownloadProgressReport( bool isOptional=false )
57  : _oldRec { Distributor::instance().getReceiver() }
58  , _isOptional { isOptional }
59  { connect(); }
60 
62  { if ( _oldRec ) Distributor::instance().setReceiver( *_oldRec ); else Distributor::instance().noReceiver(); }
63 
64  void reportbegin() override
65  { if ( _oldRec ) _oldRec->reportbegin(); }
66 
67  void reportend() override
68  { if ( _oldRec ) _oldRec->reportend(); }
69 
70  void report( const UserData & userData_r = UserData() ) override
71  { if ( _oldRec ) _oldRec->report( userData_r ); }
72 
73 
74  void start( const Url & file_r, Pathname localfile_r ) override
75  {
76  if ( not _oldRec ) return;
77  if ( _isOptional ) {
78  // delay start until first data are received.
79  _startFile = file_r;
80  _startLocalfile = std::move(localfile_r);
81  return;
82  }
83  _oldRec->start( file_r, localfile_r );
84  }
85 
86  bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override
87  {
88  if ( not _oldRec ) return true;
89  if ( notStarted() ) {
90  if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
91  return true;
92  sendStart();
93  }
94 
95  //static constexpr std::chrono::milliseconds minfequency { 1000 }; only needed if we'd avoid sending reports without change
96  static constexpr std::chrono::milliseconds maxfequency { 100 };
97  TimePoint now { TimePoint::clock::now() };
98  TimePoint::duration elapsed { now - _lastProgressSent };
99  if ( elapsed < maxfequency )
100  return true; // continue
101  _lastProgressSent = now;
102  return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
103  }
104 
105  Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override
106  {
107  if ( not _oldRec || notStarted() ) return ABORT;
108  return _oldRec->problem( file_r, error_r, description_r );
109  }
110 
111  void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override
112  {
113  if ( not _oldRec || notStarted() ) return;
114  _oldRec->finish( file_r, error_r, reason_r );
115  }
116 
117  private:
118  // _isOptional also indicates the delayed start
119  bool notStarted() const
120  { return _isOptional; }
121 
122  void sendStart()
123  {
124  if ( _isOptional ) {
125  // we know _oldRec is valid...
126  _oldRec->start( std::move(_startFile), std::move(_startLocalfile) );
127  _isOptional = false;
128  }
129  }
130 
131  private:
132  Receiver *const _oldRec;
135  Pathname _startLocalfile;
137  };
138 
140  {
141  ProgressData( CURL *curl, time_t timeout = 0, const zypp::Url & url = zypp::Url(),
142  zypp::ByteCount expectedFileSize_r = 0,
144 
145  void updateStats( curl_off_t dltotal = 0.0, curl_off_t dlnow = 0.0 );
146 
147  int reportProgress() const;
148 
149  CURL * curl()
150  { return _curl; }
151 
152  bool timeoutReached() const
153  { return _timeoutReached; }
154 
155  bool fileSizeExceeded() const
156  { return _fileSizeExceeded; }
157 
159  { return _expectedFileSize; }
160 
161  void expectedFileSize( ByteCount newval_r )
162  { _expectedFileSize = newval_r; }
163 
164  private:
165  CURL * _curl;
167  time_t _timeout;
172 
173  time_t _timeStart = 0;
174  time_t _timeLast = 0;
175  time_t _timeRcv = 0;
176  time_t _timeNow = 0;
177 
178  curl_off_t _dnlTotal = 0.0;
179  curl_off_t _dnlLast = 0.0;
180  curl_off_t _dnlNow = 0.0;
181 
182  int _dnlPercent= 0;
183 
184  double _drateTotal= 0.0;
185  double _drateLast = 0.0;
186  };
187 
188 
189 
190  ProgressData::ProgressData(CURL *curl, time_t timeout, const Url &url, ByteCount expectedFileSize_r, zypp::callback::SendReport< zypp::media::DownloadProgressReport> *_report)
191  : _curl( curl )
192  , _url( url )
193  , _timeout( timeout )
194  , _timeoutReached( false )
195  , _fileSizeExceeded ( false )
196  , _expectedFileSize( expectedFileSize_r )
197  , report( _report )
198  {}
199 
200  void ProgressData::updateStats( curl_off_t dltotal, curl_off_t dlnow )
201  {
202  time_t now = _timeNow = time(0);
203 
204  // If called without args (0.0), recompute based on the last values seen
205  if ( dltotal && dltotal != _dnlTotal )
206  _dnlTotal = dltotal;
207 
208  if ( dlnow && dlnow != _dnlNow )
209  {
210  _timeRcv = now;
211  _dnlNow = dlnow;
212  }
213 
214  // init or reset if time jumps back
215  if ( !_timeStart || _timeStart > now )
216  _timeStart = _timeLast = _timeRcv = now;
217 
218  // timeout condition
219  if ( _timeout )
220  _timeoutReached = ( (now - _timeRcv) > _timeout );
221 
222  // check if the downloaded data is already bigger than what we expected
223  _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
224 
225  // percentage:
226  if ( _dnlTotal )
227  _dnlPercent = int( _dnlNow * 100 / _dnlTotal );
228 
229  // download rates:
230  _drateTotal = double(_dnlNow) / std::max( int(now - _timeStart), 1 );
231 
232  if ( _timeLast < now )
233  {
234  _drateLast = double(_dnlNow - _dnlLast) / int(now - _timeLast);
235  // start new period
236  _timeLast = now;
237  _dnlLast = _dnlNow;
238  }
239  else if ( _timeStart == _timeLast )
241  }
242 
244  {
245  if ( _fileSizeExceeded )
246  return 1;
247  if ( _timeoutReached )
248  return 1; // no-data timeout
249  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
250  return 1; // user requested abort
251  return 0;
252  }
253 
254  const char * anonymousIdHeader()
255  {
256  // we need to add the release and identifier to the
257  // agent string.
258  // The target could be not initialized, and then this information
259  // is guessed.
260  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
261  static const std::string _value( str::trim( str::form(
262  "X-ZYpp-AnonymousId: %s",
263  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str()
264  )));
265  return _value.c_str();
266  }
267 
269  {
270  // we need to add the release and identifier to the
271  // agent string.
272  // The target could be not initialized, and then this information
273  // is guessed.
274  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
275  static const std::string _value( str::trim( str::form(
276  "X-ZYpp-DistributionFlavor: %s",
277  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str()
278  )));
279  return _value.c_str();
280  }
281 
282  const char * agentString()
283  {
284  // we need to add the release and identifier to the
285  // agent string.
286  // The target could be not initialized, and then this information
287  // is guessed.
288  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
289  static const std::string _value( str::trim( str::form(
290  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
291  , curl_version_info(CURLVERSION_NOW)->version
292  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
293  )));
294  return _value.c_str();
295  }
296 
301  {
302  public:
304  const std::string & err_r,
305  const std::string & msg_r )
306  : media::MediaCurlException( url_r, err_r, msg_r )
307  {}
308  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
309  };
310 
311 }
312 
313 
314 using namespace internal;
315 using namespace zypp::base;
316 
317 namespace zypp {
318 
319  namespace media {
320 
321 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
322 
323 // we use this define to unbloat code as this C setting option
324 // and catching exception is done frequently.
326 #define SET_OPTION(opt,val) do { \
327  ret = curl_easy_setopt ( _curl, opt, val ); \
328  if ( ret != 0) { \
329  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
330  } \
331  } while ( false )
332 
333 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
334 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
335 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
336 
337 MediaCurl::MediaCurl( const Url & url_r,
338  const Pathname & attach_point_hint_r )
339  : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
340  "/", // urlpath at attachpoint
341  true ), // does_download
342  _curl( NULL ),
343  _customHeaders(0L)
344 {
345  _curlError[0] = '\0';
346 
347  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
348 
350 
351  if( !attachPoint().empty())
352  {
353  PathInfo ainfo(attachPoint());
354  Pathname apath(attachPoint() + "XXXXXX");
355  char *atemp = ::strdup( apath.asString().c_str());
356  char *atest = NULL;
357  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
358  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
359  {
360  WAR << "attach point " << ainfo.path()
361  << " is not useable for " << url_r.getScheme() << endl;
362  setAttachPoint("", true);
363  }
364  else if( atest != NULL)
365  ::rmdir(atest);
366 
367  if( atemp != NULL)
368  ::free(atemp);
369  }
370 }
371 
373 {
375 }
376 
377 void MediaCurl::setCookieFile( const Pathname &fileName )
378 {
379  _cookieFile = fileName;
380 }
381 
383 
384 void MediaCurl::checkProtocol(const Url &url) const
385 {
386  curl_version_info_data *curl_info = NULL;
387  curl_info = curl_version_info(CURLVERSION_NOW);
388  // curl_info does not need any free (is static)
389  if (curl_info->protocols)
390  {
391  const char * const *proto;
392  std::string scheme( url.getScheme());
393  bool found = false;
394  for(proto=curl_info->protocols; !found && *proto; ++proto)
395  {
396  if( scheme == std::string((const char *)*proto))
397  found = true;
398  }
399  if( !found)
400  {
401  std::string msg("Unsupported protocol '");
402  msg += scheme;
403  msg += "'";
405  }
406  }
407 }
408 
410 {
412 
413  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
414  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
415  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
416  if ( ret != 0 ) {
417  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
418  }
419 
420  SET_OPTION(CURLOPT_FAILONERROR, 1L);
421  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
422 
423  // create non persistant settings
424  // so that we don't add headers twice
425  TransferSettings vol_settings(_settings);
426 
427  // add custom headers for download.opensuse.org (bsc#955801)
428  if ( _url.getHost() == "download.opensuse.org" )
429  {
430  vol_settings.addHeader(anonymousIdHeader());
431  vol_settings.addHeader(distributionFlavorHeader());
432  }
433  vol_settings.addHeader("Pragma:");
434 
436 
437  // fill some settings from url query parameters
438  try
439  {
441  }
442  catch ( const MediaException &e )
443  {
444  disconnectFrom();
445  ZYPP_RETHROW(e);
446  }
447  // if the proxy was not set (or explicitly unset) by url, then look...
448  if ( _settings.proxy().empty() )
449  {
450  // ...at the system proxy settings
452  }
453 
455  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
456  {
457  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
458  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
459  }
460 
464  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
465  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
466  // just in case curl does not trigger its progress callback frequently
467  // enough.
468  if ( _settings.timeout() )
469  {
470  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
471  }
472 
473  // follow any Location: header that the server sends as part of
474  // an HTTP header (#113275)
475  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
476  // 3 redirects seem to be too few in some cases (bnc #465532)
477  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
478 
479  if ( _url.getScheme() == "https" )
480  {
481  if ( :: internal::setCurlRedirProtocols ( _curl ) != CURLE_OK ) {
483  }
484 
487  {
489  }
490 
492  {
493  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
494  }
495  if( ! _settings.clientKeyPath().empty() )
496  {
497  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
498  }
499 
500 #ifdef CURLSSLOPT_ALLOW_BEAST
501  // see bnc#779177
502  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
503  if ( ret != 0 ) {
504  disconnectFrom();
506  }
507 #endif
508  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
509  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
510  // bnc#903405 - POODLE: libzypp should only talk TLS
511  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
512  }
513 
514  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
515 
516  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
517  * We should proactively add the password to the request if basic auth is configured
518  * and a password is available in the credentials but not in the URL.
519  *
520  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
521  * and ask the server first about the auth method
522  */
523  if ( _settings.authType() == "basic"
524  && _settings.username().size()
525  && !_settings.password().size() ) {
526 
527  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
528  const auto cred = cm.getCred( _url );
529  if ( cred && cred->valid() ) {
530  if ( !_settings.username().size() )
531  _settings.setUsername(cred->username());
532  _settings.setPassword(cred->password());
533  }
534  }
535 
536  /*---------------------------------------------------------------*
537  CURLOPT_USERPWD: [user name]:[password]
538 
539  Url::username/password -> CURLOPT_USERPWD
540  If not provided, anonymous FTP identification
541  *---------------------------------------------------------------*/
542 
543  if ( _settings.userPassword().size() )
544  {
545  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
546  std::string use_auth = _settings.authType();
547  if (use_auth.empty())
548  use_auth = "digest,basic"; // our default
549  long auth = CurlAuthData::auth_type_str2long(use_auth);
550  if( auth != CURLAUTH_NONE)
551  {
552  DBG << "Enabling HTTP authentication methods: " << use_auth
553  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
554  SET_OPTION(CURLOPT_HTTPAUTH, auth);
555  }
556  }
557 
558  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
559  {
560  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
561  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
562  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
563  /*---------------------------------------------------------------*
564  * CURLOPT_PROXYUSERPWD: [user name]:[password]
565  *
566  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
567  * If not provided, $HOME/.curlrc is evaluated
568  *---------------------------------------------------------------*/
569 
570  std::string proxyuserpwd = _settings.proxyUserPassword();
571 
572  if ( proxyuserpwd.empty() )
573  {
574  CurlConfig curlconf;
575  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
576  if ( curlconf.proxyuserpwd.empty() )
577  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
578  else
579  {
580  proxyuserpwd = curlconf.proxyuserpwd;
581  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
582  }
583  }
584  else
585  {
586  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
587  }
588 
589  if ( ! proxyuserpwd.empty() )
590  {
591  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
592  }
593  }
594 #if CURLVERSION_AT_LEAST(7,19,4)
595  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
596  {
597  // Explicitly disabled in URL (see fillSettingsFromUrl()).
598  // This should also prevent libcurl from looking into the environment.
599  DBG << "Proxy: explicitly NOPROXY" << endl;
600  SET_OPTION(CURLOPT_NOPROXY, "*");
601  }
602 #endif
603  else
604  {
605  DBG << "Proxy: not explicitly set" << endl;
606  DBG << "Proxy: libcurl may look into the environment" << endl;
607  }
608 
610  if ( _settings.minDownloadSpeed() != 0 )
611  {
612  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
613  // default to 10 seconds at low speed
614  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
615  }
616 
617 #if CURLVERSION_AT_LEAST(7,15,5)
618  if ( _settings.maxDownloadSpeed() != 0 )
619  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
620 #endif
621 
622  /*---------------------------------------------------------------*
623  *---------------------------------------------------------------*/
624 
626  if ( ::geteuid() == 0 || PathInfo(_currentCookieFile).owner() == ::geteuid() )
628  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
629  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
630  else
631  MIL << "No cookies requested" << endl;
632  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
633  SET_OPTION(CURLOPT_XFERINFOFUNCTION, &progressCallback );
634  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
635 
636 #if CURLVERSION_AT_LEAST(7,18,0)
637  // bnc #306272
638  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
639 #endif
640  // Append settings custom headers to curl.
641  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
642  for ( const auto &header : vol_settings.headers() ) {
643  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
644  if ( !_customHeaders )
646  }
647  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
648 }
649 
651 
652 
653 void MediaCurl::attachTo (bool next)
654 {
655  if ( next )
657 
658  if ( !_url.isValid() )
660 
663  {
665  }
666 
667  disconnectFrom(); // clean _curl if needed
668  _curl = curl_easy_init();
669  if ( !_curl ) {
671  }
672  try
673  {
674  setupEasy();
675  }
676  catch (Exception & ex)
677  {
678  disconnectFrom();
679  ZYPP_RETHROW(ex);
680  }
681 
682  // FIXME: need a derived class to propelly compare url's
684  setMediaSource(media);
685 }
686 
687 bool
689 {
690  return MediaHandler::checkAttachPoint( apoint, true, true);
691 }
692 
694 
696 {
697  if ( _customHeaders )
698  {
699  curl_slist_free_all(_customHeaders);
700  _customHeaders = 0L;
701  }
702 
703  if ( _curl )
704  {
705  // bsc#1201092: Strange but within global_dtors we may exceptions here.
706  try { curl_easy_cleanup( _curl ); }
707  catch (...) { ; }
708  _curl = NULL;
709  }
710 }
711 
713 
714 void MediaCurl::releaseFrom( const std::string & ejectDev )
715 {
716  disconnect();
717 }
718 
720 
721 void MediaCurl::getFile( const OnMediaLocation &file ) const
722 {
723  // Use absolute file name to prevent access of files outside of the
724  // hierarchy below the attach point.
725  getFileCopy( file, localPath(file.filename()).absolutename() );
726 }
727 
729 
730 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
731 {
732 
733  const auto &filename = srcFile.filename();
734 
735  // Optional files will send no report until data are actually received (we know it exists).
736  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
738 
739  Url fileurl(getFileUrl(filename));
740 
741  bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
742  unsigned internalTry = 0;
743  static constexpr unsigned maxInternalTry = 3;
744 
745  do
746  {
747  try
748  {
749  doGetFileCopy( srcFile, target, report );
750  break; // success!
751  }
752  // retry with proper authentication data
753  catch (MediaUnauthorizedException & ex_r)
754  {
755  if ( authenticate(ex_r.hint(), firstAuth) ) {
756  firstAuth = false; // must not return stored credentials again
757  continue; // retry
758  }
759 
760  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
761  ZYPP_RETHROW(ex_r);
762  }
763  // unexpected exception
764  catch (MediaException & excpt_r)
765  {
766  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
767  ++internalTry;
768  if ( internalTry < maxInternalTry ) {
769  // just report (NO_ERROR); no interactive request to the user
770  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
771  continue; // retry
772  }
773  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
774  }
775 
777  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
778  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
779  {
781  }
782  report->finish(fileurl, reason, excpt_r.asUserHistory());
783  ZYPP_RETHROW(excpt_r);
784  }
785  }
786  while ( true );
787  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
788 }
789 
791 
792 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
793 {
794  bool retry = false;
795 
796  do
797  {
798  try
799  {
800  return doGetDoesFileExist( filename );
801  }
802  // authentication problem, retry with proper authentication data
803  catch (MediaUnauthorizedException & ex_r)
804  {
805  if(authenticate(ex_r.hint(), !retry))
806  retry = true;
807  else
808  ZYPP_RETHROW(ex_r);
809  }
810  // unexpected exception
811  catch (MediaException & excpt_r)
812  {
813  ZYPP_RETHROW(excpt_r);
814  }
815  }
816  while (retry);
817 
818  return false;
819 }
820 
822 
824  CURLcode code,
825  bool timeout_reached) const
826 {
827  if ( code != 0 )
828  {
829  Url url;
830  if (filename.empty())
831  url = _url;
832  else
833  url = getFileUrl(filename);
834 
835  std::string err;
836  {
837  switch ( code )
838  {
839  case CURLE_UNSUPPORTED_PROTOCOL:
840  err = " Unsupported protocol";
841  if ( !_lastRedirect.empty() )
842  {
843  err += " or redirect (";
844  err += _lastRedirect;
845  err += ")";
846  }
847  break;
848  case CURLE_URL_MALFORMAT:
849  case CURLE_URL_MALFORMAT_USER:
850  err = " Bad URL";
851  break;
852  case CURLE_LOGIN_DENIED:
853  ZYPP_THROW(
854  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
855  break;
856  case CURLE_HTTP_RETURNED_ERROR:
857  {
858  long httpReturnCode = 0;
859  CURLcode infoRet = curl_easy_getinfo( _curl,
860  CURLINFO_RESPONSE_CODE,
861  &httpReturnCode );
862  if ( infoRet == CURLE_OK )
863  {
864  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
865  switch ( httpReturnCode )
866  {
867  case 401:
868  {
869  std::string auth_hint = getAuthHint();
870 
871  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
872  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
873 
875  url, "Login failed.", _curlError, auth_hint
876  ));
877  }
878 
879  case 502: // bad gateway (bnc #1070851)
880  case 503: // service temporarily unavailable (bnc #462545)
882  case 504: // gateway timeout
884  case 403:
885  {
886  std::string msg403;
887  if ( url.getHost().find(".suse.com") != std::string::npos )
888  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
889  else if (url.asString().find("novell.com") != std::string::npos)
890  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
892  }
893  case 404:
894  case 410:
896  }
897 
898  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
900  }
901  else
902  {
903  std::string msg = "Unable to retrieve HTTP response:";
904  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
906  }
907  }
908  break;
909  case CURLE_FTP_COULDNT_RETR_FILE:
910 #if CURLVERSION_AT_LEAST(7,16,0)
911  case CURLE_REMOTE_FILE_NOT_FOUND:
912 #endif
913  case CURLE_FTP_ACCESS_DENIED:
914  case CURLE_TFTP_NOTFOUND:
915  err = "File not found";
917  break;
918  case CURLE_BAD_PASSWORD_ENTERED:
919  case CURLE_FTP_USER_PASSWORD_INCORRECT:
920  err = "Login failed";
921  break;
922  case CURLE_COULDNT_RESOLVE_PROXY:
923  case CURLE_COULDNT_RESOLVE_HOST:
924  case CURLE_COULDNT_CONNECT:
925  case CURLE_FTP_CANT_GET_HOST:
926  err = "Connection failed";
927  break;
928  case CURLE_WRITE_ERROR:
929  err = "Write error";
930  break;
931  case CURLE_PARTIAL_FILE:
932  case CURLE_OPERATION_TIMEDOUT:
933  timeout_reached = true; // fall though to TimeoutException
934  // fall though...
935  case CURLE_ABORTED_BY_CALLBACK:
936  if( timeout_reached )
937  {
938  err = "Timeout reached";
940  }
941  else
942  {
943  err = "User abort";
944  }
945  break;
946 
947  // Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy
948  case CURLE_HTTP2:
949  case CURLE_HTTP2_STREAM:
950  err = "Curl error " + str::numstring( code );
952  break;
953 
954  default:
955  err = "Curl error " + str::numstring( code );
956  break;
957  }
958 
959  // uhm, no 0 code but unknown curl exception
961  }
962  }
963  else
964  {
965  // actually the code is 0, nothing happened
966  }
967 }
968 
970 
971 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
972 {
973  DBG << filename.asString() << endl;
974 
975  if(!_url.isValid())
977 
978  if(_url.getHost().empty())
980 
981  Url url(getFileUrl(filename));
982 
983  DBG << "URL: " << url.asString() << endl;
984  // Use URL without options and without username and passwd
985  // (some proxies dislike them in the URL).
986  // Curl seems to need the just scheme, hostname and a path;
987  // the rest was already passed as curl options (in attachTo).
988  Url curlUrl( clearQueryString(url) );
989 
990  //
991  // See also Bug #154197 and ftp url definition in RFC 1738:
992  // The url "ftp://user@host/foo/bar/file" contains a path,
993  // that is relative to the user's home.
994  // The url "ftp://user@host//foo/bar/file" (or also with
995  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
996  // contains an absolute path.
997  //
998  _lastRedirect.clear();
999  std::string urlBuffer( curlUrl.asString());
1000  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1001  urlBuffer.c_str() );
1002  if ( ret != 0 ) {
1004  }
1005 
1006  // If no head requests allowed (?head_requests=no):
1007  // Instead of returning no data with NOBODY, we return
1008  // little data, that works with broken servers, and
1009  // works for ftp as well, because retrieving only headers
1010  // ftp will return always OK code ?
1011  // See http://curl.haxx.se/docs/knownbugs.html #58
1013  struct TempSetHeadRequest
1014  {
1015  TempSetHeadRequest( CURL * curl_r, bool doHttpHeadRequest_r )
1016  : _curl { curl_r }
1017  , _doHttpHeadRequest { doHttpHeadRequest_r }
1018  {
1019  if ( _doHttpHeadRequest ) {
1020  curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1021  } else {
1022  curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1023  }
1024  }
1025  ~TempSetHeadRequest() {
1026  if ( _doHttpHeadRequest ) {
1027  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1028  /* yes, this is why we never got to get NOBODY working before,
1029  because setting it changes this option too, and we also*
1030  need to reset it
1031  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1032  */
1033  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1034  } else {
1035  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1036  }
1037  }
1038  private:
1039  CURL * _curl;
1040  bool _doHttpHeadRequest;
1041  } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
1042 
1043 
1044  AutoFILE file { ::fopen( "/dev/null", "w" ) };
1045  if ( !file ) {
1046  ERR << "fopen failed for /dev/null" << endl;
1047  ZYPP_THROW(MediaWriteException("/dev/null"));
1048  }
1049 
1050  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
1051  if ( ret != 0 ) {
1053  }
1054 
1055  CURLcode ok = executeCurl();
1056  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1057 
1058  // as we are not having user interaction, the user can't cancel
1059  // the file existence checking, a callback or timeout return code
1060  // will be always a timeout.
1061  try {
1062  evaluateCurlCode( filename, ok, true /* timeout */);
1063  }
1064  catch ( const MediaFileNotFoundException &e ) {
1065  // if the file did not exist then we can return false
1066  return false;
1067  }
1068  catch ( const MediaException &e ) {
1069  // some error, we are not sure about file existence, rethrw
1070  ZYPP_RETHROW(e);
1071  }
1072  // exists
1073  return ( ok == CURLE_OK );
1074 }
1075 
1077 
1078 void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1079 {
1080  Pathname dest = target.absolutename();
1081  if( assert_dir( dest.dirname() ) )
1082  {
1083  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1084  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1085  }
1086 
1087  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1088  AutoFILE file;
1089  {
1090  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1091  if( ! buf )
1092  {
1093  ERR << "out of memory for temp file name" << endl;
1094  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1095  }
1096 
1097  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1098  if( tmp_fd == -1 )
1099  {
1100  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1101  ZYPP_THROW(MediaWriteException(destNew));
1102  }
1103  destNew = ManagedFile( (*buf), filesystem::unlink );
1104 
1105  file = ::fdopen( tmp_fd, "we" );
1106  if ( ! file )
1107  {
1108  ERR << "fopen failed for file '" << destNew << "'" << endl;
1109  ZYPP_THROW(MediaWriteException(destNew));
1110  }
1111  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1112  }
1113 
1114  DBG << "dest: " << dest << endl;
1115  DBG << "temp: " << destNew << endl;
1116 
1117  // set IFMODSINCE time condition (no download if not modified)
1118  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1119  {
1120  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1121  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1122  }
1123  else
1124  {
1125  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1126  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1127  }
1128  try
1129  {
1130  doGetFileCopyFile( srcFile, dest, file, report, options);
1131  }
1132  catch (Exception &e)
1133  {
1134  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1135  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1136  ZYPP_RETHROW(e);
1137  }
1138 
1139  long httpReturnCode = 0;
1140  CURLcode infoRet = curl_easy_getinfo(_curl,
1141  CURLINFO_RESPONSE_CODE,
1142  &httpReturnCode);
1143  bool modified = true;
1144  if (infoRet == CURLE_OK)
1145  {
1146  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1147  if ( httpReturnCode == 304
1148  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1149  {
1150  DBG << " Not modified.";
1151  modified = false;
1152  }
1153  DBG << endl;
1154  }
1155  else
1156  {
1157  WAR << "Could not get the response code." << endl;
1158  }
1159 
1160  if (modified || infoRet != CURLE_OK)
1161  {
1162  // apply umask
1163  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1164  {
1165  ERR << "Failed to chmod file " << destNew << endl;
1166  }
1167 
1168  file.resetDispose(); // we're going to close it manually here
1169  if ( ::fclose( file ) )
1170  {
1171  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1172  ZYPP_THROW(MediaWriteException(destNew));
1173  }
1174 
1175  // move the temp file into dest
1176  if ( rename( destNew, dest ) != 0 ) {
1177  ERR << "Rename failed" << endl;
1179  }
1180  destNew.resetDispose(); // no more need to unlink it
1181  }
1182 
1183  DBG << "done: " << PathInfo(dest) << endl;
1184 }
1185 
1187 
1188 void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1189 {
1190  DBG << srcFile.filename().asString() << endl;
1191 
1192  if(!_url.isValid())
1194 
1195  if(_url.getHost().empty())
1197 
1198  Url url(getFileUrl(srcFile.filename()));
1199 
1200  DBG << "URL: " << url.asString() << endl;
1201  // Use URL without options and without username and passwd
1202  // (some proxies dislike them in the URL).
1203  // Curl seems to need the just scheme, hostname and a path;
1204  // the rest was already passed as curl options (in attachTo).
1205  Url curlUrl( clearQueryString(url) );
1206 
1207  //
1208  // See also Bug #154197 and ftp url definition in RFC 1738:
1209  // The url "ftp://user@host/foo/bar/file" contains a path,
1210  // that is relative to the user's home.
1211  // The url "ftp://user@host//foo/bar/file" (or also with
1212  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1213  // contains an absolute path.
1214  //
1215  _lastRedirect.clear();
1216  std::string urlBuffer( curlUrl.asString());
1217  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1218  urlBuffer.c_str() );
1219  if ( ret != 0 ) {
1221  }
1222 
1223  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1224  if ( ret != 0 ) {
1226  }
1227 
1228  // Set callback and perform.
1229  internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1230  if (!(options & OPTION_NO_REPORT_START))
1231  report->start(url, dest);
1232  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1233  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1234  }
1235 
1236  ret = executeCurl();
1237 #if CURLVERSION_AT_LEAST(7,19,4)
1238  // bnc#692260: If the client sends a request with an If-Modified-Since header
1239  // with a future date for the server, the server may respond 200 sending a
1240  // zero size file.
1241  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1242  if ( ftell(file) == 0 && ret == 0 )
1243  {
1244  long httpReturnCode = 33;
1245  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1246  {
1247  long conditionUnmet = 33;
1248  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1249  {
1250  WAR << "TIMECONDITION unmet - retry without." << endl;
1251  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1252  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1253  ret = executeCurl();
1254  }
1255  }
1256  }
1257 #endif
1258 
1259  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1260  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1261  }
1262 
1263  if ( ret != 0 )
1264  {
1265  ERR << "curl error: " << ret << ": " << _curlError
1266  << ", temp file size " << ftell(file)
1267  << " bytes." << endl;
1268 
1269  // the timeout is determined by the progress data object
1270  // which holds whether the timeout was reached or not,
1271  // otherwise it would be a user cancel
1272  try {
1273 
1274  if ( progressData.fileSizeExceeded() )
1275  ZYPP_THROW(MediaFileSizeExceededException(url, progressData.expectedFileSize()));
1276 
1277  evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1278  }
1279  catch ( const MediaException &e ) {
1280  // some error, we are not sure about file existence, rethrw
1281  ZYPP_RETHROW(e);
1282  }
1283  }
1284 }
1285 
1287 
1288 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1289 {
1290  filesystem::DirContent content;
1291  getDirInfo( content, dirname, /*dots*/false );
1292 
1293  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1294  Pathname filename = dirname + it->name;
1295  int res = 0;
1296 
1297  switch ( it->type ) {
1298  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1299  case filesystem::FT_FILE:
1300  getFile( OnMediaLocation( filename ) );
1301  break;
1302  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1303  if ( recurse_r ) {
1304  getDir( filename, recurse_r );
1305  } else {
1306  res = assert_dir( localPath( filename ) );
1307  if ( res ) {
1308  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1309  }
1310  }
1311  break;
1312  default:
1313  // don't provide devices, sockets, etc.
1314  break;
1315  }
1316  }
1317 }
1318 
1320 
1321 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1322  const Pathname & dirname, bool dots ) const
1323 {
1324  getDirectoryYast( retlist, dirname, dots );
1325 }
1326 
1328 
1330  const Pathname & dirname, bool dots ) const
1331 {
1332  getDirectoryYast( retlist, dirname, dots );
1333 }
1334 
1336 //
1337 int MediaCurl::aliveCallback( void *clientp, curl_off_t /*dltotal*/, curl_off_t dlnow, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/ )
1338 {
1339  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1340  if( pdata )
1341  {
1342  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1343  // prevent a percentage raise while downloading a metalink file. Download
1344  // activity however is indicated by propagating the download rate (via dlnow).
1345  pdata->updateStats( 0.0, dlnow );
1346  return pdata->reportProgress();
1347  }
1348  return 0;
1349 }
1350 
1351 int MediaCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
1352 {
1353  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1354  if( pdata )
1355  {
1356  // work around curl bug that gives us old data
1357  long httpReturnCode = 0;
1358  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1359  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1360 
1361  pdata->updateStats( dltotal, dlnow );
1362  return pdata->reportProgress();
1363  }
1364  return 0;
1365 }
1366 
1368 {
1369  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1370  return pdata ? pdata->curl() : 0;
1371 }
1372 
1374 
1375 std::string MediaCurl::getAuthHint() const
1376 {
1377  long auth_info = CURLAUTH_NONE;
1378 
1379  CURLcode infoRet =
1380  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1381 
1382  if(infoRet == CURLE_OK)
1383  {
1384  return CurlAuthData::auth_type_long2str(auth_info);
1385  }
1386 
1387  return "";
1388 }
1389 
1394 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1395 {
1396  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1397  if ( data ) {
1398  data->expectedFileSize( expectedFileSize );
1399  }
1400 }
1401 
1407 CURLcode MediaCurl::executeCurl() const
1408 {
1409  // initialize our helpers
1411  internal::CurlPollHelper::CurlPoll{ curl_multi_init() }
1412  ,[](auto &releaseMe ){ if (releaseMe._multi) curl_multi_cleanup(releaseMe._multi); }
1413  );
1414 
1415  if (!cMulti->_multi)
1417 
1418  // we could derive from that, but currently that does not make a lot of sense
1419  internal::CurlPollHelper _curlHelper(cMulti.value());
1420 
1421  // add the easy handle to the multi instance
1422  if ( curl_multi_add_handle( cMulti->_multi, _curl ) != CURLM_OK )
1423  ZYPP_THROW(MediaCurlException( _url, "curl_multi_add_handle", "unknown error"));
1424 
1425  // make sure the handle is cleanly removed from the multi handle
1426  OnScopeExit autoRemove([&](){ curl_multi_remove_handle( cMulti->_multi, _curl ); });
1427 
1428  // kickstart curl, this will cause libcurl to go over the added handles and register sockets and timeouts
1429  CURLMcode mcode = _curlHelper.handleTimout();
1430  if (mcode != CURLM_OK)
1431  ZYPP_THROW(MediaCurlException( _url, "curl_multi_socket_action", "unknown error"));
1432 
1433  bool canContinue = true;
1434  while ( canContinue ) {
1435 
1436  // copy watched sockets in case curl changes the vector as we go over the events later
1437  std::vector<GPollFD> requestedFds = _curlHelper.socks;
1438 
1439  int r = zypp_detail::zypp_poll( requestedFds, _curlHelper.timeout_ms.value_or( -1 ) );
1440  if ( r == -1 )
1441  ZYPP_THROW( MediaCurlException(_url, "zypp_poll() failed", "unknown error") );
1442 
1443  // run curl
1444  if ( r == 0 ) {
1445  CURLMcode mcode = _curlHelper.handleTimout();
1446  if (mcode != CURLM_OK)
1447  ZYPP_THROW(MediaCurlException(_url, "curl_multi_socket_action", "unknown error"));
1448  } else {
1449  CURLMcode mcode = _curlHelper.handleSocketActions( requestedFds );
1450  if (mcode != CURLM_OK)
1451  ZYPP_THROW(MediaCurlException(_url, "curl_multi_socket_action", "unknown error"));
1452  }
1453 
1454  CURLMsg *msg;
1455  int nqueue;
1456  while ((msg = curl_multi_info_read( cMulti->_multi, &nqueue)) != 0) {
1457  if ( msg->msg != CURLMSG_DONE ) continue;
1458  if ( msg->easy_handle != _curl ) continue;
1459 
1460  return msg->data.result;
1461  }
1462  }
1463  return CURLE_OK;
1464 }
1465 
1467 
1468 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1469 {
1471  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1472  CurlAuthData_Ptr credentials;
1473 
1474  // get stored credentials
1475  AuthData_Ptr cmcred = cm.getCred(_url);
1476 
1477  if (cmcred && firstTry)
1478  {
1479  credentials.reset(new CurlAuthData(*cmcred));
1480  DBG << "got stored credentials:" << endl << *credentials << endl;
1481  }
1482  // if not found, ask user
1483  else
1484  {
1485 
1486  CurlAuthData_Ptr curlcred;
1487  curlcred.reset(new CurlAuthData());
1489 
1490  // preset the username if present in current url
1491  if (!_url.getUsername().empty() && firstTry)
1492  curlcred->setUsername(_url.getUsername());
1493  // if CM has found some credentials, preset the username from there
1494  else if (cmcred)
1495  curlcred->setUsername(cmcred->username());
1496 
1497  // indicate we have no good credentials from CM
1498  cmcred.reset();
1499 
1500  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1501 
1502  // set available authentication types from the exception
1503  // might be needed in prompt
1504  curlcred->setAuthType(availAuthTypes);
1505 
1506  // ask user
1507  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1508  {
1509  DBG << "callback answer: retry" << endl
1510  << "CurlAuthData: " << *curlcred << endl;
1511 
1512  if (curlcred->valid())
1513  {
1514  credentials = curlcred;
1515  // if (credentials->username() != _url.getUsername())
1516  // _url.setUsername(credentials->username());
1524  }
1525  }
1526  else
1527  {
1528  DBG << "callback answer: cancel" << endl;
1529  }
1530  }
1531 
1532  // set username and password
1533  if (credentials)
1534  {
1535  // HACK, why is this const?
1536  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1537  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1538 
1539  // set username and password
1540  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1542 
1543  // set available authentication types from the exception
1544  if (credentials->authType() == CURLAUTH_NONE)
1545  credentials->setAuthType(availAuthTypes);
1546 
1547  // set auth type (seems this must be set _after_ setting the userpwd)
1548  if (credentials->authType() != CURLAUTH_NONE)
1549  {
1550  // FIXME: only overwrite if not empty?
1551  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1552  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1554  }
1555 
1556  if (!cmcred)
1557  {
1558  credentials->setUrl(_url);
1559  cm.addCred(*credentials);
1560  cm.save();
1561  }
1562 
1563  return true;
1564  }
1565 
1566  return false;
1567 }
1568 
1569 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1571 
1572  } // namespace media
1573 } // namespace zypp
1574 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:64
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:688
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:319
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:303
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:333
#define MIL
Definition: Logger.h:96
const Pathname & clientCertificatePath() const
SSL client certificate file.
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
Definition: MediaCurl.cc:86
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
#define _(MSG)
Definition: Gettext.h:37
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:143
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:384
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1468
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
void setPassword(const std::string &val_r)
sets the auth password
std::chrono::steady_clock::time_point TimePoint
Definition: MediaCurl.cc:54
ByteCount _expectedFileSize
Definition: MediaCurl.cc:170
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
const char * distributionFlavorHeader()
Definition: MediaCurl.cc:268
Store and operate with byte count.
Definition: ByteCount.h:30
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
int reportProgress() const
Definition: MediaCurl.cc:243
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:373
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
void start(const Url &file_r, Pathname localfile_r) override
Definition: MediaCurl.cc:74
const char * anonymousIdHeader()
Definition: MediaCurl.cc:254
const char * c_str() const
String representation.
Definition: Pathname.h:110
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:140
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
Definition: MediaCurl.cc:105
virtual void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:730
time_t _timeNow
Now.
Definition: MediaCurl.cc:176
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:409
bool timeoutReached() const
Definition: MediaCurl.cc:152
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
CURLcode executeCurl() const
Definition: MediaCurl.cc:1407
Edition * _value
Definition: SysContent.cc:311
AutoDispose<int> calling ::close
Definition: AutoDispose.h:301
std::string _currentCookieFile
Definition: MediaCurl.h:163
const std::string & password() const
auth password
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1288
virtual void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:721
#define ERR
Definition: Logger.h:98
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:155
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
const Headers & headers() const
returns a list of all added headers (trimmed)
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:377
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:714
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1394
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:660
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition: MediaCurl.cc:52
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:372
void setAuthType(const std::string &val_r)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
Definition: MediaCurl.cc:111
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:700
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:91
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:742
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:823
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:97
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:518
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:183
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1351
Receiver * _oldRec
std::string proxyuserpwd
Definition: curlconfig.h:49
bool isValid() const
Verifies the Url.
Definition: Url.cc:489
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:971
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:161
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
const Pathname & filename() const
The path to the resource on the medium.
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:653
std::string numstring(char n, int w=0)
Definition: String.h:289
Common baseclass for MediaCurl and MediaNetwork.
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:792
const char * agentString()
Definition: MediaCurl.cc:282
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1188
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:36
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:182
curl_slist * _customHeaders
Definition: MediaCurl.h:171
bool proxyEnabled() const
proxy is enabled
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:79
#define LIBZYPP_VERSION_STRING
Definition: APIConfig.h:15
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:366
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:326
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:139
Base class for Exception.
Definition: Exception.h:145
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:166
curl_off_t _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:180
Url url() const
Url used.
Definition: MediaHandler.h:503
curl_off_t _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:178
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:185
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:366
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:588
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:173
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1321
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:130
virtual void disconnectFrom() override
Definition: MediaCurl.cc:695
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1367
Typesafe passing of user data via callbacks.
Definition: UserData.h:38
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
curl_off_t _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:179
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
Definition: MediaCurl.cc:70
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:312
static Pathname _cookieFile
Definition: MediaCurl.h:164
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:331
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:184
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:789
std::string userPassword() const
returns the user and password as a user:pass string
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1375
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:175
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:23
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Definition: ZYppImpl.cc:313
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1078
void updateStats(curl_off_t dltotal=0.0, curl_off_t dlnow=0.0)
Definition: MediaCurl.cc:200
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
OptionalDownloadProgressReport(bool isOptional=false)
Definition: MediaCurl.cc:56
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:170
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:158
CURLcode setCurlRedirProtocols(CURL *curl)
Definition: curlhelper.cc:512
const std::string & proxy() const
proxy host
MediaCurlException(const Url &url_r, const std::string &err_r, const std::string &msg_r)
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:174
const char * c_str() const
Definition: IdStringType.h:105
TrueBool _guard
Definition: TargetImpl.cc:1608
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1337
bool userMayRWX() const
Definition: PathInfo.h:353
const std::string & userAgentString() const
user agent string (trimmed)
Url manipulation class.
Definition: Url.h:91
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:300
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:171
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:572