libzypp  17.31.31
downloader.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------*/
13 #include <zypp-curl/TransferSettings>
15 #include <zypp-media/MediaException>
16 #include <zypp-core/base/String.h>
17 
18 namespace zyppng {
19 
20  DownloadPrivateBase::DownloadPrivateBase(Downloader &parent, std::shared_ptr<NetworkRequestDispatcher> requestDispatcher, std::shared_ptr<MirrorControl> mirrors, DownloadSpec &&spec, Download &p)
21  : BasePrivate(p)
22  , _requestDispatcher ( std::move(requestDispatcher) )
23  , _mirrorControl( std::move(mirrors) )
24  , _spec( std::move(spec) )
25  , _parent( &parent )
26  {}
27 
29  { }
30 
31  bool DownloadPrivateBase::handleRequestAuthError( std::shared_ptr<Request> req, const zyppng::NetworkRequestError &err )
32  {
33  //Handle the auth errors explicitly, we need to give the user a way to put in new credentials
34  //if we get valid new credentials we can retry the request
35  bool retry = false;
37 
38  MIL << "Authentication failed for " << req->url() << " trying to recover." << std::endl;
39 
40  TransferSettings &ts = req->transferSettings();
41  const auto &applyCredToSettings = [&ts]( AuthData_Ptr auth, const std::string &authHint ) {
42  ts.setUsername( auth->username() );
43  ts.setPassword( auth->password() );
44  auto nwCred = dynamic_cast<NetworkAuthData *>( auth.get() );
45  if ( nwCred ) {
46  // set available authentication types from the error
47  if ( nwCred->authType() == CURLAUTH_NONE )
48  nwCred->setAuthType( authHint );
49 
50  // set auth type (seems this must be set _after_ setting the userpwd)
51  if ( nwCred->authType() != CURLAUTH_NONE ) {
52  // FIXME: only overwrite if not empty?
53  ts.setAuthType(nwCred->authTypeAsString());
54  }
55  }
56  };
57 
58  // try to find one in the cache
60  vopt = vopt
64 
65  auto cachedCred = zypp::media::CredentialManager::findIn( _credCache, req->url(), vopt );
66 
67  // only consider a cache entry if its newer than what we tried last time
68  if ( cachedCred && cachedCred->lastDatabaseUpdate() > req->_authTimestamp ) {
69  MIL << "Found a credential match in the cache!" << std::endl;
70  applyCredToSettings( cachedCred, "" );
71  _lastTriedAuthTime = req->_authTimestamp = cachedCred->lastDatabaseUpdate();
72  retry = true;
73  } else {
74 
76  credFromUser->setUrl( req->url() );
77  credFromUser->setLastDatabaseUpdate ( req->_authTimestamp );
78 
79  //in case we got a auth hint from the server the error object will contain it
80  std::string authHint = err.extraInfoValue("authHint", std::string());
81 
82  _sigAuthRequired.emit( *z_func(), *credFromUser, authHint );
83  if ( credFromUser->valid() ) {
84  // remember for next time , we don't want to ask the user again for the same URL set
85  _credCache.insert( credFromUser );
86  applyCredToSettings( credFromUser, authHint );
87  _lastTriedAuthTime = req->_authTimestamp = credFromUser->lastDatabaseUpdate();
88  retry = true;
89  }
90  }
91  }
92  return retry;
93  }
94 
95 #if ENABLE_ZCHUNK_COMPRESSION
96  bool DownloadPrivateBase::hasZckInfo() const
97  {
98  if ( zypp::indeterminate(_specHasZckInfo) )
99  _specHasZckInfo = ( _spec.headerSize() > 0 && isZchunkFile( _spec.deltaFile() ) );
100  return bool(_specHasZckInfo);
101  }
102 #endif
103 
105  {
106  _sigStartedConn.disconnect();
107  _sigProgressConn.disconnect();
108  _sigFinishedConn.disconnect();
109  }
110 
111  DownloadPrivate::DownloadPrivate(Downloader &parent, std::shared_ptr<NetworkRequestDispatcher> requestDispatcher, std::shared_ptr<MirrorControl> mirrors, DownloadSpec &&spec, Download &p)
112  : DownloadPrivateBase( parent, std::move(requestDispatcher), std::move(mirrors), std::move(spec), p )
113  { }
114 
116  {
117  Base::connectFunc( *this, &DownloadStatemachine<DownloadPrivate>::sigFinished, [this](){
118  DownloadPrivateBase::_sigFinished.emit( *z_func() );
119  } );
120 
121  Base::connectFunc( *this, &DownloadStatemachine<DownloadPrivate>::sigStateChanged, [this]( const auto state ){
122  DownloadPrivateBase::_sigStateChanged.emit( *z_func(), state );
123  } );
124  }
125 
127  {
128  auto cState = currentState();
129  if ( !cState )
131 
132  cState = currentState();
133  if ( *cState != Download::InitialState && *cState != Download::Finished ) {
134  // the state machine has advaned already, we can only restart it in a finished state
135  return;
136  }
137 
138  //reset state variables
139  _specHasZckInfo = zypp::indeterminate;
140  _emittedSigStart = false;
141  _stoppedOnMetalink = false;
142  _lastTriedAuthTime = 0;
143 
144  // restart the statemachine
145  if ( cState == Download::Finished )
147 
148  //jumpstart the process
149  state<InitialState>()->initiate();
150  }
151 
152 
154  {
155  auto buildExtraInfo = [this, &url](){
156  std::map<std::string, boost::any> extraInfo;
157  extraInfo.insert( {"requestUrl", url } );
158  extraInfo.insert( {"filepath", _spec.targetPath() } );
159  return extraInfo;
160  };
161 
163  try {
165  if ( _spec.settings().proxy().empty() )
166  ::internal::fillSettingsSystemProxy( url, set );
167 
168 #if 0
169  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
170  * We should proactively add the password to the request if basic auth is configured
171  * and a password is available in the credentials but not in the URL.
172  *
173  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
174  * and ask the server first about the auth method
175  */
176  if ( set.authType() == "basic"
177  && set.username().size()
178  && !set.password().size() ) {
180  const auto cred = cm.getCred( url );
181  if ( cred && cred->valid() ) {
182  if ( !set.username().size() )
183  set.setUsername(cred->username());
184  set.setPassword(cred->password());
185  }
186  }
187 #endif
188 
189  } catch ( const zypp::media::MediaBadUrlException & e ) {
191  } catch ( const zypp::media::MediaUnauthorizedException & e ) {
193  } catch ( const zypp::Exception & e ) {
195  }
196  return res;
197  }
198 
199  Download::Download(zyppng::Downloader &parent, std::shared_ptr<zyppng::NetworkRequestDispatcher> requestDispatcher, std::shared_ptr<zyppng::MirrorControl> mirrors, zyppng::DownloadSpec &&spec)
200  : Base( *new DownloadPrivate( parent, std::move(requestDispatcher), std::move(mirrors), std::move(spec), *this ) )
201  { }
202 
204 
206  {
207  if ( state() != InitialState && state() != Finished )
208  cancel();
209  }
210 
212  {
213  const auto &s = d_func()->currentState();
214  if ( !s )
215  return Download::InitialState;
216  return *s;
217  }
218 
220  {
221  if ( state() == Finished ) {
222  return d_func()->state<FinishedState>()->_error;
223  }
224  return NetworkRequestError();
225  }
226 
227  bool Download::hasError() const
228  {
229  return lastRequestError().isError();
230  }
231 
232  std::string Download::errorString() const
233  {
234  const auto &lReq = lastRequestError();
235  if (! lReq.isError() ) {
236  return {};
237  }
238 
239  return ( zypp::str::Format("%1%(%2%)") % lReq.toString() % lReq.nativeErrorString() );
240  }
241 
243  {
244  d_func()->start();
245  }
246 
248  {
249  Z_D();
250 
251  if ( !d->_requestDispatcher )
252  return;
253 
254  d->_defaultSubRequestPriority = NetworkRequest::Critical;
255 
256  // we only reschedule requests when we are in a state that downloads in blocks
257  d->visitState( []( auto &s ){
258  using T = std::decay_t<decltype (s)>;
259  if constexpr ( std::is_same_v<T, DlMetalinkState>
260 #if ENABLE_ZCHUNK_COMPRESSION
261  || std::is_same_v<T, DLZckState>
262 #endif
263  ) {
264  s.reschedule();
265  }
266  });
267  }
268 
270  {
271  Z_D();
272  d->forceState ( std::make_unique<FinishedState>( NetworkRequestErrorPrivate::customError( NetworkRequestError::Cancelled, "Download was cancelled explicitly" ), *d_func() ) );
273  }
274 
275  void Download::setStopOnMetalink(const bool set)
276  {
277  d_func()->_stopOnMetalink = set;
278  }
279 
281  {
282  return d_func()->_stoppedOnMetalink;
283  }
284 
286  {
287  return d_func()->_spec;
288  }
289 
291  {
292  return d_func()->_spec;
293  }
294 
296  {
297  return d_func()->_lastTriedAuthTime;
298  }
299 
300  zyppng::NetworkRequestDispatcher &Download::dispatcher() const
301  {
302  return *d_func()->_requestDispatcher;
303  }
304 
305  SignalProxy<void (Download &req)> Download::sigStarted()
306  {
307  return d_func()->_sigStarted;
308  }
309 
310  SignalProxy<void (Download &req, Download::State state)> Download::sigStateChanged()
311  {
312  return d_func()->DownloadPrivateBase::_sigStateChanged;
313  }
314 
315  SignalProxy<void (zyppng::Download &req, off_t dlnow)> zyppng::Download::sigAlive()
316  {
317  return d_func()->_sigAlive;
318  }
319 
320  SignalProxy<void (Download &req, off_t dltotal, off_t dlnow)> Download::sigProgress()
321  {
322  return d_func()->_sigProgress;
323  }
324 
325  SignalProxy<void (Download &req)> Download::sigFinished()
326  {
327  return d_func()->DownloadPrivateBase::_sigFinished;
328  }
329 
330  SignalProxy<void (zyppng::Download &req, zyppng::NetworkAuthData &auth, const std::string &availAuth)> Download::sigAuthRequired()
331  {
332  return d_func()->_sigAuthRequired;
333  }
334 
335  DownloaderPrivate::DownloaderPrivate(std::shared_ptr<MirrorControl> mc, Downloader &p)
336  : BasePrivate(p)
337  , _mirrors( std::move(mc) )
338  {
339  _requestDispatcher = std::make_shared<NetworkRequestDispatcher>( );
340  if ( !_mirrors ) {
342  }
343  }
344 
346  {
347  _sigStarted.emit( *z_func(), download );
348  }
349 
351  {
352  _sigFinished.emit( *z_func(), download );
353 
354  auto it = std::find_if( _runningDownloads.begin(), _runningDownloads.end(), [ &download ]( const std::shared_ptr<Download> &dl){
355  return dl.get() == &download;
356  });
357 
358  if ( it != _runningDownloads.end() ) {
359  //make sure this is not deleted before all user code was done
360  _runningDownloads.erase( it );
361  }
362 
363  if ( _runningDownloads.empty() )
364  _queueEmpty.emit( *z_func() );
365  }
366 
368 
370  : Base ( *new DownloaderPrivate( {}, *this ) )
371  {
372 
373  }
374 
375  Downloader::Downloader( std::shared_ptr<MirrorControl> mc )
376  : Base ( *new DownloaderPrivate( mc, *this ) )
377  { }
378 
380  {
381  Z_D();
382  while ( d->_runningDownloads.size() ) {
383  d->_runningDownloads.back()->cancel();
384  d->_runningDownloads.pop_back();
385  }
386  }
387 
388  std::shared_ptr<Download> Downloader::downloadFile(const zyppng::DownloadSpec &spec )
389  {
390  Z_D();
391  std::shared_ptr<Download> dl ( new Download ( *this, d->_requestDispatcher, d->_mirrors, DownloadSpec(spec) ) );
392 
393  d->_runningDownloads.push_back( dl );
395  d->_requestDispatcher->run();
396 
397  return dl;
398  }
399 
400  std::shared_ptr<NetworkRequestDispatcher> Downloader::requestDispatcher() const
401  {
402  return d_func()->_requestDispatcher;
403  }
404 
405  SignalProxy<void (Downloader &parent, Download &download)> Downloader::sigStarted()
406  {
407  return d_func()->_sigStarted;
408  }
409 
410  SignalProxy<void (Downloader &parent, Download &download)> Downloader::sigFinished()
411  {
412  return d_func()->_sigFinished;
413  }
414 
415  SignalProxy<void (Downloader &parent)> Downloader::queueEmpty()
416  {
417  return d_func()->_queueEmpty;
418  }
419 
420 }
bool isError() const
isError Will return true if this is a actual error
#define MIL
Definition: Logger.h:96
zypp::media::AuthData_Ptr AuthData_Ptr
Definition: authdata.h:22
static const ViewOption WITH_USERNAME
Option to include username in the URL string.
Definition: UrlBase.h:58
SignalProxy< void(Download &req, off_t dlnow)> sigAlive()
Definition: downloader.cc:315
The Downloader class.
Definition: downloader.h:38
DownloaderPrivate(std::shared_ptr< MirrorControl > mc, Downloader &p)
Definition: downloader.cc:335
Downloader * _parent
Definition: base_p.h:101
ZYPP_IMPL_PRIVATE(Provide)
void setPassword(const std::string &val_r)
sets the auth password
void init() override
Definition: downloader.cc:115
DownloadSpec _spec
Definition: base_p.h:98
void setAuthType(std::string auth_type)
Set HTTP authentication type(s) to use.
Definition: curlauthdata.h:55
bool handleRequestAuthError(std::shared_ptr< Request > req, const zyppng::NetworkRequestError &err)
Definition: downloader.cc:31
Holds transfer setting.
void setStopOnMetalink(const bool set=true)
Definition: downloader.cc:275
SignalProxy< void(Download &req, NetworkAuthData &auth, const std::string &availAuth)> sigAuthRequired()
Definition: downloader.cc:330
std::shared_ptr< NetworkRequestDispatcher > _requestDispatcher
Definition: downloader_p.h:104
DownloadPrivateBase(Downloader &parent, std::shared_ptr< NetworkRequestDispatcher > requestDispatcher, std::shared_ptr< MirrorControl > mirrors, DownloadSpec &&spec, Download &p)
Definition: downloader.cc:20
void setUsername(const std::string &val_r)
sets the auth username
zypp::TriBool _specHasZckInfo
Definition: base_p.h:99
Definition: Arch.h:363
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
Signal< void(Download &req, Download::State state)> _sigStateChanged
Definition: base_p.h:109
std::vector< std::shared_ptr< Download > > _runningDownloads
Definition: downloader_p.h:103
uint64_t lastAuthTimestamp() const
Definition: downloader.cc:295
Convenient building of std::string with boost::format.
Definition: String.h:252
SignalProxy< void(Downloader &parent, Download &download)> sigStarted()
Definition: downloader.cc:405
SignalProxy< void(Download &req)> sigFinished()
Definition: downloader.cc:325
Url::asString() view options.
Definition: UrlBase.h:39
zypp::filesystem::Pathname deltaFile() const
Signal< void(Downloader &parent, Download &download)> _sigFinished
Definition: downloader_p.h:110
virtual ~Downloader()
Definition: downloader.cc:379
void onDownloadFinished(Download &download)
Definition: downloader.cc:350
SignalProxy< void(Downloader &parent, Download &download)> sigFinished()
Definition: downloader.cc:410
Signal< void(Downloader &parent)> _queueEmpty
Definition: downloader_p.h:111
State state() const
Definition: downloader.cc:211
std::shared_ptr< MirrorControl > _mirrors
Definition: downloader_p.h:112
void setAuthType(const std::string &val_r)
set the allowed authentication types
T extraInfoValue(const std::string &key, T &&defaultVal=T()) const
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:75
Download(Downloader &parent, std::shared_ptr< NetworkRequestDispatcher > requestDispatcher, std::shared_ptr< MirrorControl > mirrors, DownloadSpec &&spec)
Definition: downloader.cc:199
bool stoppedOnMetalink() const
Definition: downloader.cc:280
The NetworkRequestError class Represents a error that occured in.
std::shared_ptr< NetworkRequestDispatcher > requestDispatcher() const
Definition: downloader.cc:400
zypp::media::CurlAuthData_Ptr NetworkAuthData_Ptr
Definition: authdata.h:25
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:183
std::shared_ptr< Download > downloadFile(const DownloadSpec &spec)
Definition: downloader.cc:388
NetworkRequestDispatcher & dispatcher() const
Definition: downloader.cc:300
const zypp::Pathname & targetPath() const
Definition: downloadspec.cc:61
NetworkRequestError lastRequestError() const
Definition: downloader.cc:219
static const ViewOption WITH_QUERY_STR
Option to include query string in the URL string.
Definition: UrlBase.h:101
zypp::media::CurlAuthData NetworkAuthData
Definition: authdata.h:24
SignalProxy< void(Downloader &parent)> queueEmpty()
Definition: downloader.cc:415
NetworkRequestError safeFillSettingsFromURL(const Url &url, TransferSettings &set)
Definition: downloader.cc:153
DownloadPrivate(Downloader &parent, std::shared_ptr< NetworkRequestDispatcher > requestDispatcher, std::shared_ptr< MirrorControl > mirrors, DownloadSpec &&spec, Download &p)
Definition: downloader.cc:111
DownloadSpec & spec()
Definition: downloader.cc:285
SignalProxy< void(Download &req)> sigStarted()
Definition: downloader.cc:305
const TransferSettings & settings() const
Base class for Exception.
Definition: Exception.h:145
static const ViewOption WITH_PASSWORD
Option to include password in the URL string.
Definition: UrlBase.h:67
void onDownloadStarted(Download &download)
Definition: downloader.cc:345
Signal< void(zyppng::Download &req, zyppng::NetworkAuthData &auth, const std::string &availAuth)> _sigAuthRequired
Definition: base_p.h:113
SignalProxy< void(Download &req, off_t dltotal, off_t dlnow)> sigProgress()
Definition: downloader.cc:320
static AuthData_Ptr findIn(const CredentialManager::CredentialSet &set, const Url &url, url::ViewOption vopt)
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:331
Type type() const
type Returns the type of the error
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string errorString() const
Definition: downloader.cc:232
zypp::ByteCount headerSize() const
SignalProxy< void(Download &req, State state)> sigStateChanged()
Definition: downloader.cc:310
const std::string & proxy() const
proxy host
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
Signal< void(Download &req)> _sigFinished
Definition: base_p.h:112
Signal< void(Downloader &parent, Download &download)> _sigStarted
Definition: downloader_p.h:109
bool hasError() const
Definition: downloader.cc:227
zypp::media::CredentialManager::CredentialSet _credCache
Definition: base_p.h:96