libzypp  17.31.31
mirrorcontrol.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------*/
10 #include "private/mediadebug_p.h"
11 #include <zypp-core/zyppng/base/EventDispatcher>
12 #include <zypp-core/zyppng/base/Signals>
13 #include <zypp-core/base/String.h>
14 #include <iostream>
15 
16 namespace zyppng {
17 
18  constexpr uint penaltyIncrease = 100;
19  constexpr uint defaultSampleTime = 2;
20  constexpr uint defaultMaxConnections = 5;
21 
22  MirrorControl::Mirror::Mirror( MirrorControl &parent ) : _parent( parent )
23  {}
24 
26  {
27  runningTransfers++;
28  }
29 
30  void MirrorControl::Mirror::finishTransfer(const bool success)
31  {
32  if ( success ) {
33  if ( penalty >= penaltyIncrease ) penalty -= penaltyIncrease;
34  successfulTransfers++;
35  failedTransfers = 0;
36  } else {
37  penalty += penaltyIncrease;
38  failedTransfers++;
39  }
40  transferUnref();
41  }
42 
44  {
45  transferUnref();
46  }
47 
49  {
50  return ( _maxConnections > 0 ? _maxConnections : defaultMaxConnections ); //max connections per mirror @todo make this configurable
51  }
52 
54  {
55  return ( runningTransfers < maxConnections() );
56  }
57 
59  {
60  const auto newCount = runningTransfers - 1;
61  if ( newCount < 0 )
62  return;
63 
64  bool stillLoaded = ( newCount ) >= maxConnections();
65  runningTransfers--;
66  if ( !stillLoaded )
67  _parent._sigNewMirrorsReady.emit();
68  }
69 
71  {
72  // set up the single shot timer, that way we can emit a signal after we finished processing all
73  // events that have queued up in the event loop instead of constantly firing the signal
74  _newMirrSigDelay = Timer::create();
75  _newMirrSigDelay->setSingleShot( true );
76  _newMirrSigDelay->connectFunc( &Timer::sigExpired, [this]( const auto & ){
77  _sigNewMirrorsReady.emit();
78  });
79 
80  _dispatcher = std::make_shared<NetworkRequestDispatcher>();
81  _queueEmptyConn = _dispatcher->connectFunc( &NetworkRequestDispatcher::sigQueueFinished, [ this ]( NetworkRequestDispatcher& ) {
82  //tell the world the queue is empty
83 
84  std::vector< std::unordered_map<std::string, MirrorHandle>::const_iterator > allOfEm;
85  for ( auto i = _handles.begin(); i != _handles.end(); i++ ) {
86  allOfEm.push_back( i );
87  }
88 
89  std::sort( allOfEm.begin(), allOfEm.end(), []( const auto &a, const auto &b ){
90  return ( zypp::str::compareCI( a->second->mirrorUrl.asString().c_str(), b->second->mirrorUrl.asString().c_str() ) < 0 );
91  });
92 
93  DBG_MEDIA << "Finished probing mirrors, these are the results: \n";
94  for ( const auto &iter : allOfEm ) {
95  DBG_MEDIA << "Mirror: " << iter->second->mirrorUrl << ", rating is: " << iter->second->rating << "\n";
96  }
97  DBG_MEDIA << "End Mirror probing results." << std::endl;
98 
99  _sigAllMirrorsReady.emit();
100  }, *this );
101  _dispatcher->run();
102  }
103 
105  {
106  return std::shared_ptr<MirrorControl>( new MirrorControl );
107  }
108 
110  {
111  // do not send signals to us while we are destructing
112  _queueEmptyConn.disconnect();
113 
114  if ( _dispatcher->count() > 0 ) {
115  MIL << "Destroying MirrorControl while measurements are still running, aborting" << std::endl;
116  for ( auto &mirr : _handles ) {
117  if ( mirr.second->_request ) {
118  mirr.second->_finishedConn.disconnect();
119  _dispatcher->cancel( *mirr.second->_request );
120  }
121  }
122  }
123 
124  }
125 
126  void MirrorControl::registerMirrors( const std::vector<zypp::media::MetalinkMirror> &urls )
127  {
128  bool doesKnowSomeMirrors = false;
129  for ( const auto &mirror : urls ) {
130 
131  const auto scheme = mirror.url.getScheme();
132  if ( scheme == "http" || scheme == "https" || scheme == "ftp" || scheme == "tftp" ) {
133 
134  const std::string urlKey = makeKey( mirror.url );
135 
136  // already there
137  const auto hndlIt = _handles.find( urlKey );
138  if ( hndlIt != _handles.end() ) {
139  doesKnowSomeMirrors = true;
140  continue;
141  }
142 
143  auto mirrorHandle = std::shared_ptr<Mirror>( new Mirror(*this) );
144  mirrorHandle->rating = mirror.priority;
145  mirrorHandle->_maxConnections = mirror.maxConnections;
146  mirrorHandle->mirrorUrl = mirror.url;
147  mirrorHandle->mirrorUrl.setPathName("/");
148 
149  mirrorHandle->_request = std::make_shared<NetworkRequest>( mirrorHandle->mirrorUrl, "/dev/null", NetworkRequest::WriteShared );
150  mirrorHandle->_request->setOptions( NetworkRequest::ConnectionTest );
151  mirrorHandle->_request->transferSettings().setTimeout( defaultSampleTime );
152  mirrorHandle->_request->transferSettings().setConnectTimeout( defaultSampleTime );
153  mirrorHandle->_finishedConn = mirrorHandle->_request->connectFunc( &NetworkRequest::sigFinished, [ mirrorHandle, &someReadyDelay = _newMirrSigDelay ]( NetworkRequest &req, const NetworkRequestError & ){
154 
155  if ( req.hasError() )
156  ERR << "Mirror request failed: " << req.error().toString() << " ; " << req.extendedErrorString() << "; for url: "<<req.url()<<std::endl;
157 
158  const auto timings = req.timings();
159  std::chrono::milliseconds connTime;
160  if ( timings ) {
161  connTime = std::chrono::duration_cast<std::chrono::milliseconds>(timings->connect - timings->namelookup);
162  } else {
163  // we can not get any measurements, maximum penalty
164  connTime = std::chrono::seconds( defaultSampleTime );
165  }
166 
167  DBG_MEDIA << "Got rating for mirror: " << mirrorHandle->mirrorUrl << ", rating was " << mirrorHandle->rating;
168  mirrorHandle->rating += connTime.count();
169  DBG_MEDIA << " rating is now " << mirrorHandle->rating << " conn time was " << connTime.count() << std::endl;
170 
171  // clean the request up
172  mirrorHandle->_finishedConn.disconnect();
173  mirrorHandle->_request.reset();
174 
175  // start the timer to emit someMirrorsReady
176  someReadyDelay->start( 0 );
177  });
178 
179  _dispatcher->enqueue( mirrorHandle->_request );
180  _handles.insert( std::make_pair(urlKey, mirrorHandle ) );
181  }
182  }
183 
184  if ( doesKnowSomeMirrors )
185  _sigNewMirrorsReady.emit();
186 
187  if ( _dispatcher->count() == 0 ) {
188  // we did know all Mirrors before, notify the outside world we are ready
189  _sigAllMirrorsReady.emit();
190  }
191  }
192 
193  MirrorControl::PickResult MirrorControl::pickBestMirror( const std::vector<Url> &mirrors )
194  {
195  bool hasPendingRating = false;
196  std::vector< MirrorPick > possibleMirrs;
197  for ( auto i = mirrors.begin(); i != mirrors.end(); i++ ) {
198  const auto key = makeKey( *i );
199  const auto hdlIt = this->_handles.find( key );
200  if ( hdlIt == _handles.end( ) )
201  continue;
202  // still waiting for the request to finish
203  if ( hdlIt->second->_request ) {
204  hasPendingRating = true;
205  continue;
206  }
207  possibleMirrs.push_back( std::make_pair( i, hdlIt->second ) );
208  }
209 
210  if ( possibleMirrs.empty() && hasPendingRating ) {
211  // still waiting return , tell the caller to try again later
212  return PickResult{ PickResult::Again, std::make_pair( mirrors.end(), MirrorHandle() ) };
213  }
214 
215  std::stable_sort( possibleMirrs.begin(), possibleMirrs.end(), []( const auto &a, const auto &b ) {
216  return a.second->rating < b.second->rating;
217  });
218 
219  bool hasLoadedOne = false; // do we have a mirror that will be ready again later?
220  for ( const auto &mirr : possibleMirrs ) {
221  if ( !mirr.second->hasFreeConnections() ) {
222  hasLoadedOne = true;
223  continue;
224  }
225  if ( mirr.second->failedTransfers >= 10 )
226  continue;
227  return PickResult{ PickResult::Ok, mirr };
228  }
229 
230  if ( hasLoadedOne ){
231  // we have mirrors, but they have reached maximum capacity, tell the caller to delay the call
232  return PickResult{ PickResult::Again, std::make_pair( mirrors.end(), MirrorHandle() ) };
233  }
234 
235  return PickResult{ PickResult::Unknown, std::make_pair( mirrors.end(), MirrorHandle() ) };
236  }
237 
238  SignalProxy<void ()> MirrorControl::sigNewMirrorsReady()
239  {
240  return _sigNewMirrorsReady;
241  }
242 
243  SignalProxy<void ()> MirrorControl::sigAllMirrorsReady()
244  {
245  return _sigAllMirrorsReady;
246  }
247 
248  std::string MirrorControl::makeKey(const zypp::Url &url) const
249  {
254  );
255  }
256 
257 #if 0
258 
259  MirrorRef::MirrorRef( MirrorControl::MirrorHandle handle )
260  {
261  _data = std::make_shared<Helper>( handle, false );
262  }
263 
264  MirrorRef::~MirrorRef()
265  { }
266 
267  void MirrorRef::startTransfer()
268  {
269  _data->_myHandle->startTransfer();
270  _data->_cancelOnDestruct = true;
271  }
272 
273  void MirrorRef::finishTransfer(const bool success)
274  {
275  _data->_cancelOnDestruct = false;
276  _data->_myHandle->finishTransfer( success );
277  }
278 
279  void MirrorRef::cancelTransfer()
280  {
281  _data->_cancelOnDestruct = false;
282  _data->_myHandle->cancelTransfer();
283  }
284 
285  MirrorRef::operator bool() const
286  {
287  return _data->_myHandle.operator bool();
288  }
289 
290  MirrorControl::MirrorHandle MirrorRef::get()
291  {
292  return _data->_myHandle;
293  }
294 
295  MirrorRef::Helper::~Helper()
296  {
297  if ( _cancelOnDestruct )
298  _myHandle->cancelTransfer();
299  }
300 #endif
301 
302 }
#define MIL
Definition: Logger.h:96
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
Definition: request.cc:845
#define DBG_MEDIA
Definition: mediadebug_p.h:28
SignalProxy< void()> sigNewMirrorsReady()
unsigned short b
constexpr uint defaultSampleTime
static const ViewOption WITH_SCHEME
Option to include scheme name in the URL string.
Definition: UrlBase.h:51
static const ViewOption WITH_HOST
Option to include hostname in the URL string.
Definition: UrlBase.h:74
std::string toString() const
toString Returns a string representation of the error
Signal< void()> _sigNewMirrorsReady
#define ERR
Definition: Logger.h:98
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1001
void registerMirrors(const std::vector< zypp::media::MetalinkMirror > &urls)
sigc::connection _queueEmptyConn
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
std::list< PublicKeyData > _data
Definition: KeyRing.cc:191
static const ViewOption EMPTY_AUTHORITY
Explicitely include the URL authority separator "//".
Definition: UrlBase.h:121
The NetworkRequestError class Represents a error that occured in.
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:985
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:993
void finishTransfer(const bool success)
Mirror(MirrorControl &parent)
constexpr uint defaultMaxConnections
int compareCI(const C_Str &lhs, const C_Str &rhs)
Definition: String.h:984
Signal< void()> _sigAllMirrorsReady
constexpr uint penaltyIncrease
std::shared_ptr< Mirror > MirrorHandle
unsigned short a
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1035
SignalProxy< void()> sigAllMirrorsReady()
std::string makeKey(const zypp::Url &url) const
static const ViewOption WITH_PORT
Option to include port number in the URL string.
Definition: UrlBase.h:81
std::unordered_map< std::string, MirrorHandle > _handles
Url manipulation class.
Definition: Url.h:91
PickResult pickBestMirror(const std::vector< Url > &mirrors)
std::shared_ptr< MirrorControl > Ptr
NetworkRequestDispatcher::Ptr _dispatcher