libzypp 17.31.28
request.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8----------------------------------------------------------------------*/
13#include <zypp-core/zyppng/base/EventDispatcher>
14#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
15#include <zypp-core/zyppng/core/String>
16#include <zypp-core/fs/PathInfo.h>
18#include <zypp-curl/CurlConfig>
19#include <zypp-curl/auth/CurlAuthData>
20#include <zypp-media/MediaConfig>
21#include <zypp-core/base/String.h>
22#include <zypp-core/base/StringV.h>
23#include <zypp-core/base/IOTools.h>
24#include <zypp-core/Pathname.h>
25#include <curl/curl.h>
26#include <stdio.h>
27#include <fcntl.h>
28#include <utility>
29
30#include <iostream>
31#include <boost/variant.hpp>
32#include <boost/variant/polymorphic_get.hpp>
33
34
35namespace zyppng {
36
37 namespace {
38 static size_t nwr_headerCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
39 if ( !userdata )
40 return 0;
41
42 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
43 return that->headerfunction( ptr, size * nmemb );
44 }
45 static size_t nwr_writeCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
46 if ( !userdata )
47 return 0;
48
49 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
50 return that->writefunction( ptr, {}, size * nmemb );
51 }
52
53 //helper for std::visit
54 template<class T> struct always_false : std::false_type {};
55 }
56
58 : _outFile( std::move(prevState._outFile) )
59 , _downloaded( prevState._downloaded )
60 , _partialHelper( std::move(prevState._partialHelper) )
61 { }
62
64 : _partialHelper( std::move(prevState._partialHelper) )
65 { }
66
68 : _outFile( std::move(prevState._outFile) )
69 , _partialHelper( std::move(prevState._partialHelper) )
70 , _downloaded( prevState._downloaded )
71 { }
72
74 : BasePrivate(p)
75 , _url ( std::move(url) )
76 , _targetFile ( std::move( targetFile) )
77 , _fMode ( std::move(fMode) )
78 , _headers( std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
79 { }
80
82 {
83 if ( _easyHandle ) {
84 //clean up for now, later we might reuse handles
85 curl_easy_cleanup( _easyHandle );
86 //reset in request but make sure the request was not enqueued again and got a new handle
87 _easyHandle = nullptr;
88 }
89 }
90
91 bool NetworkRequestPrivate::initialize( std::string &errBuf )
92 {
93 reset();
94
95 if ( _easyHandle )
96 //will reset to defaults but keep live connections, session ID and DNS caches
97 curl_easy_reset( _easyHandle );
98 else
99 _easyHandle = curl_easy_init();
100 return setupHandle ( errBuf );
101 }
102
103 bool NetworkRequestPrivate::setupHandle( std::string &errBuf )
104 {
106 curl_easy_setopt( _easyHandle, CURLOPT_ERRORBUFFER, this->_errorBuf.data() );
107
108 const std::string urlScheme = _url.getScheme();
109 if ( urlScheme == "http" || urlScheme == "https" )
111
112 try {
113
114 setCurlOption( CURLOPT_PRIVATE, this );
116 setCurlOption( CURLOPT_XFERINFODATA, this );
117 setCurlOption( CURLOPT_NOPROGRESS, 0L);
118 setCurlOption( CURLOPT_FAILONERROR, 1L);
119 setCurlOption( CURLOPT_NOSIGNAL, 1L);
120
121 std::string urlBuffer( _url.asString() );
122 setCurlOption( CURLOPT_URL, urlBuffer.c_str() );
123
124 setCurlOption( CURLOPT_WRITEFUNCTION, nwr_writeCallback );
125 setCurlOption( CURLOPT_WRITEDATA, this );
126
128 setCurlOption( CURLOPT_CONNECT_ONLY, 1L );
129 setCurlOption( CURLOPT_FRESH_CONNECT, 1L );
130 }
132 // instead of returning no data with NOBODY, we return
133 // little data, that works with broken servers, and
134 // works for ftp as well, because retrieving only headers
135 // ftp will return always OK code ?
136 // See http://curl.haxx.se/docs/knownbugs.html #58
138 setCurlOption( CURLOPT_NOBODY, 1L );
139 else
140 setCurlOption( CURLOPT_RANGE, "0-1" );
141 }
142
143 //make a local copy of the settings, so headers are not added multiple times
145
146 if ( _dispatcher ) {
147 locSet.setUserAgentString( _dispatcher->agentString().c_str() );
148
149 // add custom headers as configured (bsc#955801)
150 const auto &cHeaders = _dispatcher->hostSpecificHeaders();
151 if ( auto i = cHeaders.find(_url.getHost()); i != cHeaders.end() ) {
152 for ( const auto &[key, value] : i->second ) {
154 "%s: %s", key.c_str(), value.c_str() )
155 ));
156 }
157 }
158 }
159
160 locSet.addHeader("Pragma:");
161
164 {
165 case 4: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); break;
166 case 6: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); break;
167 default: break;
168 }
169
170 setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
171 setCurlOption( CURLOPT_HEADERDATA, this );
172
176 setCurlOption( CURLOPT_CONNECTTIMEOUT, locSet.connectTimeout() );
177 // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
178 // just in case curl does not trigger its progress callback frequently
179 // enough.
180 if ( locSet.timeout() )
181 {
182 setCurlOption( CURLOPT_TIMEOUT, 3600L );
183 }
184
185 if ( urlScheme == "https" )
186 {
187#if CURLVERSION_AT_LEAST(7,19,4)
188 // restrict following of redirections from https to https only
189 if ( _url.getHost() == "download.opensuse.org" ) {
190#if CURLVERSION_AT_LEAST(7,85,0)
191 setCurlOption( CURLOPT_REDIR_PROTOCOLS_STR, "http,https" );
192#else
193 setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
194#endif
195 } else {
196#if CURLVERSION_AT_LEAST(7,85,0)
197 setCurlOption( CURLOPT_REDIR_PROTOCOLS_STR, "https" );
198#else
199 setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
200#endif
201 }
202#endif // #if CURLVERSION_AT_LEAST(7,19,4)
203
204 if( locSet.verifyPeerEnabled() ||
205 locSet.verifyHostEnabled() )
206 {
207 setCurlOption(CURLOPT_CAPATH, locSet.certificateAuthoritiesPath().c_str());
208 }
209
210 if( ! locSet.clientCertificatePath().empty() )
211 {
212 setCurlOption(CURLOPT_SSLCERT, locSet.clientCertificatePath().c_str());
213 }
214 if( ! locSet.clientKeyPath().empty() )
215 {
216 setCurlOption(CURLOPT_SSLKEY, locSet.clientKeyPath().c_str());
217 }
218
219#ifdef CURLSSLOPT_ALLOW_BEAST
220 // see bnc#779177
221 setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
222#endif
223 setCurlOption(CURLOPT_SSL_VERIFYPEER, locSet.verifyPeerEnabled() ? 1L : 0L);
224 setCurlOption(CURLOPT_SSL_VERIFYHOST, locSet.verifyHostEnabled() ? 2L : 0L);
225 // bnc#903405 - POODLE: libzypp should only talk TLS
226 setCurlOption(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
227 }
228
229 // follow any Location: header that the server sends as part of
230 // an HTTP header (#113275)
231 setCurlOption( CURLOPT_FOLLOWLOCATION, 1L);
232 // 3 redirects seem to be too few in some cases (bnc #465532)
233 setCurlOption( CURLOPT_MAXREDIRS, 6L );
234
235 //set the user agent
236 setCurlOption(CURLOPT_USERAGENT, locSet.userAgentString().c_str() );
237
238
239 /*---------------------------------------------------------------*
240 CURLOPT_USERPWD: [user name]:[password]
241 Url::username/password -> CURLOPT_USERPWD
242 If not provided, anonymous FTP identification
243 *---------------------------------------------------------------*/
244 if ( locSet.userPassword().size() )
245 {
246 setCurlOption(CURLOPT_USERPWD, locSet.userPassword().c_str());
247 std::string use_auth = _settings.authType();
248 if (use_auth.empty())
249 use_auth = "digest,basic"; // our default
251 if( auth != CURLAUTH_NONE)
252 {
253 DBG << _easyHandle << " " << "Enabling HTTP authentication methods: " << use_auth
254 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
255 setCurlOption(CURLOPT_HTTPAUTH, auth);
256 }
257 }
258
259 if ( locSet.proxyEnabled() && ! locSet.proxy().empty() )
260 {
261 DBG << _easyHandle << " " << "Proxy: '" << locSet.proxy() << "'" << std::endl;
262 setCurlOption(CURLOPT_PROXY, locSet.proxy().c_str());
263 setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
264
265 /*---------------------------------------------------------------*
266 * CURLOPT_PROXYUSERPWD: [user name]:[password]
267 *
268 * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
269 * If not provided, $HOME/.curlrc is evaluated
270 *---------------------------------------------------------------*/
271
272 std::string proxyuserpwd = locSet.proxyUserPassword();
273
274 if ( proxyuserpwd.empty() )
275 {
277 zypp::media::CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
278 if ( curlconf.proxyuserpwd.empty() )
279 DBG << _easyHandle << " " << "Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
280 else
281 {
282 proxyuserpwd = curlconf.proxyuserpwd;
283 DBG << _easyHandle << " " << "Proxy: using proxy-user from ~/.curlrc" << std::endl;
284 }
285 }
286 else
287 {
288 DBG << _easyHandle << " " << _easyHandle << " " << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << std::endl;
289 }
290
291 if ( ! proxyuserpwd.empty() )
292 {
293 setCurlOption(CURLOPT_PROXYUSERPWD, ::internal::curlUnEscape( proxyuserpwd ).c_str());
294 }
295 }
296#if CURLVERSION_AT_LEAST(7,19,4)
297 else if ( locSet.proxy() == EXPLICITLY_NO_PROXY )
298 {
299 // Explicitly disabled in URL (see fillSettingsFromUrl()).
300 // This should also prevent libcurl from looking into the environment.
301 DBG << _easyHandle << " " << "Proxy: explicitly NOPROXY" << std::endl;
302 setCurlOption(CURLOPT_NOPROXY, "*");
303 }
304
305#endif
306 // else: Proxy: not explicitly set; libcurl may look into the environment
307
309 if ( locSet.minDownloadSpeed() != 0 )
310 {
311 setCurlOption(CURLOPT_LOW_SPEED_LIMIT, locSet.minDownloadSpeed());
312 // default to 10 seconds at low speed
313 setCurlOption(CURLOPT_LOW_SPEED_TIME, 60L);
314 }
315
316#if CURLVERSION_AT_LEAST(7,15,5)
317 if ( locSet.maxDownloadSpeed() != 0 )
318 setCurlOption(CURLOPT_MAX_RECV_SPEED_LARGE, locSet.maxDownloadSpeed());
319#endif
320
322 if ( zypp::str::strToBool( _url.getQueryParam( "cookies" ), true ) )
323 setCurlOption( CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
324 else
325 MIL << _easyHandle << " " << "No cookies requested" << std::endl;
326 setCurlOption(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
327
328#if CURLVERSION_AT_LEAST(7,18,0)
329 // bnc #306272
330 setCurlOption(CURLOPT_PROXY_TRANSFER_MODE, 1L );
331#endif
332
333 // Append settings custom headers to curl.
334 // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
335 for ( const auto &header : locSet.headers() ) {
336 if ( !z_func()->addRequestHeader( header.c_str() ) )
338 }
339
340 if ( _headers )
341 setCurlOption( CURLOPT_HTTPHEADER, _headers.get() );
342
343 // set up ranges if required
345 if ( _requestedRanges.size() ) {
346
347 std::sort( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &elem1, const auto &elem2 ){
348 return ( elem1.start < elem2.start );
349 });
350
351 CurlMultiPartHandler *helper = nullptr;
352 if ( auto initState = std::get_if<pending_t>(&_runningMode) ) {
353
355 initState->_partialHelper = std::make_unique<CurlMultiPartHandler>(
356 multiPartMode
359 , *this
360 );
361 helper = initState->_partialHelper.get();
362
363 } else if ( auto pendingState = std::get_if<prepareNextRangeBatch_t>(&_runningMode) ) {
364 helper = pendingState->_partialHelper.get();
365 } else {
366 errBuf = "Request is in invalid state to call setupHandle";
367 return false;
368 }
369
370 if ( !helper->prepare () ) {
371 errBuf = helper->lastErrorMessage ();
372 return false;
373 }
374 }
375 }
376
377 return true;
378
379 } catch ( const zypp::Exception &excp ) {
380 ZYPP_CAUGHT(excp);
381 errBuf = excp.asString();
382 }
383 return false;
384 }
385
387 {
388 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
389 if ( !rmode ) {
390 DBG << _easyHandle << "Can only create output file in running mode" << std::endl;
391 return false;
392 }
393 // if we have no open file create or open it
394 if ( !rmode->_outFile ) {
395 std::string openMode = "w+b";
397 openMode = "r+b";
398
399 rmode->_outFile = fopen( _targetFile.asString().c_str() , openMode.c_str() );
400
401 //if the file does not exist create a new one
402 if ( !rmode->_outFile && _fMode == NetworkRequest::WriteShared ) {
403 rmode->_outFile = fopen( _targetFile.asString().c_str() , "w+b" );
404 }
405
406 if ( !rmode->_outFile ) {
408 ,zypp::str::Format("Unable to open target file (%1%). Errno: (%2%:%3%)") % _targetFile.asString() % errno % strerr_cxx() );
409 return false;
410 }
411 }
412
413 return true;
414 }
415
417 {
418 // We can recover from RangeFail errors if the helper indicates it
419 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
420 if ( rmode->_partialHelper ) return rmode->_partialHelper->canRecover();
421 return false;
422 }
423
425 {
426 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
427 if ( !rmode ) {
428 errBuf = "NetworkRequestPrivate::prepareToContinue called in invalid state";
429 return false;
430 }
431
432 if ( rmode->_partialHelper && rmode->_partialHelper->hasMoreWork() ) {
433
434 bool hadRangeFail = rmode->_partialHelper->lastError () == NetworkRequestError::RangeFail;
435
436 _runningMode = prepareNextRangeBatch_t( std::move(std::get<running_t>( _runningMode )) );
437
438 auto &prepMode = std::get<prepareNextRangeBatch_t>(_runningMode);
439 if ( !prepMode._partialHelper->prepareToContinue() ) {
440 errBuf = prepMode._partialHelper->lastErrorMessage();
441 return false;
442 }
443
444 if ( hadRangeFail ) {
445 // we reset the handle to default values. We do this to not run into
446 // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
447 // we cancel a connection because of a range error to request a smaller batch.
448 // The error will still happen but much less frequently than without resetting the handle.
449 //
450 // Note: Even creating a new handle will NOT fix the issue
451 curl_easy_reset( _easyHandle );
452 }
453 if ( !setupHandle(errBuf))
454 return false;
455 return true;
456 }
457 errBuf = "Request has no more work";
458 return false;
459
460 }
461
463 {
464 // check if we have ranges that have never been requested
465 return std::any_of( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &range ){ return range._rangeState == CurlMultiPartHandler::Pending; });
466 }
467
469 {
470 bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>( _runningMode );
471 if ( isRangeContinuation ) {
472 MIL << _easyHandle << " " << "Continuing a previously started range batch." << std::endl;
473 _runningMode = running_t( std::move(std::get<prepareNextRangeBatch_t>( _runningMode )) );
474 } else {
475 _runningMode = running_t( std::move(std::get<pending_t>( _runningMode )) );
476 }
477
478 auto &m = std::get<running_t>( _runningMode );
479
480 if ( m._activityTimer ) {
481 DBG_MEDIA << _easyHandle << " Setting activity timeout to: " << _settings.timeout() << std::endl;
482 m._activityTimer->connect( &Timer::sigExpired, *this, &NetworkRequestPrivate::onActivityTimeout );
483 m._activityTimer->start( static_cast<uint64_t>( _settings.timeout() * 1000 ) );
484 }
485
486 if ( !isRangeContinuation )
487 _sigStarted.emit( *z_func() );
488 }
489
491 {
492 if ( std::holds_alternative<running_t>(_runningMode) ) {
493 auto &rmode = std::get<running_t>( _runningMode );
494 if ( rmode._partialHelper )
495 rmode._partialHelper->finalize();
496 }
497 }
498
500 {
501 finished_t resState;
502 resState._result = std::move(err);
503
504 if ( std::holds_alternative<running_t>(_runningMode) ) {
505
506 auto &rmode = std::get<running_t>( _runningMode );
507 resState._downloaded = rmode._downloaded;
508 resState._contentLenght = rmode._contentLenght;
509
511 if ( _requestedRanges.size( ) ) {
512 //we have a successful download lets see if we got everything we needed
513 if ( !rmode._partialHelper->verifyData() ){
514 NetworkRequestError::Type err = rmode._partialHelper->lastError();
515 resState._result = NetworkRequestErrorPrivate::customError( err, std::string(rmode._partialHelper->lastErrorMessage()) );
516 }
517
518 // if we have ranges we need to fill our digest from the full file
520 if ( fseek( rmode._outFile, 0, SEEK_SET ) != 0 ) {
521 resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Unable to set output file pointer." );
522 } else {
523 constexpr size_t bufSize = 4096;
524 char buf[bufSize];
525 while( auto cnt = fread(buf, 1, bufSize, rmode._outFile ) > 0 ) {
526 _fileVerification->_fileDigest.update(buf, cnt);
527 }
528 }
529 }
530 } // if ( _requestedRanges.size( ) )
531 }
532
533 // finally check the file digest if we have one
535 const UByteArray &calcSum = _fileVerification->_fileDigest.digestVector ();
536 const UByteArray &expSum = zypp::Digest::hexStringToUByteArray( _fileVerification->_fileChecksum.checksum () );
537 if ( calcSum != expSum ) {
540 , (zypp::str::Format("Invalid file checksum %1%, expected checksum %2%")
541 % _fileVerification->_fileDigest.digest()
542 % _fileVerification->_fileChecksum.checksum () ) );
543 }
544 }
545
546 rmode._outFile.reset();
547 }
548
549 _runningMode = std::move( resState );
550 _sigFinished.emit( *z_func(), std::get<finished_t>(_runningMode)._result );
551 }
552
554 {
556 _headers.reset( nullptr );
557 _errorBuf.fill( 0 );
559
560 if ( _fileVerification )
561 _fileVerification->_fileDigest.reset ();
562
563 std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( CurlMultiPartHandler::Range &range ) {
564 range.restart();
565 });
566 }
567
569 {
570 MIL_MEDIA << _easyHandle << " Request timeout interval: " << t.interval()<< " remaining: " << t.remaining() << std::endl;
571 std::map<std::string, boost::any> extraInfo;
572 extraInfo.insert( {"requestUrl", _url } );
573 extraInfo.insert( {"filepath", _targetFile } );
574 _dispatcher->cancel( *z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::Timeout, "Download timed out", std::move(extraInfo) ) );
575 }
576
578 {
579 return std::string( _errorBuf.data() );
580 }
581
583 {
584 if ( std::holds_alternative<running_t>( _runningMode ) ){
585 auto &rmode = std::get<running_t>( _runningMode );
586 if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
587 rmode._activityTimer->start();
588 }
589 }
590
591 int NetworkRequestPrivate::curlProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
592 {
593 if ( !clientp )
594 return CURLE_OK;
595 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( clientp );
596
597 if ( !std::holds_alternative<running_t>(that->_runningMode) ){
598 DBG << that->_easyHandle << " " << "Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
599 return -1;
600 }
601
602 auto &rmode = std::get<running_t>( that->_runningMode );
603
604 //reset the timer
605 that->resetActivityTimer();
606
607 rmode._isInCallback = true;
608 if ( rmode._lastProgressNow != dlnow ) {
609 rmode._lastProgressNow = dlnow;
610 that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
611 }
612 rmode._isInCallback = false;
613
614 return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
615 }
616
617 size_t NetworkRequestPrivate::headerfunction( char *ptr, size_t bytes )
618 {
619 //it is valid to call this function with no data to write, just return OK
620 if ( bytes == 0)
621 return 0;
622
624
626
627 std::string_view hdr( ptr, bytes );
628
629 hdr.remove_prefix( std::min( hdr.find_first_not_of(" \t\r\n"), hdr.size() ) );
630 const auto lastNonWhitespace = hdr.find_last_not_of(" \t\r\n");
631 if ( lastNonWhitespace != hdr.npos )
632 hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
633 else
634 hdr = std::string_view();
635
636 auto &rmode = std::get<running_t>( _runningMode );
637 if ( !hdr.size() ) {
638 return ( bytes );
639 }
640 if ( _expectedFileSize && rmode._partialHelper ) {
641 const auto &repSize = rmode._partialHelper->reportedFileSize ();
642 if ( repSize && repSize != _expectedFileSize ) {
643 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Reported downloaded data length is not what was expected." );
644 return 0;
645 }
646 }
647 if ( zypp::strv::hasPrefixCI( hdr, "HTTP/" ) ) {
648
649 long statuscode = 0;
650 (void)curl_easy_getinfo( _easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
651
652 // if we have a status 204 we need to create a empty file
653 if( statuscode == 204 && !( _options & NetworkRequest::ConnectionTest ) && !( _options & NetworkRequest::HeadRequest ) )
655
656 } else if ( zypp::strv::hasPrefixCI( hdr, "Location:" ) ) {
657 _lastRedirect = hdr.substr( 9 );
658 DBG << _easyHandle << " " << "redirecting to " << _lastRedirect << std::endl;
659
660 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Length:") ) {
661 auto lenStr = str::trim( hdr.substr( 15 ), zypp::str::TRIM );
662 auto str = std::string ( lenStr.data(), lenStr.length() );
663 auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>( str.data() );
664 if ( len > 0 ) {
665 DBG << _easyHandle << " " << "Got Content-Length Header: " << len << std::endl;
666 rmode._contentLenght = zypp::ByteCount(len, zypp::ByteCount::B);
667 }
668 }
669 }
670
671 return bytes;
672 }
673
674 size_t NetworkRequestPrivate::writefunction( char *data, std::optional<off_t> offset, size_t max )
675 {
677
678 //it is valid to call this function with no data to write, just return OK
679 if ( max == 0)
680 return 0;
681
682 //in case of a HEAD request, we do not write anything
684 return ( max );
685 }
686
687 auto &rmode = std::get<running_t>( _runningMode );
688
689 // if we have no open file create or open it
690 if ( !assertOutputFile() )
691 return 0;
692
693 if ( offset ) {
694 // seek to the given offset
695 if ( fseek( rmode._outFile, *offset, SEEK_SET ) != 0 ) {
697 "Unable to set output file pointer." );
698 return 0;
699 }
700 rmode._currentFileOffset = *offset;
701 }
702
703 if ( _expectedFileSize && rmode._partialHelper ) {
704 const auto &repSize = rmode._partialHelper->reportedFileSize ();
705 if ( repSize && repSize != _expectedFileSize ) {
706 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Reported downloaded data length is not what was expected." );
707 return 0;
708 }
709 }
710
711 //make sure we do not write after the expected file size
712 if ( _expectedFileSize && _expectedFileSize <= static_cast<zypp::ByteCount::SizeType>( rmode._currentFileOffset + max) ) {
713 rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Downloaded data exceeds expected length." );
714 return 0;
715 }
716
717 auto written = fwrite( data, 1, max, rmode._outFile );
718 if ( written == 0 )
719 return 0;
720
721 // if we are not downloading in ranges, we can update the file digest on the fly if we have one
722 if ( !rmode._partialHelper && _fileVerification ) {
723 _fileVerification->_fileDigest.update( data, written );
724 }
725
726 rmode._currentFileOffset += written;
727
728 // count the number of real bytes we have downloaded so far
729 rmode._downloaded += written;
730 _sigBytesDownloaded.emit( *z_func(), rmode._downloaded );
731
732 return written;
733 }
734
736
737 NetworkRequest::NetworkRequest(zyppng::Url url, zypp::filesystem::Pathname targetFile, zyppng::NetworkRequest::FileMode fMode)
738 : Base ( *new NetworkRequestPrivate( std::move(url), std::move(targetFile), std::move(fMode), *this ) )
739 {
740 }
741
743 {
744 Z_D();
745
746 if ( d->_dispatcher )
747 d->_dispatcher->cancel( *this, "Request destroyed while still running" );
748 }
749
751 {
752 d_func()->_expectedFileSize = std::move( expectedFileSize );
753 }
754
755 void NetworkRequest::setPriority( NetworkRequest::Priority prio, bool triggerReschedule )
756 {
757 Z_D();
758 d->_priority = prio;
759 if ( state() == Pending && triggerReschedule && d->_dispatcher )
760 d->_dispatcher->reschedule();
761 }
762
764 {
765 return d_func()->_priority;
766 }
767
768 void NetworkRequest::setOptions( Options opt )
769 {
770 d_func()->_options = opt;
771 }
772
773 NetworkRequest::Options NetworkRequest::options() const
774 {
775 return d_func()->_options;
776 }
777
778 void NetworkRequest::addRequestRange(size_t start, size_t len, std::optional<zypp::Digest> &&digest, CheckSumBytes expectedChkSum , std::any userData, std::optional<size_t> digestCompareLen, std::optional<size_t> chksumpad )
779 {
780 Z_D();
781 if ( state() == Running )
782 return;
783
784 d->_requestedRanges.push_back( Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
785 }
786
788 {
789 Z_D();
790 if ( state() == Running )
791 return;
792
793 d->_requestedRanges.push_back( std::move(range) );
794 auto &rng = d->_requestedRanges.back();
795 rng._rangeState = CurlMultiPartHandler::Pending;
796 rng.bytesWritten = 0;
797 if ( rng._digest )
798 rng._digest->reset();
799 }
800
802 {
803 Z_D();
804 if ( state() == Running )
805 return false;
806
807 zypp::Digest fDig;
808 if ( !fDig.create( expected.type () ) )
809 return false;
810
811 d->_fileVerification = NetworkRequestPrivate::FileVerifyInfo{
812 ._fileDigest = std::move(fDig),
813 ._fileChecksum = expected
814 };
815 return true;
816 }
817
819 {
820 Z_D();
821 if ( state() == Running )
822 return;
823 d->_requestedRanges.clear();
824 }
825
826 std::vector<NetworkRequest::Range> NetworkRequest::failedRanges() const
827 {
828 const auto mystate = state();
829 if ( mystate != Finished && mystate != Error )
830 return {};
831
832 Z_D();
833
834 std::vector<Range> failed;
835 for ( auto &r : d->_requestedRanges ) {
836 if ( r._rangeState != CurlMultiPartHandler::Finished ) {
837 failed.push_back( r.clone() );
838 }
839 }
840 return failed;
841 }
842
843 const std::vector<NetworkRequest::Range> &NetworkRequest::requestedRanges() const
844 {
845 return d_func()->_requestedRanges;
846 }
847
848 const std::string &NetworkRequest::lastRedirectInfo() const
849 {
850 return d_func()->_lastRedirect;
851 }
852
854 {
855 return d_func()->_easyHandle;
856 }
857
858 std::optional<zyppng::NetworkRequest::Timings> NetworkRequest::timings() const
859 {
860 const auto myerr = error();
861 const auto mystate = state();
862 if ( mystate != Finished )
863 return {};
864
865 Timings t;
866
867 auto getMeasurement = [ this ]( const CURLINFO info, std::chrono::microseconds &target ){
868 using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
869 double val = 0;
870 const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
871 if ( CURLE_OK == res ) {
872 target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
873 }
874 };
875
876 getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.namelookup );
877 getMeasurement( CURLINFO_CONNECT_TIME, t.connect);
878 getMeasurement( CURLINFO_APPCONNECT_TIME, t.appconnect);
879 getMeasurement( CURLINFO_PRETRANSFER_TIME , t.pretransfer);
880 getMeasurement( CURLINFO_TOTAL_TIME, t.total);
881 getMeasurement( CURLINFO_REDIRECT_TIME, t.redirect);
882
883 return t;
884 }
885
886 std::vector<char> NetworkRequest::peekData( off_t offset, size_t count ) const
887 {
888 Z_D();
889
890 if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
891 return {};
892
893 const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
894 return zypp::io::peek_data_fd( rmode._outFile, offset, count );
895 }
896
898 {
899 return d_func()->_url;
900 }
901
902 void NetworkRequest::setUrl(const Url &url)
903 {
904 Z_D();
906 return;
907
908 d->_url = url;
909 }
910
912 {
913 return d_func()->_targetFile;
914 }
915
917 {
918 Z_D();
920 return;
921 d->_targetFile = path;
922 }
923
925 {
926 return d_func()->_fMode;
927 }
928
930 {
931 Z_D();
933 return;
934 d->_fMode = std::move( mode );
935 }
936
937 std::string NetworkRequest::contentType() const
938 {
939 char *ptr = NULL;
940 if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
941 return std::string(ptr);
942 return std::string();
943 }
944
946 {
947 return std::visit([](auto& arg) -> zypp::ByteCount {
948 using T = std::decay_t<decltype(arg)>;
949 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
950 return zypp::ByteCount(0);
951 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
952 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
953 return arg._contentLenght;
954 else
955 static_assert(always_false<T>::value, "Unhandled state type");
956 }, d_func()->_runningMode);
957 }
958
960 {
961 return std::visit([](auto& arg) -> zypp::ByteCount {
962 using T = std::decay_t<decltype(arg)>;
963 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
964 return zypp::ByteCount();
965 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
966 || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
967 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
968 return arg._downloaded;
969 else
970 static_assert(always_false<T>::value, "Unhandled state type");
971 }, d_func()->_runningMode);
972 }
973
975 {
976 return d_func()->_settings;
977 }
978
980 {
981 return std::visit([this](auto& arg) {
982 using T = std::decay_t<decltype(arg)>;
983 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
984 return Pending;
985 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
986 return Running;
987 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
988 if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode )._result.isError() )
989 return Error;
990 else
991 return Finished;
992 }
993 else
994 static_assert(always_false<T>::value, "Unhandled state type");
995 }, d_func()->_runningMode);
996 }
997
999 {
1000 const auto s = state();
1001 if ( s != Error && s != Finished )
1002 return NetworkRequestError();
1003 return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode)._result;
1004 }
1005
1007 {
1008 if ( !hasError() )
1009 return std::string();
1010
1011 return error().nativeErrorString();
1012 }
1013
1015 {
1016 return error().isError();
1017 }
1018
1019 bool NetworkRequest::addRequestHeader( const std::string &header )
1020 {
1021 Z_D();
1022
1023 curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : nullptr, header.c_str() );
1024 if ( !res )
1025 return false;
1026
1027 if ( !d->_headers )
1028 d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1029
1030 return true;
1031 }
1032
1034 {
1035 return d_func()->_sigStarted;
1036 }
1037
1039 {
1040 return d_func()->_sigBytesDownloaded;
1041 }
1042
1043 SignalProxy<void (NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> NetworkRequest::sigProgress()
1044 {
1045 return d_func()->_sigProgress;
1046 }
1047
1049 {
1050 return d_func()->_sigFinished;
1051 }
1052
1053}
ZYppCommitResult & _result
Definition: TargetImpl.cc:1609
Store and operate with byte count.
Definition: ByteCount.h:31
static const Unit B
1 Byte
Definition: ByteCount.h:42
Unit::ValueType SizeType
Definition: ByteCount.h:37
std::string type() const
Definition: CheckSum.cc:167
Compute Message Digests (MD5, SHA1 etc)
Definition: Digest.h:38
bool create(const std::string &name)
initialize creation of a new message digest
Definition: Digest.cc:195
Base class for Exception.
Definition: Exception.h:146
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:75
const char * c_str() const
String representation.
Definition: Pathname.h:110
const std::string & asString() const
String representation.
Definition: Pathname.h:91
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
Holds transfer setting.
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
long connectTimeout() const
connection timeout
const std::string & authType() const
get the allowed authentication types
long timeout() const
transfer timeout
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::string userPassword() const
returns the user and password as a user:pass string
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
const Headers & headers() const
returns a list of all added headers (trimmed)
const std::string & proxy() const
proxy host
const Pathname & clientKeyPath() const
SSL client key file.
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & userAgentString() const
user agent string (trimmed)
bool headRequestsAllowed() const
whether HEAD requests are allowed
bool proxyEnabled() const
proxy is enabled
const std::string & proxyUsername() const
proxy auth username
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
The CurlMultiPartHandler class.
const std::string & lastErrorMessage() const
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
The NetworkRequestError class Represents a error that occured in.
Type type() const
type Returns the type of the error
std::string nativeErrorString() const
bool isError() const
isError Will return true if this is a actual error
size_t headerfunction(char *ptr, size_t bytes) override
Definition: request.cc:617
std::optional< FileVerifyInfo > _fileVerification
The digest for the full file.
Definition: request_p.h:116
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
const std::string _currentCookieFile
Definition: request_p.h:122
zypp::Pathname _targetFile
Definition: request_p.h:106
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
Definition: request_p.h:129
NetworkRequestDispatcher * _dispatcher
Definition: request_p.h:125
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
Definition: request_p.h:110
size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes) override
Definition: request.cc:674
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Definition: request.cc:591
std::string errorMessage() const
Definition: request.cc:577
Signal< void(NetworkRequest &req)> _sigStarted
Definition: request_p.h:128
NetworkRequest::FileMode _fMode
Definition: request_p.h:118
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
Definition: request_p.h:186
bool initialize(std::string &errBuf)
Definition: request.cc:91
void onActivityTimeout(Timer &)
Definition: request.cc:568
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
Definition: request_p.h:130
std::string _lastRedirect
to log/report redirections
Definition: request_p.h:121
NetworkRequest::Options _options
Definition: request_p.h:108
bool prepareToContinue(std::string &errBuf)
Definition: request.cc:424
void setResult(NetworkRequestError &&err)
Definition: request.cc:499
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
Definition: request_p.h:94
virtual ~NetworkRequestPrivate()
Definition: request.cc:81
bool setupHandle(std::string &errBuf)
Definition: request.cc:103
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
Definition: request.cc:73
TransferSettings _settings
Definition: request_p.h:107
void setCurlOption(CURLoption opt, T data)
Definition: request_p.h:97
zypp::ByteCount _expectedFileSize
Definition: request_p.h:109
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
Definition: request_p.h:131
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
Definition: request_p.h:138
bool setExpectedFileChecksum(const zypp::CheckSum &expected)
Definition: request.cc:801
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size,...
Definition: request.cc:945
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
Definition: request.cc:911
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
Definition: request.cc:959
void setUrl(const Url &url)
This will change the URL of the request.
Definition: request.cc:902
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
Definition: request.cc:750
virtual ~NetworkRequest()
Definition: request.cc:742
void setPriority(Priority prio, bool triggerReschedule=true)
Definition: request.cc:755
std::vector< char > peekData(off_t offset, size_t count) const
Definition: request.cc:886
std::string contentType() const
Returns the content type as reported from the server.
Definition: request.cc:937
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
Definition: request.cc:929
bool addRequestHeader(const std::string &header)
Definition: request.cc:1019
void setOptions(Options opt)
Definition: request.cc:768
FileMode fileOpenMode() const
Returns the currently configured file open mode.
Definition: request.cc:924
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1014
State state() const
Returns the current state the HttpDownloadRequest is in.
Definition: request.cc:979
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1048
UByteArray CheckSumBytes
Definition: request.h:49
Options options() const
Definition: request.cc:773
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition: request.cc:1038
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
Definition: request.cc:858
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:1006
Priority priority() const
Definition: request.cc:763
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:998
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
Definition: request.cc:916
void * nativeHandle() const
Definition: request.cc:853
void addRequestRange(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
Definition: request.cc:778
std::vector< Range > failedRanges() const
Definition: request.cc:826
const std::vector< Range > & requestedRanges() const
Definition: request.cc:843
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition: request.cc:1033
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
Definition: request.cc:1043
TransferSettings & transferSettings()
Definition: request.cc:974
const std::string & lastRedirectInfo() const
Definition: request.cc:848
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
#define MIL_MEDIA
Definition: mediadebug_p.h:29
#define DBG_MEDIA
Definition: mediadebug_p.h:28
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
Definition: Arch.h:364
String related utilities and Regular expression matching.
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
constexpr bool always_false
Definition: PathInfo.cc:544
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
Definition: IOTools.cc:167
@ TRIM
Definition: String.h:500
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
ZYPP_IMPL_PRIVATE(Provide)
Structure holding values of curlrc options.
Definition: curlconfig.h:27
std::string proxyuserpwd
Definition: curlconfig.h:49
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
Convenient building of std::string with boost::format.
Definition: String.h:253
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
running_t(pending_t &&prevState)
Definition: request.cc:63
std::chrono::microseconds appconnect
Definition: request.h:81
std::chrono::microseconds redirect
Definition: request.h:84
std::chrono::microseconds pretransfer
Definition: request.h:82
std::chrono::microseconds total
Definition: request.h:83
std::chrono::microseconds namelookup
Definition: request.h:79
std::chrono::microseconds connect
Definition: request.h:80
#define nullptr
Definition: Easy.h:55
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:436
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
#define DBG
Definition: Logger.h:95
#define MIL
Definition: Logger.h:96