libzypp 17.31.25
Downloader.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
9
10#include <fstream>
11#include <solv/solvversion.h>
12#include <zypp/base/String.h>
13#include <zypp/base/LogTools.h>
14#include <zypp/base/Function.h>
15#include <zypp/ZConfig.h>
16
17#include "Downloader.h"
20#include <zypp-core/base/UserRequestException>
23
24using namespace zypp::xml;
25using namespace zypp::parser::yum;
26
27namespace zypp
28{
29 namespace env {
31 {
32 static bool val = [](){
33 const char * env = getenv("ZYPP_REPOMD_WITH_OTHER");
34 return( env && zypp::str::strToBool( env, true ) );
35 }();
36 return val;
37 }
38
40 {
41 static bool val = [](){
42 const char * env = getenv("ZYPP_REPOMD_WITH_FILELISTS");
43 return( env && zypp::str::strToBool( env, true ) );
44 }();
45 return val;
46 }
47 } // namespace env
48
49namespace repo
50{
51namespace yum
52{
54 namespace
55 {
56 inline OnMediaLocation loc_with_path_prefix( OnMediaLocation loc_r, const Pathname & prefix_r )
57 {
58 if ( ! prefix_r.empty() && prefix_r != "/" )
59 loc_r.changeFilename( prefix_r / loc_r.filename() );
60 return loc_r;
61 }
62
63 // search old repository file to run the delta algorithm on
64 Pathname search_deltafile( const Pathname & dir, const Pathname & file )
65 {
67 if ( ! PathInfo(dir).isDir() )
68 return deltafile;
69
70 // Strip the checksum preceding the file stem so we can look for an
71 // old *-primary.xml which may contain some reusable blocks.
72 std::string base { file.basename() };
73 size_t hypoff = base.find( "-" );
74 if ( hypoff != std::string::npos )
75 base.replace( 0, hypoff + 1, "" );
76
77 std::list<std::string> retlist;
78 if ( ! filesystem::readdir( retlist, dir, false ) )
79 {
80 for ( const auto & fn : retlist )
81 {
82 if ( str::endsWith( fn, base ) )
83 deltafile = fn;
84 }
85 }
86 if ( !deltafile.empty() )
87 return dir/deltafile;
88
89 return deltafile;
90 }
91 } // namespace
93
108 {
111
112 Impl( Downloader & downloader_r, MediaSetAccess & media_r, const Pathname & destDir_r )
113 : _downloader { downloader_r }
114 , _media { media_r }
115 , _destDir { destDir_r }
116 {
117 addWantedLocale( ZConfig::instance().textLocale() );
118 for ( const Locale & it : ZConfig::instance().repoRefreshLocales() )
119 addWantedLocale( it );
120 }
121
126 bool operator()( const OnMediaLocation & loc_r, const std::string & typestr_r )
127 {
128 if ( str::endsWith( typestr_r, "_db" ) )
129 return true; // skip sqlitedb
130
131 bool zchk { str::endsWith( typestr_r, "_zck" ) };
132#if defined(LIBSOLVEXT_FEATURE_ZCHUNK_COMPRESSION)
133 const std::string & basetype { zchk ? typestr_r.substr( 0, typestr_r.size()-4 ) : typestr_r };
134#else
135 if ( zchk )
136 return true; // skip zchunk if not supported by libsolv
137 const std::string & basetype { typestr_r };
138#endif
139
140 // filter well known resource types
141 if ( basetype == "other" && not env::ZYPP_REPOMD_WITH_OTHER() )
142 return true; // skip it
143
144 if ( basetype == "filelists" && not env::ZYPP_REPOMD_WITH_FILELISTS() )
145 return true; // skip it
146
147 // filter localized susedata
148 if ( str::startsWith( basetype, "susedata." ) )
149 {
150 // susedata.LANG
151 if ( ! wantLocale( Locale(basetype.c_str()+9) ) )
152 return true; // skip it
153 }
154
155 // may take it... (prefer zchnk)
156 if ( zchk || !_wantedFiles.count( basetype ) )
157 _wantedFiles[basetype] = loc_r;
158
159 return true;
160 }
161
162 void finalize()
163 {
164 // schedule fileS for download
165 for ( const auto & el : _wantedFiles )
166 {
167 const OnMediaLocation & loc { el.second };
168 const OnMediaLocation & loc_with_path { loc_with_path_prefix( loc, _downloader.repoInfo().path() ) };
169 _downloader.enqueueDigested( OnMediaLocation(loc_with_path).setDeltafile( search_deltafile( deltaDir()/"repodata", loc.filename() ) ), FileChecker() );
170 }
171 }
172
173 private:
174 const Pathname & deltaDir() const
175 { return _downloader._deltaDir; }
176
177 bool wantLocale( const Locale & locale_r ) const
178 { return _wantedLocales.count( locale_r ); }
179
180 void addWantedLocale( Locale locale_r )
181 {
182 while ( locale_r )
183 {
184 _wantedLocales.insert( locale_r );
185 locale_r = locale_r.fallback();
186 }
187 }
188
189 private:
193
195 std::map<std::string,OnMediaLocation> _wantedFiles;
196 };
197
199 //
200 // class Downloader
201 //
203
204 Downloader::Downloader( const RepoInfo & info_r, const Pathname & deltaDir_r )
205 : repo::Downloader { info_r}
206 , _deltaDir { deltaDir_r }
207 {}
208
209 void Downloader::download( MediaSetAccess & media_r, const Pathname & destDir_r, const ProgressData::ReceiverFnc & progress_r )
210 {
211 downloadMediaInfo( destDir_r, media_r );
212
213 Pathname masterIndex { repoInfo().path() / "/repodata/repomd.xml" };
214 defaultDownloadMasterIndex( media_r, destDir_r, masterIndex );
215
216 //enable precache
217 setMediaSetAccess( media_r );
218
219 // setup parser
220 Impl pimpl( *this, media_r, destDir_r );
221 RepomdFileReader( destDir_r / masterIndex, std::ref(pimpl) );
222 pimpl.finalize();
223
224 // ready, go!
225 start( destDir_r );
226 }
227
229 {
230 const auto & ri = repoInfo();
231 RepoStatus ret { media_r.provideOptionalFile( ri.path() / "/repodata/repomd.xml" ) };
232 if ( !ret.empty() && ri.requireStatusWithMediaFile() ) // else: mandatory master index is missing
233 ret = ret && RepoStatus( media_r.provideOptionalFile( "/media.1/media" ) );
234 // else: mandatory master index is missing -> stay empty
235 return ret;
236 }
237} // namespace yum
238} // namespace repo
239} // namespace zypp
Pathname deltafile
Interface of repomd.xml file reader.
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:908
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:861
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition: Fetcher.cc:903
'Language[_Country]' codes.
Definition: Locale.h:50
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
Media access layer responsible for handling files distributed on a set of media with media change and...
Pathname provideOptionalFile(const Pathname &file, unsigned media_nr=1)
Provides an optional file from media media_nr.
Describes a resource file located on a medium.
const Pathname & filename() const
The path to the resource on the medium.
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: progressdata.h:140
What is known about a repository.
Definition: RepoInfo.h:72
Pathname path() const
Repository path.
Definition: RepoInfo.cc:722
Track changing files or directories.
Definition: RepoStatus.h:41
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
Wrapper class for stat/lstat.
Definition: PathInfo.h:221
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
Reads through a repomd.xml file and collects type, location, checksum and other data about metadata f...
const RepoInfo & repoInfo() const
Definition: Downloader.h:63
void defaultDownloadMasterIndex(MediaSetAccess &media_r, const Pathname &destdir_r, const Pathname &masterIndex_r)
Common workflow downloading a (signed) master index file.
Definition: Downloader.cc:152
Downloader()
Constructor.
Definition: Downloader.cc:129
Downloader for YUM (rpm-nmd) repositories Encapsulates all the knowledge of which files have to be do...
Definition: Downloader.h:41
void download(MediaSetAccess &media_r, const Pathname &destDir_r, const ProgressData::ReceiverFnc &progress_r=ProgressData::ReceiverFnc()) override
Download metadata to a local directory.
Definition: Downloader.cc:209
RepoStatus status(MediaSetAccess &media_r) override
Status of the remote repository.
Definition: Downloader.cc:228
bool ZYPP_REPOMD_WITH_FILELISTS()
Definition: Downloader.cc:39
bool ZYPP_REPOMD_WITH_OTHER()
Definition: Downloader.cc:30
int readdir(std::list< std::string > &retlist_r, const Pathname &path_r, bool dots_r)
Return content of directory via retlist.
Definition: PathInfo.cc:605
void downloadMediaInfo(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progressrcv)
Downloads the media info (/media.1) to a local directory.
bool startsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasPrefix
Definition: String.h:1085
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1092
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
std::unordered_set< Locale > LocaleSet
Definition: Locale.h:28
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:38
Helper filtering the files offered by a RepomdFileReader.
Definition: Downloader.cc:108
LocaleSet _wantedLocales
Locales do download.
Definition: Downloader.cc:194
const Pathname & deltaDir() const
Definition: Downloader.cc:174
bool wantLocale(const Locale &locale_r) const
Definition: Downloader.cc:177
void addWantedLocale(Locale locale_r)
Definition: Downloader.cc:180
std::map< std::string, OnMediaLocation > _wantedFiles
Definition: Downloader.cc:195
bool operator()(const OnMediaLocation &loc_r, const std::string &typestr_r)
The callback invoked by the RepomdFileReader.
Definition: Downloader.cc:126
Impl(Downloader &downloader_r, MediaSetAccess &media_r, const Pathname &destDir_r)
Definition: Downloader.cc:112