libzypp  17.31.31
Downloader.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
9 
10 #include <fstream>
11 #include <solv/solvversion.h>
12 #include <zypp/base/String.h>
13 #include <zypp/base/LogTools.h>
14 #include <zypp/base/Function.h>
15 #include <zypp/ZConfig.h>
16 
17 #include "Downloader.h"
20 #include <zypp-core/base/UserRequestException>
21 #include <zypp/parser/xml/Reader.h>
23 
24 using namespace zypp::xml;
25 using namespace zypp::parser::yum;
26 
27 namespace zypp
28 {
29  namespace env {
31  {
32  static bool val = [](){
33  const char * env = getenv("ZYPP_REPOMD_WITH_OTHER");
34  return( env && zypp::str::strToBool( env, true ) );
35  }();
36  return val;
37  }
38 
40  {
41  static bool val = [](){
42  const char * env = getenv("ZYPP_REPOMD_WITH_FILELISTS");
43  return( env && zypp::str::strToBool( env, true ) );
44  }();
45  return val;
46  }
47  } // namespace env
48 
49 namespace repo
50 {
51 namespace yum
52 {
54  namespace
55  {
56  inline OnMediaLocation loc_with_path_prefix( OnMediaLocation loc_r, const Pathname & prefix_r )
57  {
58  if ( ! prefix_r.empty() && prefix_r != "/" )
59  loc_r.changeFilename( prefix_r / loc_r.filename() );
60  return loc_r;
61  }
62 
63  // search old repository file to run the delta algorithm on
64  Pathname search_deltafile( const Pathname & dir, const Pathname & file )
65  {
67  if ( ! PathInfo(dir).isDir() )
68  return deltafile;
69 
70  // Strip the checksum preceding the file stem so we can look for an
71  // old *-primary.xml which may contain some reusable blocks.
72  std::string base { file.basename() };
73  size_t hypoff = base.find( "-" );
74  if ( hypoff != std::string::npos )
75  base.replace( 0, hypoff + 1, "" );
76 
77  std::list<std::string> retlist;
78  if ( ! filesystem::readdir( retlist, dir, false ) )
79  {
80  for ( const auto & fn : retlist )
81  {
82  if ( str::endsWith( fn, base ) )
83  deltafile = fn;
84  }
85  }
86  if ( !deltafile.empty() )
87  return dir/deltafile;
88 
89  return deltafile;
90  }
91  } // namespace
93 
108  {
109  NON_COPYABLE( Impl );
110  NON_MOVABLE( Impl );
111 
112  Impl( Downloader & downloader_r, MediaSetAccess & media_r, const Pathname & destDir_r )
113  : _downloader { downloader_r }
114  , _media { media_r }
115  , _destDir { destDir_r }
116  {
117  addWantedLocale( ZConfig::instance().textLocale() );
118  for ( const Locale & it : ZConfig::instance().repoRefreshLocales() )
119  addWantedLocale( it );
120  }
121 
126  bool operator()( const OnMediaLocation & loc_r, const std::string & typestr_r )
127  {
128  if ( str::endsWith( typestr_r, "_db" ) )
129  return true; // skip sqlitedb
130 
131  bool zchk { str::endsWith( typestr_r, "_zck" ) };
132 #if defined(LIBSOLVEXT_FEATURE_ZCHUNK_COMPRESSION)
133  const std::string & basetype { zchk ? typestr_r.substr( 0, typestr_r.size()-4 ) : typestr_r };
134 #else
135  if ( zchk )
136  return true; // skip zchunk if not supported by libsolv
137  const std::string & basetype { typestr_r };
138 #endif
139 
140  // filter well known resource types
141  if ( basetype == "other" && not env::ZYPP_REPOMD_WITH_OTHER() )
142  return true; // skip it
143 
144  if ( basetype == "filelists" && not env::ZYPP_REPOMD_WITH_FILELISTS() )
145  return true; // skip it
146 
147  // filter localized susedata
148  if ( str::startsWith( basetype, "susedata." ) )
149  {
150  // susedata.LANG
151  if ( ! wantLocale( Locale(basetype.c_str()+9) ) )
152  return true; // skip it
153  }
154 
155  // may take it... (prefer zchnk)
156  if ( zchk || !_wantedFiles.count( basetype ) )
157  _wantedFiles[basetype] = loc_r;
158 
159  return true;
160  }
161 
162  void finalize()
163  {
164  // schedule fileS for download
165  for ( const auto & el : _wantedFiles )
166  {
167  const OnMediaLocation & loc { el.second };
168  const OnMediaLocation & loc_with_path { loc_with_path_prefix( loc, _downloader.repoInfo().path() ) };
169  _downloader.enqueueDigested( OnMediaLocation(loc_with_path).setDeltafile( search_deltafile( deltaDir()/"repodata", loc.filename() ) ), FileChecker() );
170  }
171  }
172 
173  private:
174  const Pathname & deltaDir() const
175  { return _downloader._deltaDir; }
176 
177  bool wantLocale( const Locale & locale_r ) const
178  { return _wantedLocales.count( locale_r ); }
179 
180  void addWantedLocale( Locale locale_r )
181  {
182  while ( locale_r )
183  {
184  _wantedLocales.insert( locale_r );
185  locale_r = locale_r.fallback();
186  }
187  }
188 
189  private:
193 
195  std::map<std::string,OnMediaLocation> _wantedFiles;
196  };
197 
199  //
200  // class Downloader
201  //
203 
204  Downloader::Downloader( const RepoInfo & info_r, const Pathname & deltaDir_r )
205  : repo::Downloader { info_r}
206  , _deltaDir { deltaDir_r }
207  {}
208 
209  void Downloader::download( MediaSetAccess & media_r, const Pathname & destDir_r, const ProgressData::ReceiverFnc & progress_r )
210  {
211  downloadMediaInfo( destDir_r, media_r );
212 
213  Pathname masterIndex { repoInfo().path() / "/repodata/repomd.xml" };
214  defaultDownloadMasterIndex( media_r, destDir_r, masterIndex );
215 
216  //enable precache
217  setMediaSetAccess( media_r );
218 
219  // setup parser
220  Impl pimpl( *this, media_r, destDir_r );
221  RepomdFileReader( destDir_r / masterIndex, std::ref(pimpl) );
222  pimpl.finalize();
223 
224  // ready, go!
225  start( destDir_r );
226  }
227 
229  {
230  const auto & ri = repoInfo();
231  RepoStatus ret { media_r.provideOptionalFile( ri.path() / "/repodata/repomd.xml" ) };
232  if ( !ret.empty() && ri.requireStatusWithMediaFile() ) // else: mandatory master index is missing
233  ret = ret && RepoStatus( media_r.provideOptionalFile( "/media.1/media" ) );
234  // else: mandatory master index is missing -> stay empty
235  return ret;
236  }
237 } // namespace yum
238 } // namespace repo
239 } // namespace zypp
Pathname path() const
Repository path.
Definition: RepoInfo.cc:722
void addWantedLocale(Locale locale_r)
Definition: Downloader.cc:180
void defaultDownloadMasterIndex(MediaSetAccess &media_r, const Pathname &destdir_r, const Pathname &masterIndex_r)
Common workflow downloading a (signed) master index file.
Definition: Downloader.cc:152
Pathname deltafile
bool ZYPP_REPOMD_WITH_OTHER()
Definition: Downloader.cc:30
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
Pathname provideOptionalFile(const Pathname &file, unsigned media_nr=1)
Provides an optional file from media media_nr.
Helper filtering the files offered by a RepomdFileReader.
Definition: Downloader.cc:107
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
void downloadMediaInfo(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progressrcv)
Downloads the media info (/media.1) to a local directory.
What is known about a repository.
Definition: RepoInfo.h:71
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
bool operator()(const OnMediaLocation &loc_r, const std::string &typestr_r)
The callback invoked by the RepomdFileReader.
Definition: Downloader.cc:126
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: progressdata.h:140
LocaleSet _wantedLocales
Locales do download.
Definition: Downloader.cc:194
bool ZYPP_REPOMD_WITH_FILELISTS()
Definition: Downloader.cc:39
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Definition: Fetcher.cc:903
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
Definition: Fetcher.cc:908
std::map< std::string, OnMediaLocation > _wantedFiles
Definition: Downloader.cc:195
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
const Pathname & deltaDir() const
Definition: Downloader.cc:174
void download(MediaSetAccess &media_r, const Pathname &destDir_r, const ProgressData::ReceiverFnc &progress_r=ProgressData::ReceiverFnc()) override
Download metadata to a local directory.
Definition: Downloader.cc:209
Impl(Downloader &downloader_r, MediaSetAccess &media_r, const Pathname &destDir_r)
Definition: Downloader.cc:112
RepoStatus status(MediaSetAccess &media_r) override
Status of the remote repository.
Definition: Downloader.cc:228
bool startsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasPrefix
Definition: String.h:1085
const RepoInfo & repoInfo() const
Definition: Downloader.h:63
static Pathname search_deltafile(const Pathname &dir, const Pathname &file)
Definition: Downloader.cc:45
const Pathname & filename() const
The path to the resource on the medium.
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1092
bool wantLocale(const Locale &locale_r) const
Definition: Downloader.cc:177
&#39;Language[_Country]&#39; codes.
Definition: Locale.h:49
int readdir(std::list< std::string > &retlist_r, const Pathname &path_r, bool dots_r)
Return content of directory via retlist.
Definition: PathInfo.cc:605
#define NON_MOVABLE(CLASS)
Delete move ctor and move assign.
Definition: Easy.h:69
Reads through a repomd.xml file and collects type, location, checksum and other data about metadata f...
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
#define NON_COPYABLE(CLASS)
Delete copy ctor and copy assign.
Definition: Easy.h:59
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
Interface of repomd.xml file reader.
Track changing files or directories.
Definition: RepoStatus.h:40
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:29
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
Downloader for YUM (rpm-nmd) repositories Encapsulates all the knowledge of which files have to be do...
Definition: Downloader.h:40
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
std::unordered_set< Locale > LocaleSet
Definition: Locale.h:27