libzypp  17.31.31
zck_p.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------*/
9 
10 #if ENABLE_ZCHUNK_COMPRESSION
11 
15 #include <zypp-core/AutoDispose.h>
16 
17 #include "zck_p.h"
18 
19 #include <iostream>
20 #include <fstream>
21 #include <fcntl.h>
22 
23 extern "C" {
24 #include <zck.h>
25 }
26 
27 namespace zyppng {
28 
29  bool isZchunkFile ( const zypp::Pathname &file ) {
30  std::ifstream dFile( file.c_str() );
31  if ( !dFile.is_open() )
32  return false;
33 
34  constexpr std::string_view magic("\0ZCK1", 5);
35 
36  std::array< char, magic.size() > lead;
37  lead.fill('\0');
38  dFile.read( lead.data(), lead.size() );
39  return ( magic == std::string_view( lead.data(), lead.size()) );
40  }
41 
42  DLZckHeadState::DLZckHeadState( std::vector<Url> &&mirrors, std::shared_ptr<Request> &&oldReq, DownloadPrivate &parent )
43  : BasicDownloaderStateBase( std::move(oldReq), parent )
44  {
45  _fileMirrors = std::move(mirrors);
46  MIL << "About to enter DlZckHeadState for url " << parent._spec.url() << std::endl;
47  }
48 
49  DLZckHeadState::DLZckHeadState( std::vector<Url> &&mirrors, DownloadPrivate &parent )
50  : BasicDownloaderStateBase( parent )
51  {
52  _fileMirrors = std::move(mirrors);
53  MIL << "About to enter DlZckHeadState for url " << parent._spec.url() << std::endl;
54  }
55 
56 
57  bool DLZckHeadState::initializeRequest(std::shared_ptr<Request> &r )
58  {
60 
61  const auto &s = stateMachine()._spec;
62  if ( s.headerSize() == 0 ) {
63  ERR << "Downloading the zck header was requested, but headersize is zero." << std::endl;
64  return false;
65  }
66 
67  std::optional<zypp::Digest> digest;
69 
70  const auto &headerSum = s.headerChecksum();
71  if ( headerSum ) {
72  digest = zypp::Digest();
73  if ( !digest->create( headerSum->type() ) ) {
74  ERR << "Unknown header checksum type " << headerSum->type() << std::endl;
75  return false;
76  }
77  sum = zypp::Digest::hexStringToUByteArray( headerSum->checksum() );
78  }
79 
80  r->addRequestRange( 0, s.headerSize(), std::move(digest), sum );
81  return true;
82  }
83 
84  void DLZckHeadState::gotFinished()
85  {
86  if ( isZchunkFile( stateMachine()._spec.targetPath() ) )
88  failed ( "Downloaded header is not a zchunk header");
89  }
90 
91  std::shared_ptr<DLZckState> DLZckHeadState::transitionToDlZckState()
92  {
93  MIL_MEDIA << "Downloaded the header of size: " << _request->downloadedByteCount() << std::endl;
94  return std::make_shared<DLZckState>( std::move(_fileMirrors), stateMachine() );
95  }
96 
97  DLZckState::DLZckState(std::vector<Url> &&mirrors, DownloadPrivate &parent)
98  : RangeDownloaderBaseState( std::move(mirrors), parent )
99  {
100  MIL_MEDIA << "About to enter DLZckState for url " << parent._spec.url() << std::endl;
101  }
102 
103  void DLZckState::enter()
104  {
105  auto &sm = stateMachine();
106  const auto &spec = sm._spec;
107 
108  // setup the base downloader
109  _error = {};
110  _ranges.clear();
111  _failedRanges.clear();
112 
113  // @TODO get this from zchunk file?
114  _fileSize = spec.expectedFileSize();
115 
116  zypp::AutoFD src_fd = open( spec.deltaFile().asString().c_str(), O_RDONLY);
117  if(src_fd < 0)
118  return setFailed ( zypp::str::Format("Unable to open %1%") % spec.deltaFile() );
119 
120  zypp::AutoDispose<zckCtx *> zck_src ( zck_create(), []( auto ptr ) { if ( ptr ) zck_free( &ptr ); } );
121  if( !zck_src )
122  return setFailed ( zypp::str::Format("%1%") % zck_get_error(NULL) );
123 
124  if(!zck_init_read(zck_src, src_fd))
125  return setFailed ( zypp::str::Format( "Unable to open %1%: %2%") % spec.deltaFile() % zck_get_error(zck_src) );
126 
127  zypp::AutoFD target_fd = open( spec.targetPath().asString().c_str(), O_RDWR);
128  if(target_fd < 0)
129  return setFailed ( zypp::str::Format("Unable to open %1%") % spec.targetPath() );
130 
131  zypp::AutoDispose<zckCtx *> zckTarget ( zck_create(), []( auto ptr ) { if ( ptr ) zck_free( &ptr ); } );
132  if( !zckTarget )
133  return setFailed ( zypp::str::Format("%1%") % zck_get_error(NULL) );
134 
135  if(!zck_init_read(zckTarget, target_fd))
136  return setFailed ( zypp::str::Format( "Unable to open %1%: %2%") % spec.targetPath() % zck_get_error(zckTarget) );
137 
138  // Returns 0 for error, -1 for invalid checksum and 1 for valid checksum
139  switch ( zck_find_valid_chunks(zckTarget) ) {
140  case 0: // Returns 0 if there was a error
141  return setFailed ( zypp::str::Format( "Unable to open %1%: %2%") % spec.targetPath() % zck_get_error(zckTarget) );
142  case 1: // getting a 1 would mean the file is already complete, basically impossible but lets handle it anyway
143  return setFinished();
144  }
145 
146  const auto srcHashType = zck_get_chunk_hash_type( zckTarget );
147  const auto targetHashType = zck_get_chunk_hash_type( zckTarget );
148 
149  const size_t fLen = zck_get_length( zckTarget );
150  if ( _fileSize > 0 ) {
151  // check if the file size as reported by zchunk is equal to the one we expect
152  if ( _fileSize != fLen ) {
153  return setFailed( NetworkRequestErrorPrivate::customError(
155  zypp::str::Format("Zchunk header reports a different filesize than what was expected ( Zck: %1% != Exp: %2%).") % fLen % _fileSize )
156  );
157  }
158  } else {
159  _fileSize = fLen;
160  }
161 
162  const auto maxConns = sm._requestDispatcher->maximumConcurrentConnections();
163  if ( sm._spec.preferredChunkSize() == 0 ) {
164  const auto autoBlkSize = makeBlksize( _fileSize );
165  if ( maxConns == -1 ) {
166  _preferredChunkSize = autoBlkSize;
167  } else {
168  _preferredChunkSize = _fileSize / maxConns;
169  if ( _preferredChunkSize < autoBlkSize )
170  _preferredChunkSize = autoBlkSize;
171  else if ( _preferredChunkSize > zypp::ByteCount( 100, zypp::ByteCount::M ) )
172  _preferredChunkSize = zypp::ByteCount( 100, zypp::ByteCount::M );
173  }
174  } else {
175  // spec chunk size overrules the automatic one
176  _preferredChunkSize = sm._spec.preferredChunkSize();
177  }
178 
179  if( srcHashType != targetHashType )
180  return setFailed ( zypp::str::Format( "ERROR: Chunk hash types don't match. Source Hash: %1% vs Target Hash: %2%")
181  % zck_hash_name_from_type ( srcHashType )
182  % zck_hash_name_from_type ( targetHashType ) );
183 
184  if(!zck_copy_chunks( zck_src, zckTarget ))
185  return setFailed ( zypp::str::Format( "Unable to copy chunks from deltafile.") );
186 
187  // will reset all chunks that are marked as failed back to missing
188  zck_reset_failed_chunks( zckTarget );
189 
190 
191  // we calculate what is already downloaded by substracting the block sizes we still need to download from the full file size
192  _downloadedMultiByteCount = _fileSize;
193 
194  auto chunk = zck_get_first_chunk( zckTarget );
195  do {
196  // Get validity of current chunk: 1 = valid, 0 = missing, -1 = invalid
197  if ( zck_get_chunk_valid( chunk ) == 1 )
198  continue;
199 
200  zypp::AutoFREE<char> zckDigest( zck_get_chunk_digest( chunk ) );
201  UByteArray chksumVec = zypp::Digest::hexStringToUByteArray( std::string_view( zckDigest.value() ) );
202  std::string chksumName;
203  std::optional<size_t> chksumCompareLen;
204 
205  switch ( targetHashType ) {
206  case ZCK_HASH_SHA1: {
207  chksumName = zypp::Digest::sha1();
208  break;
209  }
210  case ZCK_HASH_SHA256: {
211  chksumName = zypp::Digest::sha256();
212  break;
213  }
214  case ZCK_HASH_SHA512: {
215  chksumName = zypp::Digest::sha512();
216  break;
217  }
218  case ZCK_HASH_SHA512_128: {
219  // defined in zchunk as
220  // SHA-512/128 (first 128 bits of SHA-512 checksum)
221  chksumName = zypp::Digest::sha512();
222  chksumCompareLen = chksumVec.size();
223  break;
224  }
225  default: {
226  return setFailed ( zypp::str::Format( "Unsupported chunk hash type: %1%.") % zck_hash_name_from_type( targetHashType ) );
227  }
228  }
229 
230  const auto s = static_cast<off_t>( zck_get_chunk_start( chunk ) );
231  const auto l = static_cast<size_t>( zck_get_chunk_comp_size ( chunk ) );
232 
233  MIL_MEDIA << "Downloading block " << s << " with length " << l << " checksum " << zckDigest.value() << " type " << chksumName << std::endl;
234 
235  _ranges.push_back( Block{
236  .start = s,
237  .len = l,
238  .chksumtype = chksumName,
239  .chksumVec = std::move( chksumVec ),
240  .chksumCompareLen = std::move(chksumCompareLen)
241  } );
242 
243  // substract the block length from the already downloaded bytes size
244  _downloadedMultiByteCount -= l;
245 
246  } while ( (chunk = zck_get_next_chunk( chunk )) );
247 
248  ensureDownloadsRunning();
249  }
250 
251  void DLZckState::exit()
252  {
253  cancelAll( NetworkRequestError() );
254  }
255 
256  std::shared_ptr<FinishedState> DLZckState::transitionToFinished()
257  {
258  return std::make_shared<FinishedState>( std::move(_error), stateMachine() );
259  }
260 
261  void DLZckState::setFinished()
262  {
263  const auto &spec = stateMachine()._spec;
264 
265  zypp::AutoFD target_fd = open( spec.targetPath().asString().c_str(), O_RDONLY );
266  if( target_fd < 0 )
267  return setFailed ( zypp::str::Format("Unable to open %1%") % spec.targetPath() );
268 
269  zypp::AutoDispose<zckCtx *> zckTarget ( zck_create(), []( auto ptr ) { if ( ptr ) zck_free( &ptr ); } );
270  if( !zckTarget )
271  return setFailed ( zypp::str::Format("%1%") % zck_get_error(nullptr) );
272 
273  if(!zck_init_read(zckTarget, target_fd))
274  return setFailed ( zypp::str::Format( "Unable to open %1%: %2%") % spec.targetPath() % zck_get_error(zckTarget) );
275 
276  /* Validate the chunk and data checksums for the current file.
277  * Returns 0 for error, -1 for invalid checksum and 1 for valid checksum */
278  const auto res = zck_validate_checksums( zckTarget );
279  if ( res == 0 || res == -1 ) {
280  if( zck_is_error(nullptr) ) {
281  std::string err = zck_get_error(NULL);
282  zck_clear_error(NULL);
283  return setFailed( std::move(err) );
284  }
285  if( zck_is_error(zckTarget) )
286  return setFailed( zck_get_error(zckTarget) );
287  return setFailed( "zck_validate_checksums returned a unknown error." );
288  }
289 
290  // everything is valid
292  }
293 
294 }
295 
296 #endif
static const std::string & sha256()
sha256
Definition: Digest.cc:50
#define MIL
Definition: Logger.h:96
static const std::string & sha1()
sha1
Definition: Digest.cc:44
Compute Message Digests (MD5, SHA1 etc)
Definition: Digest.h:37
Store and operate with byte count.
Definition: ByteCount.h:30
const char * c_str() const
String representation.
Definition: Pathname.h:110
Definition: Arch.h:363
Convenient building of std::string with boost::format.
Definition: String.h:252
AutoDispose<int> calling ::close
Definition: AutoDispose.h:301
#define ERR
Definition: Logger.h:98
static const std::string & sha512()
sha512
Definition: Digest.cc:56
UByteArray CheckSumBytes
Definition: request.h:49
virtual bool initializeRequest(std::shared_ptr< Request > &r)
#define MIL_MEDIA
Definition: mediadebug_p.h:29
static const Unit M
1024^2 Byte
Definition: ByteCount.h:48
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:93
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})