lz4diff/lz4diff_compress.cc - SHIFTPHONES/android_system_update_engine - Gitiles

 //
 // Copyright (C) 2021 The Android Open Source Project
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //

 #include "lz4diff_compress.h"

 #include "update_engine/common/utils.h"
 #include "update_engine/common/hash_calculator.h"
 #include "update_engine/payload_generator/delta_diff_generator.h"
 #include "update_engine/payload_generator/payload_generation_config.h"

 #include <base/logging.h>
 #include <lz4.h>
 #include <lz4hc.h>

 namespace chromeos_update_engine {

 bool TryCompressBlob(std::string_view blob,
                      const std::vector<CompressedBlock>& block_info,
                      const bool zero_padding_enabled,
                      const CompressionAlgorithm compression_algo,
                      const SinkFunc& sink) {
   size_t uncompressed_size = 0;
   for (const auto& block : block_info) {
     CHECK_EQ(uncompressed_size, block.uncompressed_offset)
         << "Compressed block info is expected to be sorted.";
     uncompressed_size += block.uncompressed_length;
   }
   auto hc = LZ4_createStreamHC();
   DEFER {
     if (hc) {
       LZ4_freeStreamHC(hc);
       hc = nullptr;
     }
   };
   size_t compressed_offset = 0;
   Blob block_buffer;
   for (const auto& block : block_info) {
     const auto uncompressed_block =
         blob.substr(block.uncompressed_offset, block.uncompressed_length);
     if (!block.IsCompressed()) {
       TEST_EQ(sink(reinterpret_cast<const uint8_t*>(uncompressed_block.data()),
                    uncompressed_block.size()),
               uncompressed_block.size());
       continue;
     }
     block_buffer.resize(block.compressed_length);
     // Execute the increment at end of each loop
     DEFER {
       compressed_offset += block.compressed_length;
       block_buffer.clear();
     };

     int ret = 0;
     // LZ4 spec enforces that last op of a compressed block must be an insert op
     // of at least 5 bytes. Compressors will try to conform to that requirement
     // if the input size is just right. We don't want that. So always give a
     // little bit more data.
     switch (int src_size = uncompressed_size - block.uncompressed_offset;
             compression_algo.type()) {
       case CompressionAlgorithm::LZ4HC:
         ret = LZ4_compress_HC_destSize(
             hc,
             uncompressed_block.data(),
             reinterpret_cast<char*>(block_buffer.data()),
             &src_size,
             block.compressed_length,
             compression_algo.level());
         break;
       case CompressionAlgorithm::LZ4:
         ret =
             LZ4_compress_destSize(uncompressed_block.data(),
                                   reinterpret_cast<char*>(block_buffer.data()),
                                   &src_size,
                                   block.compressed_length);
         break;
       default:
         LOG(ERROR) << "Unrecognized compression algorithm: "
                    << compression_algo.type();
         return {};
     }
     TEST_GT(ret, 0);
     const uint64_t bytes_written = ret;
     // Last block may have trailing zeros
     TEST_LE(bytes_written, block.compressed_length);
     if (bytes_written < block.compressed_length) {
       if (zero_padding_enabled) {
         const auto padding = block.compressed_length - bytes_written;
         std::memmove(
             block_buffer.data() + padding, block_buffer.data(), bytes_written);
         std::fill(block_buffer.data(), block_buffer.data() + padding, 0);

       } else {
         std::fill(block_buffer.data() + bytes_written,
                   block_buffer.data() + block.compressed_length,
                   0);
       }
     }
     TEST_EQ(sink(block_buffer.data(), block_buffer.size()),
             block_buffer.size());
   }
   // Any trailing data will be copied to the output buffer.
   TEST_EQ(
       sink(reinterpret_cast<const uint8_t*>(blob.data()) + uncompressed_size,
            blob.size() - uncompressed_size),
       blob.size() - uncompressed_size);
   return true;
 }

 Blob TryCompressBlob(std::string_view blob,
                      const std::vector<CompressedBlock>& block_info,
                      const bool zero_padding_enabled,
                      const CompressionAlgorithm compression_algo) {
   size_t uncompressed_size = 0;
   size_t compressed_size = 0;
   for (const auto& block : block_info) {
     CHECK_EQ(uncompressed_size, block.uncompressed_offset)
         << "Compressed block info is expected to be sorted.";
     uncompressed_size += block.uncompressed_length;
     compressed_size += block.compressed_length;
   }
   TEST_EQ(uncompressed_size, blob.size());
   Blob output;
   output.reserve(utils::RoundUp(compressed_size, kBlockSize));
   if (!TryCompressBlob(blob,
                        block_info,
                        zero_padding_enabled,
                        compression_algo,
                        [&output](const uint8_t* data, size_t size) {
                          output.insert(output.end(), data, data + size);
                          return size;
                        })) {
     return {};
   }

   return output;
 }

 Blob TryDecompressBlob(std::string_view blob,
                        const std::vector<CompressedBlock>& block_info,
                        const bool zero_padding_enabled) {
   if (block_info.empty()) {
     return {};
   }
   size_t uncompressed_size = 0;
   size_t compressed_size = 0;
   for (const auto& block : block_info) {
     CHECK_EQ(uncompressed_size, block.uncompressed_offset)
         << " Compressed block info is expected to be sorted, expected offset "
         << uncompressed_size << ", actual block " << block;
     uncompressed_size += block.uncompressed_length;
     compressed_size += block.compressed_length;
   }
   if (blob.size() < compressed_size) {
     LOG(INFO) << "File is chunked. Skip lz4 decompress. Expected size: "
               << compressed_size << ", actual size: " << blob.size();
     return {};
   }
   Blob output;
   output.reserve(uncompressed_size);
   size_t compressed_offset = 0;
   for (const auto& block : block_info) {
     std::string_view cluster =
         blob.substr(compressed_offset, block.compressed_length);
     if (!block.IsCompressed()) {
       CHECK_NE(cluster.size(), 0UL);
       output.insert(output.end(), cluster.begin(), cluster.end());
       compressed_offset += cluster.size();
       continue;
     }
     size_t inputmargin = 0;
     if (zero_padding_enabled) {
       while (inputmargin < std::min(kBlockSize, cluster.size()) &&
              cluster[inputmargin] == 0) {
         inputmargin++;
       }
     }
     output.resize(output.size() + block.uncompressed_length);

     const auto bytes_decompressed = LZ4_decompress_safe_partial(
         cluster.data() + inputmargin,
         reinterpret_cast<char*>(output.data()) + output.size() -
             block.uncompressed_length,
         cluster.size() - inputmargin,
         block.uncompressed_length,
         block.uncompressed_length);
     if (bytes_decompressed < 0) {
       LOG(FATAL) << "Failed to decompress, " << bytes_decompressed
                  << ", output_cursor = "
                  << output.size() - block.uncompressed_length
                  << ", input_cursor = " << compressed_offset
                  << ", blob.size() = " << blob.size()
                  << ", cluster_size = " << block.compressed_length
                  << ", dest capacity = " << block.uncompressed_length
                  << ", input margin = " << inputmargin << " "
                  << HashCalculator::SHA256Digest(cluster) << " "
                  << HashCalculator::SHA256Digest(blob);
       return {};
     }
     compressed_offset += block.compressed_length;
     CHECK_EQ(static_cast<uint64_t>(bytes_decompressed),
              block.uncompressed_length);
   }
   CHECK_EQ(output.size(), uncompressed_size);

   // Trailing data not recorded by compressed block info will be treated as
   // uncompressed, most of the time these are xattrs or trailing zeros.
   CHECK_EQ(blob.size(), compressed_offset)
       << " Unexpected data the end of compressed data ";
   if (compressed_offset < blob.size()) {
     output.insert(output.end(), blob.begin() + compressed_offset, blob.end());
   }

   return output;
 }

 Blob TryDecompressBlob(const Blob& blob,
                        const std::vector<CompressedBlock>& block_info,
                        const bool zero_padding_enabled) {
   return TryDecompressBlob(
       ToStringView(blob), block_info, zero_padding_enabled);
 }

 std::ostream& operator<<(std::ostream& out, const CompressedBlock& block) {
   out << "CompressedBlock{.uncompressed_offset = " << block.uncompressed_offset
       << ", .compressed_length = " << block.compressed_length
       << ", .uncompressed_length = " << block.uncompressed_length << "}";
   return out;
 }

 std::ostream& operator<<(std::ostream& out, const CompressedBlockInfo& info) {
   out << "BlockInfo { compressed_length: " << info.compressed_length()
       << ", uncompressed_length: " << info.uncompressed_length()
       << ", uncompressed_offset: " << info.uncompressed_offset();
   if (!info.sha256_hash().empty()) {
     out << ", sha256_hash: " << HexEncode(info.sha256_hash());
   }
   if (!info.postfix_bspatch().empty()) {
     out << ", postfix_bspatch: " << info.postfix_bspatch().size();
   }
   out << "}";
   return out;
 }

 }  // namespace chromeos_update_engine
	//
	// Copyright (C) 2021 The Android Open Source Project
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	//

	#include "lz4diff_compress.h"

	#include "update_engine/common/utils.h"
	#include "update_engine/common/hash_calculator.h"
	#include "update_engine/payload_generator/delta_diff_generator.h"
	#include "update_engine/payload_generator/payload_generation_config.h"

	#include <base/logging.h>
	#include <lz4.h>
	#include <lz4hc.h>

	namespace chromeos_update_engine {

	bool TryCompressBlob(std::string_view blob,
	const std::vector<CompressedBlock>& block_info,
	const bool zero_padding_enabled,
	const CompressionAlgorithm compression_algo,
	const SinkFunc& sink) {
	size_t uncompressed_size = 0;
	for (const auto& block : block_info) {
	CHECK_EQ(uncompressed_size, block.uncompressed_offset)
	<< "Compressed block info is expected to be sorted.";
	uncompressed_size += block.uncompressed_length;
	}
	auto hc = LZ4_createStreamHC();
	DEFER {
	if (hc) {
	LZ4_freeStreamHC(hc);
	hc = nullptr;
	}
	};
	size_t compressed_offset = 0;
	Blob block_buffer;
	for (const auto& block : block_info) {
	const auto uncompressed_block =
	blob.substr(block.uncompressed_offset, block.uncompressed_length);
	if (!block.IsCompressed()) {
	TEST_EQ(sink(reinterpret_cast<const uint8_t*>(uncompressed_block.data()),
	uncompressed_block.size()),
	uncompressed_block.size());
	continue;
	}
	block_buffer.resize(block.compressed_length);
	// Execute the increment at end of each loop
	DEFER {
	compressed_offset += block.compressed_length;
	block_buffer.clear();
	};

	int ret = 0;
	// LZ4 spec enforces that last op of a compressed block must be an insert op
	// of at least 5 bytes. Compressors will try to conform to that requirement
	// if the input size is just right. We don't want that. So always give a
	// little bit more data.
	switch (int src_size = uncompressed_size - block.uncompressed_offset;
	compression_algo.type()) {
	case CompressionAlgorithm::LZ4HC:
	ret = LZ4_compress_HC_destSize(
	hc,
	uncompressed_block.data(),
	reinterpret_cast<char*>(block_buffer.data()),
	&src_size,
	block.compressed_length,
	compression_algo.level());
	break;
	case CompressionAlgorithm::LZ4:
	ret =
	LZ4_compress_destSize(uncompressed_block.data(),
	reinterpret_cast<char*>(block_buffer.data()),
	&src_size,
	block.compressed_length);
	break;
	default:
	LOG(ERROR) << "Unrecognized compression algorithm: "
	<< compression_algo.type();
	return {};
	}
	TEST_GT(ret, 0);
	const uint64_t bytes_written = ret;
	// Last block may have trailing zeros
	TEST_LE(bytes_written, block.compressed_length);
	if (bytes_written < block.compressed_length) {
	if (zero_padding_enabled) {
	const auto padding = block.compressed_length - bytes_written;
	std::memmove(
	block_buffer.data() + padding, block_buffer.data(), bytes_written);
	std::fill(block_buffer.data(), block_buffer.data() + padding, 0);

	} else {
	std::fill(block_buffer.data() + bytes_written,
	block_buffer.data() + block.compressed_length,
	0);
	}
	}
	TEST_EQ(sink(block_buffer.data(), block_buffer.size()),
	block_buffer.size());
	}
	// Any trailing data will be copied to the output buffer.
	TEST_EQ(
	sink(reinterpret_cast<const uint8_t*>(blob.data()) + uncompressed_size,
	blob.size() - uncompressed_size),
	blob.size() - uncompressed_size);
	return true;
	}

	Blob TryCompressBlob(std::string_view blob,
	const std::vector<CompressedBlock>& block_info,
	const bool zero_padding_enabled,
	const CompressionAlgorithm compression_algo) {
	size_t uncompressed_size = 0;
	size_t compressed_size = 0;
	for (const auto& block : block_info) {
	CHECK_EQ(uncompressed_size, block.uncompressed_offset)
	<< "Compressed block info is expected to be sorted.";
	uncompressed_size += block.uncompressed_length;
	compressed_size += block.compressed_length;
	}
	TEST_EQ(uncompressed_size, blob.size());
	Blob output;
	output.reserve(utils::RoundUp(compressed_size, kBlockSize));
	if (!TryCompressBlob(blob,
	block_info,
	zero_padding_enabled,
	compression_algo,
	[&output](const uint8_t* data, size_t size) {
	output.insert(output.end(), data, data + size);
	return size;
	})) {
	return {};
	}

	return output;
	}

	Blob TryDecompressBlob(std::string_view blob,
	const std::vector<CompressedBlock>& block_info,
	const bool zero_padding_enabled) {
	if (block_info.empty()) {
	return {};
	}
	size_t uncompressed_size = 0;
	size_t compressed_size = 0;
	for (const auto& block : block_info) {
	CHECK_EQ(uncompressed_size, block.uncompressed_offset)
	<< " Compressed block info is expected to be sorted, expected offset "
	<< uncompressed_size << ", actual block " << block;
	uncompressed_size += block.uncompressed_length;
	compressed_size += block.compressed_length;
	}
	if (blob.size() < compressed_size) {
	LOG(INFO) << "File is chunked. Skip lz4 decompress. Expected size: "
	<< compressed_size << ", actual size: " << blob.size();
	return {};
	}
	Blob output;
	output.reserve(uncompressed_size);
	size_t compressed_offset = 0;
	for (const auto& block : block_info) {
	std::string_view cluster =
	blob.substr(compressed_offset, block.compressed_length);
	if (!block.IsCompressed()) {
	CHECK_NE(cluster.size(), 0UL);
	output.insert(output.end(), cluster.begin(), cluster.end());
	compressed_offset += cluster.size();
	continue;
	}
	size_t inputmargin = 0;
	if (zero_padding_enabled) {
	while (inputmargin < std::min(kBlockSize, cluster.size()) &&
	cluster[inputmargin] == 0) {
	inputmargin++;
	}
	}
	output.resize(output.size() + block.uncompressed_length);

	const auto bytes_decompressed = LZ4_decompress_safe_partial(
	cluster.data() + inputmargin,
	reinterpret_cast<char*>(output.data()) + output.size() -
	block.uncompressed_length,
	cluster.size() - inputmargin,
	block.uncompressed_length,
	block.uncompressed_length);
	if (bytes_decompressed < 0) {
	LOG(FATAL) << "Failed to decompress, " << bytes_decompressed
	<< ", output_cursor = "
	<< output.size() - block.uncompressed_length
	<< ", input_cursor = " << compressed_offset
	<< ", blob.size() = " << blob.size()
	<< ", cluster_size = " << block.compressed_length
	<< ", dest capacity = " << block.uncompressed_length
	<< ", input margin = " << inputmargin << " "
	<< HashCalculator::SHA256Digest(cluster) << " "
	<< HashCalculator::SHA256Digest(blob);
	return {};
	}
	compressed_offset += block.compressed_length;
	CHECK_EQ(static_cast<uint64_t>(bytes_decompressed),
	block.uncompressed_length);
	}
	CHECK_EQ(output.size(), uncompressed_size);

	// Trailing data not recorded by compressed block info will be treated as
	// uncompressed, most of the time these are xattrs or trailing zeros.
	CHECK_EQ(blob.size(), compressed_offset)
	<< " Unexpected data the end of compressed data ";
	if (compressed_offset < blob.size()) {
	output.insert(output.end(), blob.begin() + compressed_offset, blob.end());
	}

	return output;
	}

	Blob TryDecompressBlob(const Blob& blob,
	const std::vector<CompressedBlock>& block_info,
	const bool zero_padding_enabled) {
	return TryDecompressBlob(
	ToStringView(blob), block_info, zero_padding_enabled);
	}

	std::ostream& operator<<(std::ostream& out, const CompressedBlock& block) {
	out << "CompressedBlock{.uncompressed_offset = " << block.uncompressed_offset
	<< ", .compressed_length = " << block.compressed_length
	<< ", .uncompressed_length = " << block.uncompressed_length << "}";
	return out;
	}

	std::ostream& operator<<(std::ostream& out, const CompressedBlockInfo& info) {
	out << "BlockInfo { compressed_length: " << info.compressed_length()
	<< ", uncompressed_length: " << info.uncompressed_length()
	<< ", uncompressed_offset: " << info.uncompressed_offset();
	if (!info.sha256_hash().empty()) {
	out << ", sha256_hash: " << HexEncode(info.sha256_hash());
	}
	if (!info.postfix_bspatch().empty()) {
	out << ", postfix_bspatch: " << info.postfix_bspatch().size();
	}
	out << "}";
	return out;
	}

	} // namespace chromeos_update_engine