Support incremental OTA in ota_extractor am: 91e839cf19 am: fec3837e1b am: a83d45a9fa am: 68b45aa077
Original change: https://android-review.googlesource.com/c/platform/system/update_engine/+/2056749
Change-Id: Ia54a84e2c6ac6a1c035f41840cd9260c57676f0f
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/aosp/ota_extractor.cc b/aosp/ota_extractor.cc
index 80bb74b..7492bc5 100644
--- a/aosp/ota_extractor.cc
+++ b/aosp/ota_extractor.cc
@@ -14,13 +14,13 @@
// limitations under the License.
//
-#include <fcntl.h>
-
+#include <array>
#include <cstdint>
#include <cstdio>
#include <iterator>
#include <memory>
+#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
@@ -28,16 +28,22 @@
#include <base/files/file_path.h>
#include <gflags/gflags.h>
#include <unistd.h>
+#include <xz.h>
#include "update_engine/common/utils.h"
#include "update_engine/common/hash_calculator.h"
#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_consumer/file_descriptor_utils.h"
#include "update_engine/payload_consumer/install_operation_executor.h"
#include "update_engine/payload_consumer/payload_metadata.h"
+#include "update_engine/payload_consumer/verity_writer_android.h"
#include "update_engine/update_metadata.pb.h"
-#include "xz.h"
DEFINE_string(payload, "", "Path to payload.bin");
+DEFINE_string(
+ input_dir,
+ "",
+ "Directory to read input images. Only required for incremental OTAs");
DEFINE_string(output_dir, "", "Directory to put output images");
DEFINE_int64(payload_offset,
0,
@@ -53,18 +59,55 @@
namespace chromeos_update_engine {
+void WriteVerity(const PartitionUpdate& partition,
+ FileDescriptorPtr fd,
+ const size_t block_size) {
+ // 512KB buffer, arbitrary value. Larger buffers may improve performance.
+ static constexpr size_t BUFFER_SIZE = 1024 * 512;
+ if (partition.hash_tree_extent().num_blocks() == 0 &&
+ partition.fec_extent().num_blocks() == 0) {
+ return;
+ }
+ InstallPlan::Partition install_part;
+ install_part.block_size = block_size;
+ CHECK(install_part.ParseVerityConfig(partition));
+ VerityWriterAndroid writer;
+ CHECK(writer.Init(install_part));
+ std::array<uint8_t, BUFFER_SIZE> buffer;
+ const auto data_size =
+ install_part.hash_tree_data_offset + install_part.hash_tree_data_size;
+ size_t offset = 0;
+ while (offset < data_size) {
+ const auto bytes_to_read =
+ static_cast<ssize_t>(std::min(BUFFER_SIZE, data_size - offset));
+ ssize_t bytes_read;
+ CHECK(
+ utils::ReadAll(fd, buffer.data(), bytes_to_read, offset, &bytes_read));
+ CHECK_EQ(bytes_read, bytes_to_read)
+ << " Failed to read at offset " << offset << " "
+ << android::base::ErrnoNumberAsString(errno);
+ writer.Update(offset, buffer.data(), bytes_read);
+ offset += bytes_read;
+ }
+ CHECK(writer.Finalize(fd.get(), fd.get()));
+ return;
+}
+
bool ExtractImagesFromOTA(const DeltaArchiveManifest& manifest,
const PayloadMetadata& metadata,
int payload_fd,
size_t payload_offset,
+ std::string_view input_dir,
std::string_view output_dir,
const std::set<std::string>& partitions) {
InstallOperationExecutor executor(manifest.block_size());
const size_t data_begin = metadata.GetMetadataSize() +
metadata.GetMetadataSignatureSize() +
payload_offset;
- const base::FilePath path(
+ const base::FilePath output_dir_path(
base::StringPiece(output_dir.data(), output_dir.size()));
+ const base::FilePath input_dir_path(
+ base::StringPiece(input_dir.data(), input_dir.size()));
std::vector<unsigned char> blob;
for (const auto& partition : manifest.partitions()) {
if (!partitions.empty() &&
@@ -74,18 +117,39 @@
LOG(INFO) << "Extracting partition " << partition.partition_name()
<< " size: " << partition.new_partition_info().size();
const auto output_path =
- path.Append(partition.partition_name() + ".img").value();
- auto fd =
+ output_dir_path.Append(partition.partition_name() + ".img").value();
+ const auto input_path =
+ input_dir_path.Append(partition.partition_name() + ".img").value();
+ auto out_fd =
std::make_shared<chromeos_update_engine::EintrSafeFileDescriptor>();
TEST_AND_RETURN_FALSE_ERRNO(
- fd->Open(output_path.c_str(), O_RDWR | O_CREAT, 0644));
+ out_fd->Open(output_path.c_str(), O_RDWR | O_CREAT, 0644));
+ auto in_fd =
+ std::make_shared<chromeos_update_engine::EintrSafeFileDescriptor>();
+ TEST_AND_RETURN_FALSE_ERRNO(in_fd->Open(input_path.c_str(), O_RDONLY));
+
for (const auto& op : partition.operations()) {
+ if (op.has_src_sha256_hash()) {
+ brillo::Blob actual_hash;
+ TEST_AND_RETURN_FALSE(fd_utils::ReadAndHashExtents(
+ in_fd, op.src_extents(), manifest.block_size(), &actual_hash));
+ CHECK_EQ(HexEncode(ToStringView(actual_hash)),
+ HexEncode(op.src_sha256_hash()));
+ }
+
blob.resize(op.data_length());
const auto op_data_offset = data_begin + op.data_offset();
ssize_t bytes_read = 0;
TEST_AND_RETURN_FALSE(utils::PReadAll(
payload_fd, blob.data(), blob.size(), op_data_offset, &bytes_read));
- auto direct_writer = std::make_unique<DirectExtentWriter>(fd);
+ if (op.has_data_sha256_hash()) {
+ brillo::Blob actual_hash;
+ TEST_AND_RETURN_FALSE(
+ HashCalculator::RawHashOfData(blob, &actual_hash));
+ CHECK_EQ(HexEncode(ToStringView(actual_hash)),
+ HexEncode(op.data_sha256_hash()));
+ }
+ auto direct_writer = std::make_unique<DirectExtentWriter>(out_fd);
if (op.type() == InstallOperation::ZERO) {
TEST_AND_RETURN_FALSE(executor.ExecuteZeroOrDiscardOperation(
op, std::move(direct_writer)));
@@ -94,12 +158,15 @@
op.type() == InstallOperation::REPLACE_XZ) {
TEST_AND_RETURN_FALSE(executor.ExecuteReplaceOperation(
op, std::move(direct_writer), blob.data(), blob.size()));
+ } else if (op.type() == InstallOperation::SOURCE_COPY) {
+ TEST_AND_RETURN_FALSE(executor.ExecuteSourceCopyOperation(
+ op, std::move(direct_writer), in_fd));
} else {
- LOG(ERROR) << "Unsupported operation type: " << op.type() << ", "
- << InstallOperation::Type_Name(op.type());
- return false;
+ TEST_AND_RETURN_FALSE(executor.ExecuteDiffOperation(
+ op, std::move(direct_writer), in_fd, blob.data(), blob.size()));
}
}
+ WriteVerity(partition, out_fd, manifest.block_size());
int err =
truncate64(output_path.c_str(), partition.new_partition_info().size());
if (err) {
@@ -110,14 +177,32 @@
TEST_AND_RETURN_FALSE(
HashCalculator::RawHashOfFile(output_path, &actual_hash));
CHECK_EQ(HexEncode(ToStringView(actual_hash)),
- HexEncode(partition.new_partition_info().hash()));
+ HexEncode(partition.new_partition_info().hash()))
+ << " Partition " << partition.partition_name()
+ << " hash mismatches. Either the source image or OTA package is "
+ "corrupted.";
}
return true;
}
} // namespace chromeos_update_engine
+namespace {
+
+bool IsIncrementalOTA(const DeltaArchiveManifest& manifest) {
+ for (const auto& part : manifest.partitions()) {
+ if (part.has_old_partition_info()) {
+ return true;
+ }
+ }
+ return false;
+}
+
+} // namespace
+
int main(int argc, char* argv[]) {
+ gflags::SetUsageMessage(
+ "A tool to extract device images from Android OTA packages");
gflags::ParseCommandLineFlags(&argc, &argv, true);
xz_crc32_init();
auto tokens = android::base::Tokenize(FLAGS_partitions, ",");
@@ -172,10 +257,16 @@
LOG(ERROR) << "Failed to parse manifest!";
return 1;
}
+ if (IsIncrementalOTA(manifest) && FLAGS_input_dir.empty()) {
+ LOG(ERROR) << FLAGS_payload
+ << " is an incremental OTA, --input_dir parameter is required.";
+ return 1;
+ }
return !ExtractImagesFromOTA(manifest,
payload_metadata,
payload_fd,
FLAGS_payload_offset,
+ FLAGS_input_dir,
FLAGS_output_dir,
partitions);
}
diff --git a/payload_consumer/install_operation_executor.cc b/payload_consumer/install_operation_executor.cc
index 69ef9c1..cd6546f 100644
--- a/payload_consumer/install_operation_executor.cc
+++ b/payload_consumer/install_operation_executor.cc
@@ -255,6 +255,7 @@
operation, std::move(writer), source_fd, data, count);
default:
LOG(ERROR) << "Unexpected operation type when executing diff ops "
+ << operation.type() << " "
<< operation.Type_Name(operation.type());
return false;
}
diff --git a/payload_consumer/install_plan.cc b/payload_consumer/install_plan.cc
index db0af4e..91eb53b 100644
--- a/payload_consumer/install_plan.cc
+++ b/payload_consumer/install_plan.cc
@@ -187,6 +187,44 @@
postinstall_optional == that.postinstall_optional);
}
+bool InstallPlan::Partition::ParseVerityConfig(
+ const PartitionUpdate& partition) {
+ if (partition.has_hash_tree_extent()) {
+ Extent extent = partition.hash_tree_data_extent();
+ hash_tree_data_offset = extent.start_block() * block_size;
+ hash_tree_data_size = extent.num_blocks() * block_size;
+ extent = partition.hash_tree_extent();
+ hash_tree_offset = extent.start_block() * block_size;
+ hash_tree_size = extent.num_blocks() * block_size;
+ uint64_t hash_tree_data_end = hash_tree_data_offset + hash_tree_data_size;
+ if (hash_tree_offset < hash_tree_data_end) {
+ LOG(ERROR) << "Invalid hash tree extents, hash tree data ends at "
+ << hash_tree_data_end << ", but hash tree starts at "
+ << hash_tree_offset;
+ return false;
+ }
+ hash_tree_algorithm = partition.hash_tree_algorithm();
+ hash_tree_salt.assign(partition.hash_tree_salt().begin(),
+ partition.hash_tree_salt().end());
+ }
+ if (partition.has_fec_extent()) {
+ Extent extent = partition.fec_data_extent();
+ fec_data_offset = extent.start_block() * block_size;
+ fec_data_size = extent.num_blocks() * block_size;
+ extent = partition.fec_extent();
+ fec_offset = extent.start_block() * block_size;
+ fec_size = extent.num_blocks() * block_size;
+ uint64_t fec_data_end = fec_data_offset + fec_data_size;
+ if (fec_offset < fec_data_end) {
+ LOG(ERROR) << "Invalid fec extents, fec data ends at " << fec_data_end
+ << ", but fec starts at " << fec_offset;
+ return false;
+ }
+ fec_roots = partition.fec_roots();
+ }
+ return true;
+}
+
template <typename PartitinoUpdateArray>
bool InstallPlan::ParseManifestToInstallPlan(
const PartitinoUpdateArray& partitions,
@@ -226,42 +264,11 @@
install_part.target_hash.assign(info.hash().begin(), info.hash().end());
install_part.block_size = block_size;
- if (partition.has_hash_tree_extent()) {
- Extent extent = partition.hash_tree_data_extent();
- install_part.hash_tree_data_offset = extent.start_block() * block_size;
- install_part.hash_tree_data_size = extent.num_blocks() * block_size;
- extent = partition.hash_tree_extent();
- install_part.hash_tree_offset = extent.start_block() * block_size;
- install_part.hash_tree_size = extent.num_blocks() * block_size;
- uint64_t hash_tree_data_end =
- install_part.hash_tree_data_offset + install_part.hash_tree_data_size;
- if (install_part.hash_tree_offset < hash_tree_data_end) {
- LOG(ERROR) << "Invalid hash tree extents, hash tree data ends at "
- << hash_tree_data_end << ", but hash tree starts at "
- << install_part.hash_tree_offset;
- *error = ErrorCode::kDownloadNewPartitionInfoError;
- return false;
- }
- install_part.hash_tree_algorithm = partition.hash_tree_algorithm();
- install_part.hash_tree_salt.assign(partition.hash_tree_salt().begin(),
- partition.hash_tree_salt().end());
- }
- if (partition.has_fec_extent()) {
- Extent extent = partition.fec_data_extent();
- install_part.fec_data_offset = extent.start_block() * block_size;
- install_part.fec_data_size = extent.num_blocks() * block_size;
- extent = partition.fec_extent();
- install_part.fec_offset = extent.start_block() * block_size;
- install_part.fec_size = extent.num_blocks() * block_size;
- uint64_t fec_data_end =
- install_part.fec_data_offset + install_part.fec_data_size;
- if (install_part.fec_offset < fec_data_end) {
- LOG(ERROR) << "Invalid fec extents, fec data ends at " << fec_data_end
- << ", but fec starts at " << install_part.fec_offset;
- *error = ErrorCode::kDownloadNewPartitionInfoError;
- return false;
- }
- install_part.fec_roots = partition.fec_roots();
+ if (!install_part.ParseVerityConfig(partition)) {
+ *error = ErrorCode::kDownloadNewPartitionInfoError;
+ LOG(INFO) << "Failed to parse partition `" << partition.partition_name()
+ << "` verity configs";
+ return false;
}
install_plan->partitions.push_back(install_part);
diff --git a/payload_consumer/install_plan.h b/payload_consumer/install_plan.h
index 0278ea5..883aa60 100644
--- a/payload_consumer/install_plan.h
+++ b/payload_consumer/install_plan.h
@@ -156,6 +156,8 @@
uint64_t fec_offset{0};
uint64_t fec_size{0};
uint32_t fec_roots{0};
+
+ bool ParseVerityConfig(const PartitionUpdate&);
};
std::vector<Partition> partitions;
diff --git a/payload_consumer/verity_writer_android.cc b/payload_consumer/verity_writer_android.cc
index ffa2944..91efa3e 100644
--- a/payload_consumer/verity_writer_android.cc
+++ b/payload_consumer/verity_writer_android.cc
@@ -116,7 +116,8 @@
return false;
}
// All hash tree data blocks has been hashed, write hash tree to disk.
- LOG(INFO) << "Writing verity hash tree to " << partition_->target_path;
+ LOG(INFO) << "Writing verity hash tree to "
+ << partition_->readonly_target_path;
if (hash_tree_builder_) {
TEST_AND_RETURN_FALSE(hash_tree_builder_->BuildHashTree());
TEST_AND_RETURN_FALSE_ERRNO(
@@ -130,7 +131,7 @@
hash_tree_builder_.reset();
}
if (partition_->fec_size != 0) {
- LOG(INFO) << "Writing verity FEC to " << partition_->target_path;
+ LOG(INFO) << "Writing verity FEC to " << partition_->readonly_target_path;
TEST_AND_RETURN_FALSE(EncodeFEC(read_fd,
write_fd,
partition_->fec_data_offset,