Incremental OTA support

.gitignore

@@ -32,9 +32,12 @@
 
 # End of https://www.toptal.com/developers/gitignore/api/go,vscode
 
+chromeos_update_engine/build/
 *.zip
-**/payload.bin
-
+**/payload*.bin
+payload-dumper-go
 extracted_*/
+*.img
+
 
 .DS_Store

.gitmodules

@@ -0,0 +1,18 @@
+[submodule "chromeos_update_engine/brotli"]
+	path = chromeos_update_engine/brotli
+	url = https://github.com/anonymix007/brotli
+[submodule "chromeos_update_engine/bsdiff"]
+	path = chromeos_update_engine/bsdiff
+	url = https://github.com/anonymix007/bsdiff
+[submodule "chromeos_update_engine/bzip2"]
+	path = chromeos_update_engine/bzip2
+	url = https://github.com/anonymix007/bzip2
+[submodule "chromeos_update_engine/libchrome"]
+	path = chromeos_update_engine/libchrome
+	url = https://github.com/anonymix007/libchrome
+[submodule "chromeos_update_engine/puffin"]
+	path = chromeos_update_engine/puffin
+	url = https://github.com/anonymix007/puffin
+[submodule "chromeos_update_engine/zucchini"]
+	path = chromeos_update_engine/zucchini
+	url = https://github.com/anonymix007/zucchini

README.md

@@ -18,7 +18,7 @@ See how fast payload-dumper-go is: https://imgur.com/a/X6HKJT4. (MacBook Pro 16-
 
 ### Limitations
 
-- Incremental OTA (delta) payload is not supported.
+- Incremental OTA (delta) payload is not fully tested and probably broken.
 
 ## Installation
 
@@ -56,6 +56,20 @@ payload-dumper-go /path/to/payload.bin
 
 https://android.googlesource.com/platform/system/update_engine/+/master/update_metadata.proto
 
+## Compilation
+
+### Linux and OSX
+```
+git submodule update --init --recursive
+go generate chromeos_update_engine/update_engine.go 
+go build .
+```
+
+### Windows
+
+#### TODO
+
+
 ## License
 
 This source code is licensed under the Apache License 2.0 as described in the LICENSE file.

chromeos_update_engine/Makefile

@@ -0,0 +1,60 @@
+CXXFLAGS=-std=gnu++17
+CFLAGS=-I. -Ibsdiff/include -Ipuffin/src/include -Izucchini/aosp/include -Ilibchrome -Igtest/include -I/usr/include/glib-2.0 -I/usr/lib/glib-2.0/include -Ibuild -ggdb
+LDFLAGS=-Wl,--copy-dt-needed-entries -labsl_log_internal_check_op
+
+libs: build build/libbrotli.a build/libbspatch.a build/libbz2.a build/libpuffpatch.a build/libzucchini.a build/libchrome.a
+
+build:
+	mkdir -p $@
+
+build/%.o: %.cc
+	@ mkdir -p $(dir $@)
+	$(CXX) $(CFLAGS) $(CXXFLAGS) -c -o $@ $< $(LDFLAGS)
+
+build/%.o: %.c
+	@ mkdir -p $(dir $@)
+	$(CC) $(CFLAGS) -c -o $@ $< $(LDFLAGS)
+
+BSPATCH_SRC=bsdiff/brotli_decompressor.cc bsdiff/bspatch.cc bsdiff/bz2_decompressor.cc bsdiff/buffer_file.cc bsdiff/decompressor_interface.cc bsdiff/extents.cc bsdiff/extents_file.cc bsdiff/file.cc bsdiff/logging.cc bsdiff/memory_file.cc bsdiff/patch_reader.cc bsdiff/sink_file.cc bsdiff/utils.cc
+BSPATCH_OBJ=$(addprefix build/,$(BSPATCH_SRC:.cc=.o))
+
+build/libbspatch.a: $(BSPATCH_OBJ)
+	$(AR) rcs $@ $^
+
+BROTLI_SRC=$(wildcard brotli/c/common/*.c brotli/c/dec/*.c brotli/c/enc/*.c)
+BROTLI_OBJ=$(addprefix build/,$(BROTLI_SRC:.c=.o))
+
+build/libbrotli.a: $(BROTLI_OBJ)
+	$(AR) rcs $@ $^
+
+BZ2_SRC=bzip2/blocksort.c bzip2/huffman.c bzip2/crctable.c bzip2/randtable.c bzip2/compress.c bzip2/decompress.c bzip2/bzlib.c
+BZ2_OBJ=$(addprefix build/,$(BZ2_SRC:.c=.o))
+
+build/libbz2.a: $(BZ2_OBJ)
+	$(AR) rcs $@ $^
+
+PUFFPATCH_SRC=puffin/src/bit_reader.cc puffin/src/bit_writer.cc puffin/src/brotli_util.cc puffin/src/huffer.cc puffin/src/huffman_table.cc puffin/src/memory_stream.cc puffin/src/puff_reader.cc puffin/src/puff_writer.cc puffin/src/puffer.cc puffin/src/puffin_stream.cc puffin/src/puffpatch.cc puffin/src/puffin.pb.cc
+PUFFPATCH_OBJ=$(addprefix build/,$(PUFFPATCH_SRC:.cc=.o))
+
+puffin/src/puffpatch.cc: build/puffin/src/puffin.pb.cc
+build/puffin/src/puffin.pb.cc: puffin/src/puffin.proto
+	protoc -I=puffin/src/ --cpp_out=build/puffin/src/ $^
+
+build/libpuffpatch.a: $(PUFFPATCH_OBJ)
+	$(AR) rcs $@ $^
+
+ZUCHIINI_SRC=zucchini/abs32_utils.cc zucchini/address_translator.cc zucchini/arm_utils.cc zucchini/binary_data_histogram.cc zucchini/buffer_sink.cc zucchini/buffer_source.cc zucchini/crc32.cc zucchini/disassembler.cc zucchini/disassembler_dex.cc zucchini/disassembler_elf.cc zucchini/disassembler_no_op.cc zucchini/disassembler_win32.cc zucchini/disassembler_ztf.cc zucchini/element_detection.cc zucchini/encoded_view.cc zucchini/ensemble_matcher.cc zucchini/equivalence_map.cc zucchini/heuristic_ensemble_matcher.cc zucchini/image_index.cc zucchini/imposed_ensemble_matcher.cc zucchini/io_utils.cc zucchini/patch_reader.cc zucchini/patch_writer.cc zucchini/reference_bytes_mixer.cc zucchini/reference_set.cc zucchini/rel32_finder.cc zucchini/rel32_utils.cc zucchini/reloc_elf.cc zucchini/reloc_win32.cc zucchini/target_pool.cc zucchini/targets_affinity.cc zucchini/zucchini_apply.cc zucchini/zucchini_gen.cc zucchini/zucchini_tools.cc
+ZUCCHINI_OBJ=$(addprefix build/,$(ZUCHIINI_SRC:.cc=.o))
+
+build/libzucchini.a: $(ZUCCHINI_OBJ)
+	$(AR) rcs $@ $^
+#CHROME_SRC=$(wildcard libchrome/base/third_party/*/*.cc libchrome/base/synchronization/*.cc libchrome/base/strings/*.cc libchrome/base/metrics/*.c libchrome/base/timer/*.cc libchrome/base/time/*.cc libchrome/base/process/*.cc libchrome/base/posix/*.cc libchrome/base/memory/*.cc libchrome/base/debug/*.cc libchrome/base/files/*.cc libchrome/base/*.cc )
+CHROME_SRC=$(wildcard libchrome/base/third_party/*/*.cc libchrome/base/*/*.cc libchrome/base/*.cc)
+CHROME_OBJ=$(addprefix build/,$(CHROME_SRC:.cc=.o))
+
+build/libchrome.a: $(CHROME_OBJ)
+	$(AR) rcs $@ $^
+
+ALL_OBJ=build/
+clean:
+	rm -rf $(ALL_OBJ)

chromeos_update_engine/glog/gtest_prod_util.h

@@ -0,0 +1,11 @@
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BASE_GTEST_PROD_UTIL_H_
+#define BASE_GTEST_PROD_UTIL_H_
+
+#define FRIEND_TEST_ALL_PREFIXES(...)
+#define FORWARD_DECLARE_TEST(...)
+
+#endif  // BASE_GTEST_PROD_UTIL_H_

chromeos_update_engine/glog/logging.h

@@ -0,0 +1,150 @@
+#ifndef CPP_LOGGING_H_
+#define CPP_LOGGING_H_
+
+#include <iostream>
+#include <cassert>
+#include <cstring>
+#include <cstdint>
+#include <cmath>
+
+//#include "base/template_util.h"
+
+#ifndef DCHECK_IS_ON
+#define DCHECK_IS_ON() 1
+#endif
+
+#ifndef EAT_STREAM_PARAMETERS
+#define EAT_STREAM_PARAMETERS
+#endif
+
+
+#ifndef NOTREACHED
+#define NOTREACHED() DCHECK(false)
+#endif
+
+#ifndef ANALYZER_SKIP_THIS_PATH
+#define ANALYZER_SKIP_THIS_PATH()
+#endif
+
+#ifndef DISALLOW_IMPLICIT_CONSTRUCTORS
+#define DISALLOW_IMPLICIT_CONSTRUCTORS(...)
+#endif
+
+#ifndef LOG
+#define LOG(level) std::cout << #level << ": "
+#endif
+
+#ifndef LOG_IF
+#define LOG_IF(level, cond) if (cond) {} else LOG(level)
+#endif
+
+#ifndef DLOG_IF
+#define DLOG_IF(level, cond) if (cond) {} else LOG(level)
+#endif
+
+#ifndef CHECK
+#define CHECK(...) if (__VA_ARGS__) {} else std::cout << "Check failed"
+#endif
+
+#ifndef PCHECK
+#define PCHECK CHECK
+#endif
+
+#ifndef DPCHECK
+#define DPCHECK CHECK
+#endif
+
+#ifndef DLOG
+#define DLOG(level) LOG(level)
+#endif
+
+#ifndef VLOG
+#define VLOG(level) LOG(level)
+#endif
+
+#ifndef PLOG
+#define PLOG(level) LOG(level)
+#endif
+
+#ifndef DVLOG
+#define DVLOG(level) LOG(level)
+#endif
+
+#ifndef DPLOG
+#define DPLOG(level) LOG(level)
+#endif
+
+#ifndef DVPLOG
+#define DVPLOG(level) LOG(level)
+#endif
+
+#ifndef VPLOG
+#define VPLOG(level) LOG(level)
+#endif
+
+#ifndef RAW_CHECK
+#define RAW_CHECK(...) assert(__VA_ARGS__)
+#endif
+
+#ifndef RAW_LOG
+#define RAW_LOG(level, ...) LOG(level) << __VA_ARGS__
+#endif
+
+#ifndef DCHECK
+#define DCHECK CHECK
+#endif
+
+#ifndef DCHECK_GT
+#define DCHECK_GT CHECK_GT
+#endif
+
+#ifndef DCHECK_GE
+#define DCHECK_GE CHECK_GE
+#endif
+
+#ifndef DCHECK_LT
+#define DCHECK_LT CHECK_LT
+#endif
+
+#ifndef DCHECK_LE
+#define DCHECK_LE CHECK_LE
+#endif
+
+#ifndef DCHECK_NE
+#define DCHECK_NE CHECK_NE
+#endif
+
+#ifndef DCHECK_EQ
+#define DCHECK_EQ CHECK_EQ
+#endif
+
+#ifndef CHECK_OP
+#define CHECK_OP(a, b, op) CHECK((a) op (b))
+#endif
+
+
+#ifndef CHECK_GT
+#define CHECK_GT(a, b) CHECK_OP(a, b, >)
+#endif
+
+#ifndef CHECK_GE
+#define CHECK_GE(a, b) CHECK_OP(a, b, >=)
+#endif
+
+#ifndef CHECK_LT
+#define CHECK_LT(a, b) CHECK_OP(a, b, <)
+#endif
+
+#ifndef CHECK_LE
+#define CHECK_LE(a, b) CHECK_OP(a, b, <=)
+#endif
+
+#ifndef CHECK_NE
+#define CHECK_NE(a, b) CHECK_OP(a, b, !=)
+#endif
+
+#ifndef CHECK_EQ
+#define CHECK_EQ(a, b) CHECK_OP(a, b, ==)
+#endif
+
+#endif /* CPP_LOGGING_H_ */

chromeos_update_engine/update_engine.cc

@@ -0,0 +1,95 @@
+#include "update_engine.h"
+#include <bsdiff/bspatch.h>
+#include <puffin/puffpatch.h>
+#include <cstring>
+#include <utility>
+
+class PuffinDataStream : public puffin::StreamInterface {
+ public:
+   PuffinDataStream(void *data, uint64_t size, bool is_read)
+      : data_(data),
+        size_(size),
+        offset_(0),
+        is_read_(is_read) {}
+
+  ~PuffinDataStream() override = default;
+
+  bool GetSize(uint64_t* size) const override {
+    *size = size_;
+    return true;
+  }
+
+  bool GetOffset(uint64_t* offset) const override {
+    *offset = offset_;
+    return true;
+  }
+
+  bool Seek(uint64_t offset) override {
+    if (is_read_) {
+      offset_ = offset;
+    } else {
+      // For writes technically there should be no change of position, or it
+      // should equivalent of current offset.
+      return offset_ != offset;
+    }
+    return true;
+  }
+
+  bool Read(void* buffer, size_t count) override {
+    if (offset_ + count >= size_ || !is_read_) return false;
+    std::memcpy(buffer, data_, count);
+    offset_ += count;
+    return true;
+  }
+
+  bool Write(const void* buffer, size_t count) override {
+    if (offset_ + count >= size_ || is_read_) return false;
+    std::memcpy(data_, buffer, count);
+    offset_ += count;
+    return true;
+  }
+
+  bool Close() override { return true; }
+
+ private:
+
+  void *data_;
+  uint64_t size_;
+  uint64_t offset_;
+  bool is_read_;
+
+  DISALLOW_COPY_AND_ASSIGN(PuffinDataStream);
+};
+
+extern "C" int64_t ExecuteSourcePuffDiffOperation(void *data, size_t data_size,
+                                              void *patch, size_t patch_size,
+                                              void *output, size_t output_size) {
+    constexpr size_t kMaxCacheSize = 5 * 1024 * 1024;  // Total 5MB cache.
+
+    puffin::UniqueStreamPtr src(new PuffinDataStream(data, data_size, true));
+    puffin::UniqueStreamPtr dst(new PuffinDataStream(output, output_size, true));
+
+    return puffin::PuffPatch(std::move(src), std::move(dst), (const uint8_t *) patch, patch_size, kMaxCacheSize) ? -1 : output_size;
+}
+
+
+extern "C" int64_t ExecuteSourceBsdiffOperation(void *data, size_t data_size,
+                                            void *patch, size_t patch_size,
+                                            void *output, size_t output_size) {
+
+    size_t written = 0;
+
+    auto sink = [output, &written, output_size](const uint8_t *data, size_t count) -> size_t {
+        written += count;
+        if (written >= output_size) {
+            return 0;
+        }
+        std::memcpy(output, data, count);
+        return count;
+    };
+
+    int result = bsdiff::bspatch((const uint8_t *) data, data_size, (const uint8_t *) patch, patch_size, sink);
+
+    return result == 0 ? written : -result;
+
+}