blob: 37715bc703abb9c9c1f80075e1f35ac2b40360b3 [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build. An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage: ota_from_target_files [flags] input_target_files output_ota_package
23
24 -b (--board_config) <file>
Doug Zongkerfdd8e692009-08-03 17:27:48 -070025 Deprecated.
Doug Zongkereef39442009-04-02 12:14:19 -070026
27 -k (--package_key) <key>
28 Key to use to sign the package (default is
29 "build/target/product/security/testkey").
30
31 -i (--incremental_from) <file>
32 Generate an incremental OTA using the given target-files zip as
33 the starting build.
34
Doug Zongkerdbfaae52009-04-21 17:12:54 -070035 -w (--wipe_user_data)
36 Generate an OTA package that will wipe the user data partition
37 when installed.
38
Doug Zongker962069c2009-04-23 11:41:58 -070039 -n (--no_prereq)
40 Omit the timestamp prereq check normally included at the top of
41 the build scripts (used for developer OTA packages which
42 legitimately need to go back and forth).
43
Doug Zongker1c390a22009-05-14 19:06:36 -070044 -e (--extra_script) <file>
45 Insert the contents of file at the end of the update script.
46
Doug Zongkereef39442009-04-02 12:14:19 -070047"""
48
49import sys
50
51if sys.hexversion < 0x02040000:
52 print >> sys.stderr, "Python 2.4 or newer is required."
53 sys.exit(1)
54
55import copy
Doug Zongkerc18736b2009-09-30 09:20:32 -070056import errno
Doug Zongkereef39442009-04-02 12:14:19 -070057import os
58import re
59import sha
60import subprocess
61import tempfile
62import time
63import zipfile
64
65import common
Doug Zongkerc494d7c2009-06-18 08:43:44 -070066import edify_generator
Doug Zongkereef39442009-04-02 12:14:19 -070067
68OPTIONS = common.OPTIONS
69OPTIONS.package_key = "build/target/product/security/testkey"
70OPTIONS.incremental_source = None
71OPTIONS.require_verbatim = set()
72OPTIONS.prohibit_verbatim = set(("system/build.prop",))
73OPTIONS.patch_threshold = 0.95
Doug Zongkerdbfaae52009-04-21 17:12:54 -070074OPTIONS.wipe_user_data = False
Doug Zongker962069c2009-04-23 11:41:58 -070075OPTIONS.omit_prereq = False
Doug Zongker1c390a22009-05-14 19:06:36 -070076OPTIONS.extra_script = None
Doug Zongker761e6422009-09-25 10:45:39 -070077OPTIONS.worker_threads = 3
Doug Zongkereef39442009-04-02 12:14:19 -070078
79def MostPopularKey(d, default):
80 """Given a dict, return the key corresponding to the largest
81 value. Returns 'default' if the dict is empty."""
82 x = [(v, k) for (k, v) in d.iteritems()]
83 if not x: return default
84 x.sort()
85 return x[-1][1]
86
87
88def IsSymlink(info):
89 """Return true if the zipfile.ZipInfo object passed in represents a
90 symlink."""
91 return (info.external_attr >> 16) == 0120777
92
93
94
95class Item:
96 """Items represent the metadata (user, group, mode) of files and
97 directories in the system image."""
98 ITEMS = {}
99 def __init__(self, name, dir=False):
100 self.name = name
101 self.uid = None
102 self.gid = None
103 self.mode = None
104 self.dir = dir
105
106 if name:
107 self.parent = Item.Get(os.path.dirname(name), dir=True)
108 self.parent.children.append(self)
109 else:
110 self.parent = None
111 if dir:
112 self.children = []
113
114 def Dump(self, indent=0):
115 if self.uid is not None:
116 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode)
117 else:
118 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode)
119 if self.dir:
120 print "%s%s" % (" "*indent, self.descendants)
121 print "%s%s" % (" "*indent, self.best_subtree)
122 for i in self.children:
123 i.Dump(indent=indent+1)
124
125 @classmethod
126 def Get(cls, name, dir=False):
127 if name not in cls.ITEMS:
128 cls.ITEMS[name] = Item(name, dir=dir)
129 return cls.ITEMS[name]
130
131 @classmethod
Doug Zongker283e2a12010-03-15 17:52:32 -0700132 def GetMetadata(cls, input_zip):
133
134 try:
135 # See if the target_files contains a record of what the uid,
136 # gid, and mode is supposed to be.
137 output = input_zip.read("META/filesystem_config.txt")
138 except KeyError:
139 # Run the external 'fs_config' program to determine the desired
140 # uid, gid, and mode for every Item object. Note this uses the
141 # one in the client now, which might not be the same as the one
142 # used when this target_files was built.
143 p = common.Run(["fs_config"], stdin=subprocess.PIPE,
144 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
145 suffix = { False: "", True: "/" }
146 input = "".join(["%s%s\n" % (i.name, suffix[i.dir])
147 for i in cls.ITEMS.itervalues() if i.name])
Doug Zongker3475d362010-03-17 16:39:30 -0700148 output, error = p.communicate(input)
Doug Zongker283e2a12010-03-15 17:52:32 -0700149 assert not error
Doug Zongkereef39442009-04-02 12:14:19 -0700150
151 for line in output.split("\n"):
152 if not line: continue
153 name, uid, gid, mode = line.split()
Doug Zongker283e2a12010-03-15 17:52:32 -0700154 i = cls.ITEMS.get(name, None)
155 if i is not None:
156 i.uid = int(uid)
157 i.gid = int(gid)
158 i.mode = int(mode, 8)
159 if i.dir:
160 i.children.sort(key=lambda i: i.name)
161
162 # set metadata for the files generated by this script.
163 i = cls.ITEMS.get("system/recovery-from-boot.p", None)
164 if i: i.uid, i.gid, i.mode = 0, 0, 0644
165 i = cls.ITEMS.get("system/etc/install-recovery.sh", None)
166 if i: i.uid, i.gid, i.mode = 0, 0, 0544
Doug Zongkereef39442009-04-02 12:14:19 -0700167
168 def CountChildMetadata(self):
169 """Count up the (uid, gid, mode) tuples for all children and
170 determine the best strategy for using set_perm_recursive and
171 set_perm to correctly chown/chmod all the files to their desired
172 values. Recursively calls itself for all descendants.
173
174 Returns a dict of {(uid, gid, dmode, fmode): count} counting up
175 all descendants of this node. (dmode or fmode may be None.) Also
176 sets the best_subtree of each directory Item to the (uid, gid,
177 dmode, fmode) tuple that will match the most descendants of that
178 Item.
179 """
180
181 assert self.dir
182 d = self.descendants = {(self.uid, self.gid, self.mode, None): 1}
183 for i in self.children:
184 if i.dir:
185 for k, v in i.CountChildMetadata().iteritems():
186 d[k] = d.get(k, 0) + v
187 else:
188 k = (i.uid, i.gid, None, i.mode)
189 d[k] = d.get(k, 0) + 1
190
191 # Find the (uid, gid, dmode, fmode) tuple that matches the most
192 # descendants.
193
194 # First, find the (uid, gid) pair that matches the most
195 # descendants.
196 ug = {}
197 for (uid, gid, _, _), count in d.iteritems():
198 ug[(uid, gid)] = ug.get((uid, gid), 0) + count
199 ug = MostPopularKey(ug, (0, 0))
200
201 # Now find the dmode and fmode that match the most descendants
202 # with that (uid, gid), and choose those.
203 best_dmode = (0, 0755)
204 best_fmode = (0, 0644)
205 for k, count in d.iteritems():
206 if k[:2] != ug: continue
207 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
208 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
209 self.best_subtree = ug + (best_dmode[1], best_fmode[1])
210
211 return d
212
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700213 def SetPermissions(self, script):
Doug Zongkereef39442009-04-02 12:14:19 -0700214 """Append set_perm/set_perm_recursive commands to 'script' to
215 set all permissions, users, and groups for the tree of files
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700216 rooted at 'self'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700217
218 self.CountChildMetadata()
219
220 def recurse(item, current):
221 # current is the (uid, gid, dmode, fmode) tuple that the current
222 # item (and all its children) have already been set to. We only
223 # need to issue set_perm/set_perm_recursive commands if we're
224 # supposed to be something different.
225 if item.dir:
226 if current != item.best_subtree:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700227 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree)
Doug Zongkereef39442009-04-02 12:14:19 -0700228 current = item.best_subtree
229
230 if item.uid != current[0] or item.gid != current[1] or \
231 item.mode != current[2]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700232 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700233
234 for i in item.children:
235 recurse(i, current)
236 else:
237 if item.uid != current[0] or item.gid != current[1] or \
238 item.mode != current[3]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700239 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700240
241 recurse(self, (-1, -1, -1, -1))
242
243
244def CopySystemFiles(input_zip, output_zip=None,
245 substitute=None):
246 """Copies files underneath system/ in the input zip to the output
247 zip. Populates the Item class with their metadata, and returns a
248 list of symlinks. output_zip may be None, in which case the copy is
249 skipped (but the other side effects still happen). substitute is an
250 optional dict of {output filename: contents} to be output instead of
251 certain input files.
252 """
253
254 symlinks = []
255
256 for info in input_zip.infolist():
257 if info.filename.startswith("SYSTEM/"):
258 basefilename = info.filename[7:]
259 if IsSymlink(info):
260 symlinks.append((input_zip.read(info.filename),
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700261 "/system/" + basefilename))
Doug Zongkereef39442009-04-02 12:14:19 -0700262 else:
263 info2 = copy.copy(info)
264 fn = info2.filename = "system/" + basefilename
265 if substitute and fn in substitute and substitute[fn] is None:
266 continue
267 if output_zip is not None:
268 if substitute and fn in substitute:
269 data = substitute[fn]
270 else:
271 data = input_zip.read(info.filename)
272 output_zip.writestr(info2, data)
273 if fn.endswith("/"):
274 Item.Get(fn[:-1], dir=True)
275 else:
276 Item.Get(fn, dir=False)
277
278 symlinks.sort()
279 return symlinks
280
281
Doug Zongkereef39442009-04-02 12:14:19 -0700282def SignOutput(temp_zip_name, output_zip_name):
283 key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
284 pw = key_passwords[OPTIONS.package_key]
285
Doug Zongker951495f2009-08-14 12:44:19 -0700286 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
287 whole_file=True)
Doug Zongkereef39442009-04-02 12:14:19 -0700288
289
Doug Zongkereef39442009-04-02 12:14:19 -0700290def AppendAssertions(script, input_zip):
Doug Zongkereef39442009-04-02 12:14:19 -0700291 device = GetBuildProp("ro.product.device", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700292 script.AssertDevice(device)
Doug Zongkereef39442009-04-02 12:14:19 -0700293
Doug Zongkereef39442009-04-02 12:14:19 -0700294
Doug Zongker73ef8252009-07-23 15:12:53 -0700295def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
296 """Generate a binary patch that creates the recovery image starting
297 with the boot image. (Most of the space in these images is just the
298 kernel, which is identical for the two, so the resulting patch
299 should be efficient.) Add it to the output zip, along with a shell
300 script that is run from init.rc on first boot to actually do the
301 patching and install the new recovery image.
302
303 recovery_img and boot_img should be File objects for the
304 corresponding images.
305
306 Returns an Item for the shell script, which must be made
307 executable.
308 """
309
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700310 d = common.Difference(recovery_img, boot_img)
Doug Zongker761e6422009-09-25 10:45:39 -0700311 _, _, patch = d.ComputePatch()
Doug Zongkercfd7db62009-10-07 11:35:53 -0700312 common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch)
Doug Zongker73ef8252009-07-23 15:12:53 -0700313 Item.Get("system/recovery-from-boot.p", dir=False)
314
315 # Images with different content will have a different first page, so
316 # we check to see if this recovery has already been installed by
317 # testing just the first 2k.
318 HEADER_SIZE = 2048
319 header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest()
320 sh = """#!/system/bin/sh
321if ! applypatch -c MTD:recovery:%(header_size)d:%(header_sha1)s; then
322 log -t recovery "Installing new recovery image"
323 applypatch MTD:boot:%(boot_size)d:%(boot_sha1)s MTD:recovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p
324else
325 log -t recovery "Recovery image already installed"
326fi
327""" % { 'boot_size': boot_img.size,
328 'boot_sha1': boot_img.sha1,
329 'header_size': HEADER_SIZE,
330 'header_sha1': header_sha1,
331 'recovery_size': recovery_img.size,
332 'recovery_sha1': recovery_img.sha1 }
Doug Zongkercfd7db62009-10-07 11:35:53 -0700333 common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh)
Doug Zongker73ef8252009-07-23 15:12:53 -0700334 return Item.Get("system/etc/install-recovery.sh", dir=False)
335
336
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700337def WriteFullOTAPackage(input_zip, output_zip):
Doug Zongkerc637db12010-04-21 14:08:44 -0700338 # TODO: how to determine this? We don't know what version it will
339 # be installed on top of. For now, we expect the API just won't
340 # change very often.
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700341 script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict)
Doug Zongkereef39442009-04-02 12:14:19 -0700342
Doug Zongker2ea21062010-04-28 16:05:21 -0700343 metadata = {"post-build": GetBuildProp("ro.build.fingerprint", input_zip),
344 "pre-device": GetBuildProp("ro.product.device", input_zip),
Doug Zongker3b852692010-06-21 15:30:45 -0700345 "post-timestamp": GetBuildProp("ro.build.date.utc", input_zip),
Doug Zongker2ea21062010-04-28 16:05:21 -0700346 }
347
Doug Zongker05d3dea2009-06-22 11:32:31 -0700348 device_specific = common.DeviceSpecificParams(
349 input_zip=input_zip,
Doug Zongker37974732010-09-16 17:44:38 -0700350 input_version=OPTIONS.info_dict["recovery_api_version"],
Doug Zongker05d3dea2009-06-22 11:32:31 -0700351 output_zip=output_zip,
352 script=script,
Doug Zongker2ea21062010-04-28 16:05:21 -0700353 input_tmp=OPTIONS.input_tmp,
354 metadata=metadata)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700355
Doug Zongker962069c2009-04-23 11:41:58 -0700356 if not OPTIONS.omit_prereq:
357 ts = GetBuildProp("ro.build.date.utc", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700358 script.AssertOlderBuild(ts)
Doug Zongkereef39442009-04-02 12:14:19 -0700359
360 AppendAssertions(script, input_zip)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700361 device_specific.FullOTA_Assertions()
Doug Zongker171f1cd2009-06-15 22:36:37 -0700362
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700363 script.ShowProgress(0.5, 0)
Doug Zongkereef39442009-04-02 12:14:19 -0700364
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700365 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700366 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700367
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700368 script.FormatPartition("system")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700369 script.Mount("system", "/system")
Doug Zongkercfd7db62009-10-07 11:35:53 -0700370 script.UnpackPackageDir("recovery", "/system")
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700371 script.UnpackPackageDir("system", "/system")
Doug Zongkereef39442009-04-02 12:14:19 -0700372
373 symlinks = CopySystemFiles(input_zip, output_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700374 script.MakeSymlinks(symlinks)
Doug Zongkereef39442009-04-02 12:14:19 -0700375
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700376 boot_img = common.File("boot.img", common.BuildBootableImage(
Doug Zongker73ef8252009-07-23 15:12:53 -0700377 os.path.join(OPTIONS.input_tmp, "BOOT")))
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700378 recovery_img = common.File("recovery.img", common.BuildBootableImage(
Doug Zongker73ef8252009-07-23 15:12:53 -0700379 os.path.join(OPTIONS.input_tmp, "RECOVERY")))
Doug Zongker283e2a12010-03-15 17:52:32 -0700380 MakeRecoveryPatch(output_zip, recovery_img, boot_img)
Doug Zongkereef39442009-04-02 12:14:19 -0700381
Doug Zongker283e2a12010-03-15 17:52:32 -0700382 Item.GetMetadata(input_zip)
Doug Zongker73ef8252009-07-23 15:12:53 -0700383 Item.Get("system").SetPermissions(script)
384
Doug Zongker37974732010-09-16 17:44:38 -0700385 common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict)
Doug Zongker73ef8252009-07-23 15:12:53 -0700386 common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700387 script.ShowProgress(0.2, 0)
388
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700389 script.ShowProgress(0.2, 10)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700390 script.WriteRawImage("boot", "boot.img")
391
392 script.ShowProgress(0.1, 0)
393 device_specific.FullOTA_InstallEnd()
Doug Zongkereef39442009-04-02 12:14:19 -0700394
Doug Zongker1c390a22009-05-14 19:06:36 -0700395 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700396 script.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700397
Doug Zongker14833602010-02-02 13:12:04 -0800398 script.UnmountAll()
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700399 script.AddToZip(input_zip, output_zip)
Doug Zongker2ea21062010-04-28 16:05:21 -0700400 WriteMetadata(metadata, output_zip)
401
402
403def WriteMetadata(metadata, output_zip):
404 common.ZipWriteStr(output_zip, "META-INF/com/android/metadata",
405 "".join(["%s=%s\n" % kv
406 for kv in sorted(metadata.iteritems())]))
Doug Zongkereef39442009-04-02 12:14:19 -0700407
408
Doug Zongkereef39442009-04-02 12:14:19 -0700409
410
411def LoadSystemFiles(z):
412 """Load all the files from SYSTEM/... in a given target-files
413 ZipFile, and return a dict of {filename: File object}."""
414 out = {}
415 for info in z.infolist():
416 if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
417 fn = "system/" + info.filename[7:]
418 data = z.read(info.filename)
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700419 out[fn] = common.File(fn, data)
Doug Zongkereef39442009-04-02 12:14:19 -0700420 return out
421
422
Doug Zongkereef39442009-04-02 12:14:19 -0700423def GetBuildProp(property, z):
424 """Return the fingerprint of the build of a given target-files
425 ZipFile object."""
426 bp = z.read("SYSTEM/build.prop")
427 if not property:
428 return bp
429 m = re.search(re.escape(property) + r"=(.*)\n", bp)
430 if not m:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700431 raise common.ExternalError("couldn't find %s in build.prop" % (property,))
Doug Zongkereef39442009-04-02 12:14:19 -0700432 return m.group(1).strip()
433
434
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700435def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
Doug Zongker37974732010-09-16 17:44:38 -0700436 source_version = OPTIONS.source_info_dict["recovery_api_version"]
437 target_version = OPTIONS.target_info_dict["recovery_api_version"]
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700438
Doug Zongkerc637db12010-04-21 14:08:44 -0700439 if source_version == 0:
440 print ("WARNING: generating edify script for a source that "
441 "can't install it.")
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700442 script = edify_generator.EdifyGenerator(source_version, OPTIONS.info_dict)
Doug Zongkereef39442009-04-02 12:14:19 -0700443
Doug Zongker2ea21062010-04-28 16:05:21 -0700444 metadata = {"pre-device": GetBuildProp("ro.product.device", source_zip),
Doug Zongker3b852692010-06-21 15:30:45 -0700445 "post-timestamp": GetBuildProp("ro.build.date.utc", target_zip),
Doug Zongker2ea21062010-04-28 16:05:21 -0700446 }
447
Doug Zongker05d3dea2009-06-22 11:32:31 -0700448 device_specific = common.DeviceSpecificParams(
449 source_zip=source_zip,
Doug Zongker14833602010-02-02 13:12:04 -0800450 source_version=source_version,
Doug Zongker05d3dea2009-06-22 11:32:31 -0700451 target_zip=target_zip,
Doug Zongker14833602010-02-02 13:12:04 -0800452 target_version=target_version,
Doug Zongker05d3dea2009-06-22 11:32:31 -0700453 output_zip=output_zip,
Doug Zongker2ea21062010-04-28 16:05:21 -0700454 script=script,
455 metadata=metadata)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700456
Doug Zongkereef39442009-04-02 12:14:19 -0700457 print "Loading target..."
458 target_data = LoadSystemFiles(target_zip)
459 print "Loading source..."
460 source_data = LoadSystemFiles(source_zip)
461
462 verbatim_targets = []
463 patch_list = []
Doug Zongker761e6422009-09-25 10:45:39 -0700464 diffs = []
Doug Zongkereef39442009-04-02 12:14:19 -0700465 largest_source_size = 0
466 for fn in sorted(target_data.keys()):
467 tf = target_data[fn]
Doug Zongker761e6422009-09-25 10:45:39 -0700468 assert fn == tf.name
Doug Zongkereef39442009-04-02 12:14:19 -0700469 sf = source_data.get(fn, None)
470
471 if sf is None or fn in OPTIONS.require_verbatim:
472 # This file should be included verbatim
473 if fn in OPTIONS.prohibit_verbatim:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700474 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,))
Doug Zongkereef39442009-04-02 12:14:19 -0700475 print "send", fn, "verbatim"
476 tf.AddToZip(output_zip)
477 verbatim_targets.append((fn, tf.size))
478 elif tf.sha1 != sf.sha1:
479 # File is different; consider sending as a patch
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700480 diffs.append(common.Difference(tf, sf))
Doug Zongkereef39442009-04-02 12:14:19 -0700481 else:
482 # Target file identical to source.
483 pass
484
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700485 common.ComputeDifferences(diffs)
Doug Zongker761e6422009-09-25 10:45:39 -0700486
487 for diff in diffs:
488 tf, sf, d = diff.GetPatch()
489 if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
490 # patch is almost as big as the file; don't bother patching
491 tf.AddToZip(output_zip)
492 verbatim_targets.append((tf.name, tf.size))
493 else:
494 common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d)
Doug Zongker5a482092010-02-17 16:09:18 -0800495 patch_list.append((tf.name, tf, sf, tf.size, sha.sha(d).hexdigest()))
Doug Zongker761e6422009-09-25 10:45:39 -0700496 largest_source_size = max(largest_source_size, sf.size)
Doug Zongkereef39442009-04-02 12:14:19 -0700497
498 source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
499 target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
Doug Zongker2ea21062010-04-28 16:05:21 -0700500 metadata["pre-build"] = source_fp
501 metadata["post-build"] = target_fp
Doug Zongkereef39442009-04-02 12:14:19 -0700502
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700503 script.Mount("system", "/system")
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700504 script.AssertSomeFingerprint(source_fp, target_fp)
Doug Zongkereef39442009-04-02 12:14:19 -0700505
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700506 source_boot = common.File("/tmp/boot.img",
507 common.BuildBootableImage(
508 os.path.join(OPTIONS.source_tmp, "BOOT")))
509 target_boot = common.File("/tmp/boot.img",
510 common.BuildBootableImage(
511 os.path.join(OPTIONS.target_tmp, "BOOT")))
Doug Zongker5da317e2009-06-02 13:38:17 -0700512 updating_boot = (source_boot.data != target_boot.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700513
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700514 source_recovery = common.File("system/recovery.img",
515 common.BuildBootableImage(
516 os.path.join(OPTIONS.source_tmp, "RECOVERY")))
517 target_recovery = common.File("system/recovery.img",
518 common.BuildBootableImage(
519 os.path.join(OPTIONS.target_tmp, "RECOVERY")))
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700520 updating_recovery = (source_recovery.data != target_recovery.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700521
Doug Zongker881dd402009-09-20 14:03:55 -0700522 # Here's how we divide up the progress bar:
523 # 0.1 for verifying the start state (PatchCheck calls)
524 # 0.8 for applying patches (ApplyPatch calls)
525 # 0.1 for unpacking verbatim files, symlinking, and doing the
526 # device-specific commands.
Doug Zongkereef39442009-04-02 12:14:19 -0700527
528 AppendAssertions(script, target_zip)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700529 device_specific.IncrementalOTA_Assertions()
Doug Zongkereef39442009-04-02 12:14:19 -0700530
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700531 script.Print("Verifying current system...")
532
Doug Zongker881dd402009-09-20 14:03:55 -0700533 script.ShowProgress(0.1, 0)
534 total_verify_size = float(sum([i[2].size for i in patch_list]) + 1)
535 if updating_boot:
536 total_verify_size += source_boot.size
537 so_far = 0
Doug Zongkereef39442009-04-02 12:14:19 -0700538
Doug Zongker5a482092010-02-17 16:09:18 -0800539 for fn, tf, sf, size, patch_sha in patch_list:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700540 script.PatchCheck("/"+fn, tf.sha1, sf.sha1)
Doug Zongker881dd402009-09-20 14:03:55 -0700541 so_far += sf.size
542 script.SetProgress(so_far / total_verify_size)
Doug Zongkereef39442009-04-02 12:14:19 -0700543
Doug Zongker5da317e2009-06-02 13:38:17 -0700544 if updating_boot:
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700545 d = common.Difference(target_boot, source_boot)
Doug Zongker761e6422009-09-25 10:45:39 -0700546 _, _, d = d.ComputePatch()
Doug Zongker5da317e2009-06-02 13:38:17 -0700547 print "boot target: %d source: %d diff: %d" % (
548 target_boot.size, source_boot.size, len(d))
549
Doug Zongker048e7ca2009-06-15 14:31:53 -0700550 common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
Doug Zongker5da317e2009-06-02 13:38:17 -0700551
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700552 script.PatchCheck("MTD:boot:%d:%s:%d:%s" %
553 (source_boot.size, source_boot.sha1,
554 target_boot.size, target_boot.sha1))
Doug Zongker881dd402009-09-20 14:03:55 -0700555 so_far += source_boot.size
556 script.SetProgress(so_far / total_verify_size)
Doug Zongker5da317e2009-06-02 13:38:17 -0700557
558 if patch_list or updating_recovery or updating_boot:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700559 script.CacheFreeSpaceCheck(largest_source_size)
Doug Zongker5a482092010-02-17 16:09:18 -0800560
Doug Zongker05d3dea2009-06-22 11:32:31 -0700561 device_specific.IncrementalOTA_VerifyEnd()
562
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700563 script.Comment("---- start making changes here ----")
Doug Zongkereef39442009-04-02 12:14:19 -0700564
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700565 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700566 script.Print("Erasing user data...")
567 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700568
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700569 script.Print("Removing unneeded files...")
Doug Zongker0f3298a2009-06-30 08:16:58 -0700570 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] +
571 ["/"+i for i in sorted(source_data)
Doug Zongker3b949f02009-08-24 10:24:32 -0700572 if i not in target_data] +
573 ["/system/recovery.img"])
Doug Zongkereef39442009-04-02 12:14:19 -0700574
Doug Zongker881dd402009-09-20 14:03:55 -0700575 script.ShowProgress(0.8, 0)
576 total_patch_size = float(sum([i[1].size for i in patch_list]) + 1)
577 if updating_boot:
578 total_patch_size += target_boot.size
579 so_far = 0
580
581 script.Print("Patching system files...")
Doug Zongker5a482092010-02-17 16:09:18 -0800582 for fn, tf, sf, size, _ in patch_list:
Doug Zongkerc8d446b2010-02-22 15:41:53 -0800583 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1, sf.sha1, "patch/"+fn+".p")
Doug Zongker881dd402009-09-20 14:03:55 -0700584 so_far += tf.size
585 script.SetProgress(so_far / total_patch_size)
586
Doug Zongkereef39442009-04-02 12:14:19 -0700587 if updating_boot:
Doug Zongker5da317e2009-06-02 13:38:17 -0700588 # Produce the boot image by applying a patch to the current
589 # contents of the boot partition, and write it back to the
590 # partition.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700591 script.Print("Patching boot image...")
592 script.ApplyPatch("MTD:boot:%d:%s:%d:%s"
593 % (source_boot.size, source_boot.sha1,
594 target_boot.size, target_boot.sha1),
595 "-",
596 target_boot.size, target_boot.sha1,
Doug Zongkerc8d446b2010-02-22 15:41:53 -0800597 source_boot.sha1, "patch/boot.img.p")
Doug Zongker881dd402009-09-20 14:03:55 -0700598 so_far += target_boot.size
599 script.SetProgress(so_far / total_patch_size)
Doug Zongkereef39442009-04-02 12:14:19 -0700600 print "boot image changed; including."
601 else:
602 print "boot image unchanged; skipping."
603
604 if updating_recovery:
Doug Zongker73ef8252009-07-23 15:12:53 -0700605 # Is it better to generate recovery as a patch from the current
606 # boot image, or from the previous recovery image? For large
607 # updates with significant kernel changes, probably the former.
608 # For small updates where the kernel hasn't changed, almost
609 # certainly the latter. We pick the first option. Future
610 # complicated schemes may let us effectively use both.
611 #
612 # A wacky possibility: as long as there is room in the boot
613 # partition, include the binaries and image files from recovery in
614 # the boot image (though not in the ramdisk) so they can be used
615 # as fodder for constructing the recovery image.
Doug Zongker283e2a12010-03-15 17:52:32 -0700616 MakeRecoveryPatch(output_zip, target_recovery, target_boot)
Doug Zongker42265392010-02-12 10:21:00 -0800617 script.DeleteFiles(["/system/recovery-from-boot.p",
618 "/system/etc/install-recovery.sh"])
Doug Zongker73ef8252009-07-23 15:12:53 -0700619 print "recovery image changed; including as patch from boot."
Doug Zongkereef39442009-04-02 12:14:19 -0700620 else:
621 print "recovery image unchanged; skipping."
622
Doug Zongker881dd402009-09-20 14:03:55 -0700623 script.ShowProgress(0.1, 10)
Doug Zongkereef39442009-04-02 12:14:19 -0700624
625 target_symlinks = CopySystemFiles(target_zip, None)
626
627 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700628 temp_script = script.MakeTemporary()
Doug Zongker283e2a12010-03-15 17:52:32 -0700629 Item.GetMetadata(target_zip)
Doug Zongker73ef8252009-07-23 15:12:53 -0700630 Item.Get("system").SetPermissions(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700631
632 # Note that this call will mess up the tree of Items, so make sure
633 # we're done with it.
634 source_symlinks = CopySystemFiles(source_zip, None)
635 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
636
637 # Delete all the symlinks in source that aren't in target. This
638 # needs to happen before verbatim files are unpacked, in case a
639 # symlink in the source is replaced by a real file in the target.
640 to_delete = []
641 for dest, link in source_symlinks:
642 if link not in target_symlinks_d:
643 to_delete.append(link)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700644 script.DeleteFiles(to_delete)
Doug Zongkereef39442009-04-02 12:14:19 -0700645
646 if verbatim_targets:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700647 script.Print("Unpacking new files...")
648 script.UnpackPackageDir("system", "/system")
649
Doug Zongker42265392010-02-12 10:21:00 -0800650 if updating_recovery:
651 script.Print("Unpacking new recovery...")
652 script.UnpackPackageDir("recovery", "/system")
653
Doug Zongker05d3dea2009-06-22 11:32:31 -0700654 script.Print("Symlinks and permissions...")
Doug Zongkereef39442009-04-02 12:14:19 -0700655
656 # Create all the symlinks that don't already exist, or point to
657 # somewhere different than what we want. Delete each symlink before
658 # creating it, since the 'symlink' command won't overwrite.
659 to_create = []
660 for dest, link in target_symlinks:
661 if link in source_symlinks_d:
662 if dest != source_symlinks_d[link]:
663 to_create.append((dest, link))
664 else:
665 to_create.append((dest, link))
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700666 script.DeleteFiles([i[1] for i in to_create])
667 script.MakeSymlinks(to_create)
Doug Zongkereef39442009-04-02 12:14:19 -0700668
669 # Now that the symlinks are created, we can set all the
670 # permissions.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700671 script.AppendScript(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700672
Doug Zongker881dd402009-09-20 14:03:55 -0700673 # Do device-specific installation (eg, write radio image).
Doug Zongker05d3dea2009-06-22 11:32:31 -0700674 device_specific.IncrementalOTA_InstallEnd()
675
Doug Zongker1c390a22009-05-14 19:06:36 -0700676 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700677 scirpt.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700678
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700679 script.AddToZip(target_zip, output_zip)
Doug Zongker2ea21062010-04-28 16:05:21 -0700680 WriteMetadata(metadata, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700681
682
683def main(argv):
684
685 def option_handler(o, a):
686 if o in ("-b", "--board_config"):
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700687 pass # deprecated
Doug Zongkereef39442009-04-02 12:14:19 -0700688 elif o in ("-k", "--package_key"):
689 OPTIONS.package_key = a
Doug Zongkereef39442009-04-02 12:14:19 -0700690 elif o in ("-i", "--incremental_from"):
691 OPTIONS.incremental_source = a
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700692 elif o in ("-w", "--wipe_user_data"):
693 OPTIONS.wipe_user_data = True
Doug Zongker962069c2009-04-23 11:41:58 -0700694 elif o in ("-n", "--no_prereq"):
695 OPTIONS.omit_prereq = True
Doug Zongker1c390a22009-05-14 19:06:36 -0700696 elif o in ("-e", "--extra_script"):
697 OPTIONS.extra_script = a
Doug Zongker761e6422009-09-25 10:45:39 -0700698 elif o in ("--worker_threads"):
699 OPTIONS.worker_threads = int(a)
Doug Zongkereef39442009-04-02 12:14:19 -0700700 else:
701 return False
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700702 return True
Doug Zongkereef39442009-04-02 12:14:19 -0700703
704 args = common.ParseOptions(argv, __doc__,
Doug Zongkerc637db12010-04-21 14:08:44 -0700705 extra_opts="b:k:i:d:wne:",
Doug Zongkereef39442009-04-02 12:14:19 -0700706 extra_long_opts=["board_config=",
707 "package_key=",
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700708 "incremental_from=",
Doug Zongker962069c2009-04-23 11:41:58 -0700709 "wipe_user_data",
Doug Zongker1c390a22009-05-14 19:06:36 -0700710 "no_prereq",
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700711 "extra_script=",
Doug Zongker761e6422009-09-25 10:45:39 -0700712 "worker_threads="],
Doug Zongkereef39442009-04-02 12:14:19 -0700713 extra_option_handler=option_handler)
714
715 if len(args) != 2:
716 common.Usage(__doc__)
717 sys.exit(1)
718
Doug Zongker1c390a22009-05-14 19:06:36 -0700719 if OPTIONS.extra_script is not None:
720 OPTIONS.extra_script = open(OPTIONS.extra_script).read()
721
Doug Zongkereef39442009-04-02 12:14:19 -0700722 print "unzipping target target-files..."
723 OPTIONS.input_tmp = common.UnzipTemp(args[0])
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700724
Doug Zongkereef39442009-04-02 12:14:19 -0700725 OPTIONS.target_tmp = OPTIONS.input_tmp
726 input_zip = zipfile.ZipFile(args[0], "r")
Doug Zongker37974732010-09-16 17:44:38 -0700727 OPTIONS.info_dict = common.LoadInfoDict(input_zip)
728 if OPTIONS.verbose:
729 print "--- target info ---"
730 common.DumpInfoDict(OPTIONS.info_dict)
731
732 if OPTIONS.device_specific is None:
733 OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None)
734 if OPTIONS.device_specific is not None:
735 OPTIONS.device_specific = os.path.normpath(OPTIONS.device_specific)
736 print "using device-specific extensions in", OPTIONS.device_specific
737
Doug Zongkereef39442009-04-02 12:14:19 -0700738 if OPTIONS.package_key:
739 temp_zip_file = tempfile.NamedTemporaryFile()
740 output_zip = zipfile.ZipFile(temp_zip_file, "w",
741 compression=zipfile.ZIP_DEFLATED)
742 else:
743 output_zip = zipfile.ZipFile(args[1], "w",
744 compression=zipfile.ZIP_DEFLATED)
745
746 if OPTIONS.incremental_source is None:
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700747 WriteFullOTAPackage(input_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700748 else:
749 print "unzipping source target-files..."
750 OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source)
751 source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r")
Doug Zongker37974732010-09-16 17:44:38 -0700752 OPTIONS.target_info_dict = OPTIONS.info_dict
753 OPTIONS.source_info_dict = common.LoadInfoDict(source_zip)
754 if OPTIONS.verbose:
755 print "--- source info ---"
756 common.DumpInfoDict(OPTIONS.source_info_dict)
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700757 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700758
759 output_zip.close()
760 if OPTIONS.package_key:
761 SignOutput(temp_zip_file.name, args[1])
762 temp_zip_file.close()
763
764 common.Cleanup()
765
766 print "done."
767
768
769if __name__ == '__main__':
770 try:
771 main(sys.argv[1:])
772 except common.ExternalError, e:
773 print
774 print " ERROR: %s" % (e,)
775 print
776 sys.exit(1)