blob: d414bc1f32c3d0725042114f544b560ccbeb96ca [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
33import shutil
34import six
35import subprocess
36import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080037import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070038import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000039
40# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
41COMMON_MK_USES = [
42 'asan',
43 'coverage',
44 'cros_host',
45 'fuzzer',
46 'fuzzer',
47 'msan',
48 'profiling',
49 'tcmalloc',
50 'test',
51 'ubsan',
52]
53
54# Default use flags.
55USE_DEFAULTS = {
56 'android': False,
57 'bt_nonstandard_codecs': False,
58 'test': False,
59}
60
61VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000062 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080063 'clean', # Clean up output directory
64 'docs', # Build Rust docs
65 'main', # Build the main C++ codebase
66 'prepare', # Prepare the output directory (gn gen + rust setup)
Ivan Podogovcd9fe992022-08-10 11:07:12 +010067 'rootcanal', # Build Rust targets for RootCanal
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080068 'rust', # Build only the rust components + copy artifacts to output dir
69 'test', # Run the unit tests
70 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000071]
72
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000073# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000074HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000075 # 'bluetooth_test_common',
76 # 'bluetoothtbd_test',
77 # 'net_test_avrcp',
78 # 'net_test_btcore',
79 # 'net_test_types',
80 # 'net_test_btm_iso',
81 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000082]
83
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070084BOOTSTRAP_GIT_REPOS = {
85 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
86 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
87 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
88}
89
90# List of packages required for linux build
91REQUIRED_APT_PACKAGES = [
92 'bison',
93 'build-essential',
94 'curl',
95 'debmake',
96 'flatbuffers-compiler',
97 'flex',
98 'g++-multilib',
99 'gcc-multilib',
100 'generate-ninja',
101 'gnupg',
102 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000103 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700104 'libc++-dev',
105 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000106 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700107 'libevent-dev',
108 'libevent-dev',
109 'libflatbuffers-dev',
110 'libflatbuffers1',
111 'libgl1-mesa-dev',
112 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000113 'libgtest-dev',
114 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700115 'liblz4-tool',
116 'libncurses5',
117 'libnss3-dev',
118 'libprotobuf-dev',
119 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000120 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700121 'libssl-dev',
122 'libtinyxml2-dev',
123 'libx11-dev',
124 'libxml2-utils',
125 'ninja-build',
126 'openssl',
127 'protobuf-compiler',
128 'unzip',
129 'x11proto-core-dev',
130 'xsltproc',
131 'zip',
132 'zlib1g-dev',
133]
134
135# List of cargo packages required for linux build
136REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
137
138APT_PKG_LIST = ['apt', '-qq', 'list']
139CARGO_PKG_LIST = ['cargo', 'install', '--list']
140
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000141
142class UseFlags():
143
144 def __init__(self, use_flags):
145 """ Construct the use flags.
146
147 Args:
148 use_flags: List of use flags parsed from the command.
149 """
150 self.flags = {}
151
152 # Import use flags required by common-mk
153 for use in COMMON_MK_USES:
154 self.set_flag(use, False)
155
156 # Set our defaults
157 for use, value in USE_DEFAULTS.items():
158 self.set_flag(use, value)
159
160 # Set use flags - value is set to True unless the use starts with -
161 # All given use flags always override the defaults
162 for use in use_flags:
163 value = not use.startswith('-')
164 self.set_flag(use, value)
165
166 def set_flag(self, key, value=True):
167 setattr(self, key, value)
168 self.flags[key] = value
169
170
171class HostBuild():
172
173 def __init__(self, args):
174 """ Construct the builder.
175
176 Args:
177 args: Parsed arguments from ArgumentParser
178 """
179 self.args = args
180
181 # Set jobs to number of cpus unless explicitly set
182 self.jobs = self.args.jobs
183 if not self.jobs:
184 self.jobs = multiprocessing.cpu_count()
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000185 print("Number of jobs = {}".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000186
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700187 # Normalize bootstrap dir and make sure it exists
188 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
189 os.makedirs(self.bootstrap_dir, exist_ok=True)
190
191 # Output and platform directories are based on bootstrap
192 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
193 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000194 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000195 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800196 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000197
198 # If default target isn't set, build everything
199 self.target = 'all'
200 if hasattr(self.args, 'target') and self.args.target:
201 self.target = self.args.target
202
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000203 target_use = self.args.use if self.args.use else []
204
205 # Unless set, always build test code
206 if not self.args.notest:
207 target_use.append('test')
208
209 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000210
211 # Validate platform directory
212 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
213 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
214
215 # Make sure output directory exists (or create it)
216 os.makedirs(self.output_dir, exist_ok=True)
217
218 # Set some default attributes
219 self.libbase_ver = None
220
221 self.configure_environ()
222
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000223 def _generate_rustflags(self):
224 """ Rustflags to include for the build.
225 """
226 rust_flags = [
227 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700228 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000229 '-C',
230 'link-arg=-Wl,--allow-multiple-definition',
231 ]
232
233 return ' '.join(rust_flags)
234
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000235 def configure_environ(self):
236 """ Configure environment variables for GN and Cargo.
237 """
238 self.env = os.environ.copy()
239
240 # Make sure cargo home dir exists and has a bin directory
241 cargo_home = os.path.join(self.output_dir, 'cargo_home')
242 os.makedirs(cargo_home, exist_ok=True)
243 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
244
245 # Configure Rust env variables
246 self.env['CARGO_TARGET_DIR'] = self.output_dir
247 self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000248 self.env['RUSTFLAGS'] = self._generate_rustflags()
Abhishek Pandit-Subedi1927afa2021-04-28 21:16:18 -0700249 self.env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000250
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000251 def run_command(self, target, args, cwd=None, env=None):
252 """ Run command and stream the output.
253 """
254 # Set some defaults
255 if not cwd:
256 cwd = self.platform_dir
257 if not env:
258 env = self.env
259
260 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
261 with open(log_file, 'wb') as lf:
262 rc = 0
263 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
264 while True:
265 line = process.stdout.readline()
266 print(line.decode('utf-8'), end="")
267 lf.write(line)
268 if not line:
269 rc = process.poll()
270 if rc is not None:
271 break
272
273 time.sleep(0.1)
274
275 if rc != 0:
276 raise Exception("Return code is {}".format(rc))
277
278 def _get_basever(self):
279 if self.libbase_ver:
280 return self.libbase_ver
281
282 self.libbase_ver = os.environ.get('BASE_VER', '')
283 if not self.libbase_ver:
284 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
285 try:
286 with open(base_file, 'r') as f:
287 self.libbase_ver = f.read().strip('\n')
288 except:
289 self.libbase_ver = 'NOT-INSTALLED'
290
291 return self.libbase_ver
292
293 def _gn_default_output(self):
294 return os.path.join(self.output_dir, 'out/Default')
295
296 def _gn_configure(self):
297 """ Configure all required parameters for platform2.
298
299 Mostly copied from //common-mk/platform2.py
300 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700301 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000302
303 def to_gn_string(s):
304 return '"%s"' % s.replace('"', '\\"')
305
306 def to_gn_list(strs):
307 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
308
309 def to_gn_args_args(gn_args):
310 for k, v in gn_args.items():
311 if isinstance(v, bool):
312 v = str(v).lower()
313 elif isinstance(v, list):
314 v = to_gn_list(v)
315 elif isinstance(v, six.string_types):
316 v = to_gn_string(v)
317 else:
318 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
319 yield '%s=%s' % (k.replace('-', '_'), v)
320
321 gn_args = {
322 'platform_subdir': 'bt',
323 'cc': 'clang' if clang else 'gcc',
324 'cxx': 'clang++' if clang else 'g++',
325 'ar': 'llvm-ar' if clang else 'ar',
326 'pkg-config': 'pkg-config',
327 'clang_cc': clang,
328 'clang_cxx': clang,
329 'OS': 'linux',
330 'sysroot': self.sysroot,
331 'libdir': os.path.join(self.sysroot, self.libdir),
332 'build_root': self.output_dir,
333 'platform2_root': self.platform_dir,
334 'libbase_ver': self._get_basever(),
335 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
336 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800337 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000338 'enable_werror': False,
339 }
340
341 if clang:
342 # Make sure to mark the clang use flag as true
343 self.use.set_flag('clang', True)
344 gn_args['external_cxxflags'] += ['-I/usr/include/']
345
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000346 gn_args_args = list(to_gn_args_args(gn_args))
347 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
348 gn_args_args += ['use={%s}' % (' '.join(use_args))]
349
350 gn_args = [
351 'gn',
352 'gen',
353 ]
354
355 if self.args.verbose:
356 gn_args.append('-v')
357
358 gn_args += [
359 '--root=%s' % self.platform_dir,
360 '--args=%s' % ' '.join(gn_args_args),
361 self._gn_default_output(),
362 ]
363
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700364 if 'PKG_CONFIG_PATH' in self.env:
365 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000366
367 self.run_command('configure', gn_args)
368
369 def _gn_build(self, target):
370 """ Generate the ninja command for the target and run it.
371 """
372 args = ['%s:%s' % ('bt', target)]
373 ninja_args = ['ninja', '-C', self._gn_default_output()]
374 if self.jobs:
375 ninja_args += ['-j', str(self.jobs)]
376 ninja_args += args
377
378 if self.args.verbose:
379 ninja_args.append('-v')
380
381 self.run_command('build', ninja_args)
382
383 def _rust_configure(self):
384 """ Generate config file at cargo_home so we use vendored crates.
385 """
386 template = """
387 [source.systembt]
388 directory = "{}/external/rust/vendor"
389
390 [source.crates-io]
391 replace-with = "systembt"
392 local-registry = "/nonexistent"
393 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700394
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700395 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700396 contents = template.format(self.platform_dir)
397 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
398 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000399
400 def _rust_build(self):
401 """ Run `cargo build` from platform2/bt directory.
402 """
403 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
404
405 def _target_prepare(self):
406 """ Target to prepare the output directory for building.
407
408 This runs gn gen to generate all rquired files and set up the Rust
409 config properly. This will be run
410 """
411 self._gn_configure()
412 self._rust_configure()
413
414 def _target_tools(self):
415 """ Build the tools target in an already prepared environment.
416 """
417 self._gn_build('tools')
418
419 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
420 shutil.copy(
421 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
422
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800423 def _target_docs(self):
424 """Build the Rust docs."""
425 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
426
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000427 def _target_rust(self):
428 """ Build rust artifacts in an already prepared environment.
429 """
430 self._rust_build()
431
Ivan Podogovcd9fe992022-08-10 11:07:12 +0100432 def _target_rootcanal(self):
433 """ Build rust artifacts for RootCanal in an already prepared environment.
434 """
435 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
436
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000437 def _target_main(self):
438 """ Build the main GN artifacts in an already prepared environment.
439 """
440 self._gn_build('all')
441
442 def _target_test(self):
443 """ Runs the host tests.
444 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000445 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800446 rust_test_cmd = ['cargo', 'test']
447 if self.args.test_name:
448 rust_test_cmd = rust_test_cmd + [self.args.test_name]
449
450 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Ivan Podogovcd9fe992022-08-10 11:07:12 +0100451 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000452
453 # Host tests second based on host test list
454 for t in HOST_TESTS:
455 self.run_command(
456 'test', [os.path.join(self.output_dir, 'out/Default', t)],
457 cwd=os.path.join(self.output_dir),
458 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000459
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800460 def _target_install(self):
461 """ Installs files required to run Floss to install directory.
462 """
463 # First make the install directory
464 prefix = self.install_dir
465 os.makedirs(prefix, exist_ok=True)
466
467 # Next save the cwd and change to install directory
468 last_cwd = os.getcwd()
469 os.chdir(prefix)
470
471 bindir = os.path.join(self.output_dir, 'debug')
472 srcdir = os.path.dirname(__file__)
473
474 install_map = [
475 {
476 'src': os.path.join(bindir, 'btadapterd'),
477 'dst': 'usr/libexec/bluetooth/btadapterd',
478 'strip': True
479 },
480 {
481 'src': os.path.join(bindir, 'btmanagerd'),
482 'dst': 'usr/libexec/bluetooth/btmanagerd',
483 'strip': True
484 },
485 {
486 'src': os.path.join(bindir, 'btclient'),
487 'dst': 'usr/local/bin/btclient',
488 'strip': True
489 },
490 ]
491
492 for v in install_map:
493 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
494 dst = os.path.join(prefix, partial_dst)
495
496 # Create dst directory first and copy file there
497 os.makedirs(os.path.dirname(dst), exist_ok=True)
498 print('Installing {}'.format(dst))
499 shutil.copy(src, dst)
500
501 # Binary should be marked for strip and no-strip option shouldn't be
502 # set. No-strip is useful while debugging.
503 if strip and not self.args.no_strip:
504 self.run_command('install', ['llvm-strip', dst])
505
506 # Put all files into a tar.gz for easier installation
507 tar_location = os.path.join(prefix, 'floss.tar.gz')
508 with tarfile.open(tar_location, 'w:gz') as tar:
509 for v in install_map:
510 tar.add(v['dst'])
511
512 print('Tarball created at {}'.format(tar_location))
513
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000514 def _target_clean(self):
515 """ Delete the output directory entirely.
516 """
517 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800518
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700519 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800520 try:
521 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
522 except FileNotFoundError:
523 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000524
525 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800526 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000527 """
528 self._target_prepare()
529 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000530 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700531 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000532
533 def build(self):
534 """ Builds according to self.target
535 """
536 print('Building target ', self.target)
537
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800538 # Validate that the target is valid
539 if self.target not in VALID_TARGETS:
540 print('Target {} is not valid. Must be in {}', self.target, VALID_TARGETS)
541 return
542
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000543 if self.target == 'prepare':
544 self._target_prepare()
545 elif self.target == 'tools':
546 self._target_tools()
Ivan Podogovcd9fe992022-08-10 11:07:12 +0100547 elif self.target == 'rootcanal':
548 self._target_rootcanal()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000549 elif self.target == 'rust':
550 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800551 elif self.target == 'docs':
552 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000553 elif self.target == 'main':
554 self._target_main()
555 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000556 self._target_test()
557 elif self.target == 'clean':
558 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800559 elif self.target == 'install':
560 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000561 elif self.target == 'all':
562 self._target_all()
563
564
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700565class Bootstrap():
566
567 def __init__(self, base_dir, bt_dir):
568 """ Construct bootstrapper.
569
570 Args:
571 base_dir: Where to stage everything.
572 bt_dir: Where bluetooth source is kept (will be symlinked)
573 """
574 self.base_dir = os.path.abspath(base_dir)
575 self.bt_dir = os.path.abspath(bt_dir)
576
577 # Create base directory if it doesn't already exist
578 os.makedirs(self.base_dir, exist_ok=True)
579
580 if not os.path.isdir(self.bt_dir):
581 raise Exception('{} is not a valid directory'.format(self.bt_dir))
582
583 self.git_dir = os.path.join(self.base_dir, 'repos')
584 self.staging_dir = os.path.join(self.base_dir, 'staging')
585 self.output_dir = os.path.join(self.base_dir, 'output')
586 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
587
588 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
589
590 def _update_platform2(self):
591 """Updates repositories used for build."""
592 for repo in BOOTSTRAP_GIT_REPOS.keys():
593 cwd = os.path.join(self.git_dir, repo)
594 subprocess.check_call(['git', 'pull'], cwd=cwd)
595
596 def _setup_platform2(self):
597 """ Set up platform2.
598
599 This will check out all the git repos and symlink everything correctly.
600 """
601
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800602 # Create all directories we will need to use
603 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
604 os.makedirs(dirpath, exist_ok=True)
605
606 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700607 if os.path.isfile(self.dir_setup_complete):
608 print('{} already set-up. Updating instead.'.format(self.base_dir))
609 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800610 else:
611 # Check out all repos in git directory
612 for repo in BOOTSTRAP_GIT_REPOS.values():
613 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700614
615 # Symlink things
616 symlinks = [
617 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
618 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
619 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
620 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
621 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
622 ]
623
624 # Create symlinks
625 for pairs in symlinks:
626 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000627 try:
628 os.unlink(dst)
629 except Exception as e:
630 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700631 os.symlink(src, dst)
632
633 # Write to setup complete file so we don't repeat this step
634 with open(self.dir_setup_complete, 'w') as f:
635 f.write('Setup complete.')
636
637 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
638 """ Pretty print an install command.
639
640 Args:
641 install_cmd: Prefixed install command.
642 packages: Enumerate packages and append them to install command.
643 line_limit: Number of characters per line.
644
645 Return:
646 Array of lines to join and print.
647 """
648 install = [install_cmd]
649 line = ' '
650 # Remainder needed = space + len(pkg) + space + \
651 # Assuming 80 character lines, that's 80 - 3 = 77
652 line_limit = line_limit - 3
653 for pkg in packages:
654 if len(line) + len(pkg) < line_limit:
655 line = '{}{} '.format(line, pkg)
656 else:
657 install.append(line)
658 line = ' {} '.format(pkg)
659
660 if len(line) > 0:
661 install.append(line)
662
663 return install
664
665 def _check_package_installed(self, package, cmd, predicate):
666 """Check that the given package is installed.
667
668 Args:
669 package: Check that this package is installed.
670 cmd: Command prefix to check if installed (package appended to end)
671 predicate: Function/lambda to check if package is installed based
672 on output. Takes string output and returns boolean.
673
674 Return:
675 True if package is installed.
676 """
677 try:
678 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
679 is_installed = predicate(output.decode('utf-8'))
680 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
681
682 return is_installed
683 except Exception as e:
684 print(e)
685 return False
686
687 def _get_command_output(self, cmd):
688 """Runs the command and gets the output.
689
690 Args:
691 cmd: Command to run.
692
693 Return:
694 Tuple (Success, Output). Success represents if the command ran ok.
695 """
696 try:
697 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
698 return (True, output.decode('utf-8').split('\n'))
699 except Exception as e:
700 print(e)
701 return (False, "")
702
703 def _print_missing_packages(self):
704 """Print any missing packages found via apt.
705
706 This will find any missing packages necessary for build using apt and
707 print it out as an apt-get install printf.
708 """
709 print('Checking for any missing packages...')
710
711 (success, output) = self._get_command_output(APT_PKG_LIST)
712 if not success:
713 raise Exception("Could not query apt for packages.")
714
715 packages_installed = {}
716 for line in output:
717 if 'installed' in line:
718 split = line.split('/', 2)
719 packages_installed[split[0]] = True
720
721 need_packages = []
722 for pkg in REQUIRED_APT_PACKAGES:
723 if pkg not in packages_installed:
724 need_packages.append(pkg)
725
726 # No packages need to be installed
727 if len(need_packages) == 0:
728 print('+ All required packages are installed')
729 return
730
731 install = self._pretty_print_install('sudo apt-get install', need_packages)
732
733 # Print all lines so they can be run in cmdline
734 print('Missing system packages. Run the following command: ')
735 print(' \\\n'.join(install))
736
737 def _print_missing_rust_packages(self):
738 """Print any missing packages found via cargo.
739
740 This will find any missing packages necessary for build using cargo and
741 print it out as a cargo-install printf.
742 """
743 print('Checking for any missing cargo packages...')
744
745 (success, output) = self._get_command_output(CARGO_PKG_LIST)
746 if not success:
747 raise Exception("Could not query cargo for packages.")
748
749 packages_installed = {}
750 for line in output:
751 # Cargo installed packages have this format
752 # <crate name> <version>:
753 # <binary name>
754 # We only care about the crates themselves
755 if ':' not in line:
756 continue
757
758 split = line.split(' ', 2)
759 packages_installed[split[0]] = True
760
761 need_packages = []
762 for pkg in REQUIRED_CARGO_PACKAGES:
763 if pkg not in packages_installed:
764 need_packages.append(pkg)
765
766 # No packages to be installed
767 if len(need_packages) == 0:
768 print('+ All required cargo packages are installed')
769 return
770
771 install = self._pretty_print_install('cargo install', need_packages)
772 print('Missing cargo packages. Run the following command: ')
773 print(' \\\n'.join(install))
774
775 def bootstrap(self):
776 """ Bootstrap the Linux build."""
777 self._setup_platform2()
778 self._print_missing_packages()
779 self._print_missing_rust_packages()
780
781
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000782if __name__ == '__main__':
783 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700784 parser.add_argument(
785 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
786 parser.add_argument(
787 '--run-bootstrap',
788 help='Run bootstrap code to verify build env is ok to build.',
789 default=False,
790 action='store_true')
791 parser.add_argument('--no-clang', help='Use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800792 parser.add_argument(
793 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000794 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800795 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
796 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000797 parser.add_argument('--target', help='Run specific build target')
798 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700799 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000800 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700801 parser.add_argument(
802 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000803 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000804 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700805
806 # Make sure we get absolute path + expanded path for bootstrap directory
807 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
808
809 if args.run_bootstrap:
810 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
811 bootstrap.bootstrap()
812 else:
813 build = HostBuild(args)
814 build.build()