Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # Copyright 2021 Google, Inc. |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at: |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | """ Build BT targets on the host system. |
| 17 | |
| 18 | For building, you will first have to stage a platform directory that has the |
| 19 | following structure: |
| 20 | |-common-mk |
| 21 | |-bt |
| 22 | |-external |
| 23 | |-|-rust |
| 24 | |-|-|-vendor |
| 25 | |
| 26 | The simplest way to do this is to check out platform2 to another directory (that |
| 27 | is not a subdir of this bt directory), symlink bt there and symlink the rust |
| 28 | vendor repository as well. |
| 29 | """ |
| 30 | import argparse |
| 31 | import multiprocessing |
| 32 | import os |
| 33 | import shutil |
| 34 | import six |
| 35 | import subprocess |
| 36 | import sys |
Abhishek Pandit-Subedi | a1a94d1 | 2021-11-18 13:14:04 -0800 | [diff] [blame] | 37 | import tarfile |
Chris Manton | e7ad633 | 2021-09-30 22:55:39 -0700 | [diff] [blame] | 38 | import time |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 39 | |
| 40 | # Use flags required by common-mk (find -type f | grep -nE 'use[.]' {}) |
| 41 | COMMON_MK_USES = [ |
| 42 | 'asan', |
| 43 | 'coverage', |
| 44 | 'cros_host', |
| 45 | 'fuzzer', |
| 46 | 'fuzzer', |
| 47 | 'msan', |
| 48 | 'profiling', |
| 49 | 'tcmalloc', |
| 50 | 'test', |
| 51 | 'ubsan', |
| 52 | ] |
| 53 | |
| 54 | # Default use flags. |
| 55 | USE_DEFAULTS = { |
| 56 | 'android': False, |
| 57 | 'bt_nonstandard_codecs': False, |
| 58 | 'test': False, |
| 59 | } |
| 60 | |
| 61 | VALID_TARGETS = [ |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 62 | 'all', # All targets except test and clean |
Abhishek Pandit-Subedi | 9476414 | 2022-02-07 17:18:00 -0800 | [diff] [blame] | 63 | 'clean', # Clean up output directory |
| 64 | 'docs', # Build Rust docs |
| 65 | 'main', # Build the main C++ codebase |
| 66 | 'prepare', # Prepare the output directory (gn gen + rust setup) |
Ivan Podogov | cd9fe99 | 2022-08-10 11:07:12 +0100 | [diff] [blame] | 67 | 'rootcanal', # Build Rust targets for RootCanal |
Abhishek Pandit-Subedi | 9476414 | 2022-02-07 17:18:00 -0800 | [diff] [blame] | 68 | 'rust', # Build only the rust components + copy artifacts to output dir |
| 69 | 'test', # Run the unit tests |
| 70 | 'tools', # Build the host tools (i.e. packetgen) |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 71 | ] |
| 72 | |
Abhishek Pandit-Subedi | 70e4304 | 2021-06-10 21:16:52 +0000 | [diff] [blame] | 73 | # TODO(b/190750167) - Host tests are disabled until we are full bazel build |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 74 | HOST_TESTS = [ |
Abhishek Pandit-Subedi | 70e4304 | 2021-06-10 21:16:52 +0000 | [diff] [blame] | 75 | # 'bluetooth_test_common', |
| 76 | # 'bluetoothtbd_test', |
| 77 | # 'net_test_avrcp', |
| 78 | # 'net_test_btcore', |
| 79 | # 'net_test_types', |
| 80 | # 'net_test_btm_iso', |
| 81 | # 'net_test_btpackets', |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 82 | ] |
| 83 | |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 84 | BOOTSTRAP_GIT_REPOS = { |
| 85 | 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2', |
| 86 | 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates', |
| 87 | 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging' |
| 88 | } |
| 89 | |
| 90 | # List of packages required for linux build |
| 91 | REQUIRED_APT_PACKAGES = [ |
| 92 | 'bison', |
| 93 | 'build-essential', |
| 94 | 'curl', |
| 95 | 'debmake', |
| 96 | 'flatbuffers-compiler', |
| 97 | 'flex', |
| 98 | 'g++-multilib', |
| 99 | 'gcc-multilib', |
| 100 | 'generate-ninja', |
| 101 | 'gnupg', |
| 102 | 'gperf', |
Martin Brabham | ba22adf | 2022-02-04 19:51:21 +0000 | [diff] [blame] | 103 | 'libc++abi-dev', |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 104 | 'libc++-dev', |
| 105 | 'libdbus-1-dev', |
Martin Brabham | 996f150 | 2022-02-14 17:39:23 +0000 | [diff] [blame] | 106 | 'libdouble-conversion-dev', |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 107 | 'libevent-dev', |
| 108 | 'libevent-dev', |
| 109 | 'libflatbuffers-dev', |
| 110 | 'libflatbuffers1', |
| 111 | 'libgl1-mesa-dev', |
| 112 | 'libglib2.0-dev', |
Martin Brabham | 996f150 | 2022-02-14 17:39:23 +0000 | [diff] [blame] | 113 | 'libgtest-dev', |
| 114 | 'libgmock-dev', |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 115 | 'liblz4-tool', |
| 116 | 'libncurses5', |
| 117 | 'libnss3-dev', |
| 118 | 'libprotobuf-dev', |
| 119 | 'libre2-9', |
Martin Brabham | 996f150 | 2022-02-14 17:39:23 +0000 | [diff] [blame] | 120 | 'libre2-dev', |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 121 | 'libssl-dev', |
| 122 | 'libtinyxml2-dev', |
| 123 | 'libx11-dev', |
| 124 | 'libxml2-utils', |
| 125 | 'ninja-build', |
| 126 | 'openssl', |
| 127 | 'protobuf-compiler', |
| 128 | 'unzip', |
| 129 | 'x11proto-core-dev', |
| 130 | 'xsltproc', |
| 131 | 'zip', |
| 132 | 'zlib1g-dev', |
| 133 | ] |
| 134 | |
| 135 | # List of cargo packages required for linux build |
| 136 | REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd'] |
| 137 | |
| 138 | APT_PKG_LIST = ['apt', '-qq', 'list'] |
| 139 | CARGO_PKG_LIST = ['cargo', 'install', '--list'] |
| 140 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 141 | |
| 142 | class UseFlags(): |
| 143 | |
| 144 | def __init__(self, use_flags): |
| 145 | """ Construct the use flags. |
| 146 | |
| 147 | Args: |
| 148 | use_flags: List of use flags parsed from the command. |
| 149 | """ |
| 150 | self.flags = {} |
| 151 | |
| 152 | # Import use flags required by common-mk |
| 153 | for use in COMMON_MK_USES: |
| 154 | self.set_flag(use, False) |
| 155 | |
| 156 | # Set our defaults |
| 157 | for use, value in USE_DEFAULTS.items(): |
| 158 | self.set_flag(use, value) |
| 159 | |
| 160 | # Set use flags - value is set to True unless the use starts with - |
| 161 | # All given use flags always override the defaults |
| 162 | for use in use_flags: |
| 163 | value = not use.startswith('-') |
| 164 | self.set_flag(use, value) |
| 165 | |
| 166 | def set_flag(self, key, value=True): |
| 167 | setattr(self, key, value) |
| 168 | self.flags[key] = value |
| 169 | |
| 170 | |
| 171 | class HostBuild(): |
| 172 | |
| 173 | def __init__(self, args): |
| 174 | """ Construct the builder. |
| 175 | |
| 176 | Args: |
| 177 | args: Parsed arguments from ArgumentParser |
| 178 | """ |
| 179 | self.args = args |
| 180 | |
| 181 | # Set jobs to number of cpus unless explicitly set |
| 182 | self.jobs = self.args.jobs |
| 183 | if not self.jobs: |
| 184 | self.jobs = multiprocessing.cpu_count() |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 185 | print("Number of jobs = {}".format(self.jobs)) |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 186 | |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 187 | # Normalize bootstrap dir and make sure it exists |
| 188 | self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir) |
| 189 | os.makedirs(self.bootstrap_dir, exist_ok=True) |
| 190 | |
| 191 | # Output and platform directories are based on bootstrap |
| 192 | self.output_dir = os.path.join(self.bootstrap_dir, 'output') |
| 193 | self.platform_dir = os.path.join(self.bootstrap_dir, 'staging') |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 194 | self.sysroot = self.args.sysroot |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 195 | self.libdir = self.args.libdir |
Abhishek Pandit-Subedi | a1a94d1 | 2021-11-18 13:14:04 -0800 | [diff] [blame] | 196 | self.install_dir = os.path.join(self.output_dir, 'install') |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 197 | |
| 198 | # If default target isn't set, build everything |
| 199 | self.target = 'all' |
| 200 | if hasattr(self.args, 'target') and self.args.target: |
| 201 | self.target = self.args.target |
| 202 | |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 203 | target_use = self.args.use if self.args.use else [] |
| 204 | |
| 205 | # Unless set, always build test code |
| 206 | if not self.args.notest: |
| 207 | target_use.append('test') |
| 208 | |
| 209 | self.use = UseFlags(target_use) |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 210 | |
| 211 | # Validate platform directory |
| 212 | assert os.path.isdir(self.platform_dir), 'Platform dir does not exist' |
| 213 | assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root' |
| 214 | |
| 215 | # Make sure output directory exists (or create it) |
| 216 | os.makedirs(self.output_dir, exist_ok=True) |
| 217 | |
| 218 | # Set some default attributes |
| 219 | self.libbase_ver = None |
| 220 | |
| 221 | self.configure_environ() |
| 222 | |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 223 | def _generate_rustflags(self): |
| 224 | """ Rustflags to include for the build. |
| 225 | """ |
| 226 | rust_flags = [ |
| 227 | '-L', |
Martin Brabham | 1c24fda | 2021-09-16 11:19:46 -0700 | [diff] [blame] | 228 | '{}/out/Default'.format(self.output_dir), |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 229 | '-C', |
| 230 | 'link-arg=-Wl,--allow-multiple-definition', |
| 231 | ] |
| 232 | |
| 233 | return ' '.join(rust_flags) |
| 234 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 235 | def configure_environ(self): |
| 236 | """ Configure environment variables for GN and Cargo. |
| 237 | """ |
| 238 | self.env = os.environ.copy() |
| 239 | |
| 240 | # Make sure cargo home dir exists and has a bin directory |
| 241 | cargo_home = os.path.join(self.output_dir, 'cargo_home') |
| 242 | os.makedirs(cargo_home, exist_ok=True) |
| 243 | os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True) |
| 244 | |
| 245 | # Configure Rust env variables |
| 246 | self.env['CARGO_TARGET_DIR'] = self.output_dir |
| 247 | self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home') |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 248 | self.env['RUSTFLAGS'] = self._generate_rustflags() |
Abhishek Pandit-Subedi | 1927afa | 2021-04-28 21:16:18 -0700 | [diff] [blame] | 249 | self.env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt') |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 250 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 251 | def run_command(self, target, args, cwd=None, env=None): |
| 252 | """ Run command and stream the output. |
| 253 | """ |
| 254 | # Set some defaults |
| 255 | if not cwd: |
| 256 | cwd = self.platform_dir |
| 257 | if not env: |
| 258 | env = self.env |
| 259 | |
| 260 | log_file = os.path.join(self.output_dir, '{}.log'.format(target)) |
| 261 | with open(log_file, 'wb') as lf: |
| 262 | rc = 0 |
| 263 | process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE) |
| 264 | while True: |
| 265 | line = process.stdout.readline() |
| 266 | print(line.decode('utf-8'), end="") |
| 267 | lf.write(line) |
| 268 | if not line: |
| 269 | rc = process.poll() |
| 270 | if rc is not None: |
| 271 | break |
| 272 | |
| 273 | time.sleep(0.1) |
| 274 | |
| 275 | if rc != 0: |
| 276 | raise Exception("Return code is {}".format(rc)) |
| 277 | |
| 278 | def _get_basever(self): |
| 279 | if self.libbase_ver: |
| 280 | return self.libbase_ver |
| 281 | |
| 282 | self.libbase_ver = os.environ.get('BASE_VER', '') |
| 283 | if not self.libbase_ver: |
| 284 | base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER') |
| 285 | try: |
| 286 | with open(base_file, 'r') as f: |
| 287 | self.libbase_ver = f.read().strip('\n') |
| 288 | except: |
| 289 | self.libbase_ver = 'NOT-INSTALLED' |
| 290 | |
| 291 | return self.libbase_ver |
| 292 | |
| 293 | def _gn_default_output(self): |
| 294 | return os.path.join(self.output_dir, 'out/Default') |
| 295 | |
| 296 | def _gn_configure(self): |
| 297 | """ Configure all required parameters for platform2. |
| 298 | |
| 299 | Mostly copied from //common-mk/platform2.py |
| 300 | """ |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 301 | clang = not self.args.no_clang |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 302 | |
| 303 | def to_gn_string(s): |
| 304 | return '"%s"' % s.replace('"', '\\"') |
| 305 | |
| 306 | def to_gn_list(strs): |
| 307 | return '[%s]' % ','.join([to_gn_string(s) for s in strs]) |
| 308 | |
| 309 | def to_gn_args_args(gn_args): |
| 310 | for k, v in gn_args.items(): |
| 311 | if isinstance(v, bool): |
| 312 | v = str(v).lower() |
| 313 | elif isinstance(v, list): |
| 314 | v = to_gn_list(v) |
| 315 | elif isinstance(v, six.string_types): |
| 316 | v = to_gn_string(v) |
| 317 | else: |
| 318 | raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v)) |
| 319 | yield '%s=%s' % (k.replace('-', '_'), v) |
| 320 | |
| 321 | gn_args = { |
| 322 | 'platform_subdir': 'bt', |
| 323 | 'cc': 'clang' if clang else 'gcc', |
| 324 | 'cxx': 'clang++' if clang else 'g++', |
| 325 | 'ar': 'llvm-ar' if clang else 'ar', |
| 326 | 'pkg-config': 'pkg-config', |
| 327 | 'clang_cc': clang, |
| 328 | 'clang_cxx': clang, |
| 329 | 'OS': 'linux', |
| 330 | 'sysroot': self.sysroot, |
| 331 | 'libdir': os.path.join(self.sysroot, self.libdir), |
| 332 | 'build_root': self.output_dir, |
| 333 | 'platform2_root': self.platform_dir, |
| 334 | 'libbase_ver': self._get_basever(), |
| 335 | 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1', |
| 336 | 'external_cflags': [], |
Abhishek Pandit-Subedi | 852dc3a | 2022-02-14 15:12:36 -0800 | [diff] [blame] | 337 | 'external_cxxflags': ["-DNDEBUG"], |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 338 | 'enable_werror': False, |
| 339 | } |
| 340 | |
| 341 | if clang: |
| 342 | # Make sure to mark the clang use flag as true |
| 343 | self.use.set_flag('clang', True) |
| 344 | gn_args['external_cxxflags'] += ['-I/usr/include/'] |
| 345 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 346 | gn_args_args = list(to_gn_args_args(gn_args)) |
| 347 | use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()] |
| 348 | gn_args_args += ['use={%s}' % (' '.join(use_args))] |
| 349 | |
| 350 | gn_args = [ |
| 351 | 'gn', |
| 352 | 'gen', |
| 353 | ] |
| 354 | |
| 355 | if self.args.verbose: |
| 356 | gn_args.append('-v') |
| 357 | |
| 358 | gn_args += [ |
| 359 | '--root=%s' % self.platform_dir, |
| 360 | '--args=%s' % ' '.join(gn_args_args), |
| 361 | self._gn_default_output(), |
| 362 | ] |
| 363 | |
Sonny Sasaka | 706ec3b | 2021-03-25 05:39:20 -0700 | [diff] [blame] | 364 | if 'PKG_CONFIG_PATH' in self.env: |
| 365 | print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH']) |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 366 | |
| 367 | self.run_command('configure', gn_args) |
| 368 | |
| 369 | def _gn_build(self, target): |
| 370 | """ Generate the ninja command for the target and run it. |
| 371 | """ |
| 372 | args = ['%s:%s' % ('bt', target)] |
| 373 | ninja_args = ['ninja', '-C', self._gn_default_output()] |
| 374 | if self.jobs: |
| 375 | ninja_args += ['-j', str(self.jobs)] |
| 376 | ninja_args += args |
| 377 | |
| 378 | if self.args.verbose: |
| 379 | ninja_args.append('-v') |
| 380 | |
| 381 | self.run_command('build', ninja_args) |
| 382 | |
| 383 | def _rust_configure(self): |
| 384 | """ Generate config file at cargo_home so we use vendored crates. |
| 385 | """ |
| 386 | template = """ |
| 387 | [source.systembt] |
| 388 | directory = "{}/external/rust/vendor" |
| 389 | |
| 390 | [source.crates-io] |
| 391 | replace-with = "systembt" |
| 392 | local-registry = "/nonexistent" |
| 393 | """ |
Sonny Sasaka | c1335a2 | 2021-03-25 07:10:47 -0700 | [diff] [blame] | 394 | |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 395 | if not self.args.no_vendored_rust: |
Sonny Sasaka | c1335a2 | 2021-03-25 07:10:47 -0700 | [diff] [blame] | 396 | contents = template.format(self.platform_dir) |
| 397 | with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f: |
| 398 | f.write(contents) |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 399 | |
| 400 | def _rust_build(self): |
| 401 | """ Run `cargo build` from platform2/bt directory. |
| 402 | """ |
| 403 | self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) |
| 404 | |
| 405 | def _target_prepare(self): |
| 406 | """ Target to prepare the output directory for building. |
| 407 | |
| 408 | This runs gn gen to generate all rquired files and set up the Rust |
| 409 | config properly. This will be run |
| 410 | """ |
| 411 | self._gn_configure() |
| 412 | self._rust_configure() |
| 413 | |
| 414 | def _target_tools(self): |
| 415 | """ Build the tools target in an already prepared environment. |
| 416 | """ |
| 417 | self._gn_build('tools') |
| 418 | |
| 419 | # Also copy bluetooth_packetgen to CARGO_HOME so it's available |
| 420 | shutil.copy( |
| 421 | os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin')) |
| 422 | |
Abhishek Pandit-Subedi | 9476414 | 2022-02-07 17:18:00 -0800 | [diff] [blame] | 423 | def _target_docs(self): |
| 424 | """Build the Rust docs.""" |
| 425 | self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) |
| 426 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 427 | def _target_rust(self): |
| 428 | """ Build rust artifacts in an already prepared environment. |
| 429 | """ |
| 430 | self._rust_build() |
| 431 | |
Ivan Podogov | cd9fe99 | 2022-08-10 11:07:12 +0100 | [diff] [blame] | 432 | def _target_rootcanal(self): |
| 433 | """ Build rust artifacts for RootCanal in an already prepared environment. |
| 434 | """ |
| 435 | self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env) |
| 436 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 437 | def _target_main(self): |
| 438 | """ Build the main GN artifacts in an already prepared environment. |
| 439 | """ |
| 440 | self._gn_build('all') |
| 441 | |
| 442 | def _target_test(self): |
| 443 | """ Runs the host tests. |
| 444 | """ |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 445 | # Rust tests first |
Abhishek Pandit-Subedi | 9476414 | 2022-02-07 17:18:00 -0800 | [diff] [blame] | 446 | rust_test_cmd = ['cargo', 'test'] |
| 447 | if self.args.test_name: |
| 448 | rust_test_cmd = rust_test_cmd + [self.args.test_name] |
| 449 | |
| 450 | self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) |
Ivan Podogov | cd9fe99 | 2022-08-10 11:07:12 +0100 | [diff] [blame] | 451 | self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env) |
Abhishek Pandit-Subedi | d801b12 | 2021-04-10 00:41:07 +0000 | [diff] [blame] | 452 | |
| 453 | # Host tests second based on host test list |
| 454 | for t in HOST_TESTS: |
| 455 | self.run_command( |
| 456 | 'test', [os.path.join(self.output_dir, 'out/Default', t)], |
| 457 | cwd=os.path.join(self.output_dir), |
| 458 | env=self.env) |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 459 | |
Abhishek Pandit-Subedi | a1a94d1 | 2021-11-18 13:14:04 -0800 | [diff] [blame] | 460 | def _target_install(self): |
| 461 | """ Installs files required to run Floss to install directory. |
| 462 | """ |
| 463 | # First make the install directory |
| 464 | prefix = self.install_dir |
| 465 | os.makedirs(prefix, exist_ok=True) |
| 466 | |
| 467 | # Next save the cwd and change to install directory |
| 468 | last_cwd = os.getcwd() |
| 469 | os.chdir(prefix) |
| 470 | |
| 471 | bindir = os.path.join(self.output_dir, 'debug') |
| 472 | srcdir = os.path.dirname(__file__) |
| 473 | |
| 474 | install_map = [ |
| 475 | { |
| 476 | 'src': os.path.join(bindir, 'btadapterd'), |
| 477 | 'dst': 'usr/libexec/bluetooth/btadapterd', |
| 478 | 'strip': True |
| 479 | }, |
| 480 | { |
| 481 | 'src': os.path.join(bindir, 'btmanagerd'), |
| 482 | 'dst': 'usr/libexec/bluetooth/btmanagerd', |
| 483 | 'strip': True |
| 484 | }, |
| 485 | { |
| 486 | 'src': os.path.join(bindir, 'btclient'), |
| 487 | 'dst': 'usr/local/bin/btclient', |
| 488 | 'strip': True |
| 489 | }, |
| 490 | ] |
| 491 | |
| 492 | for v in install_map: |
| 493 | src, partial_dst, strip = (v['src'], v['dst'], v['strip']) |
| 494 | dst = os.path.join(prefix, partial_dst) |
| 495 | |
| 496 | # Create dst directory first and copy file there |
| 497 | os.makedirs(os.path.dirname(dst), exist_ok=True) |
| 498 | print('Installing {}'.format(dst)) |
| 499 | shutil.copy(src, dst) |
| 500 | |
| 501 | # Binary should be marked for strip and no-strip option shouldn't be |
| 502 | # set. No-strip is useful while debugging. |
| 503 | if strip and not self.args.no_strip: |
| 504 | self.run_command('install', ['llvm-strip', dst]) |
| 505 | |
| 506 | # Put all files into a tar.gz for easier installation |
| 507 | tar_location = os.path.join(prefix, 'floss.tar.gz') |
| 508 | with tarfile.open(tar_location, 'w:gz') as tar: |
| 509 | for v in install_map: |
| 510 | tar.add(v['dst']) |
| 511 | |
| 512 | print('Tarball created at {}'.format(tar_location)) |
| 513 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 514 | def _target_clean(self): |
| 515 | """ Delete the output directory entirely. |
| 516 | """ |
| 517 | shutil.rmtree(self.output_dir) |
Abhishek Pandit-Subedi | a1a94d1 | 2021-11-18 13:14:04 -0800 | [diff] [blame] | 518 | |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 519 | # Remove Cargo.lock that may have become generated |
Abhishek Pandit-Subedi | a1a94d1 | 2021-11-18 13:14:04 -0800 | [diff] [blame] | 520 | try: |
| 521 | os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock')) |
| 522 | except FileNotFoundError: |
| 523 | pass |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 524 | |
| 525 | def _target_all(self): |
Abhishek Pandit-Subedi | 9476414 | 2022-02-07 17:18:00 -0800 | [diff] [blame] | 526 | """ Build all common targets (skipping doc, test, and clean). |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 527 | """ |
| 528 | self._target_prepare() |
| 529 | self._target_tools() |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 530 | self._target_main() |
Abhishek Pandit-Subedi | a7b57b7 | 2021-04-01 15:33:05 -0700 | [diff] [blame] | 531 | self._target_rust() |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 532 | |
| 533 | def build(self): |
| 534 | """ Builds according to self.target |
| 535 | """ |
| 536 | print('Building target ', self.target) |
| 537 | |
Abhishek Pandit-Subedi | 9476414 | 2022-02-07 17:18:00 -0800 | [diff] [blame] | 538 | # Validate that the target is valid |
| 539 | if self.target not in VALID_TARGETS: |
| 540 | print('Target {} is not valid. Must be in {}', self.target, VALID_TARGETS) |
| 541 | return |
| 542 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 543 | if self.target == 'prepare': |
| 544 | self._target_prepare() |
| 545 | elif self.target == 'tools': |
| 546 | self._target_tools() |
Ivan Podogov | cd9fe99 | 2022-08-10 11:07:12 +0100 | [diff] [blame] | 547 | elif self.target == 'rootcanal': |
| 548 | self._target_rootcanal() |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 549 | elif self.target == 'rust': |
| 550 | self._target_rust() |
Abhishek Pandit-Subedi | 9476414 | 2022-02-07 17:18:00 -0800 | [diff] [blame] | 551 | elif self.target == 'docs': |
| 552 | self._target_docs() |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 553 | elif self.target == 'main': |
| 554 | self._target_main() |
| 555 | elif self.target == 'test': |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 556 | self._target_test() |
| 557 | elif self.target == 'clean': |
| 558 | self._target_clean() |
Abhishek Pandit-Subedi | a1a94d1 | 2021-11-18 13:14:04 -0800 | [diff] [blame] | 559 | elif self.target == 'install': |
| 560 | self._target_install() |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 561 | elif self.target == 'all': |
| 562 | self._target_all() |
| 563 | |
| 564 | |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 565 | class Bootstrap(): |
| 566 | |
| 567 | def __init__(self, base_dir, bt_dir): |
| 568 | """ Construct bootstrapper. |
| 569 | |
| 570 | Args: |
| 571 | base_dir: Where to stage everything. |
| 572 | bt_dir: Where bluetooth source is kept (will be symlinked) |
| 573 | """ |
| 574 | self.base_dir = os.path.abspath(base_dir) |
| 575 | self.bt_dir = os.path.abspath(bt_dir) |
| 576 | |
| 577 | # Create base directory if it doesn't already exist |
| 578 | os.makedirs(self.base_dir, exist_ok=True) |
| 579 | |
| 580 | if not os.path.isdir(self.bt_dir): |
| 581 | raise Exception('{} is not a valid directory'.format(self.bt_dir)) |
| 582 | |
| 583 | self.git_dir = os.path.join(self.base_dir, 'repos') |
| 584 | self.staging_dir = os.path.join(self.base_dir, 'staging') |
| 585 | self.output_dir = os.path.join(self.base_dir, 'output') |
| 586 | self.external_dir = os.path.join(self.base_dir, 'staging', 'external') |
| 587 | |
| 588 | self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete') |
| 589 | |
| 590 | def _update_platform2(self): |
| 591 | """Updates repositories used for build.""" |
| 592 | for repo in BOOTSTRAP_GIT_REPOS.keys(): |
| 593 | cwd = os.path.join(self.git_dir, repo) |
| 594 | subprocess.check_call(['git', 'pull'], cwd=cwd) |
| 595 | |
| 596 | def _setup_platform2(self): |
| 597 | """ Set up platform2. |
| 598 | |
| 599 | This will check out all the git repos and symlink everything correctly. |
| 600 | """ |
| 601 | |
Abhishek Pandit-Subedi | b987eb6 | 2021-11-17 17:27:31 -0800 | [diff] [blame] | 602 | # Create all directories we will need to use |
| 603 | for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]: |
| 604 | os.makedirs(dirpath, exist_ok=True) |
| 605 | |
| 606 | # If already set up, only update platform2 |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 607 | if os.path.isfile(self.dir_setup_complete): |
| 608 | print('{} already set-up. Updating instead.'.format(self.base_dir)) |
| 609 | self._update_platform2() |
Abhishek Pandit-Subedi | b987eb6 | 2021-11-17 17:27:31 -0800 | [diff] [blame] | 610 | else: |
| 611 | # Check out all repos in git directory |
| 612 | for repo in BOOTSTRAP_GIT_REPOS.values(): |
| 613 | subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir) |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 614 | |
| 615 | # Symlink things |
| 616 | symlinks = [ |
| 617 | (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')), |
| 618 | (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')), |
| 619 | (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')), |
| 620 | (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')), |
| 621 | (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')), |
| 622 | ] |
| 623 | |
| 624 | # Create symlinks |
| 625 | for pairs in symlinks: |
| 626 | (src, dst) = pairs |
Martin Brabham | 247d80b | 2022-02-04 19:42:49 +0000 | [diff] [blame] | 627 | try: |
| 628 | os.unlink(dst) |
| 629 | except Exception as e: |
| 630 | print(e) |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 631 | os.symlink(src, dst) |
| 632 | |
| 633 | # Write to setup complete file so we don't repeat this step |
| 634 | with open(self.dir_setup_complete, 'w') as f: |
| 635 | f.write('Setup complete.') |
| 636 | |
| 637 | def _pretty_print_install(self, install_cmd, packages, line_limit=80): |
| 638 | """ Pretty print an install command. |
| 639 | |
| 640 | Args: |
| 641 | install_cmd: Prefixed install command. |
| 642 | packages: Enumerate packages and append them to install command. |
| 643 | line_limit: Number of characters per line. |
| 644 | |
| 645 | Return: |
| 646 | Array of lines to join and print. |
| 647 | """ |
| 648 | install = [install_cmd] |
| 649 | line = ' ' |
| 650 | # Remainder needed = space + len(pkg) + space + \ |
| 651 | # Assuming 80 character lines, that's 80 - 3 = 77 |
| 652 | line_limit = line_limit - 3 |
| 653 | for pkg in packages: |
| 654 | if len(line) + len(pkg) < line_limit: |
| 655 | line = '{}{} '.format(line, pkg) |
| 656 | else: |
| 657 | install.append(line) |
| 658 | line = ' {} '.format(pkg) |
| 659 | |
| 660 | if len(line) > 0: |
| 661 | install.append(line) |
| 662 | |
| 663 | return install |
| 664 | |
| 665 | def _check_package_installed(self, package, cmd, predicate): |
| 666 | """Check that the given package is installed. |
| 667 | |
| 668 | Args: |
| 669 | package: Check that this package is installed. |
| 670 | cmd: Command prefix to check if installed (package appended to end) |
| 671 | predicate: Function/lambda to check if package is installed based |
| 672 | on output. Takes string output and returns boolean. |
| 673 | |
| 674 | Return: |
| 675 | True if package is installed. |
| 676 | """ |
| 677 | try: |
| 678 | output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT) |
| 679 | is_installed = predicate(output.decode('utf-8')) |
| 680 | print(' {} is {}'.format(package, 'installed' if is_installed else 'missing')) |
| 681 | |
| 682 | return is_installed |
| 683 | except Exception as e: |
| 684 | print(e) |
| 685 | return False |
| 686 | |
| 687 | def _get_command_output(self, cmd): |
| 688 | """Runs the command and gets the output. |
| 689 | |
| 690 | Args: |
| 691 | cmd: Command to run. |
| 692 | |
| 693 | Return: |
| 694 | Tuple (Success, Output). Success represents if the command ran ok. |
| 695 | """ |
| 696 | try: |
| 697 | output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) |
| 698 | return (True, output.decode('utf-8').split('\n')) |
| 699 | except Exception as e: |
| 700 | print(e) |
| 701 | return (False, "") |
| 702 | |
| 703 | def _print_missing_packages(self): |
| 704 | """Print any missing packages found via apt. |
| 705 | |
| 706 | This will find any missing packages necessary for build using apt and |
| 707 | print it out as an apt-get install printf. |
| 708 | """ |
| 709 | print('Checking for any missing packages...') |
| 710 | |
| 711 | (success, output) = self._get_command_output(APT_PKG_LIST) |
| 712 | if not success: |
| 713 | raise Exception("Could not query apt for packages.") |
| 714 | |
| 715 | packages_installed = {} |
| 716 | for line in output: |
| 717 | if 'installed' in line: |
| 718 | split = line.split('/', 2) |
| 719 | packages_installed[split[0]] = True |
| 720 | |
| 721 | need_packages = [] |
| 722 | for pkg in REQUIRED_APT_PACKAGES: |
| 723 | if pkg not in packages_installed: |
| 724 | need_packages.append(pkg) |
| 725 | |
| 726 | # No packages need to be installed |
| 727 | if len(need_packages) == 0: |
| 728 | print('+ All required packages are installed') |
| 729 | return |
| 730 | |
| 731 | install = self._pretty_print_install('sudo apt-get install', need_packages) |
| 732 | |
| 733 | # Print all lines so they can be run in cmdline |
| 734 | print('Missing system packages. Run the following command: ') |
| 735 | print(' \\\n'.join(install)) |
| 736 | |
| 737 | def _print_missing_rust_packages(self): |
| 738 | """Print any missing packages found via cargo. |
| 739 | |
| 740 | This will find any missing packages necessary for build using cargo and |
| 741 | print it out as a cargo-install printf. |
| 742 | """ |
| 743 | print('Checking for any missing cargo packages...') |
| 744 | |
| 745 | (success, output) = self._get_command_output(CARGO_PKG_LIST) |
| 746 | if not success: |
| 747 | raise Exception("Could not query cargo for packages.") |
| 748 | |
| 749 | packages_installed = {} |
| 750 | for line in output: |
| 751 | # Cargo installed packages have this format |
| 752 | # <crate name> <version>: |
| 753 | # <binary name> |
| 754 | # We only care about the crates themselves |
| 755 | if ':' not in line: |
| 756 | continue |
| 757 | |
| 758 | split = line.split(' ', 2) |
| 759 | packages_installed[split[0]] = True |
| 760 | |
| 761 | need_packages = [] |
| 762 | for pkg in REQUIRED_CARGO_PACKAGES: |
| 763 | if pkg not in packages_installed: |
| 764 | need_packages.append(pkg) |
| 765 | |
| 766 | # No packages to be installed |
| 767 | if len(need_packages) == 0: |
| 768 | print('+ All required cargo packages are installed') |
| 769 | return |
| 770 | |
| 771 | install = self._pretty_print_install('cargo install', need_packages) |
| 772 | print('Missing cargo packages. Run the following command: ') |
| 773 | print(' \\\n'.join(install)) |
| 774 | |
| 775 | def bootstrap(self): |
| 776 | """ Bootstrap the Linux build.""" |
| 777 | self._setup_platform2() |
| 778 | self._print_missing_packages() |
| 779 | self._print_missing_rust_packages() |
| 780 | |
| 781 | |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 782 | if __name__ == '__main__': |
| 783 | parser = argparse.ArgumentParser(description='Simple build for host.') |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 784 | parser.add_argument( |
| 785 | '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss") |
| 786 | parser.add_argument( |
| 787 | '--run-bootstrap', |
| 788 | help='Run bootstrap code to verify build env is ok to build.', |
| 789 | default=False, |
| 790 | action='store_true') |
| 791 | parser.add_argument('--no-clang', help='Use clang compiler.', default=False, action='store_true') |
Abhishek Pandit-Subedi | a1a94d1 | 2021-11-18 13:14:04 -0800 | [diff] [blame] | 792 | parser.add_argument( |
| 793 | '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true') |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 794 | parser.add_argument('--use', help='Set a specific use flag.') |
Abhishek Pandit-Subedi | 9476414 | 2022-02-07 17:18:00 -0800 | [diff] [blame] | 795 | parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true') |
| 796 | parser.add_argument('--test-name', help='Run test with this string in the name.', default=None) |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 797 | parser.add_argument('--target', help='Run specific build target') |
| 798 | parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/') |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 799 | parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib') |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 800 | parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int) |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 801 | parser.add_argument( |
| 802 | '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true') |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 803 | parser.add_argument('--verbose', help='Verbose logs for build.') |
Abhishek Pandit-Subedi | f52cf66 | 2021-03-02 22:33:25 +0000 | [diff] [blame] | 804 | args = parser.parse_args() |
Abhishek Pandit-Subedi | 0db7701 | 2021-09-20 17:54:26 -0700 | [diff] [blame] | 805 | |
| 806 | # Make sure we get absolute path + expanded path for bootstrap directory |
| 807 | args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir)) |
| 808 | |
| 809 | if args.run_bootstrap: |
| 810 | bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__)) |
| 811 | bootstrap.bootstrap() |
| 812 | else: |
| 813 | build = HostBuild(args) |
| 814 | build.build() |