tests/sbom_test.sh - SHIFTPHONES/android_build_soong - Gitiles

 #!/bin/bash

 # Copyright (C) 2023 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 set -uo pipefail

 # Integration test for verifying generated SBOM for cuttlefish device.

 if [ ! -e "build/make/core/Makefile" ]; then
   echo "$0 must be run from the top of the Android source tree."
   exit 1
 fi

 tmp_dir="$(mktemp -d tmp.XXXXXX)"
 function cleanup {
   rm -rf "${tmp_dir}"
 }
 trap cleanup EXIT

 out_dir=$tmp_dir
 droid_target=droid

 debug=false
 if [ $debug = "true" ]; then
   out_dir=out
   droid_target=
 fi

 function run_soong {
   TARGET_PRODUCT="aosp_cf_x86_64_phone" TARGET_BUILD_VARIANT=userdebug OUT_DIR=$out_dir \
     build/soong/soong_ui.bash --make-mode "$@"
 }

 # m droid, build sbom later in case additional dependencies might be built and included in partition images.
 run_soong $droid_target dump.erofs lz4

 product_out=$out_dir/target/product/vsoc_x86_64
 sbom_test=$product_out/sbom_test
 mkdir $sbom_test
 cp $product_out/*.img $sbom_test

 # m sbom
 run_soong sbom

 # Generate installed file list from .img files in PRODUCT_OUT
 dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs
 lz4=$out_dir/host/linux-x86/bin/lz4

 declare -A diff_excludes
 diff_excludes[odm]="-I /odm/lib/modules"
 diff_excludes[vendor]=\
 "-I /vendor/lib64/libkeystore2_crypto.so \
  -I /vendor/lib/modules \
  -I /vendor/odm"
 diff_excludes[system]=\
 "-I /bin \
  -I /bugreports \
  -I /cache \
  -I /d \
  -I /etc \
  -I /init \
  -I /odm/app \
  -I /odm/bin \
  -I /odm_dlkm/etc \
  -I /odm/etc \
  -I /odm/firmware \
  -I /odm/framework \
  -I /odm/lib \
  -I /odm/lib64 \
  -I /odm/overlay \
  -I /odm/priv-app \
  -I /odm/usr \
  -I /sdcard \
  -I /system/lib64/android.hardware.confirmationui@1.0.so \
  -I /system/lib64/android.hardware.confirmationui-V1-ndk.so \
  -I /system/lib64/android.hardware.keymaster@4.1.so \
  -I /system/lib64/android.hardware.security.rkp-V3-ndk.so \
  -I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \
  -I /system/lib64/android.security.compat-ndk.so \
  -I /system/lib64/libkeymaster4_1support.so \
  -I /system/lib64/libkeymint.so \
  -I /system/lib64/libkeystore2_aaid.so \
  -I /system/lib64/libkeystore2_apc_compat.so \
  -I /system/lib64/libkeystore2_crypto.so \
  -I /system/lib64/libkm_compat_service.so \
  -I /system/lib64/libkm_compat.so \
  -I /system/lib64/vndk-29 \
  -I /system/lib64/vndk-sp-29 \
  -I /system/lib/vndk-29 \
  -I /system/lib/vndk-sp-29 \
  -I /system/usr/icu \
  -I /vendor_dlkm/etc"

 function diff_files {
    file_list_file="$1"; shift
    files_in_spdx_file="$1"; shift
    partition_name="$1"; shift
    exclude=
    if [ -v 'diff_excludes[$partition_name]' ]; then
      exclude=${diff_excludes[$partition_name]}
    fi

    diff "$file_list_file" "$files_in_spdx_file" $exclude
    if [ $? != "0" ]; then
      echo Found diffs in $f and SBOM.
      exit 1
    else
      echo No diffs.
    fi
  }

 # Example output of dump.erofs is as below, and the data used in the test start
 # at line 11. Column 1 is inode id, column 2 is inode type and column 3 is name.
 # Each line is captured in variable "entry", awk is used to get type and name.
 # Output of dump.erofs:
 #     File : /
 #     Size: 160  On-disk size: 160  directory
 #     NID: 39   Links: 10   Layout: 2   Compression ratio: 100.00%
 #     Inode size: 64   Extent size: 0   Xattr size: 16
 #     Uid: 0   Gid: 0  Access: 0755/rwxr-xr-x
 #     Timestamp: 2023-02-14 01:15:54.000000000
 #
 #            NID TYPE  FILENAME
 #             39    2  .
 #             39    2  ..
 #             47    2  app
 #        1286748    2  bin
 #        1286754    2  etc
 #        5304814    2  lib
 #        5309056    2  lib64
 #        5309130    2  media
 #        5388910    2  overlay
 #        5479537    2  priv-app
 EROFS_IMAGES="\
   $sbom_test/product.img \
   $sbom_test/system.img \
   $sbom_test/system_ext.img \
   $sbom_test/system_dlkm.img \
   $sbom_test/system_other.img \
   $sbom_test/odm.img \
   $sbom_test/odm_dlkm.img \
   $sbom_test/vendor.img \
   $sbom_test/vendor_dlkm.img"
 for f in $EROFS_IMAGES; do
   partition_name=$(basename $f | cut -d. -f1)
   file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
   files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
   rm "$file_list_file" > /dev/null 2>&1
   all_dirs="/"
   while [ ! -z "$all_dirs" ]; do
     dir=$(echo "$all_dirs" | cut -d ' ' -f1)
     all_dirs=$(echo "$all_dirs" | cut -d ' ' -f1 --complement -s)
     entries=$($dump_erofs --ls --path "$dir" $f | tail -n +11)
     while read -r entry; do
       inode_type=$(echo $entry | awk -F ' ' '{print $2}')
       name=$(echo $entry | awk -F ' ' '{print $3}')
       case $inode_type in
         "2")  # directory
           all_dirs=$(echo "$all_dirs $dir/$name" | sed 's/^\s*//')
           ;;
         "1"|"7")  # 1: file, 7: symlink
           (
           if [ "$partition_name" != "system" ]; then
             # system partition is mounted to /, not to prepend partition name.
             printf %s "/$partition_name"
           fi
           echo "$dir/$name" | sed 's#^//#/#'
           ) >> "$file_list_file"
           ;;
       esac
     done <<< "$entries"
   done
   sort -n -o "$file_list_file" "$file_list_file"

   grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file"
   if [ "$partition_name" = "system" ]; then
     # system partition is mounted to /, so include FileName starts with /root/ too.
     grep "FileName: /root/" $product_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_spdx_file"
   fi
   sort -n -o "$files_in_spdx_file" "$files_in_spdx_file"

   echo ============ Diffing files in $f and SBOM
   diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name"
 done

 RAMDISK_IMAGES="$product_out/ramdisk.img"
 for f in $RAMDISK_IMAGES; do
   partition_name=$(basename $f | cut -d. -f1)
   file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
   files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
   # lz4 decompress $f to stdout
   # cpio list all entries like ls -l
   # grep filter normal files and symlinks
   # awk get entry names
   # sed remove partition name from entry names
   $lz4 -c -d $f | cpio -tv 2>/dev/null | grep '^[-l]' | awk -F ' ' '{print $9}' | sed "s:^:/$partition_name/:" | sort -n > "$file_list_file"

   grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file"

   echo ============ Diffing files in $f and SBOM
   diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name"
 done
	#!/bin/bash

	# Copyright (C) 2023 The Android Open Source Project
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	set -uo pipefail

	# Integration test for verifying generated SBOM for cuttlefish device.

	if [ ! -e "build/make/core/Makefile" ]; then
	echo "$0 must be run from the top of the Android source tree."
	exit 1
	fi

	tmp_dir="$(mktemp -d tmp.XXXXXX)"
	function cleanup {
	rm -rf "${tmp_dir}"
	}
	trap cleanup EXIT

	out_dir=$tmp_dir
	droid_target=droid

	debug=false
	if [ $debug = "true" ]; then
	out_dir=out
	droid_target=
	fi

	function run_soong {
	TARGET_PRODUCT="aosp_cf_x86_64_phone" TARGET_BUILD_VARIANT=userdebug OUT_DIR=$out_dir \
	build/soong/soong_ui.bash --make-mode "$@"
	}

	# m droid, build sbom later in case additional dependencies might be built and included in partition images.
	run_soong $droid_target dump.erofs lz4

	product_out=$out_dir/target/product/vsoc_x86_64
	sbom_test=$product_out/sbom_test
	mkdir $sbom_test
	cp $product_out/*.img $sbom_test

	# m sbom
	run_soong sbom

	# Generate installed file list from .img files in PRODUCT_OUT
	dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs
	lz4=$out_dir/host/linux-x86/bin/lz4

	declare -A diff_excludes
	diff_excludes[odm]="-I /odm/lib/modules"
	diff_excludes[vendor]=\
	"-I /vendor/lib64/libkeystore2_crypto.so \
	-I /vendor/lib/modules \
	-I /vendor/odm"
	diff_excludes[system]=\
	"-I /bin \
	-I /bugreports \
	-I /cache \
	-I /d \
	-I /etc \
	-I /init \
	-I /odm/app \
	-I /odm/bin \
	-I /odm_dlkm/etc \
	-I /odm/etc \
	-I /odm/firmware \
	-I /odm/framework \
	-I /odm/lib \
	-I /odm/lib64 \
	-I /odm/overlay \
	-I /odm/priv-app \
	-I /odm/usr \
	-I /sdcard \
	-I /system/lib64/android.hardware.confirmationui@1.0.so \
	-I /system/lib64/android.hardware.confirmationui-V1-ndk.so \
	-I /system/lib64/android.hardware.keymaster@4.1.so \
	-I /system/lib64/android.hardware.security.rkp-V3-ndk.so \
	-I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \
	-I /system/lib64/android.security.compat-ndk.so \
	-I /system/lib64/libkeymaster4_1support.so \
	-I /system/lib64/libkeymint.so \
	-I /system/lib64/libkeystore2_aaid.so \
	-I /system/lib64/libkeystore2_apc_compat.so \
	-I /system/lib64/libkeystore2_crypto.so \
	-I /system/lib64/libkm_compat_service.so \
	-I /system/lib64/libkm_compat.so \
	-I /system/lib64/vndk-29 \
	-I /system/lib64/vndk-sp-29 \
	-I /system/lib/vndk-29 \
	-I /system/lib/vndk-sp-29 \
	-I /system/usr/icu \
	-I /vendor_dlkm/etc"

	function diff_files {
	file_list_file="$1"; shift
	files_in_spdx_file="$1"; shift
	partition_name="$1"; shift
	exclude=
	if [ -v 'diff_excludes[$partition_name]' ]; then
	exclude=${diff_excludes[$partition_name]}
	fi

	diff "$file_list_file" "$files_in_spdx_file" $exclude
	if [ $? != "0" ]; then
	echo Found diffs in $f and SBOM.
	exit 1
	else
	echo No diffs.
	fi
	}

	# Example output of dump.erofs is as below, and the data used in the test start
	# at line 11. Column 1 is inode id, column 2 is inode type and column 3 is name.
	# Each line is captured in variable "entry", awk is used to get type and name.
	# Output of dump.erofs:
	# File : /
	# Size: 160 On-disk size: 160 directory
	# NID: 39 Links: 10 Layout: 2 Compression ratio: 100.00%
	# Inode size: 64 Extent size: 0 Xattr size: 16
	# Uid: 0 Gid: 0 Access: 0755/rwxr-xr-x
	# Timestamp: 2023-02-14 01:15:54.000000000
	#
	# NID TYPE FILENAME
	# 39 2 .
	# 39 2 ..
	# 47 2 app
	# 1286748 2 bin
	# 1286754 2 etc
	# 5304814 2 lib
	# 5309056 2 lib64
	# 5309130 2 media
	# 5388910 2 overlay
	# 5479537 2 priv-app
	EROFS_IMAGES="\
	$sbom_test/product.img \
	$sbom_test/system.img \
	$sbom_test/system_ext.img \
	$sbom_test/system_dlkm.img \
	$sbom_test/system_other.img \
	$sbom_test/odm.img \
	$sbom_test/odm_dlkm.img \
	$sbom_test/vendor.img \
	$sbom_test/vendor_dlkm.img"
	for f in $EROFS_IMAGES; do
	partition_name=$(basename $f \| cut -d. -f1)
	file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
	files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
	rm "$file_list_file" > /dev/null 2>&1
	all_dirs="/"
	while [ ! -z "$all_dirs" ]; do
	dir=$(echo "$all_dirs" \| cut -d ' ' -f1)
	all_dirs=$(echo "$all_dirs" \| cut -d ' ' -f1 --complement -s)
	entries=$($dump_erofs --ls --path "$dir" $f \| tail -n +11)
	while read -r entry; do
	inode_type=$(echo $entry \| awk -F ' ' '{print $2}')
	name=$(echo $entry \| awk -F ' ' '{print $3}')
	case $inode_type in
	"2") # directory
	all_dirs=$(echo "$all_dirs $dir/$name" \| sed 's/^\s*//')
	;;
	"1"\|"7") # 1: file, 7: symlink
	(
	if [ "$partition_name" != "system" ]; then
	# system partition is mounted to /, not to prepend partition name.
	printf %s "/$partition_name"
	fi
	echo "$dir/$name" \| sed 's#^//#/#'
	) >> "$file_list_file"
	;;
	esac
	done <<< "$entries"
	done
	sort -n -o "$file_list_file" "$file_list_file"

	grep "FileName: /${partition_name}/" $product_out/sbom.spdx \| sed 's/^FileName: //' > "$files_in_spdx_file"
	if [ "$partition_name" = "system" ]; then
	# system partition is mounted to /, so include FileName starts with /root/ too.
	grep "FileName: /root/" $product_out/sbom.spdx \| sed 's/^FileName: \/root//' >> "$files_in_spdx_file"
	fi
	sort -n -o "$files_in_spdx_file" "$files_in_spdx_file"

	echo ============ Diffing files in $f and SBOM
	diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name"
	done

	RAMDISK_IMAGES="$product_out/ramdisk.img"
	for f in $RAMDISK_IMAGES; do
	partition_name=$(basename $f \| cut -d. -f1)
	file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
	files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
	# lz4 decompress $f to stdout
	# cpio list all entries like ls -l
	# grep filter normal files and symlinks
	# awk get entry names
	# sed remove partition name from entry names
	$lz4 -c -d $f \| cpio -tv 2>/dev/null \| grep '^[-l]' \| awk -F ' ' '{print $9}' \| sed "s:^:/$partition_name/:" \| sort -n > "$file_list_file"

	grep "FileName: /${partition_name}/" $product_out/sbom.spdx \| sed 's/^FileName: //' \| sort -n > "$files_in_spdx_file"

	echo ============ Diffing files in $f and SBOM
	diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name"
	done