Rewrite hidden API list generation in Python
Generating hidden API lists has grown in complexity and the original
Makefile rule has become a bottleneck for build times. Rewrite the
logic in Python.
Bug: 113278235
Bug: 73736106
Test: frameworks/base/tools/hiddenapi/generate_hiddenapi_lists_test.py
Change-Id: I63f03133d70260d06c55f482b844a4980dc6f734
diff --git a/Android.mk b/Android.mk
index 84b708e..988c009 100644
--- a/Android.mk
+++ b/Android.mk
@@ -322,131 +322,34 @@
( unzip -qo $< -d $(OUT_DOCS)/offline-sdk && touch -f $@ ) || exit 1
# ==== hiddenapi lists =======================================
-include $(CLEAR_VARS)
-
-# File names of final API lists
-LOCAL_WHITELIST := $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST)
-LOCAL_LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST)
-LOCAL_DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST)
-LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)
-
-# File names of source files we will use to generate the final API lists.
-LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt
-LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt
-LOCAL_SRC_FORCE_BLACKLIST := frameworks/base/config/hiddenapi-force-blacklist.txt
-LOCAL_SRC_PUBLIC_API := $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST)
-LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)
-LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)
-
-LOCAL_SRC_ALL := \
- $(LOCAL_SRC_GREYLIST) \
- $(LOCAL_SRC_VENDOR_LIST) \
- $(LOCAL_SRC_FORCE_BLACKLIST) \
- $(LOCAL_SRC_PUBLIC_API) \
- $(LOCAL_SRC_PRIVATE_API) \
- $(LOCAL_SRC_REMOVED_API)
-
-define assert-has-no-overlap
-if [ ! -z "`comm -12 <(sort $(1)) <(sort $(2))`" ]; then \
- echo "$(1) and $(2) should not overlap" 1>&2; \
- comm -12 <(sort $(1)) <(sort $(2)) 1>&2; \
- exit 1; \
-fi
-endef
-
-define assert-is-subset
-if [ ! -z "`comm -23 <(sort $(1)) <(sort $(2))`" ]; then \
- echo "$(1) must be a subset of $(2)" 1>&2; \
- comm -23 <(sort $(1)) <(sort $(2)) 1>&2; \
- exit 1; \
-fi
-endef
-
-define assert-has-no-duplicates
-if [ ! -z "`sort $(1) | uniq -D`" ]; then \
- echo "$(1) has duplicate entries" 1>&2; \
- sort $(1) | uniq -D 1>&2; \
- exit 1; \
-fi
-endef
-
-# The following rules build API lists in the build folder.
-# By not using files from the source tree, ART buildbots can mock these lists
-# or have alternative rules for building them. Other rules in the build system
-# should depend on the files in the build folder.
-
-# Merge whitelist from:
-# (1) public API stubs
-# (2) whitelist entries generated by class2greylist (PRIVATE_WHITELIST_INPUTS)
-$(LOCAL_WHITELIST): $(LOCAL_SRC_PUBLIC_API)
- sort $(LOCAL_SRC_PUBLIC_API) $(PRIVATE_WHITELIST_INPUTS) > $@
- $(call assert-has-no-duplicates,$@)
-
-# Merge light greylist from multiple files:
-# (1) manual greylist LOCAL_SRC_GREYLIST
-# (2) list of usages from vendor apps LOCAL_SRC_VENDOR_LIST
-# (3) list of removed APIs in LOCAL_SRC_REMOVED_API
-# @removed does not imply private in Doclava. We must take the subset also
-# in LOCAL_SRC_PRIVATE_API.
-# (4) list of serialization APIs
-# Automatically adds all methods which match the signatures in
-# REGEX_SERIALIZATION. These are greylisted in order to allow applications
-# to write their own serializers.
-# (5) greylist entries generated by class2greylist (PRIVATE_GREYLIST_INPUTS)
-$(LOCAL_LIGHT_GREYLIST): REGEX_SERIALIZATION := \
- "readObject\(Ljava/io/ObjectInputStream;\)V" \
- "readObjectNoData\(\)V" \
- "readResolve\(\)Ljava/lang/Object;" \
- "serialVersionUID:J" \
- "serialPersistentFields:\[Ljava/io/ObjectStreamField;" \
- "writeObject\(Ljava/io/ObjectOutputStream;\)V" \
- "writeReplace\(\)Ljava/lang/Object;"
-$(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST)
- sort $(LOCAL_SRC_GREYLIST) $(LOCAL_SRC_VENDOR_LIST) $(PRIVATE_GREYLIST_INPUTS) \
- <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" \
- $(LOCAL_SRC_PRIVATE_API)) \
- <(comm -12 <(sort $(LOCAL_SRC_REMOVED_API)) <(sort $(LOCAL_SRC_PRIVATE_API))) \
- > $@
- $(call assert-has-no-duplicates,$@)
- $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))
- $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))
- $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST))
-
-# Generate dark greylist as remaining classes and class members in the same
-# package as classes listed in the light greylist.
-# The algorithm is as follows:
-# (1) extract the class descriptor from each entry in LOCAL_LIGHT_GREYLIST
-# (2) strip everything after the last forward-slash,
-# e.g. 'Lpackage/subpackage/class$inner;' turns into 'Lpackage/subpackage/'
-# (3) insert all entries from LOCAL_SRC_PRIVATE_API which begin with the package
-# name but do not contain another forward-slash in the class name, e.g.
-# matching '^Lpackage/subpackage/[^/;]*;'
-# (4) subtract entries shared with LOCAL_LIGHT_GREYLIST
-$(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST)
- comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_SRC_FORCE_BLACKLIST)) \
- <(cat $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) | \
- sed 's/\->.*//' | sed 's/\(.*\/\).*/\1/' | sort | uniq | \
- while read PKG_NAME; do \
- grep -E "^$${PKG_NAME}[^/;]*;" $(LOCAL_SRC_PRIVATE_API); \
- done | sort | uniq) \
- > $@
- $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))
- $(call assert-has-no-duplicates,$@)
- $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))
- $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST))
- $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST))
-
-# Generate blacklist as private API minus (light greylist plus dark greylist).
-$(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)
- comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)) \
- <(sort $(LOCAL_SRC_PRIVATE_API)) \
- > $@
- $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))
- $(call assert-has-no-duplicates,$@)
- $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))
- $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST))
- $(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST))
- $(call assert-is-subset,$(LOCAL_SRC_FORCE_BLACKLIST),$@)
+$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \
+ .KATI_IMPLICIT_OUTPUTS := \
+ $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \
+ $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \
+ $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)
+$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \
+ frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \
+ frameworks/base/config/hiddenapi-light-greylist.txt \
+ frameworks/base/config/hiddenapi-vendor-list.txt \
+ frameworks/base/config/hiddenapi-force-blacklist.txt \
+ $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \
+ $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \
+ $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)
+ frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \
+ --input-public $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \
+ --input-private $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \
+ --input-whitelists $(PRIVATE_WHITELIST_INPUTS) \
+ --input-greylists \
+ frameworks/base/config/hiddenapi-light-greylist.txt \
+ frameworks/base/config/hiddenapi-vendor-list.txt \
+ <(comm -12 <(sort $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)) \
+ $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)) \
+ $(PRIVATE_GREYLIST_INPUTS) \
+ --input-blacklists frameworks/base/config/hiddenapi-force-blacklist.txt \
+ --output-whitelist $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) \
+ --output-light-greylist $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \
+ --output-dark-greylist $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \
+ --output-blacklist $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)
# Include subdirectory makefiles
# ============================================================