Revert "Build notice files from license metadata."
This reverts commit 43c2dcaef609c4a268bfab6c95ed924af4ead6b1.
Reason for revert: suspect build break
Bug: 227682036
Test: TARGET_BUILD_VARIANT=userdebug UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true vendor/google/build/mainline_modules_bundles.sh -j97
Change-Id: Ibfb8b4fefc264f52f32ba661c269a9cd625d800a
diff --git a/android/notices.go b/android/notices.go
index 2a4c17c..194a734 100644
--- a/android/notices.go
+++ b/android/notices.go
@@ -15,9 +15,93 @@
package android
import (
+ "path/filepath"
"strings"
+
+ "github.com/google/blueprint"
)
+func init() {
+ pctx.SourcePathVariable("merge_notices", "build/soong/scripts/mergenotice.py")
+ pctx.SourcePathVariable("generate_notice", "build/soong/scripts/generate-notice-files.py")
+
+ pctx.HostBinToolVariable("minigzip", "minigzip")
+}
+
+type NoticeOutputs struct {
+ Merged OptionalPath
+ TxtOutput OptionalPath
+ HtmlOutput OptionalPath
+ HtmlGzOutput OptionalPath
+}
+
+var (
+ mergeNoticesRule = pctx.AndroidStaticRule("mergeNoticesRule", blueprint.RuleParams{
+ Command: `${merge_notices} --output $out $in`,
+ CommandDeps: []string{"${merge_notices}"},
+ Description: "merge notice files into $out",
+ })
+
+ generateNoticeRule = pctx.AndroidStaticRule("generateNoticeRule", blueprint.RuleParams{
+ Command: `rm -rf $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` +
+ `mkdir -p $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` +
+ `${generate_notice} --text-output $txtOut --html-output $htmlOut -t "$title" -s $inputDir && ` +
+ `${minigzip} -c $htmlOut > $out`,
+ CommandDeps: []string{"${generate_notice}", "${minigzip}"},
+ Description: "produce notice file $out",
+ }, "txtOut", "htmlOut", "title", "inputDir")
+)
+
+func MergeNotices(ctx ModuleContext, mergedNotice WritablePath, noticePaths []Path) {
+ ctx.Build(pctx, BuildParams{
+ Rule: mergeNoticesRule,
+ Description: "merge notices",
+ Inputs: noticePaths,
+ Output: mergedNotice,
+ })
+}
+
+func BuildNoticeOutput(ctx ModuleContext, installPath InstallPath, installFilename string,
+ noticePaths []Path) NoticeOutputs {
+ // Merge all NOTICE files into one.
+ // TODO(jungjw): We should just produce a well-formatted NOTICE.html file in a single pass.
+ //
+ // generate-notice-files.py, which processes the merged NOTICE file, has somewhat strict rules
+ // about input NOTICE file paths.
+ // 1. Their relative paths to the src root become their NOTICE index titles. We want to use
+ // on-device paths as titles, and so output the merged NOTICE file the corresponding location.
+ // 2. They must end with .txt extension. Otherwise, they're ignored.
+ noticeRelPath := InstallPathToOnDevicePath(ctx, installPath.Join(ctx, installFilename+".txt"))
+ mergedNotice := PathForModuleOut(ctx, filepath.Join("NOTICE_FILES/src", noticeRelPath))
+ MergeNotices(ctx, mergedNotice, noticePaths)
+
+ // Transform the merged NOTICE file into a gzipped HTML file.
+ txtOuptut := PathForModuleOut(ctx, "NOTICE_txt", "NOTICE.txt")
+ htmlOutput := PathForModuleOut(ctx, "NOTICE_html", "NOTICE.html")
+ htmlGzOutput := PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz")
+ title := "Notices for " + ctx.ModuleName()
+ ctx.Build(pctx, BuildParams{
+ Rule: generateNoticeRule,
+ Description: "generate notice output",
+ Input: mergedNotice,
+ Output: htmlGzOutput,
+ ImplicitOutputs: WritablePaths{txtOuptut, htmlOutput},
+ Args: map[string]string{
+ "txtOut": txtOuptut.String(),
+ "htmlOut": htmlOutput.String(),
+ "title": title,
+ "inputDir": PathForModuleOut(ctx, "NOTICE_FILES/src").String(),
+ },
+ })
+
+ return NoticeOutputs{
+ Merged: OptionalPathForPath(mergedNotice),
+ TxtOutput: OptionalPathForPath(txtOuptut),
+ HtmlOutput: OptionalPathForPath(htmlOutput),
+ HtmlGzOutput: OptionalPathForPath(htmlGzOutput),
+ }
+}
+
// BuildNoticeTextOutputFromLicenseMetadata writes out a notice text file based on the module's
// generated license metadata file.
func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) {
@@ -28,18 +112,5 @@
FlagWithOutput("-o ", outputFile).
FlagWithDepFile("-d ", depsFile).
Input(ctx.Module().base().licenseMetadataFile)
- rule.Build("text_notice", "container notice file")
-}
-
-// BuildNoticeHtmlOutputFromLicenseMetadata writes out a notice text file based on the module's
-// generated license metadata file.
-func BuildNoticeHtmlOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) {
- depsFile := outputFile.ReplaceExtension(ctx, strings.TrimPrefix(outputFile.Ext()+".d", "."))
- rule := NewRuleBuilder(pctx, ctx)
- rule.Command().
- BuiltTool("htmlnotice").
- FlagWithOutput("-o ", outputFile).
- FlagWithDepFile("-d ", depsFile).
- Input(ctx.Module().base().licenseMetadataFile)
- rule.Build("html_notice", "container notice file")
+ rule.Build("container_notice", "container notice file")
}
diff --git a/apex/androidmk.go b/apex/androidmk.go
index e094a12..059b4d7 100644
--- a/apex/androidmk.go
+++ b/apex/androidmk.go
@@ -396,6 +396,10 @@
}
a.writeRequiredModules(w, moduleNames)
+ if a.mergedNotices.Merged.Valid() {
+ fmt.Fprintln(w, "LOCAL_NOTICE_FILE :=", a.mergedNotices.Merged.Path().String())
+ }
+
fmt.Fprintln(w, "include $(BUILD_PREBUILT)")
if apexType == imageApex {
diff --git a/apex/apex.go b/apex/apex.go
index cb88f02..6d8a67a 100644
--- a/apex/apex.go
+++ b/apex/apex.go
@@ -414,8 +414,8 @@
// Processed file_contexts files
fileContexts android.WritablePath
- // Path to notice file in html.gz format.
- htmlGzNotice android.WritablePath
+ // Struct holding the merged notice file paths in different formats
+ mergedNotices android.NoticeOutputs
// The built APEX file. This is the main product.
// Could be .apex or .capex
@@ -487,10 +487,11 @@
// for each of the files in case when the APEX is flattened.
type apexFile struct {
// buildFile is put in the installDir inside the APEX.
- builtFile android.Path
- installDir string
- customStem string
- symlinks []string // additional symlinks
+ builtFile android.Path
+ noticeFiles android.Paths
+ installDir string
+ customStem string
+ symlinks []string // additional symlinks
// Info for Android.mk Module name of `module` in AndroidMk. Note the generated AndroidMk
// module for apexFile is named something like <AndroidMk module name>.<apex name>[<apex
@@ -527,6 +528,7 @@
module: module,
}
if module != nil {
+ ret.noticeFiles = module.NoticeFiles()
ret.moduleDir = ctx.OtherModuleDir(module)
ret.requiredModuleNames = module.RequiredModuleNames()
ret.targetRequiredModuleNames = module.TargetRequiredModuleNames()
diff --git a/apex/apex_test.go b/apex/apex_test.go
index ec815c4..85bd595 100644
--- a/apex/apex_test.go
+++ b/apex/apex_test.go
@@ -591,6 +591,15 @@
t.Errorf("Could not find all expected symlinks! foo: %t, foo_link_64: %t. Command was %s", found_foo, found_foo_link_64, copyCmds)
}
+ mergeNoticesRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("mergeNoticesRule")
+ noticeInputs := mergeNoticesRule.Inputs.Strings()
+ if len(noticeInputs) != 3 {
+ t.Errorf("number of input notice files: expected = 3, actual = %q", len(noticeInputs))
+ }
+ ensureListContains(t, noticeInputs, "NOTICE")
+ ensureListContains(t, noticeInputs, "custom_notice")
+ ensureListContains(t, noticeInputs, "custom_notice_for_static_lib")
+
fullDepsInfo := strings.Split(ctx.ModuleForTests("myapex", "android_common_myapex_image").Output("depsinfo/fulllist.txt").Args["content"], "\\n")
ensureListContains(t, fullDepsInfo, " myjar(minSdkVersion:(no version)) <- myapex")
ensureListContains(t, fullDepsInfo, " mylib2(minSdkVersion:(no version)) <- mylib")
diff --git a/apex/builder.go b/apex/builder.go
index 50c8dd1..8c5f99b 100644
--- a/apex/builder.go
+++ b/apex/builder.go
@@ -305,6 +305,32 @@
return output.OutputPath
}
+// buildNoticeFiles creates a buile rule for aggregating notice files from the modules that
+// contributes to this APEX. The notice files are merged into a big notice file.
+func (a *apexBundle) buildNoticeFiles(ctx android.ModuleContext, apexFileName string) android.NoticeOutputs {
+ var noticeFiles android.Paths
+
+ a.WalkPayloadDeps(ctx, func(ctx android.ModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool) bool {
+ if externalDep {
+ // As soon as the dependency graph crosses the APEX boundary, don't go further.
+ return false
+ }
+ noticeFiles = append(noticeFiles, to.NoticeFiles()...)
+ return true
+ })
+
+ // TODO(jiyong): why do we need this? WalkPayloadDeps should have already covered this.
+ for _, fi := range a.filesInfo {
+ noticeFiles = append(noticeFiles, fi.noticeFiles...)
+ }
+
+ if len(noticeFiles) == 0 {
+ return android.NoticeOutputs{}
+ }
+
+ return android.BuildNoticeOutput(ctx, a.installDir, apexFileName, android.SortedUniquePaths(noticeFiles))
+}
+
// buildInstalledFilesFile creates a build rule for the installed-files.txt file where the list of
// files included in this APEX is shown. The text file is dist'ed so that people can see what's
// included in the APEX without actually downloading and extracting it.
@@ -616,11 +642,12 @@
optFlags = append(optFlags, "--logging_parent ", a.overridableProperties.Logging_parent)
}
- // Create a NOTICE file, and embed it as an asset file in the APEX.
- a.htmlGzNotice = android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz")
- android.BuildNoticeHtmlOutputFromLicenseMetadata(ctx, a.htmlGzNotice)
- implicitInputs = append(implicitInputs, a.htmlGzNotice)
- optFlags = append(optFlags, "--assets_dir "+filepath.Dir(a.htmlGzNotice.String()))
+ a.mergedNotices = a.buildNoticeFiles(ctx, a.Name()+suffix)
+ if a.mergedNotices.HtmlGzOutput.Valid() {
+ // If there's a NOTICE file, embed it as an asset file in the APEX.
+ implicitInputs = append(implicitInputs, a.mergedNotices.HtmlGzOutput.Path())
+ optFlags = append(optFlags, "--assets_dir "+filepath.Dir(a.mergedNotices.HtmlGzOutput.String()))
+ }
if (moduleMinSdkVersion.GreaterThan(android.SdkVersion_Android10) && !a.shouldGenerateHashtree()) && !compressionEnabled {
// Apexes which are supposed to be installed in builtin dirs(/system, etc)
diff --git a/java/androidmk.go b/java/androidmk.go
index 80b828d..b930441 100644
--- a/java/androidmk.go
+++ b/java/androidmk.go
@@ -409,6 +409,22 @@
entries.SetOptionalPaths("LOCAL_SOONG_LINT_REPORTS", app.linter.reports)
},
},
+ ExtraFooters: []android.AndroidMkExtraFootersFunc{
+ func(w io.Writer, name, prefix, moduleDir string) {
+ if app.noticeOutputs.Merged.Valid() {
+ fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n",
+ app.installApkName, app.noticeOutputs.Merged.String(), app.installApkName+"_NOTICE")
+ }
+ if app.noticeOutputs.TxtOutput.Valid() {
+ fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n",
+ app.installApkName, app.noticeOutputs.TxtOutput.String(), app.installApkName+"_NOTICE.txt")
+ }
+ if app.noticeOutputs.HtmlOutput.Valid() {
+ fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n",
+ app.installApkName, app.noticeOutputs.HtmlOutput.String(), app.installApkName+"_NOTICE.html")
+ }
+ },
+ },
}}
}
diff --git a/java/app.go b/java/app.go
index 5b1daa4..8728df6 100755
--- a/java/app.go
+++ b/java/app.go
@@ -19,6 +19,7 @@
import (
"path/filepath"
+ "sort"
"strings"
"github.com/google/blueprint"
@@ -163,6 +164,8 @@
additionalAaptFlags []string
+ noticeOutputs android.NoticeOutputs
+
overriddenManifestPackageName string
android.ApexBundleDepsInfo
@@ -520,6 +523,53 @@
return jniSymbols
}
+func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext) {
+ // Collect NOTICE files from all dependencies.
+ seenModules := make(map[android.Module]bool)
+ noticePathSet := make(map[android.Path]bool)
+
+ ctx.WalkDeps(func(child android.Module, parent android.Module) bool {
+ // Have we already seen this?
+ if _, ok := seenModules[child]; ok {
+ return false
+ }
+ seenModules[child] = true
+
+ // Skip host modules.
+ if child.Target().Os.Class == android.Host {
+ return false
+ }
+
+ paths := child.(android.Module).NoticeFiles()
+ if len(paths) > 0 {
+ for _, path := range paths {
+ noticePathSet[path] = true
+ }
+ }
+ return true
+ })
+
+ // If the app has one, add it too.
+ if len(a.NoticeFiles()) > 0 {
+ for _, path := range a.NoticeFiles() {
+ noticePathSet[path] = true
+ }
+ }
+
+ if len(noticePathSet) == 0 {
+ return
+ }
+ var noticePaths []android.Path
+ for path := range noticePathSet {
+ noticePaths = append(noticePaths, path)
+ }
+ sort.Slice(noticePaths, func(i, j int) bool {
+ return noticePaths[i].String() < noticePaths[j].String()
+ })
+
+ a.noticeOutputs = android.BuildNoticeOutput(ctx, a.installDir, a.installApkName+".apk", noticePaths)
+}
+
// Reads and prepends a main cert from the default cert dir if it hasn't been set already, i.e. it
// isn't a cert module reference. Also checks and enforces system cert restriction if applicable.
func processMainCert(m android.ModuleBase, certPropValue string, certificates []Certificate, ctx android.ModuleContext) []Certificate {
@@ -586,10 +636,9 @@
}
a.onDeviceDir = android.InstallPathToOnDevicePath(ctx, a.installDir)
- noticeFile := android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz")
- android.BuildNoticeHtmlOutputFromLicenseMetadata(ctx, noticeFile)
+ a.noticeBuildActions(ctx)
if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") {
- a.aapt.noticeFile = android.OptionalPathForPath(noticeFile)
+ a.aapt.noticeFile = a.noticeOutputs.HtmlGzOutput
}
a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx)
diff --git a/java/app_test.go b/java/app_test.go
index 08baf54..16bbec1 100644
--- a/java/app_test.go
+++ b/java/app_test.go
@@ -27,6 +27,7 @@
"android/soong/android"
"android/soong/cc"
"android/soong/dexpreopt"
+ "android/soong/genrule"
)
// testApp runs tests using the prepareForJavaTest
@@ -2721,6 +2722,116 @@
}
}
+func TestEmbedNotice(t *testing.T) {
+ result := android.GroupFixturePreparers(
+ PrepareForTestWithJavaDefaultModules,
+ cc.PrepareForTestWithCcDefaultModules,
+ genrule.PrepareForTestWithGenRuleBuildComponents,
+ android.MockFS{
+ "APP_NOTICE": nil,
+ "GENRULE_NOTICE": nil,
+ "LIB_NOTICE": nil,
+ "TOOL_NOTICE": nil,
+ }.AddToFixture(),
+ ).RunTestWithBp(t, `
+ android_app {
+ name: "foo",
+ srcs: ["a.java"],
+ static_libs: ["javalib"],
+ jni_libs: ["libjni"],
+ notice: "APP_NOTICE",
+ embed_notices: true,
+ sdk_version: "current",
+ }
+
+ // No embed_notice flag
+ android_app {
+ name: "bar",
+ srcs: ["a.java"],
+ jni_libs: ["libjni"],
+ notice: "APP_NOTICE",
+ sdk_version: "current",
+ }
+
+ // No NOTICE files
+ android_app {
+ name: "baz",
+ srcs: ["a.java"],
+ embed_notices: true,
+ sdk_version: "current",
+ }
+
+ cc_library {
+ name: "libjni",
+ system_shared_libs: [],
+ stl: "none",
+ notice: "LIB_NOTICE",
+ sdk_version: "current",
+ }
+
+ java_library {
+ name: "javalib",
+ srcs: [
+ ":gen",
+ ],
+ sdk_version: "current",
+ }
+
+ genrule {
+ name: "gen",
+ tools: ["gentool"],
+ out: ["gen.java"],
+ notice: "GENRULE_NOTICE",
+ }
+
+ java_binary_host {
+ name: "gentool",
+ srcs: ["b.java"],
+ notice: "TOOL_NOTICE",
+ }
+ `)
+
+ // foo has NOTICE files to process, and embed_notices is true.
+ foo := result.ModuleForTests("foo", "android_common")
+ // verify merge notices rule.
+ mergeNotices := foo.Rule("mergeNoticesRule")
+ noticeInputs := mergeNotices.Inputs.Strings()
+ // TOOL_NOTICE should be excluded as it's a host module.
+ if len(mergeNotices.Inputs) != 3 {
+ t.Errorf("number of input notice files: expected = 3, actual = %q", noticeInputs)
+ }
+ if !inList("APP_NOTICE", noticeInputs) {
+ t.Errorf("APP_NOTICE is missing from notice files, %q", noticeInputs)
+ }
+ if !inList("LIB_NOTICE", noticeInputs) {
+ t.Errorf("LIB_NOTICE is missing from notice files, %q", noticeInputs)
+ }
+ if !inList("GENRULE_NOTICE", noticeInputs) {
+ t.Errorf("GENRULE_NOTICE is missing from notice files, %q", noticeInputs)
+ }
+ // aapt2 flags should include -A <NOTICE dir> so that its contents are put in the APK's /assets.
+ res := foo.Output("package-res.apk")
+ aapt2Flags := res.Args["flags"]
+ e := "-A out/soong/.intermediates/foo/android_common/NOTICE"
+ android.AssertStringDoesContain(t, "expected.apkPath", aapt2Flags, e)
+
+ // bar has NOTICE files to process, but embed_notices is not set.
+ bar := result.ModuleForTests("bar", "android_common")
+ res = bar.Output("package-res.apk")
+ aapt2Flags = res.Args["flags"]
+ e = "-A out/soong/.intermediates/bar/android_common/NOTICE"
+ android.AssertStringDoesNotContain(t, "bar shouldn't have the asset dir flag for NOTICE", aapt2Flags, e)
+
+ // baz's embed_notice is true, but it doesn't have any NOTICE files.
+ baz := result.ModuleForTests("baz", "android_common")
+ res = baz.Output("package-res.apk")
+ aapt2Flags = res.Args["flags"]
+ e = "-A out/soong/.intermediates/baz/android_common/NOTICE"
+ if strings.Contains(aapt2Flags, e) {
+ t.Errorf("baz shouldn't have the asset dir flag for NOTICE: %q", e)
+ }
+}
+
func TestUncompressDex(t *testing.T) {
testCases := []struct {
name string
diff --git a/scripts/generate-notice-files.py b/scripts/generate-notice-files.py
new file mode 100755
index 0000000..1b4acfa
--- /dev/null
+++ b/scripts/generate-notice-files.py
@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: generate-notice-files --text-output [plain text output file] \
+ --html-output [html output file] \
+ --xml-output [xml output file] \
+ -t [file title] -s [directory of notices]
+
+Generate the Android notice files, including both text and html files.
+
+-h to display this usage message and exit.
+"""
+from collections import defaultdict
+import argparse
+import hashlib
+import itertools
+import os
+import os.path
+import re
+import struct
+import sys
+
+MD5_BLOCKSIZE = 1024 * 1024
+HTML_ESCAPE_TABLE = {
+ b"&": b"&",
+ b'"': b""",
+ b"'": b"'",
+ b">": b">",
+ b"<": b"<",
+ }
+
+def md5sum(filename):
+ """Calculate an MD5 of the file given by FILENAME,
+ and return hex digest as a string.
+ Output should be compatible with md5sum command"""
+
+ f = open(filename, "rb")
+ sum = hashlib.md5()
+ while 1:
+ block = f.read(MD5_BLOCKSIZE)
+ if not block:
+ break
+ sum.update(block)
+ f.close()
+ return sum.hexdigest()
+
+
+def html_escape(text):
+ """Produce entities within text."""
+ # Using for i in text doesn't work since i will be an int, not a byte.
+ # There are multiple ways to solve this, but the most performant way
+ # to iterate over a byte array is to use unpack. Using the
+ # for i in range(len(text)) and using that to get a byte using array
+ # slices is twice as slow as this method.
+ return b"".join(HTML_ESCAPE_TABLE.get(i,i) for i in struct.unpack(str(len(text)) + 'c', text))
+
+HTML_OUTPUT_CSS=b"""
+<style type="text/css">
+body { padding: 0; font-family: sans-serif; }
+.same-license { background-color: #eeeeee; border-top: 20px solid white; padding: 10px; }
+.label { font-weight: bold; }
+.file-list { margin-left: 1em; color: blue; }
+</style>
+
+"""
+
+def combine_notice_files_html(file_hash, input_dir, output_filename):
+ """Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME."""
+
+ SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
+
+ # Set up a filename to row id table (anchors inside tables don't work in
+ # most browsers, but href's to table row ids do)
+ id_table = {}
+ id_count = 0
+ for value in file_hash:
+ for filename in value:
+ id_table[filename] = id_count
+ id_count += 1
+
+ # Open the output file, and output the header pieces
+ output_file = open(output_filename, "wb")
+
+ output_file.write(b"<html><head>\n")
+ output_file.write(HTML_OUTPUT_CSS)
+ output_file.write(b'</head><body topmargin="0" leftmargin="0" rightmargin="0" bottommargin="0">\n')
+
+ # Output our table of contents
+ output_file.write(b'<div class="toc">\n')
+ output_file.write(b"<ul>\n")
+
+ # Flatten the list of lists into a single list of filenames
+ sorted_filenames = sorted(itertools.chain.from_iterable(file_hash))
+
+ # Print out a nice table of contents
+ for filename in sorted_filenames:
+ stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename)
+ output_file.write(('<li><a href="#id%d">%s</a></li>\n' % (id_table.get(filename), stripped_filename)).encode())
+
+ output_file.write(b"</ul>\n")
+ output_file.write(b"</div><!-- table of contents -->\n")
+ # Output the individual notice file lists
+ output_file.write(b'<table cellpadding="0" cellspacing="0" border="0">\n')
+ for value in file_hash:
+ output_file.write(('<tr id="id%d"><td class="same-license">\n' % id_table.get(value[0])).encode())
+ output_file.write(b'<div class="label">Notices for file(s):</div>\n')
+ output_file.write(b'<div class="file-list">\n')
+ for filename in value:
+ output_file.write(("%s <br/>\n" % (SRC_DIR_STRIP_RE.sub(r"\1", filename))).encode())
+ output_file.write(b"</div><!-- file-list -->\n\n")
+ output_file.write(b'<pre class="license-text">\n')
+ with open(value[0], "rb") as notice_file:
+ output_file.write(html_escape(notice_file.read()))
+ output_file.write(b"\n</pre><!-- license-text -->\n")
+ output_file.write(b"</td></tr><!-- same-license -->\n\n\n\n")
+
+ # Finish off the file output
+ output_file.write(b"</table>\n")
+ output_file.write(b"</body></html>\n")
+ output_file.close()
+
+def combine_notice_files_text(file_hash, input_dir, output_filename, file_title):
+ """Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME."""
+
+ SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
+ output_file = open(output_filename, "wb")
+ output_file.write(file_title.encode())
+ output_file.write(b"\n")
+ for value in file_hash:
+ output_file.write(b"============================================================\n")
+ output_file.write(b"Notices for file(s):\n")
+ for filename in value:
+ output_file.write(SRC_DIR_STRIP_RE.sub(r"\1", filename).encode())
+ output_file.write(b"\n")
+ output_file.write(b"------------------------------------------------------------\n")
+ with open(value[0], "rb") as notice_file:
+ output_file.write(notice_file.read())
+ output_file.write(b"\n")
+ output_file.close()
+
+def combine_notice_files_xml(files_with_same_hash, input_dir, output_filename):
+ """Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME."""
+
+ SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt")
+
+ # Set up a filename to row id table (anchors inside tables don't work in
+ # most browsers, but href's to table row ids do)
+ id_table = {}
+ for file_key, files in files_with_same_hash.items():
+ for filename in files:
+ id_table[filename] = file_key
+
+ # Open the output file, and output the header pieces
+ output_file = open(output_filename, "wb")
+
+ output_file.write(b'<?xml version="1.0" encoding="utf-8"?>\n')
+ output_file.write(b"<licenses>\n")
+
+ # Flatten the list of lists into a single list of filenames
+ sorted_filenames = sorted(list(id_table))
+
+ # Print out a nice table of contents
+ for filename in sorted_filenames:
+ stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename)
+ output_file.write(('<file-name contentId="%s">%s</file-name>\n' % (id_table.get(filename), stripped_filename)).encode())
+ output_file.write(b"\n\n")
+
+ processed_file_keys = []
+ # Output the individual notice file lists
+ for filename in sorted_filenames:
+ file_key = id_table.get(filename)
+ if file_key in processed_file_keys:
+ continue
+ processed_file_keys.append(file_key)
+
+ output_file.write(('<file-content contentId="%s"><![CDATA[' % file_key).encode())
+ with open(filename, "rb") as notice_file:
+ output_file.write(html_escape(notice_file.read()))
+ output_file.write(b"]]></file-content>\n\n")
+
+ # Finish off the file output
+ output_file.write(b"</licenses>\n")
+ output_file.close()
+
+def get_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--text-output', required=True,
+ help='The text output file path.')
+ parser.add_argument(
+ '--html-output',
+ help='The html output file path.')
+ parser.add_argument(
+ '--xml-output',
+ help='The xml output file path.')
+ parser.add_argument(
+ '-t', '--title', required=True,
+ help='The file title.')
+ parser.add_argument(
+ '-s', '--source-dir', required=True,
+ help='The directory containing notices.')
+ parser.add_argument(
+ '-i', '--included-subdirs', action='append',
+ help='The sub directories which should be included.')
+ parser.add_argument(
+ '-e', '--excluded-subdirs', action='append',
+ help='The sub directories which should be excluded.')
+ return parser.parse_args()
+
+def main(argv):
+ args = get_args()
+
+ txt_output_file = args.text_output
+ html_output_file = args.html_output
+ xml_output_file = args.xml_output
+ file_title = args.title
+ included_subdirs = []
+ excluded_subdirs = []
+ if args.included_subdirs is not None:
+ included_subdirs = args.included_subdirs
+ if args.excluded_subdirs is not None:
+ excluded_subdirs = args.excluded_subdirs
+
+ # Find all the notice files and md5 them
+ input_dir = os.path.normpath(args.source_dir)
+ files_with_same_hash = defaultdict(list)
+ for root, dir, files in os.walk(input_dir):
+ for file in files:
+ matched = True
+ if len(included_subdirs) > 0:
+ matched = False
+ for subdir in included_subdirs:
+ if (root == (input_dir + '/' + subdir) or
+ root.startswith(input_dir + '/' + subdir + '/')):
+ matched = True
+ break
+ elif len(excluded_subdirs) > 0:
+ for subdir in excluded_subdirs:
+ if (root == (input_dir + '/' + subdir) or
+ root.startswith(input_dir + '/' + subdir + '/')):
+ matched = False
+ break
+ if matched and file.endswith(".txt"):
+ filename = os.path.join(root, file)
+ file_md5sum = md5sum(filename)
+ files_with_same_hash[file_md5sum].append(filename)
+
+ filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(list(files_with_same_hash))]
+
+ combine_notice_files_text(filesets, input_dir, txt_output_file, file_title)
+
+ if html_output_file is not None:
+ combine_notice_files_html(filesets, input_dir, html_output_file)
+
+ if xml_output_file is not None:
+ combine_notice_files_xml(files_with_same_hash, input_dir, xml_output_file)
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/scripts/mergenotice.py b/scripts/mergenotice.py
new file mode 100755
index 0000000..fe99073
--- /dev/null
+++ b/scripts/mergenotice.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Merges input notice files to the output file while ignoring duplicated files
+This script shouldn't be confused with build/soong/scripts/generate-notice-files.py
+which is responsible for creating the final notice file for all artifacts
+installed. This script has rather limited scope; it is meant to create a merged
+notice file for a set of modules that are packaged together, e.g. in an APEX.
+The merged notice file does not reveal the individual files in the package.
+"""
+
+import sys
+import argparse
+
+def get_args():
+ parser = argparse.ArgumentParser(description='Merge notice files.')
+ parser.add_argument('--output', help='output file path.')
+ parser.add_argument('inputs', metavar='INPUT', nargs='+',
+ help='input notice file')
+ return parser.parse_args()
+
+def main(argv):
+ args = get_args()
+
+ processed = set()
+ with open(args.output, 'w+') as output:
+ for input in args.inputs:
+ with open(input, 'r') as f:
+ data = f.read().strip()
+ if data not in processed:
+ processed.add(data)
+ output.write('%s\n\n' % data)
+
+if __name__ == '__main__':
+ main(sys.argv)