Refactor java compileDex

We want to support a compile_dex property for java_import. This splits
dex-related properties into a dexer struct which can be embedded in
relevant modules.

Test: m
Test: soong tests
Bug: 160455085
Change-Id: If56a51dac43f630d49483a36db29cd50e9ccd529
diff --git a/java/androidmk.go b/java/androidmk.go
index 25dd329..a140b7d 100644
--- a/java/androidmk.go
+++ b/java/androidmk.go
@@ -127,8 +127,8 @@
 						entries.AddStrings("LOCAL_ADDITIONAL_CHECKED_MODULE", library.additionalCheckedModules.Strings()...)
 					}
 
-					if library.proguardDictionary != nil {
-						entries.SetPath("LOCAL_SOONG_PROGUARD_DICT", library.proguardDictionary)
+					if library.dexer.proguardDictionary.Valid() {
+						entries.SetPath("LOCAL_SOONG_PROGUARD_DICT", library.dexer.proguardDictionary.Path())
 					}
 					entries.SetString("LOCAL_MODULE_STEM", library.Stem())
 
@@ -332,8 +332,8 @@
 				if app.jacocoReportClassesFile != nil {
 					entries.SetPath("LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR", app.jacocoReportClassesFile)
 				}
-				if app.proguardDictionary != nil {
-					entries.SetPath("LOCAL_SOONG_PROGUARD_DICT", app.proguardDictionary)
+				if app.dexer.proguardDictionary.Valid() {
+					entries.SetPath("LOCAL_SOONG_PROGUARD_DICT", app.dexer.proguardDictionary.Path())
 				}
 
 				if app.Name() == "framework-res" {
diff --git a/java/app.go b/java/app.go
index 4031cfe..f77f476 100755
--- a/java/app.go
+++ b/java/app.go
@@ -588,11 +588,11 @@
 
 func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path {
 	a.dexpreopter.installPath = a.installPath(ctx)
-	if a.deviceProperties.Uncompress_dex == nil {
+	if a.dexProperties.Uncompress_dex == nil {
 		// If the value was not force-set by the user, use reasonable default based on the module.
-		a.deviceProperties.Uncompress_dex = proptools.BoolPtr(a.shouldUncompressDex(ctx))
+		a.dexProperties.Uncompress_dex = proptools.BoolPtr(a.shouldUncompressDex(ctx))
 	}
-	a.dexpreopter.uncompressedDex = *a.deviceProperties.Uncompress_dex
+	a.dexpreopter.uncompressedDex = *a.dexProperties.Uncompress_dex
 	a.dexpreopter.enforceUsesLibs = a.usesLibrary.enforceUsesLibraries()
 	a.dexpreopter.usesLibs = a.usesLibrary.usesLibraryProperties.Uses_libs
 	a.dexpreopter.optionalUsesLibs = a.usesLibrary.presentOptionalUsesLibs(ctx)
@@ -995,8 +995,8 @@
 func AndroidAppFactory() android.Module {
 	module := &AndroidApp{}
 
-	module.Module.deviceProperties.Optimize.EnabledByDefault = true
-	module.Module.deviceProperties.Optimize.Shrink = proptools.BoolPtr(true)
+	module.Module.dexProperties.Optimize.EnabledByDefault = true
+	module.Module.dexProperties.Optimize.Shrink = proptools.BoolPtr(true)
 
 	module.Module.properties.Instrument = true
 	module.Module.properties.Installable = proptools.BoolPtr(true)
@@ -1110,7 +1110,7 @@
 func AndroidTestFactory() android.Module {
 	module := &AndroidTest{}
 
-	module.Module.deviceProperties.Optimize.EnabledByDefault = true
+	module.Module.dexProperties.Optimize.EnabledByDefault = true
 
 	module.Module.properties.Instrument = true
 	module.Module.properties.Installable = proptools.BoolPtr(true)
@@ -1161,7 +1161,7 @@
 func AndroidTestHelperAppFactory() android.Module {
 	module := &AndroidTestHelperApp{}
 
-	module.Module.deviceProperties.Optimize.EnabledByDefault = true
+	module.Module.dexProperties.Optimize.EnabledByDefault = true
 
 	module.Module.properties.Installable = proptools.BoolPtr(true)
 	module.appProperties.Use_embedded_native_libs = proptools.BoolPtr(true)
diff --git a/java/dex.go b/java/dex.go
index 9e61e95..cd45a93 100644
--- a/java/dex.go
+++ b/java/dex.go
@@ -24,6 +24,60 @@
 	"android/soong/remoteexec"
 )
 
+type DexProperties struct {
+	// If set to true, compile dex regardless of installable.  Defaults to false.
+	Compile_dex *bool
+
+	// list of module-specific flags that will be used for dex compiles
+	Dxflags []string `android:"arch_variant"`
+
+	Optimize struct {
+		// If false, disable all optimization.  Defaults to true for android_app and android_test
+		// modules, false for java_library and java_test modules.
+		Enabled *bool
+		// True if the module containing this has it set by default.
+		EnabledByDefault bool `blueprint:"mutated"`
+
+		// If true, optimize for size by removing unused code.  Defaults to true for apps,
+		// false for libraries and tests.
+		Shrink *bool
+
+		// If true, optimize bytecode.  Defaults to false.
+		Optimize *bool
+
+		// If true, obfuscate bytecode.  Defaults to false.
+		Obfuscate *bool
+
+		// If true, do not use the flag files generated by aapt that automatically keep
+		// classes referenced by the app manifest.  Defaults to false.
+		No_aapt_flags *bool
+
+		// Flags to pass to proguard.
+		Proguard_flags []string
+
+		// Specifies the locations of files containing proguard flags.
+		Proguard_flags_files []string `android:"path"`
+	}
+
+	// Keep the data uncompressed. We always need uncompressed dex for execution,
+	// so this might actually save space by avoiding storing the same data twice.
+	// This defaults to reasonable value based on module and should not be set.
+	// It exists only to support ART tests.
+	Uncompress_dex *bool
+}
+
+type dexer struct {
+	dexProperties DexProperties
+
+	// list of extra proguard flag files
+	extraProguardFlagFiles android.Paths
+	proguardDictionary     android.OptionalPath
+}
+
+func (d *dexer) effectiveOptimizeEnabled() bool {
+	return BoolDefault(d.dexProperties.Optimize.Enabled, d.dexProperties.Optimize.EnabledByDefault)
+}
+
 var d8, d8RE = remoteexec.MultiCommandStaticRules(pctx, "d8",
 	blueprint.RuleParams{
 		Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
@@ -86,8 +140,8 @@
 		},
 	}, []string{"outDir", "outDict", "r8Flags", "zipFlags"}, []string{"implicits"})
 
-func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string {
-	flags := j.deviceProperties.Dxflags
+func (d *dexer) dexCommonFlags(ctx android.ModuleContext, minSdkVersion sdkSpec) []string {
+	flags := d.dexProperties.Dxflags
 	// Translate all the DX flags to D8 ones until all the build files have been migrated
 	// to D8 flags. See: b/69377755
 	flags = android.RemoveListFromList(flags,
@@ -103,30 +157,27 @@
 			"--verbose")
 	}
 
-	minSdkVersion, err := j.minSdkVersion().effectiveVersion(ctx)
+	effectiveVersion, err := minSdkVersion.effectiveVersion(ctx)
 	if err != nil {
 		ctx.PropertyErrorf("min_sdk_version", "%s", err)
 	}
 
-	flags = append(flags, "--min-api "+minSdkVersion.asNumberString())
+	flags = append(flags, "--min-api "+effectiveVersion.asNumberString())
 	return flags
 }
 
-func (j *Module) d8Flags(ctx android.ModuleContext, flags javaBuilderFlags) ([]string, android.Paths) {
-	d8Flags := j.dexCommonFlags(ctx)
-
+func d8Flags(flags javaBuilderFlags) (d8Flags []string, d8Deps android.Paths) {
 	d8Flags = append(d8Flags, flags.bootClasspath.FormRepeatedClassPath("--lib ")...)
 	d8Flags = append(d8Flags, flags.classpath.FormRepeatedClassPath("--lib ")...)
 
-	var d8Deps android.Paths
 	d8Deps = append(d8Deps, flags.bootClasspath...)
 	d8Deps = append(d8Deps, flags.classpath...)
 
 	return d8Flags, d8Deps
 }
 
-func (j *Module) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8Flags []string, r8Deps android.Paths) {
-	opt := j.deviceProperties.Optimize
+func (d *dexer) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8Flags []string, r8Deps android.Paths) {
+	opt := d.dexProperties.Optimize
 
 	// When an app contains references to APIs that are not in the SDK specified by
 	// its LOCAL_SDK_VERSION for example added by support library or by runtime
@@ -140,8 +191,6 @@
 		proguardRaiseDeps = append(proguardRaiseDeps, dep.(Dependency).HeaderJars()...)
 	})
 
-	r8Flags = append(r8Flags, j.dexCommonFlags(ctx)...)
-
 	r8Flags = append(r8Flags, proguardRaiseDeps.FormJavaClassPath("-libraryjars"))
 	r8Flags = append(r8Flags, flags.bootClasspath.FormJavaClassPath("-libraryjars"))
 	r8Flags = append(r8Flags, flags.classpath.FormJavaClassPath("-libraryjars"))
@@ -154,15 +203,10 @@
 		android.PathForSource(ctx, "build/make/core/proguard.flags"),
 	}
 
-	if j.shouldInstrumentStatic(ctx) {
-		flagFiles = append(flagFiles,
-			android.PathForSource(ctx, "build/make/core/proguard.jacoco.flags"))
-	}
-
-	flagFiles = append(flagFiles, j.extraProguardFlagFiles...)
+	flagFiles = append(flagFiles, d.extraProguardFlagFiles...)
 	// TODO(ccross): static android library proguard files
 
-	flagFiles = append(flagFiles, android.PathsForModuleSrc(ctx, j.deviceProperties.Optimize.Proguard_flags_files)...)
+	flagFiles = append(flagFiles, android.PathsForModuleSrc(ctx, opt.Proguard_flags_files)...)
 
 	r8Flags = append(r8Flags, android.JoinWithPrefix(flagFiles.Strings(), "-include "))
 	r8Deps = append(r8Deps, flagFiles...)
@@ -171,7 +215,7 @@
 	r8Deps = append(r8Deps, android.PathForSource(ctx,
 		"build/make/core/proguard_basic_keeps.flags"))
 
-	r8Flags = append(r8Flags, j.deviceProperties.Optimize.Proguard_flags...)
+	r8Flags = append(r8Flags, opt.Proguard_flags...)
 
 	// TODO(ccross): Don't shrink app instrumentation tests by default.
 	if !Bool(opt.Shrink) {
@@ -197,29 +241,30 @@
 	return r8Flags, r8Deps
 }
 
-func (j *Module) compileDex(ctx android.ModuleContext, flags javaBuilderFlags,
+func (d *dexer) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, minSdkVersion sdkSpec,
 	classesJar android.Path, jarName string) android.ModuleOutPath {
 
-	useR8 := j.deviceProperties.EffectiveOptimizeEnabled()
-
 	// Compile classes.jar into classes.dex and then javalib.jar
 	javalibJar := android.PathForModuleOut(ctx, "dex", jarName)
 	outDir := android.PathForModuleOut(ctx, "dex")
 
 	zipFlags := "--ignore_missing_files"
-	if proptools.Bool(j.deviceProperties.Uncompress_dex) {
+	if proptools.Bool(d.dexProperties.Uncompress_dex) {
 		zipFlags += " -L 0"
 	}
 
+	commonFlags := d.dexCommonFlags(ctx, minSdkVersion)
+
+	useR8 := d.effectiveOptimizeEnabled()
 	if useR8 {
 		proguardDictionary := android.PathForModuleOut(ctx, "proguard_dictionary")
-		j.proguardDictionary = proguardDictionary
-		r8Flags, r8Deps := j.r8Flags(ctx, flags)
+		d.proguardDictionary = android.OptionalPathForPath(proguardDictionary)
+		r8Flags, r8Deps := d.r8Flags(ctx, flags)
 		rule := r8
 		args := map[string]string{
-			"r8Flags":  strings.Join(r8Flags, " "),
+			"r8Flags":  strings.Join(append(commonFlags, r8Flags...), " "),
 			"zipFlags": zipFlags,
-			"outDict":  j.proguardDictionary.String(),
+			"outDict":  proguardDictionary.String(),
 			"outDir":   outDir.String(),
 		}
 		if ctx.Config().IsEnvTrue("RBE_R8") {
@@ -236,7 +281,7 @@
 			Args:           args,
 		})
 	} else {
-		d8Flags, d8Deps := j.d8Flags(ctx, flags)
+		d8Flags, d8Deps := d8Flags(flags)
 		rule := d8
 		if ctx.Config().IsEnvTrue("RBE_D8") {
 			rule = d8RE
@@ -248,13 +293,13 @@
 			Input:       classesJar,
 			Implicits:   d8Deps,
 			Args: map[string]string{
-				"d8Flags":  strings.Join(d8Flags, " "),
+				"d8Flags":  strings.Join(append(commonFlags, d8Flags...), " "),
 				"zipFlags": zipFlags,
 				"outDir":   outDir.String(),
 			},
 		})
 	}
-	if proptools.Bool(j.deviceProperties.Uncompress_dex) {
+	if proptools.Bool(d.dexProperties.Uncompress_dex) {
 		alignedJavalibJar := android.PathForModuleOut(ctx, "aligned", jarName)
 		TransformZipAlign(ctx, alignedJavalibJar, javalibJar)
 		javalibJar = alignedJavalibJar
diff --git a/java/java.go b/java/java.go
index bd476bc..d5375a5 100644
--- a/java/java.go
+++ b/java/java.go
@@ -264,9 +264,6 @@
 }
 
 type CompilerDeviceProperties struct {
-	// list of module-specific flags that will be used for dex compiles
-	Dxflags []string `android:"arch_variant"`
-
 	// if not blank, set to the version of the sdk to compile against.
 	// Defaults to compiling against the current platform.
 	Sdk_version *string
@@ -312,37 +309,6 @@
 		}
 	}
 
-	// If set to true, compile dex regardless of installable.  Defaults to false.
-	Compile_dex *bool
-
-	Optimize struct {
-		// If false, disable all optimization.  Defaults to true for android_app and android_test
-		// modules, false for java_library and java_test modules.
-		Enabled *bool
-		// True if the module containing this has it set by default.
-		EnabledByDefault bool `blueprint:"mutated"`
-
-		// If true, optimize for size by removing unused code.  Defaults to true for apps,
-		// false for libraries and tests.
-		Shrink *bool
-
-		// If true, optimize bytecode.  Defaults to false.
-		Optimize *bool
-
-		// If true, obfuscate bytecode.  Defaults to false.
-		Obfuscate *bool
-
-		// If true, do not use the flag files generated by aapt that automatically keep
-		// classes referenced by the app manifest.  Defaults to false.
-		No_aapt_flags *bool
-
-		// Flags to pass to proguard.
-		Proguard_flags []string
-
-		// Specifies the locations of files containing proguard flags.
-		Proguard_flags_files []string `android:"path"`
-	}
-
 	// When targeting 1.9 and above, override the modules to use with --system,
 	// otherwise provides defaults libraries to add to the bootclasspath.
 	System_modules *string
@@ -356,19 +322,9 @@
 	// set the name of the output
 	Stem *string
 
-	// Keep the data uncompressed. We always need uncompressed dex for execution,
-	// so this might actually save space by avoiding storing the same data twice.
-	// This defaults to reasonable value based on module and should not be set.
-	// It exists only to support ART tests.
-	Uncompress_dex *bool
-
 	IsSDKLibrary bool `blueprint:"mutated"`
 }
 
-func (me *CompilerDeviceProperties) EffectiveOptimizeEnabled() bool {
-	return BoolDefault(me.Optimize.Enabled, me.Optimize.EnabledByDefault)
-}
-
 // Functionality common to Module and Import
 //
 // It is embedded in Module so its functionality can be used by methods in Module
@@ -437,9 +393,6 @@
 	// output file containing uninstrumented classes that will be instrumented by jacoco
 	jacocoReportClassesFile android.Path
 
-	// output file containing mapping of obfuscated names
-	proguardDictionary android.Path
-
 	// output file of the module, which may be a classes jar or a dex jar
 	outputFile       android.Path
 	extraOutputFiles android.Paths
@@ -455,9 +408,6 @@
 	compiledJavaSrcs android.Paths
 	compiledSrcJars  android.Paths
 
-	// list of extra progurad flag files
-	extraProguardFlagFiles android.Paths
-
 	// manifest file to use instead of properties.Manifest
 	overrideManifest android.OptionalPath
 
@@ -484,6 +434,7 @@
 	extraResources android.Paths
 
 	hiddenAPI
+	dexer
 	dexpreopter
 	linter
 
@@ -507,6 +458,7 @@
 	j.addHostProperties()
 	j.AddProperties(
 		&j.deviceProperties,
+		&j.dexer.dexProperties,
 		&j.dexpreoptProperties,
 		&j.linter.properties,
 	)
@@ -519,7 +471,10 @@
 	case ".jar":
 		return android.Paths{j.implementationAndResourcesJar}, nil
 	case ".proguard_map":
-		return android.Paths{j.proguardDictionary}, nil
+		if j.dexer.proguardDictionary.Valid() {
+			return android.Paths{j.dexer.proguardDictionary.Path()}, nil
+		}
+		return nil, fmt.Errorf("%q was requested, but no output file was found.", tag)
 	default:
 		return nil, fmt.Errorf("unsupported module reference tag %q", tag)
 	}
@@ -728,10 +683,10 @@
 			ctx.AddVariationDependencies(nil, bootClasspathTag, sdkDep.bootclasspath...)
 			ctx.AddVariationDependencies(nil, java9LibTag, sdkDep.java9Classpath...)
 			ctx.AddVariationDependencies(nil, libTag, sdkDep.classpath...)
-			if j.deviceProperties.EffectiveOptimizeEnabled() && sdkDep.hasStandardLibs() {
+			if j.effectiveOptimizeEnabled() && sdkDep.hasStandardLibs() {
 				ctx.AddVariationDependencies(nil, proguardRaiseTag, config.LegacyCorePlatformBootclasspathLibraries...)
 			}
-			if j.deviceProperties.EffectiveOptimizeEnabled() && sdkDep.hasFrameworkLibs() {
+			if j.effectiveOptimizeEnabled() && sdkDep.hasFrameworkLibs() {
 				ctx.AddVariationDependencies(nil, proguardRaiseTag, config.FrameworkLibraries...)
 			}
 		}
@@ -1647,8 +1602,8 @@
 
 	// Enable dex compilation for the APEX variants, unless it is disabled explicitly
 	if android.DirectlyInAnyApex(ctx, ctx.ModuleName()) && !j.IsForPlatform() {
-		if j.deviceProperties.Compile_dex == nil {
-			j.deviceProperties.Compile_dex = proptools.BoolPtr(true)
+		if j.dexProperties.Compile_dex == nil {
+			j.dexProperties.Compile_dex = proptools.BoolPtr(true)
 		}
 		if j.deviceProperties.Hostdex == nil {
 			j.deviceProperties.Hostdex = proptools.BoolPtr(true)
@@ -1656,10 +1611,14 @@
 	}
 
 	if ctx.Device() && j.hasCode(ctx) &&
-		(Bool(j.properties.Installable) || Bool(j.deviceProperties.Compile_dex)) {
+		(Bool(j.properties.Installable) || Bool(j.dexProperties.Compile_dex)) {
+		if j.shouldInstrumentStatic(ctx) {
+			j.dexer.extraProguardFlagFiles = append(j.dexer.extraProguardFlagFiles,
+				android.PathForSource(ctx, "build/make/core/proguard.jacoco.flags"))
+		}
 		// Dex compilation
 		var dexOutputFile android.ModuleOutPath
-		dexOutputFile = j.compileDex(ctx, flags, outputFile, jarName)
+		dexOutputFile = j.dexer.compileDex(ctx, flags, j.minSdkVersion(), outputFile, jarName)
 		if ctx.Failed() {
 			return
 		}
@@ -1669,7 +1628,7 @@
 
 		// Hidden API CSV generation and dex encoding
 		dexOutputFile = j.hiddenAPI.hiddenAPI(ctx, configurationName, primary, dexOutputFile, j.implementationJarFile,
-			proptools.Bool(j.deviceProperties.Uncompress_dex))
+			proptools.Bool(j.dexProperties.Uncompress_dex))
 
 		// merge dex jar with resources if necessary
 		if j.resourceJar != nil {
@@ -1677,7 +1636,7 @@
 			combinedJar := android.PathForModuleOut(ctx, "dex-withres", jarName)
 			TransformJarsToJar(ctx, combinedJar, "for dex resources", jars, android.OptionalPath{},
 				false, nil, nil)
-			if *j.deviceProperties.Uncompress_dex {
+			if *j.dexProperties.Uncompress_dex {
 				combinedAlignedJar := android.PathForModuleOut(ctx, "dex-withres-aligned", jarName)
 				TransformZipAlign(ctx, combinedAlignedJar, combinedJar)
 				dexOutputFile = combinedAlignedJar
@@ -2008,11 +1967,11 @@
 	j.checkSdkVersions(ctx)
 	j.dexpreopter.installPath = android.PathForModuleInstall(ctx, "framework", j.Stem()+".jar")
 	j.dexpreopter.isSDKLibrary = j.deviceProperties.IsSDKLibrary
-	if j.deviceProperties.Uncompress_dex == nil {
+	if j.dexProperties.Uncompress_dex == nil {
 		// If the value was not force-set by the user, use reasonable default based on the module.
-		j.deviceProperties.Uncompress_dex = proptools.BoolPtr(shouldUncompressDex(ctx, &j.dexpreopter))
+		j.dexProperties.Uncompress_dex = proptools.BoolPtr(shouldUncompressDex(ctx, &j.dexpreopter))
 	}
-	j.dexpreopter.uncompressedDex = *j.deviceProperties.Uncompress_dex
+	j.dexpreopter.uncompressedDex = *j.dexProperties.Uncompress_dex
 	j.compile(ctx, nil)
 
 	// Collect the module directory for IDE info in java/jdeps.go.
@@ -2970,6 +2929,7 @@
 	module.AddProperties(
 		&CompilerProperties{},
 		&CompilerDeviceProperties{},
+		&DexProperties{},
 		&DexpreoptProperties{},
 		&android.ProtoProperties{},
 		&aaptProperties{},
diff --git a/java/sdk_library.go b/java/sdk_library.go
index 8a8d0c9..0379a31 100644
--- a/java/sdk_library.go
+++ b/java/sdk_library.go
@@ -1112,6 +1112,7 @@
 		&module.properties,
 		&module.protoProperties,
 		&module.deviceProperties,
+		&module.dexProperties,
 		&module.dexpreoptProperties,
 		&module.linter.properties,
 		&props,
@@ -1171,8 +1172,8 @@
 	// We compile the stubs for 1.8 in line with the main android.jar stubs, and potential
 	// interop with older developer tools that don't support 1.9.
 	props.Java_version = proptools.StringPtr("1.8")
-	if module.deviceProperties.Compile_dex != nil {
-		props.Compile_dex = module.deviceProperties.Compile_dex
+	if module.dexProperties.Compile_dex != nil {
+		props.Compile_dex = module.dexProperties.Compile_dex
 	}
 
 	// Dist the class jar artifact for sdk builds.