Change storage behavior of multiproduct_kati
Instead of deleting artifacts/logs from successful build (unless -keep
is set), and keeping unsuccessful artifacts, keep all logs and remove
all artifacts (unless -keep is set, then we'll compress the artifacts).
If -dist is passed in, we'll put an archive of the logs into the
DIST_DIR. Even compressed, the rest of the artifacts are still a bit too
large to dist (~5.6GB on AOSP).
Test: build/soong/build_test.bash
Test: build/soong/build_test.bash -keep
Test: build/soong/build_test.bash -dist
Change-Id: I87f55978c18c8ff2e517b8271554ba383003742f
diff --git a/cmd/multiproduct_kati/Android.bp b/cmd/multiproduct_kati/Android.bp
index b264c35..04a5802 100644
--- a/cmd/multiproduct_kati/Android.bp
+++ b/cmd/multiproduct_kati/Android.bp
@@ -18,6 +18,7 @@
"soong-ui-build",
"soong-ui-logger",
"soong-ui-tracer",
+ "soong-zip",
],
srcs: [
"main.go",
diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go
index 1c853d6..e4a05fc 100644
--- a/cmd/multiproduct_kati/main.go
+++ b/cmd/multiproduct_kati/main.go
@@ -30,6 +30,7 @@
"android/soong/ui/build"
"android/soong/ui/logger"
"android/soong/ui/tracer"
+ "android/soong/zip"
)
// We default to number of cpus / 4, which seems to be the sweet spot for my
@@ -45,7 +46,7 @@
var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs")
-var keep = flag.Bool("keep", false, "keep successful output files")
+var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts")
var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)")
var alternateResultDir = flag.Bool("dist", false, "write select results to $DIST_DIR (or <out>/dist when empty)")
@@ -200,24 +201,19 @@
if err := os.MkdirAll(*outDir, 0777); err != nil {
log.Fatalf("Failed to create tempdir: %v", err)
}
-
- if !*keep {
- defer func() {
- if status.Finished() == 0 {
- os.RemoveAll(*outDir)
- }
- }()
- }
}
config.Environment().Set("OUT_DIR", *outDir)
log.Println("Output directory:", *outDir)
+ logsDir := filepath.Join(config.OutDir(), "logs")
+ os.MkdirAll(logsDir, 0777)
+
build.SetupOutDir(buildCtx, config)
if *alternateResultDir {
- logsDir := filepath.Join(config.DistDir(), "logs")
- os.MkdirAll(logsDir, 0777)
- log.SetOutput(filepath.Join(logsDir, "soong.log"))
- trace.SetOutput(filepath.Join(logsDir, "build.trace"))
+ distLogsDir := filepath.Join(config.DistDir(), "logs")
+ os.MkdirAll(distLogsDir, 0777)
+ log.SetOutput(filepath.Join(distLogsDir, "soong.log"))
+ trace.SetOutput(filepath.Join(distLogsDir, "build.trace"))
} else {
log.SetOutput(filepath.Join(config.OutDir(), "soong.log"))
trace.SetOutput(filepath.Join(config.OutDir(), "build.trace"))
@@ -269,17 +265,14 @@
})
productOutDir := filepath.Join(config.OutDir(), product)
- productLogDir := productOutDir
- if *alternateResultDir {
- productLogDir = filepath.Join(config.DistDir(), product)
- if err := os.MkdirAll(productLogDir, 0777); err != nil {
- log.Fatalf("Error creating log directory: %v", err)
- }
- }
+ productLogDir := filepath.Join(logsDir, product)
if err := os.MkdirAll(productOutDir, 0777); err != nil {
log.Fatalf("Error creating out directory: %v", err)
}
+ if err := os.MkdirAll(productLogDir, 0777); err != nil {
+ log.Fatalf("Error creating log directory: %v", err)
+ }
stdLog = filepath.Join(productLogDir, "std.log")
f, err := os.Create(stdLog)
@@ -324,6 +317,26 @@
status.Fail(product.config.TargetProduct(), err, product.logFile)
})
+ defer func() {
+ if *keepArtifacts {
+ args := zip.ZipArgs{
+ FileArgs: []zip.FileArg{
+ {
+ GlobDir: product.config.OutDir(),
+ SourcePrefixToStrip: product.config.OutDir(),
+ },
+ },
+ OutputFilePath: filepath.Join(config.OutDir(), product.config.TargetProduct()+".zip"),
+ NumParallelJobs: runtime.NumCPU(),
+ CompressionLevel: 5,
+ }
+ if err := zip.Run(args); err != nil {
+ log.Fatalf("Error zipping artifacts: %v", err)
+ }
+ }
+ os.RemoveAll(product.config.OutDir())
+ }()
+
buildWhat := 0
if !*onlyConfig {
buildWhat |= build.BuildSoong
@@ -332,9 +345,6 @@
}
}
build.Build(product.ctx, product.config, buildWhat)
- if !*keep {
- os.RemoveAll(product.config.OutDir())
- }
status.Finish(product.config.TargetProduct())
}()
}
@@ -342,6 +352,20 @@
}
wg2.Wait()
+ if *alternateResultDir {
+ args := zip.ZipArgs{
+ FileArgs: []zip.FileArg{
+ {GlobDir: logsDir, SourcePrefixToStrip: logsDir},
+ },
+ OutputFilePath: filepath.Join(config.DistDir(), "logs.zip"),
+ NumParallelJobs: runtime.NumCPU(),
+ CompressionLevel: 5,
+ }
+ if err := zip.Run(args); err != nil {
+ log.Fatalf("Error zipping logs: %v", err)
+ }
+ }
+
if count := status.Finished(); count > 0 {
log.Fatalln(count, "products failed")
}