Merge "Add upload-only mode and manual build-started-time flag."
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index 713ccbe..8c99988 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -58,7 +58,7 @@
stdio func() terminal.StdioInterface
// run the command
- run func(ctx build.Context, config build.Config, args []string, logsDir string)
+ run func(ctx build.Context, config build.Config, args []string)
}
// list of supported commands (flags) supported by soong ui
@@ -91,6 +91,15 @@
config: buildActionConfig,
stdio: stdio,
run: runMake,
+ }, {
+ flag: "--upload-metrics-only",
+ description: "upload metrics without building anything",
+ config: uploadOnlyConfig,
+ stdio: stdio,
+ // Upload-only mode mostly skips to the metrics-uploading phase of soong_ui.
+ // However, this invocation marks the true "end of the build", and thus we
+ // need to update the total runtime of the build to include this upload step.
+ run: updateTotalRealTime,
},
}
@@ -182,18 +191,49 @@
}}
config := c.config(buildCtx, args...)
+ config.SetLogsPrefix(c.logsPrefix)
+ logsDir := config.LogsDir()
+ buildStarted = config.BuildStartedTimeOrDefault(buildStarted)
+ buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
+ soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
+ rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
+ bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
+ metricsFiles := []string{
+ buildErrorFile, // build error strings
+ rbeMetricsFile, // high level metrics related to remote build execution.
+ bp2buildMetricsFile, // high level metrics related to bp2build.
+ soongMetricsFile, // high level metrics related to this build system.
+ config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
+ }
+
+ os.MkdirAll(logsDir, 0777)
+
+ log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
+
+ trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
+
+ c.run(buildCtx, config, args)
+
+ defer met.Dump(soongMetricsFile)
+ if !config.SkipMetricsUpload() {
+ defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, metricsFiles...)
+ }
+
+}
+
+func logAndSymlinkSetup(buildCtx build.Context, config build.Config) {
+ log := buildCtx.ContextImpl.Logger
+ logsPrefix := config.GetLogsPrefix()
build.SetupOutDir(buildCtx, config)
-
- // Set up files to be outputted in the log directory.
logsDir := config.LogsDir()
// Common list of metric file definition.
- buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
- rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
- soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
- bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
- soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb")
+ buildErrorFile := filepath.Join(logsDir, logsPrefix+"build_error")
+ rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb")
+ soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics")
+ bp2buildMetricsFile := filepath.Join(logsDir, logsPrefix+"bp2build_metrics.pb")
+ soongBuildMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_build_metrics.pb")
//Delete the stale metrics files
staleFileSlice := []string{buildErrorFile, rbeMetricsFile, soongMetricsFile, bp2buildMetricsFile, soongBuildMetricsFile}
@@ -203,14 +243,12 @@
build.PrintOutDirWarning(buildCtx, config)
- os.MkdirAll(logsDir, 0777)
- log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
- trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
- stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log")))
- stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log")))
+ stat := buildCtx.Status
+ stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, logsPrefix+"verbose.log")))
+ stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, logsPrefix+"error.log")))
stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile))
stat.AddOutput(status.NewCriticalPath(log))
- stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, c.logsPrefix+"build_progress.pb")))
+ stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, logsPrefix+"build_progress.pb")))
buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(1024*1024*1024))
buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",
@@ -218,27 +256,7 @@
setMaxFiles(buildCtx)
- {
- // The order of the function calls is important. The last defer function call
- // is the first one that is executed to save the rbe metrics to a protobuf
- // file. The soong metrics file is then next. Bazel profiles are written
- // before the uploadMetrics is invoked. The written files are then uploaded
- // if the uploading of the metrics is enabled.
- files := []string{
- buildErrorFile, // build error strings
- rbeMetricsFile, // high level metrics related to remote build execution.
- soongBuildMetricsFile, // high level metrics related to soong build(except bp2build).
- bp2buildMetricsFile, // high level metrics related to bp2build.
- soongMetricsFile, // high level metrics related to this build system.
- config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
- }
-
- if !config.SkipMetricsUpload() {
- defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, files...)
- }
- defer met.Dump(soongMetricsFile)
- defer build.CheckProdCreds(buildCtx, config)
- }
+ defer build.CheckProdCreds(buildCtx, config)
// Read the time at the starting point.
if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
@@ -253,7 +271,7 @@
}
if executable, err := os.Executable(); err == nil {
- trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
+ buildCtx.ContextImpl.Tracer.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
}
}
@@ -266,8 +284,6 @@
f := build.NewSourceFinder(buildCtx, config)
defer f.Shutdown()
build.FindSources(buildCtx, config, f)
-
- c.run(buildCtx, config, args, logsDir)
}
func fixBadDanglingLink(ctx build.Context, name string) {
@@ -284,7 +300,8 @@
}
}
-func dumpVar(ctx build.Context, config build.Config, args []string, _ string) {
+func dumpVar(ctx build.Context, config build.Config, args []string) {
+ logAndSymlinkSetup(ctx, config)
flags := flag.NewFlagSet("dumpvar", flag.ExitOnError)
flags.SetOutput(ctx.Writer)
@@ -336,7 +353,9 @@
}
}
-func dumpVars(ctx build.Context, config build.Config, args []string, _ string) {
+func dumpVars(ctx build.Context, config build.Config, args []string) {
+ logAndSymlinkSetup(ctx, config)
+
flags := flag.NewFlagSet("dumpvars", flag.ExitOnError)
flags.SetOutput(ctx.Writer)
@@ -421,6 +440,14 @@
return build.NewConfig(ctx)
}
+// uploadOnlyConfig explicitly requires no arguments.
+func uploadOnlyConfig(ctx build.Context, args ...string) build.Config {
+ if len(args) > 0 {
+ fmt.Printf("--upload-only does not require arguments.")
+ }
+ return build.UploadOnlyConfig(ctx)
+}
+
func buildActionConfig(ctx build.Context, args ...string) build.Config {
flags := flag.NewFlagSet("build-mode", flag.ContinueOnError)
flags.SetOutput(ctx.Writer)
@@ -514,7 +541,9 @@
return build.NewBuildActionConfig(buildAction, *dir, ctx, args...)
}
-func runMake(ctx build.Context, config build.Config, _ []string, logsDir string) {
+func runMake(ctx build.Context, config build.Config, _ []string) {
+ logAndSymlinkSetup(ctx, config)
+ logsDir := config.LogsDir()
if config.IsVerbose() {
writer := ctx.Writer
fmt.Fprintln(writer, "! The argument `showcommands` is no longer supported.")
@@ -659,3 +688,19 @@
ctx.Println("Failed to increase file limit:", err)
}
}
+
+func updateTotalRealTime(ctx build.Context, config build.Config, args []string) {
+ soongMetricsFile := filepath.Join(config.LogsDir(), "soong_metrics")
+
+ //read file into proto
+ data, err := os.ReadFile(soongMetricsFile)
+ if err != nil {
+ ctx.Fatal(err)
+ }
+ met := ctx.ContextImpl.Metrics
+
+ err = met.UpdateTotalRealTime(data)
+ if err != nil {
+ ctx.Fatal(err)
+ }
+}
diff --git a/ui/build/config.go b/ui/build/config.go
index a6bba15..61f6b1c 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -63,6 +63,7 @@
environ *Environment
distDir string
buildDateTime string
+ logsPrefix string
// From the arguments
parallel int
@@ -84,6 +85,7 @@
skipSoongTests bool
searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
skipMetricsUpload bool
+ buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
// From the product config
katiArgs []string
@@ -255,6 +257,14 @@
return true
}
+func UploadOnlyConfig(ctx Context, _ ...string) Config {
+ ret := &configImpl{
+ environ: OsEnvironment(),
+ sandboxConfig: &SandboxConfig{},
+ }
+ return Config{ret}
+}
+
func NewConfig(ctx Context, args ...string) Config {
ret := &configImpl{
environ: OsEnvironment(),
@@ -266,9 +276,7 @@
ret.keepGoing = 1
ret.totalRAM = detectTotalRAM(ctx)
-
ret.parseArgs(ctx, args)
-
// Make sure OUT_DIR is set appropriately
if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
@@ -756,6 +764,14 @@
ctx.Metrics.SetBuildCommand([]string{buildCmd})
} else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
+ } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
+ buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
+ val, err := strconv.ParseInt(buildTimeStr, 10, 64)
+ if err == nil {
+ c.buildStartedTime = val
+ } else {
+ ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
+ }
} else if len(arg) > 0 && arg[0] == '-' {
parseArgNum := func(def int) int {
if len(arg) > 2 {
@@ -1092,6 +1108,14 @@
c.includeTags = i
}
+func (c *configImpl) GetLogsPrefix() string {
+ return c.logsPrefix
+}
+
+func (c *configImpl) SetLogsPrefix(prefix string) {
+ c.logsPrefix = prefix
+}
+
func (c *configImpl) HighmemParallel() int {
if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
return i
@@ -1519,6 +1543,15 @@
return c.skipMetricsUpload
}
+// Returns a Time object if one was passed via a command-line flag.
+// Otherwise returns the passed default.
+func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
+ if c.buildStartedTime == 0 {
+ return defaultTime
+ }
+ return time.UnixMilli(c.buildStartedTime)
+}
+
func GetMetricsUploader(topDir string, env *Environment) string {
if p, ok := env.Get("METRICS_UPLOADER"); ok {
metricsUploader := filepath.Join(topDir, p)
diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go
index ce2d946..717530c 100644
--- a/ui/metrics/metrics.go
+++ b/ui/metrics/metrics.go
@@ -32,13 +32,13 @@
// of what an event is and how the metrics system is a stack based system.
import (
+ "fmt"
"os"
"runtime"
"strings"
"time"
"android/soong/shared"
-
"google.golang.org/protobuf/proto"
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
@@ -223,6 +223,17 @@
m.metrics.BuildDateTimestamp = proto.Int64(buildTimestamp.UnixNano() / int64(time.Second))
}
+func (m *Metrics) UpdateTotalRealTime(data []byte) error {
+ if err := proto.Unmarshal(data, &m.metrics); err != nil {
+ return fmt.Errorf("Failed to unmarshal proto", err)
+ }
+ startTime := *m.metrics.Total.StartTime
+ endTime := uint64(time.Now().UnixNano())
+
+ *m.metrics.Total.RealTime = *proto.Uint64(endTime - startTime)
+ return nil
+}
+
// SetBuildCommand adds the build command specified by the user to the
// list of collected metrics.
func (m *Metrics) SetBuildCommand(cmd []string) {