| // Copyright 2020 Google Inc. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package android |
| |
| import ( |
| "bytes" |
| "errors" |
| "fmt" |
| "io/ioutil" |
| "os" |
| "os/exec" |
| "path/filepath" |
| "runtime" |
| "strings" |
| "sync" |
| |
| "github.com/google/blueprint/bootstrap" |
| |
| "android/soong/bazel" |
| "android/soong/shared" |
| ) |
| |
| type CqueryRequestType int |
| |
| const ( |
| getAllFiles CqueryRequestType = iota |
| ) |
| |
| // Map key to describe bazel cquery requests. |
| type cqueryKey struct { |
| label string |
| requestType CqueryRequestType |
| } |
| |
| type BazelContext interface { |
| // The below methods involve queuing cquery requests to be later invoked |
| // by bazel. If any of these methods return (_, false), then the request |
| // has been queued to be run later. |
| |
| // Returns result files built by building the given bazel target label. |
| GetAllFiles(label string) ([]string, bool) |
| |
| // TODO(cparsons): Other cquery-related methods should be added here. |
| // ** End cquery methods |
| |
| // Issues commands to Bazel to receive results for all cquery requests |
| // queued in the BazelContext. |
| InvokeBazel() error |
| |
| // Returns true if bazel is enabled for the given configuration. |
| BazelEnabled() bool |
| |
| // Returns the bazel output base (the root directory for all bazel intermediate outputs). |
| OutputBase() string |
| |
| // Returns build statements which should get registered to reflect Bazel's outputs. |
| BuildStatementsToRegister() []bazel.BuildStatement |
| } |
| |
| // A context object which tracks queued requests that need to be made to Bazel, |
| // and their results after the requests have been made. |
| type bazelContext struct { |
| homeDir string |
| bazelPath string |
| outputBase string |
| workspaceDir string |
| buildDir string |
| metricsDir string |
| |
| requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel |
| requestMutex sync.Mutex // requests can be written in parallel |
| |
| results map[cqueryKey]string // Results of cquery requests after Bazel invocations |
| |
| // Build statements which should get registered to reflect Bazel's outputs. |
| buildStatements []bazel.BuildStatement |
| } |
| |
| var _ BazelContext = &bazelContext{} |
| |
| // A bazel context to use when Bazel is disabled. |
| type noopBazelContext struct{} |
| |
| var _ BazelContext = noopBazelContext{} |
| |
| // A bazel context to use for tests. |
| type MockBazelContext struct { |
| AllFiles map[string][]string |
| } |
| |
| func (m MockBazelContext) GetAllFiles(label string) ([]string, bool) { |
| result, ok := m.AllFiles[label] |
| return result, ok |
| } |
| |
| func (m MockBazelContext) InvokeBazel() error { |
| panic("unimplemented") |
| } |
| |
| func (m MockBazelContext) BazelEnabled() bool { |
| return true |
| } |
| |
| func (m MockBazelContext) OutputBase() string { |
| return "outputbase" |
| } |
| |
| func (m MockBazelContext) BuildStatementsToRegister() []bazel.BuildStatement { |
| return []bazel.BuildStatement{} |
| } |
| |
| var _ BazelContext = MockBazelContext{} |
| |
| func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) { |
| result, ok := bazelCtx.cquery(label, getAllFiles) |
| if ok { |
| bazelOutput := strings.TrimSpace(result) |
| return strings.Split(bazelOutput, ", "), true |
| } else { |
| return nil, false |
| } |
| } |
| |
| func (n noopBazelContext) GetAllFiles(label string) ([]string, bool) { |
| panic("unimplemented") |
| } |
| |
| func (n noopBazelContext) InvokeBazel() error { |
| panic("unimplemented") |
| } |
| |
| func (m noopBazelContext) OutputBase() string { |
| return "" |
| } |
| |
| func (n noopBazelContext) BazelEnabled() bool { |
| return false |
| } |
| |
| func (m noopBazelContext) BuildStatementsToRegister() []bazel.BuildStatement { |
| return []bazel.BuildStatement{} |
| } |
| |
| func NewBazelContext(c *config) (BazelContext, error) { |
| // TODO(cparsons): Assess USE_BAZEL=1 instead once "mixed Soong/Bazel builds" |
| // are production ready. |
| if c.Getenv("USE_BAZEL_ANALYSIS") != "1" { |
| return noopBazelContext{}, nil |
| } |
| |
| bazelCtx := bazelContext{buildDir: c.buildDir, requests: make(map[cqueryKey]bool)} |
| missingEnvVars := []string{} |
| if len(c.Getenv("BAZEL_HOME")) > 1 { |
| bazelCtx.homeDir = c.Getenv("BAZEL_HOME") |
| } else { |
| missingEnvVars = append(missingEnvVars, "BAZEL_HOME") |
| } |
| if len(c.Getenv("BAZEL_PATH")) > 1 { |
| bazelCtx.bazelPath = c.Getenv("BAZEL_PATH") |
| } else { |
| missingEnvVars = append(missingEnvVars, "BAZEL_PATH") |
| } |
| if len(c.Getenv("BAZEL_OUTPUT_BASE")) > 1 { |
| bazelCtx.outputBase = c.Getenv("BAZEL_OUTPUT_BASE") |
| } else { |
| missingEnvVars = append(missingEnvVars, "BAZEL_OUTPUT_BASE") |
| } |
| if len(c.Getenv("BAZEL_WORKSPACE")) > 1 { |
| bazelCtx.workspaceDir = c.Getenv("BAZEL_WORKSPACE") |
| } else { |
| missingEnvVars = append(missingEnvVars, "BAZEL_WORKSPACE") |
| } |
| if len(c.Getenv("BAZEL_METRICS_DIR")) > 1 { |
| bazelCtx.metricsDir = c.Getenv("BAZEL_METRICS_DIR") |
| } else { |
| missingEnvVars = append(missingEnvVars, "BAZEL_METRICS_DIR") |
| } |
| if len(missingEnvVars) > 0 { |
| return nil, errors.New(fmt.Sprintf("missing required env vars to use bazel: %s", missingEnvVars)) |
| } else { |
| return &bazelCtx, nil |
| } |
| } |
| |
| func (context *bazelContext) BazelMetricsDir() string { |
| return context.metricsDir |
| } |
| |
| func (context *bazelContext) BazelEnabled() bool { |
| return true |
| } |
| |
| // Adds a cquery request to the Bazel request queue, to be later invoked, or |
| // returns the result of the given request if the request was already made. |
| // If the given request was already made (and the results are available), then |
| // returns (result, true). If the request is queued but no results are available, |
| // then returns ("", false). |
| func (context *bazelContext) cquery(label string, requestType CqueryRequestType) (string, bool) { |
| key := cqueryKey{label, requestType} |
| if result, ok := context.results[key]; ok { |
| return result, true |
| } else { |
| context.requestMutex.Lock() |
| defer context.requestMutex.Unlock() |
| context.requests[key] = true |
| return "", false |
| } |
| } |
| |
| func pwdPrefix() string { |
| // Darwin doesn't have /proc |
| if runtime.GOOS != "darwin" { |
| return "PWD=/proc/self/cwd" |
| } |
| return "" |
| } |
| |
| func (context *bazelContext) issueBazelCommand(runName bazel.RunName, command string, labels []string, |
| extraFlags ...string) (string, error) { |
| |
| cmdFlags := []string{"--output_base=" + context.outputBase, command} |
| cmdFlags = append(cmdFlags, labels...) |
| cmdFlags = append(cmdFlags, "--package_path=%workspace%/"+context.intermediatesDir()) |
| cmdFlags = append(cmdFlags, "--profile="+shared.BazelMetricsFilename(context, runName)) |
| cmdFlags = append(cmdFlags, extraFlags...) |
| |
| bazelCmd := exec.Command(context.bazelPath, cmdFlags...) |
| bazelCmd.Dir = context.workspaceDir |
| bazelCmd.Env = append(os.Environ(), "HOME="+context.homeDir, pwdPrefix()) |
| |
| stderr := &bytes.Buffer{} |
| bazelCmd.Stderr = stderr |
| |
| if output, err := bazelCmd.Output(); err != nil { |
| return "", fmt.Errorf("bazel command failed. command: [%s], error [%s]", bazelCmd, stderr) |
| } else { |
| return string(output), nil |
| } |
| } |
| |
| // Returns the string contents of a workspace file that should be output |
| // adjacent to the main bzl file and build file. |
| // This workspace file allows, via local_repository rule, sourcetree-level |
| // BUILD targets to be referenced via @sourceroot. |
| func (context *bazelContext) workspaceFileContents() []byte { |
| formatString := ` |
| # This file is generated by soong_build. Do not edit. |
| local_repository( |
| name = "sourceroot", |
| path = "%s", |
| ) |
| ` |
| return []byte(fmt.Sprintf(formatString, context.workspaceDir)) |
| } |
| |
| func (context *bazelContext) mainBzlFileContents() []byte { |
| contents := ` |
| ##################################################### |
| # This file is generated by soong_build. Do not edit. |
| ##################################################### |
| |
| def _mixed_build_root_impl(ctx): |
| return [DefaultInfo(files = depset(ctx.files.deps))] |
| |
| # Rule representing the root of the build, to depend on all Bazel targets that |
| # are required for the build. Building this target will build the entire Bazel |
| # build tree. |
| mixed_build_root = rule( |
| implementation = _mixed_build_root_impl, |
| attrs = {"deps" : attr.label_list()}, |
| ) |
| |
| def _phony_root_impl(ctx): |
| return [] |
| |
| # Rule to depend on other targets but build nothing. |
| # This is useful as follows: building a target of this rule will generate |
| # symlink forests for all dependencies of the target, without executing any |
| # actions of the build. |
| phony_root = rule( |
| implementation = _phony_root_impl, |
| attrs = {"deps" : attr.label_list()}, |
| ) |
| ` |
| return []byte(contents) |
| } |
| |
| // Returns a "canonicalized" corresponding to the given sourcetree-level label. |
| // This abstraction is required because a sourcetree label such as //foo/bar:baz |
| // must be referenced via the local repository prefix, such as |
| // @sourceroot//foo/bar:baz. |
| func canonicalizeLabel(label string) string { |
| if strings.HasPrefix(label, "//") { |
| return "@sourceroot" + label |
| } else { |
| return "@sourceroot//" + label |
| } |
| } |
| |
| func (context *bazelContext) mainBuildFileContents() []byte { |
| formatString := ` |
| # This file is generated by soong_build. Do not edit. |
| load(":main.bzl", "mixed_build_root", "phony_root") |
| |
| mixed_build_root(name = "buildroot", |
| deps = [%s], |
| ) |
| |
| phony_root(name = "phonyroot", |
| deps = [":buildroot"], |
| ) |
| ` |
| var buildRootDeps []string = nil |
| for val, _ := range context.requests { |
| buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", canonicalizeLabel(val.label))) |
| } |
| buildRootDepsString := strings.Join(buildRootDeps, ",\n ") |
| |
| return []byte(fmt.Sprintf(formatString, buildRootDepsString)) |
| } |
| |
| func (context *bazelContext) cqueryStarlarkFileContents() []byte { |
| formatString := ` |
| # This file is generated by soong_build. Do not edit. |
| getAllFilesLabels = { |
| %s |
| } |
| |
| def format(target): |
| if str(target.label) in getAllFilesLabels: |
| return str(target.label) + ">>" + ', '.join([f.path for f in target.files.to_list()]) |
| else: |
| # This target was not requested via cquery, and thus must be a dependency |
| # of a requested target. |
| return "" |
| ` |
| var buildRootDeps []string = nil |
| // TODO(cparsons): Sort by request type instead of assuming all requests |
| // are of GetAllFiles type. |
| for val, _ := range context.requests { |
| buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", canonicalizeLabel(val.label))) |
| } |
| buildRootDepsString := strings.Join(buildRootDeps, ",\n ") |
| |
| return []byte(fmt.Sprintf(formatString, buildRootDepsString)) |
| } |
| |
| // Returns a workspace-relative path containing build-related metadata required |
| // for interfacing with Bazel. Example: out/soong/bazel. |
| func (context *bazelContext) intermediatesDir() string { |
| return filepath.Join(context.buildDir, "bazel") |
| } |
| |
| // Issues commands to Bazel to receive results for all cquery requests |
| // queued in the BazelContext. |
| func (context *bazelContext) InvokeBazel() error { |
| context.results = make(map[cqueryKey]string) |
| |
| var cqueryOutput string |
| var err error |
| |
| err = os.Mkdir(absolutePath(context.intermediatesDir()), 0777) |
| if err != nil { |
| return err |
| } |
| err = ioutil.WriteFile( |
| absolutePath(filepath.Join(context.intermediatesDir(), "main.bzl")), |
| context.mainBzlFileContents(), 0666) |
| if err != nil { |
| return err |
| } |
| err = ioutil.WriteFile( |
| absolutePath(filepath.Join(context.intermediatesDir(), "BUILD.bazel")), |
| context.mainBuildFileContents(), 0666) |
| if err != nil { |
| return err |
| } |
| cqueryFileRelpath := filepath.Join(context.intermediatesDir(), "buildroot.cquery") |
| err = ioutil.WriteFile( |
| absolutePath(cqueryFileRelpath), |
| context.cqueryStarlarkFileContents(), 0666) |
| if err != nil { |
| return err |
| } |
| workspaceFileRelpath := filepath.Join(context.intermediatesDir(), "WORKSPACE.bazel") |
| err = ioutil.WriteFile( |
| absolutePath(workspaceFileRelpath), |
| context.workspaceFileContents(), 0666) |
| if err != nil { |
| return err |
| } |
| buildrootLabel := "//:buildroot" |
| cqueryOutput, err = context.issueBazelCommand(bazel.CqueryBuildRootRunName, "cquery", |
| []string{fmt.Sprintf("deps(%s)", buildrootLabel)}, |
| "--output=starlark", |
| "--starlark:file="+cqueryFileRelpath) |
| |
| if err != nil { |
| return err |
| } |
| |
| cqueryResults := map[string]string{} |
| for _, outputLine := range strings.Split(cqueryOutput, "\n") { |
| if strings.Contains(outputLine, ">>") { |
| splitLine := strings.SplitN(outputLine, ">>", 2) |
| cqueryResults[splitLine[0]] = splitLine[1] |
| } |
| } |
| |
| for val, _ := range context.requests { |
| if cqueryResult, ok := cqueryResults[canonicalizeLabel(val.label)]; ok { |
| context.results[val] = string(cqueryResult) |
| } else { |
| return fmt.Errorf("missing result for bazel target %s", val.label) |
| } |
| } |
| |
| // Issue an aquery command to retrieve action information about the bazel build tree. |
| // |
| // TODO(cparsons): Use --target_pattern_file to avoid command line limits. |
| var aqueryOutput string |
| aqueryOutput, err = context.issueBazelCommand(bazel.AqueryBuildRootRunName, "aquery", |
| []string{fmt.Sprintf("deps(%s)", buildrootLabel), |
| // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's |
| // proto sources, which would add a number of unnecessary dependencies. |
| "--output=jsonproto"}) |
| |
| if err != nil { |
| return err |
| } |
| |
| context.buildStatements = bazel.AqueryBuildStatements([]byte(aqueryOutput)) |
| |
| // Issue a build command of the phony root to generate symlink forests for dependencies of the |
| // Bazel build. This is necessary because aquery invocations do not generate this symlink forest, |
| // but some of symlinks may be required to resolve source dependencies of the build. |
| _, err = context.issueBazelCommand(bazel.BazelBuildPhonyRootRunName, "build", |
| []string{"//:phonyroot"}) |
| |
| if err != nil { |
| return err |
| } |
| |
| fmt.Printf("Build statements %s", context.buildStatements) |
| // Clear requests. |
| context.requests = map[cqueryKey]bool{} |
| return nil |
| } |
| |
| func (context *bazelContext) BuildStatementsToRegister() []bazel.BuildStatement { |
| return context.buildStatements |
| } |
| |
| func (context *bazelContext) OutputBase() string { |
| return context.outputBase |
| } |
| |
| // Singleton used for registering BUILD file ninja dependencies (needed |
| // for correctness of builds which use Bazel. |
| func BazelSingleton() Singleton { |
| return &bazelSingleton{} |
| } |
| |
| type bazelSingleton struct{} |
| |
| func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { |
| // bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled. |
| if !ctx.Config().BazelContext.BazelEnabled() { |
| return |
| } |
| |
| // Add ninja file dependencies for files which all bazel invocations require. |
| bazelBuildList := absolutePath(filepath.Join( |
| filepath.Dir(bootstrap.ModuleListFile), "bazel.list")) |
| ctx.AddNinjaFileDeps(bazelBuildList) |
| |
| data, err := ioutil.ReadFile(bazelBuildList) |
| if err != nil { |
| ctx.Errorf(err.Error()) |
| } |
| files := strings.Split(strings.TrimSpace(string(data)), "\n") |
| for _, file := range files { |
| ctx.AddNinjaFileDeps(file) |
| } |
| |
| // Register bazel-owned build statements (obtained from the aquery invocation). |
| for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() { |
| rule := NewRuleBuilder(pctx, ctx) |
| cmd := rule.Command() |
| cmd.Text(fmt.Sprintf("cd %s/execroot/__main__ && %s", |
| ctx.Config().BazelContext.OutputBase(), buildStatement.Command)) |
| |
| for _, outputPath := range buildStatement.OutputPaths { |
| cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath)) |
| } |
| for _, inputPath := range buildStatement.InputPaths { |
| cmd.Implicit(PathForBazelOut(ctx, inputPath)) |
| } |
| |
| // This is required to silence warnings pertaining to unexpected timestamps. Particularly, |
| // some Bazel builtins (such as files in the bazel_tools directory) have far-future |
| // timestamps. Without restat, Ninja would emit warnings that the input files of a |
| // build statement have later timestamps than the outputs. |
| rule.Restat() |
| |
| rule.Build(fmt.Sprintf("bazel %s", index), buildStatement.Mnemonic) |
| } |
| } |