blob: 357b99fc2049a5030cefc1fa92bfbd84b19fc6be [file] [log] [blame]
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
"github.com/google/blueprint/bootstrap"
"android/soong/bazel"
"android/soong/shared"
)
type CqueryRequestType int
const (
getAllFiles CqueryRequestType = iota
)
// Map key to describe bazel cquery requests.
type cqueryKey struct {
label string
requestType CqueryRequestType
}
type BazelContext interface {
// The below methods involve queuing cquery requests to be later invoked
// by bazel. If any of these methods return (_, false), then the request
// has been queued to be run later.
// Returns result files built by building the given bazel target label.
GetAllFiles(label string) ([]string, bool)
// TODO(cparsons): Other cquery-related methods should be added here.
// ** End cquery methods
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext.
InvokeBazel() error
// Returns true if bazel is enabled for the given configuration.
BazelEnabled() bool
// Returns the bazel output base (the root directory for all bazel intermediate outputs).
OutputBase() string
// Returns build statements which should get registered to reflect Bazel's outputs.
BuildStatementsToRegister() []bazel.BuildStatement
}
// A context object which tracks queued requests that need to be made to Bazel,
// and their results after the requests have been made.
type bazelContext struct {
homeDir string
bazelPath string
outputBase string
workspaceDir string
buildDir string
metricsDir string
requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel
requestMutex sync.Mutex // requests can be written in parallel
results map[cqueryKey]string // Results of cquery requests after Bazel invocations
// Build statements which should get registered to reflect Bazel's outputs.
buildStatements []bazel.BuildStatement
}
var _ BazelContext = &bazelContext{}
// A bazel context to use when Bazel is disabled.
type noopBazelContext struct{}
var _ BazelContext = noopBazelContext{}
// A bazel context to use for tests.
type MockBazelContext struct {
AllFiles map[string][]string
}
func (m MockBazelContext) GetAllFiles(label string) ([]string, bool) {
result, ok := m.AllFiles[label]
return result, ok
}
func (m MockBazelContext) InvokeBazel() error {
panic("unimplemented")
}
func (m MockBazelContext) BazelEnabled() bool {
return true
}
func (m MockBazelContext) OutputBase() string {
return "outputbase"
}
func (m MockBazelContext) BuildStatementsToRegister() []bazel.BuildStatement {
return []bazel.BuildStatement{}
}
var _ BazelContext = MockBazelContext{}
func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
result, ok := bazelCtx.cquery(label, getAllFiles)
if ok {
bazelOutput := strings.TrimSpace(result)
return strings.Split(bazelOutput, ", "), true
} else {
return nil, false
}
}
func (n noopBazelContext) GetAllFiles(label string) ([]string, bool) {
panic("unimplemented")
}
func (n noopBazelContext) InvokeBazel() error {
panic("unimplemented")
}
func (m noopBazelContext) OutputBase() string {
return ""
}
func (n noopBazelContext) BazelEnabled() bool {
return false
}
func (m noopBazelContext) BuildStatementsToRegister() []bazel.BuildStatement {
return []bazel.BuildStatement{}
}
func NewBazelContext(c *config) (BazelContext, error) {
// TODO(cparsons): Assess USE_BAZEL=1 instead once "mixed Soong/Bazel builds"
// are production ready.
if c.Getenv("USE_BAZEL_ANALYSIS") != "1" {
return noopBazelContext{}, nil
}
bazelCtx := bazelContext{buildDir: c.buildDir, requests: make(map[cqueryKey]bool)}
missingEnvVars := []string{}
if len(c.Getenv("BAZEL_HOME")) > 1 {
bazelCtx.homeDir = c.Getenv("BAZEL_HOME")
} else {
missingEnvVars = append(missingEnvVars, "BAZEL_HOME")
}
if len(c.Getenv("BAZEL_PATH")) > 1 {
bazelCtx.bazelPath = c.Getenv("BAZEL_PATH")
} else {
missingEnvVars = append(missingEnvVars, "BAZEL_PATH")
}
if len(c.Getenv("BAZEL_OUTPUT_BASE")) > 1 {
bazelCtx.outputBase = c.Getenv("BAZEL_OUTPUT_BASE")
} else {
missingEnvVars = append(missingEnvVars, "BAZEL_OUTPUT_BASE")
}
if len(c.Getenv("BAZEL_WORKSPACE")) > 1 {
bazelCtx.workspaceDir = c.Getenv("BAZEL_WORKSPACE")
} else {
missingEnvVars = append(missingEnvVars, "BAZEL_WORKSPACE")
}
if len(c.Getenv("BAZEL_METRICS_DIR")) > 1 {
bazelCtx.metricsDir = c.Getenv("BAZEL_METRICS_DIR")
} else {
missingEnvVars = append(missingEnvVars, "BAZEL_METRICS_DIR")
}
if len(missingEnvVars) > 0 {
return nil, errors.New(fmt.Sprintf("missing required env vars to use bazel: %s", missingEnvVars))
} else {
return &bazelCtx, nil
}
}
func (context *bazelContext) BazelMetricsDir() string {
return context.metricsDir
}
func (context *bazelContext) BazelEnabled() bool {
return true
}
// Adds a cquery request to the Bazel request queue, to be later invoked, or
// returns the result of the given request if the request was already made.
// If the given request was already made (and the results are available), then
// returns (result, true). If the request is queued but no results are available,
// then returns ("", false).
func (context *bazelContext) cquery(label string, requestType CqueryRequestType) (string, bool) {
key := cqueryKey{label, requestType}
if result, ok := context.results[key]; ok {
return result, true
} else {
context.requestMutex.Lock()
defer context.requestMutex.Unlock()
context.requests[key] = true
return "", false
}
}
func pwdPrefix() string {
// Darwin doesn't have /proc
if runtime.GOOS != "darwin" {
return "PWD=/proc/self/cwd"
}
return ""
}
func (context *bazelContext) issueBazelCommand(runName bazel.RunName, command string, labels []string,
extraFlags ...string) (string, error) {
cmdFlags := []string{"--output_base=" + context.outputBase, command}
cmdFlags = append(cmdFlags, labels...)
cmdFlags = append(cmdFlags, "--package_path=%workspace%/"+context.intermediatesDir())
cmdFlags = append(cmdFlags, "--profile="+shared.BazelMetricsFilename(context, runName))
cmdFlags = append(cmdFlags, extraFlags...)
bazelCmd := exec.Command(context.bazelPath, cmdFlags...)
bazelCmd.Dir = context.workspaceDir
bazelCmd.Env = append(os.Environ(), "HOME="+context.homeDir, pwdPrefix())
stderr := &bytes.Buffer{}
bazelCmd.Stderr = stderr
if output, err := bazelCmd.Output(); err != nil {
return "", fmt.Errorf("bazel command failed. command: [%s], error [%s]", bazelCmd, stderr)
} else {
return string(output), nil
}
}
// Returns the string contents of a workspace file that should be output
// adjacent to the main bzl file and build file.
// This workspace file allows, via local_repository rule, sourcetree-level
// BUILD targets to be referenced via @sourceroot.
func (context *bazelContext) workspaceFileContents() []byte {
formatString := `
# This file is generated by soong_build. Do not edit.
local_repository(
name = "sourceroot",
path = "%s",
)
`
return []byte(fmt.Sprintf(formatString, context.workspaceDir))
}
func (context *bazelContext) mainBzlFileContents() []byte {
contents := `
#####################################################
# This file is generated by soong_build. Do not edit.
#####################################################
def _mixed_build_root_impl(ctx):
return [DefaultInfo(files = depset(ctx.files.deps))]
# Rule representing the root of the build, to depend on all Bazel targets that
# are required for the build. Building this target will build the entire Bazel
# build tree.
mixed_build_root = rule(
implementation = _mixed_build_root_impl,
attrs = {"deps" : attr.label_list()},
)
def _phony_root_impl(ctx):
return []
# Rule to depend on other targets but build nothing.
# This is useful as follows: building a target of this rule will generate
# symlink forests for all dependencies of the target, without executing any
# actions of the build.
phony_root = rule(
implementation = _phony_root_impl,
attrs = {"deps" : attr.label_list()},
)
`
return []byte(contents)
}
// Returns a "canonicalized" corresponding to the given sourcetree-level label.
// This abstraction is required because a sourcetree label such as //foo/bar:baz
// must be referenced via the local repository prefix, such as
// @sourceroot//foo/bar:baz.
func canonicalizeLabel(label string) string {
if strings.HasPrefix(label, "//") {
return "@sourceroot" + label
} else {
return "@sourceroot//" + label
}
}
func (context *bazelContext) mainBuildFileContents() []byte {
formatString := `
# This file is generated by soong_build. Do not edit.
load(":main.bzl", "mixed_build_root", "phony_root")
mixed_build_root(name = "buildroot",
deps = [%s],
)
phony_root(name = "phonyroot",
deps = [":buildroot"],
)
`
var buildRootDeps []string = nil
for val, _ := range context.requests {
buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", canonicalizeLabel(val.label)))
}
buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
return []byte(fmt.Sprintf(formatString, buildRootDepsString))
}
func (context *bazelContext) cqueryStarlarkFileContents() []byte {
formatString := `
# This file is generated by soong_build. Do not edit.
getAllFilesLabels = {
%s
}
def format(target):
if str(target.label) in getAllFilesLabels:
return str(target.label) + ">>" + ', '.join([f.path for f in target.files.to_list()])
else:
# This target was not requested via cquery, and thus must be a dependency
# of a requested target.
return ""
`
var buildRootDeps []string = nil
// TODO(cparsons): Sort by request type instead of assuming all requests
// are of GetAllFiles type.
for val, _ := range context.requests {
buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", canonicalizeLabel(val.label)))
}
buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
return []byte(fmt.Sprintf(formatString, buildRootDepsString))
}
// Returns a workspace-relative path containing build-related metadata required
// for interfacing with Bazel. Example: out/soong/bazel.
func (context *bazelContext) intermediatesDir() string {
return filepath.Join(context.buildDir, "bazel")
}
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext.
func (context *bazelContext) InvokeBazel() error {
context.results = make(map[cqueryKey]string)
var cqueryOutput string
var err error
err = os.Mkdir(absolutePath(context.intermediatesDir()), 0777)
if err != nil {
return err
}
err = ioutil.WriteFile(
absolutePath(filepath.Join(context.intermediatesDir(), "main.bzl")),
context.mainBzlFileContents(), 0666)
if err != nil {
return err
}
err = ioutil.WriteFile(
absolutePath(filepath.Join(context.intermediatesDir(), "BUILD.bazel")),
context.mainBuildFileContents(), 0666)
if err != nil {
return err
}
cqueryFileRelpath := filepath.Join(context.intermediatesDir(), "buildroot.cquery")
err = ioutil.WriteFile(
absolutePath(cqueryFileRelpath),
context.cqueryStarlarkFileContents(), 0666)
if err != nil {
return err
}
workspaceFileRelpath := filepath.Join(context.intermediatesDir(), "WORKSPACE.bazel")
err = ioutil.WriteFile(
absolutePath(workspaceFileRelpath),
context.workspaceFileContents(), 0666)
if err != nil {
return err
}
buildrootLabel := "//:buildroot"
cqueryOutput, err = context.issueBazelCommand(bazel.CqueryBuildRootRunName, "cquery",
[]string{fmt.Sprintf("deps(%s)", buildrootLabel)},
"--output=starlark",
"--starlark:file="+cqueryFileRelpath)
if err != nil {
return err
}
cqueryResults := map[string]string{}
for _, outputLine := range strings.Split(cqueryOutput, "\n") {
if strings.Contains(outputLine, ">>") {
splitLine := strings.SplitN(outputLine, ">>", 2)
cqueryResults[splitLine[0]] = splitLine[1]
}
}
for val, _ := range context.requests {
if cqueryResult, ok := cqueryResults[canonicalizeLabel(val.label)]; ok {
context.results[val] = string(cqueryResult)
} else {
return fmt.Errorf("missing result for bazel target %s", val.label)
}
}
// Issue an aquery command to retrieve action information about the bazel build tree.
//
// TODO(cparsons): Use --target_pattern_file to avoid command line limits.
var aqueryOutput string
aqueryOutput, err = context.issueBazelCommand(bazel.AqueryBuildRootRunName, "aquery",
[]string{fmt.Sprintf("deps(%s)", buildrootLabel),
// Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's
// proto sources, which would add a number of unnecessary dependencies.
"--output=jsonproto"})
if err != nil {
return err
}
context.buildStatements = bazel.AqueryBuildStatements([]byte(aqueryOutput))
// Issue a build command of the phony root to generate symlink forests for dependencies of the
// Bazel build. This is necessary because aquery invocations do not generate this symlink forest,
// but some of symlinks may be required to resolve source dependencies of the build.
_, err = context.issueBazelCommand(bazel.BazelBuildPhonyRootRunName, "build",
[]string{"//:phonyroot"})
if err != nil {
return err
}
fmt.Printf("Build statements %s", context.buildStatements)
// Clear requests.
context.requests = map[cqueryKey]bool{}
return nil
}
func (context *bazelContext) BuildStatementsToRegister() []bazel.BuildStatement {
return context.buildStatements
}
func (context *bazelContext) OutputBase() string {
return context.outputBase
}
// Singleton used for registering BUILD file ninja dependencies (needed
// for correctness of builds which use Bazel.
func BazelSingleton() Singleton {
return &bazelSingleton{}
}
type bazelSingleton struct{}
func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
// bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled.
if !ctx.Config().BazelContext.BazelEnabled() {
return
}
// Add ninja file dependencies for files which all bazel invocations require.
bazelBuildList := absolutePath(filepath.Join(
filepath.Dir(bootstrap.ModuleListFile), "bazel.list"))
ctx.AddNinjaFileDeps(bazelBuildList)
data, err := ioutil.ReadFile(bazelBuildList)
if err != nil {
ctx.Errorf(err.Error())
}
files := strings.Split(strings.TrimSpace(string(data)), "\n")
for _, file := range files {
ctx.AddNinjaFileDeps(file)
}
// Register bazel-owned build statements (obtained from the aquery invocation).
for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
rule := NewRuleBuilder(pctx, ctx)
cmd := rule.Command()
cmd.Text(fmt.Sprintf("cd %s/execroot/__main__ && %s",
ctx.Config().BazelContext.OutputBase(), buildStatement.Command))
for _, outputPath := range buildStatement.OutputPaths {
cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath))
}
for _, inputPath := range buildStatement.InputPaths {
cmd.Implicit(PathForBazelOut(ctx, inputPath))
}
// This is required to silence warnings pertaining to unexpected timestamps. Particularly,
// some Bazel builtins (such as files in the bazel_tools directory) have far-future
// timestamps. Without restat, Ninja would emit warnings that the input files of a
// build statement have later timestamps than the outputs.
rule.Restat()
rule.Build(fmt.Sprintf("bazel %s", index), buildStatement.Mnemonic)
}
}