bazel/aquery.go - SHIFTPHONES/android_build_soong - Gitiles

 // Copyright 2020 Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package bazel

 import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"strings"

 	"github.com/google/blueprint/proptools"
 )

 // artifact contains relevant portions of Bazel's aquery proto, Artifact.
 // Represents a single artifact, whether it's a source file or a derived output file.
 type artifact struct {
 	Id             int
 	PathFragmentId int
 }

 type pathFragment struct {
 	Id       int
 	Label    string
 	ParentId int
 }

 // KeyValuePair represents Bazel's aquery proto, KeyValuePair.
 type KeyValuePair struct {
 	Key   string
 	Value string
 }

 // depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles.
 // Represents a data structure containing one or more files. Depsets in Bazel are an efficient
 // data structure for storing large numbers of file paths.
 type depSetOfFiles struct {
 	Id                  int
 	DirectArtifactIds   []int
 	TransitiveDepSetIds []int
 }

 // action contains relevant portions of Bazel's aquery proto, Action.
 // Represents a single command line invocation in the Bazel build graph.
 type action struct {
 	Arguments            []string
 	EnvironmentVariables []KeyValuePair
 	InputDepSetIds       []int
 	Mnemonic             string
 	OutputIds            []int
 }

 // actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
 // An aquery response from Bazel contains a single ActionGraphContainer proto.
 type actionGraphContainer struct {
 	Artifacts     []artifact
 	Actions       []action
 	DepSetOfFiles []depSetOfFiles
 	PathFragments []pathFragment
 }

 // BuildStatement contains information to register a build statement corresponding (one to one)
 // with a Bazel action from Bazel's action graph.
 type BuildStatement struct {
 	Command     string
 	Depfile     *string
 	OutputPaths []string
 	InputPaths  []string
 	Env         []KeyValuePair
 	Mnemonic    string
 }

 // AqueryBuildStatements returns an array of BuildStatements which should be registered (and output
 // to a ninja file) to correspond one-to-one with the given action graph json proto (from a bazel
 // aquery invocation).
 func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
 	buildStatements := []BuildStatement{}

 	var aqueryResult actionGraphContainer
 	err := json.Unmarshal(aqueryJsonProto, &aqueryResult)

 	if err != nil {
 		return nil, err
 	}

 	pathFragments := map[int]pathFragment{}
 	for _, pathFragment := range aqueryResult.PathFragments {
 		pathFragments[pathFragment.Id] = pathFragment
 	}
 	artifactIdToPath := map[int]string{}
 	for _, artifact := range aqueryResult.Artifacts {
 		artifactPath, err := expandPathFragment(artifact.PathFragmentId, pathFragments)
 		if err != nil {
 			return nil, err
 		}
 		artifactIdToPath[artifact.Id] = artifactPath
 	}

 	depsetIdToDepset := map[int]depSetOfFiles{}
 	for _, depset := range aqueryResult.DepSetOfFiles {
 		depsetIdToDepset[depset.Id] = depset
 	}

 	// depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening
 	// may be an expensive operation.
 	depsetIdToArtifactIdsCache := map[int][]int{}

 	// Do a pass through all actions to identify which artifacts are middleman artifacts.
 	// These will be omitted from the inputs of other actions.
 	// TODO(b/180945500): Handle middleman actions; without proper handling, depending on generated
 	// headers may cause build failures.
 	middlemanArtifactIds := map[int]bool{}
 	for _, actionEntry := range aqueryResult.Actions {
 		if actionEntry.Mnemonic == "Middleman" {
 			for _, outputId := range actionEntry.OutputIds {
 				middlemanArtifactIds[outputId] = true
 			}
 		}
 	}

 	for _, actionEntry := range aqueryResult.Actions {
 		if shouldSkipAction(actionEntry) {
 			continue
 		}
 		outputPaths := []string{}
 		var depfile *string
 		for _, outputId := range actionEntry.OutputIds {
 			outputPath, exists := artifactIdToPath[outputId]
 			if !exists {
 				return nil, fmt.Errorf("undefined outputId %d", outputId)
 			}
 			ext := filepath.Ext(outputPath)
 			if ext == ".d" {
 				if depfile != nil {
 					return nil, fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath)
 				} else {
 					depfile = &outputPath
 				}
 			} else {
 				outputPaths = append(outputPaths, outputPath)
 			}
 		}
 		inputPaths := []string{}
 		for _, inputDepSetId := range actionEntry.InputDepSetIds {
 			inputArtifacts, err :=
 				artifactIdsFromDepsetId(depsetIdToDepset, depsetIdToArtifactIdsCache, inputDepSetId)
 			if err != nil {
 				return nil, err
 			}
 			for _, inputId := range inputArtifacts {
 				if _, isMiddlemanArtifact := middlemanArtifactIds[inputId]; isMiddlemanArtifact {
 					// Omit middleman artifacts.
 					continue
 				}
 				inputPath, exists := artifactIdToPath[inputId]
 				if !exists {
 					return nil, fmt.Errorf("undefined input artifactId %d", inputId)
 				}
 				inputPaths = append(inputPaths, inputPath)
 			}
 		}
 		buildStatement := BuildStatement{
 			Command:     strings.Join(proptools.ShellEscapeList(actionEntry.Arguments), " "),
 			Depfile:     depfile,
 			OutputPaths: outputPaths,
 			InputPaths:  inputPaths,
 			Env:         actionEntry.EnvironmentVariables,
 			Mnemonic:    actionEntry.Mnemonic}
 		if len(actionEntry.Arguments) < 1 {
 			return nil, fmt.Errorf("received action with no command: [%v]", buildStatement)
 			continue
 		}
 		buildStatements = append(buildStatements, buildStatement)
 	}

 	return buildStatements, nil
 }

 func shouldSkipAction(a action) bool {
 	// TODO(b/180945121): Handle symlink actions.
 	if a.Mnemonic == "Symlink" || a.Mnemonic == "SourceSymlinkManifest" || a.Mnemonic == "SymlinkTree" {
 		return true
 	}
 	// TODO(b/180945500): Handle middleman actions; without proper handling, depending on generated
 	// headers may cause build failures.
 	if a.Mnemonic == "Middleman" {
 		return true
 	}
 	// Skip "Fail" actions, which are placeholder actions designed to always fail.
 	if a.Mnemonic == "Fail" {
 		return true
 	}
 	// TODO(b/180946980): Handle FileWrite. The aquery proto currently contains no information
 	// about the contents that are written.
 	if a.Mnemonic == "FileWrite" {
 		return true
 	}
 	return false
 }

 func artifactIdsFromDepsetId(depsetIdToDepset map[int]depSetOfFiles,
 	depsetIdToArtifactIdsCache map[int][]int, depsetId int) ([]int, error) {
 	if result, exists := depsetIdToArtifactIdsCache[depsetId]; exists {
 		return result, nil
 	}
 	if depset, exists := depsetIdToDepset[depsetId]; exists {
 		result := depset.DirectArtifactIds
 		for _, childId := range depset.TransitiveDepSetIds {
 			childArtifactIds, err :=
 				artifactIdsFromDepsetId(depsetIdToDepset, depsetIdToArtifactIdsCache, childId)
 			if err != nil {
 				return nil, err
 			}
 			result = append(result, childArtifactIds...)
 		}
 		depsetIdToArtifactIdsCache[depsetId] = result
 		return result, nil
 	} else {
 		return nil, fmt.Errorf("undefined input depsetId %d", depsetId)
 	}
 }

 func expandPathFragment(id int, pathFragmentsMap map[int]pathFragment) (string, error) {
 	labels := []string{}
 	currId := id
 	// Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node.
 	for currId > 0 {
 		currFragment, ok := pathFragmentsMap[currId]
 		if !ok {
 			return "", fmt.Errorf("undefined path fragment id %d", currId)
 		}
 		labels = append([]string{currFragment.Label}, labels...)
 		currId = currFragment.ParentId
 	}
 	return filepath.Join(labels...), nil
 }
	// Copyright 2020 Google Inc. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package bazel

	import (
	"encoding/json"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/google/blueprint/proptools"
	)

	// artifact contains relevant portions of Bazel's aquery proto, Artifact.
	// Represents a single artifact, whether it's a source file or a derived output file.
	type artifact struct {
	Id int
	PathFragmentId int
	}

	type pathFragment struct {
	Id int
	Label string
	ParentId int
	}

	// KeyValuePair represents Bazel's aquery proto, KeyValuePair.
	type KeyValuePair struct {
	Key string
	Value string
	}

	// depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles.
	// Represents a data structure containing one or more files. Depsets in Bazel are an efficient
	// data structure for storing large numbers of file paths.
	type depSetOfFiles struct {
	Id int
	DirectArtifactIds []int
	TransitiveDepSetIds []int
	}

	// action contains relevant portions of Bazel's aquery proto, Action.
	// Represents a single command line invocation in the Bazel build graph.
	type action struct {
	Arguments []string
	EnvironmentVariables []KeyValuePair
	InputDepSetIds []int
	Mnemonic string
	OutputIds []int
	}

	// actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
	// An aquery response from Bazel contains a single ActionGraphContainer proto.
	type actionGraphContainer struct {
	Artifacts []artifact
	Actions []action
	DepSetOfFiles []depSetOfFiles
	PathFragments []pathFragment
	}

	// BuildStatement contains information to register a build statement corresponding (one to one)
	// with a Bazel action from Bazel's action graph.
	type BuildStatement struct {
	Command string
	Depfile *string
	OutputPaths []string
	InputPaths []string
	Env []KeyValuePair
	Mnemonic string
	}

	// AqueryBuildStatements returns an array of BuildStatements which should be registered (and output
	// to a ninja file) to correspond one-to-one with the given action graph json proto (from a bazel
	// aquery invocation).
	func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
	buildStatements := []BuildStatement{}

	var aqueryResult actionGraphContainer
	err := json.Unmarshal(aqueryJsonProto, &aqueryResult)

	if err != nil {
	return nil, err
	}

	pathFragments := map[int]pathFragment{}
	for _, pathFragment := range aqueryResult.PathFragments {
	pathFragments[pathFragment.Id] = pathFragment
	}
	artifactIdToPath := map[int]string{}
	for _, artifact := range aqueryResult.Artifacts {
	artifactPath, err := expandPathFragment(artifact.PathFragmentId, pathFragments)
	if err != nil {
	return nil, err
	}
	artifactIdToPath[artifact.Id] = artifactPath
	}

	depsetIdToDepset := map[int]depSetOfFiles{}
	for _, depset := range aqueryResult.DepSetOfFiles {
	depsetIdToDepset[depset.Id] = depset
	}

	// depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening
	// may be an expensive operation.
	depsetIdToArtifactIdsCache := map[int][]int{}

	// Do a pass through all actions to identify which artifacts are middleman artifacts.
	// These will be omitted from the inputs of other actions.
	// TODO(b/180945500): Handle middleman actions; without proper handling, depending on generated
	// headers may cause build failures.
	middlemanArtifactIds := map[int]bool{}
	for _, actionEntry := range aqueryResult.Actions {
	if actionEntry.Mnemonic == "Middleman" {
	for _, outputId := range actionEntry.OutputIds {
	middlemanArtifactIds[outputId] = true
	}
	}
	}

	for _, actionEntry := range aqueryResult.Actions {
	if shouldSkipAction(actionEntry) {
	continue
	}
	outputPaths := []string{}
	var depfile *string
	for _, outputId := range actionEntry.OutputIds {
	outputPath, exists := artifactIdToPath[outputId]
	if !exists {
	return nil, fmt.Errorf("undefined outputId %d", outputId)
	}
	ext := filepath.Ext(outputPath)
	if ext == ".d" {
	if depfile != nil {
	return nil, fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath)
	} else {
	depfile = &outputPath
	}
	} else {
	outputPaths = append(outputPaths, outputPath)
	}
	}
	inputPaths := []string{}
	for _, inputDepSetId := range actionEntry.InputDepSetIds {
	inputArtifacts, err :=
	artifactIdsFromDepsetId(depsetIdToDepset, depsetIdToArtifactIdsCache, inputDepSetId)
	if err != nil {
	return nil, err
	}
	for _, inputId := range inputArtifacts {
	if _, isMiddlemanArtifact := middlemanArtifactIds[inputId]; isMiddlemanArtifact {
	// Omit middleman artifacts.
	continue
	}
	inputPath, exists := artifactIdToPath[inputId]
	if !exists {
	return nil, fmt.Errorf("undefined input artifactId %d", inputId)
	}
	inputPaths = append(inputPaths, inputPath)
	}
	}
	buildStatement := BuildStatement{
	Command: strings.Join(proptools.ShellEscapeList(actionEntry.Arguments), " "),
	Depfile: depfile,
	OutputPaths: outputPaths,
	InputPaths: inputPaths,
	Env: actionEntry.EnvironmentVariables,
	Mnemonic: actionEntry.Mnemonic}
	if len(actionEntry.Arguments) < 1 {
	return nil, fmt.Errorf("received action with no command: [%v]", buildStatement)
	continue
	}
	buildStatements = append(buildStatements, buildStatement)
	}

	return buildStatements, nil
	}

	func shouldSkipAction(a action) bool {
	// TODO(b/180945121): Handle symlink actions.
	if a.Mnemonic == "Symlink" \|\| a.Mnemonic == "SourceSymlinkManifest" \|\| a.Mnemonic == "SymlinkTree" {
	return true
	}
	// TODO(b/180945500): Handle middleman actions; without proper handling, depending on generated
	// headers may cause build failures.
	if a.Mnemonic == "Middleman" {
	return true
	}
	// Skip "Fail" actions, which are placeholder actions designed to always fail.
	if a.Mnemonic == "Fail" {
	return true
	}
	// TODO(b/180946980): Handle FileWrite. The aquery proto currently contains no information
	// about the contents that are written.
	if a.Mnemonic == "FileWrite" {
	return true
	}
	return false
	}

	func artifactIdsFromDepsetId(depsetIdToDepset map[int]depSetOfFiles,
	depsetIdToArtifactIdsCache map[int][]int, depsetId int) ([]int, error) {
	if result, exists := depsetIdToArtifactIdsCache[depsetId]; exists {
	return result, nil
	}
	if depset, exists := depsetIdToDepset[depsetId]; exists {
	result := depset.DirectArtifactIds
	for _, childId := range depset.TransitiveDepSetIds {
	childArtifactIds, err :=
	artifactIdsFromDepsetId(depsetIdToDepset, depsetIdToArtifactIdsCache, childId)
	if err != nil {
	return nil, err
	}
	result = append(result, childArtifactIds...)
	}
	depsetIdToArtifactIdsCache[depsetId] = result
	return result, nil
	} else {
	return nil, fmt.Errorf("undefined input depsetId %d", depsetId)
	}
	}

	func expandPathFragment(id int, pathFragmentsMap map[int]pathFragment) (string, error) {
	labels := []string{}
	currId := id
	// Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node.
	for currId > 0 {
	currFragment, ok := pathFragmentsMap[currId]
	if !ok {
	return "", fmt.Errorf("undefined path fragment id %d", currId)
	}
	labels = append([]string{currFragment.Label}, labels...)
	currId = currFragment.ParentId
	}
	return filepath.Join(labels...), nil
	}