Handle aquery build statements in a goroutine
Creation of build statements is largely parallelizable because each
action is independent apart from updates/reads to
depsetHashToArtifactPathsCache. Locally resulted in build statements
taking ~.45 seconds on staging mode to ~.02 seconds
Test: CI
Change-Id: Iab00c8394a9eab17353f71230885ff0870e17f24
diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go
index 68fdd68..19a584f 100644
--- a/bazel/aquery_test.go
+++ b/bazel/aquery_test.go
@@ -139,10 +139,10 @@
return
}
actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
- var expectedBuildStatements []BuildStatement
+ var expectedBuildStatements []*BuildStatement
for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} {
expectedBuildStatements = append(expectedBuildStatements,
- BuildStatement{
+ &BuildStatement{
Command: fmt.Sprintf(
"/bin/bash -c 'source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py %s ../sourceroot/bionic/libc/SYSCALLS.TXT > bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S'",
arch, arch),
@@ -487,8 +487,8 @@
}
actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{})
- expectedBuildStatements := []BuildStatement{
- {
+ expectedBuildStatements := []*BuildStatement{
+ &BuildStatement{
Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'",
OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"},
Mnemonic: "Action",
@@ -545,8 +545,8 @@
if err != nil {
t.Errorf("Unexpected error %q", err)
}
- assertBuildStatements(t, []BuildStatement{
- {
+ assertBuildStatements(t, []*BuildStatement{
+ &BuildStatement{
Command: "",
OutputPaths: []string{"foo.runfiles/MANIFEST"},
Mnemonic: "SymlinkTree",
@@ -615,7 +615,7 @@
t.Errorf("dependency ../dep2 expected but not found")
}
- expectedBuildStatement := BuildStatement{
+ expectedBuildStatement := &BuildStatement{
Command: "bogus command",
OutputPaths: []string{"output"},
Mnemonic: "x",
@@ -692,7 +692,7 @@
return
}
- expectedBuildStatement := BuildStatement{
+ expectedBuildStatement := &BuildStatement{
Command: "bogus command",
OutputPaths: []string{"output"},
Mnemonic: "x",
@@ -757,8 +757,8 @@
if err != nil {
t.Errorf("Unexpected error %q", err)
}
- if expected := 1; len(actualBuildStatements) != expected {
- t.Fatalf("Expected %d build statements, got %d", expected, len(actualBuildStatements))
+ if expected := 2; len(actualBuildStatements) != expected {
+ t.Fatalf("Expected %d build statements, got %d %#v", expected, len(actualBuildStatements), actualBuildStatements)
}
expectedDepsetFiles := [][]string{
@@ -783,6 +783,11 @@
if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) {
t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs)
}
+
+ bs = actualBuildStatements[1]
+ if bs != nil {
+ t.Errorf("Expected nil action for skipped")
+ }
}
// Returns the contents of given depsets in concatenated post order.
@@ -856,8 +861,8 @@
t.Errorf("Unexpected error %q", err)
}
- expectedBuildStatements := []BuildStatement{
- {
+ expectedBuildStatements := []*BuildStatement{
+ &BuildStatement{
Command: "mkdir -p one/symlink_subdir && " +
"rm -f one/symlink_subdir/symlink && " +
"ln -sf $PWD/one/file_subdir/file one/symlink_subdir/symlink",
@@ -904,8 +909,8 @@
t.Errorf("Unexpected error %q", err)
}
- expectedBuildStatements := []BuildStatement{
- {
+ expectedBuildStatements := []*BuildStatement{
+ &BuildStatement{
Command: "mkdir -p 'one/symlink subdir' && " +
"rm -f 'one/symlink subdir/symlink' && " +
"ln -sf $PWD/'one/file subdir/file' 'one/symlink subdir/symlink'",
@@ -1014,8 +1019,8 @@
t.Errorf("Unexpected error %q", err)
}
- expectedBuildStatements := []BuildStatement{
- {
+ expectedBuildStatements := []*BuildStatement{
+ &BuildStatement{
Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > template_file && " +
"chmod a+x template_file'",
OutputPaths: []string{"template_file"},
@@ -1084,8 +1089,8 @@
if err != nil {
t.Errorf("Unexpected error %q", err)
}
- assertBuildStatements(t, []BuildStatement{
- {
+ assertBuildStatements(t, []*BuildStatement{
+ &BuildStatement{
OutputPaths: []string{"foo.manifest"},
Mnemonic: "FileWrite",
FileContents: "file data\n",
@@ -1122,8 +1127,8 @@
if err != nil {
t.Errorf("Unexpected error %q", err)
}
- assertBuildStatements(t, []BuildStatement{
- {
+ assertBuildStatements(t, []*BuildStatement{
+ &BuildStatement{
OutputPaths: []string{"foo.manifest"},
Mnemonic: "SourceSymlinkManifest",
SymlinkPaths: []string{},
@@ -1142,7 +1147,7 @@
// Asserts that the given actual build statements match the given expected build statements.
// Build statement equivalence is determined using buildStatementEquals.
-func assertBuildStatements(t *testing.T, expected []BuildStatement, actual []BuildStatement) {
+func assertBuildStatements(t *testing.T, expected []*BuildStatement, actual []*BuildStatement) {
t.Helper()
if len(expected) != len(actual) {
t.Errorf("expected %d build statements, but got %d,\n expected: %#v,\n actual: %#v",
@@ -1150,8 +1155,13 @@
return
}
type compareFn = func(i int, j int) bool
- byCommand := func(slice []BuildStatement) compareFn {
+ byCommand := func(slice []*BuildStatement) compareFn {
return func(i int, j int) bool {
+ if slice[i] == nil {
+ return false
+ } else if slice[j] == nil {
+ return false
+ }
return slice[i].Command < slice[j].Command
}
}
@@ -1167,7 +1177,10 @@
}
}
-func buildStatementEquals(first BuildStatement, second BuildStatement) string {
+func buildStatementEquals(first *BuildStatement, second *BuildStatement) string {
+ if (first == nil) != (second == nil) {
+ return "Nil"
+ }
if first.Mnemonic != second.Mnemonic {
return "Mnemonic"
}