Skip to content

Commit

Permalink
use cquery for resolving files and dependencies
Browse files Browse the repository at this point in the history 
This takes antoher stub at #594 by introducing cquery back in. In
difference to the previous attempt this splits up the resolving of
buildfiles. CQuery does not support buildfiles but without cquery
bazel-watcher could watch files which are not reflected in the build
for e.g. dependencies behind a select statement. In order to avoid
depending on such cases first a cquery is issued which resolves
dependencies of selected targets. These dependencies are then injected
into the buildfiles query in order to get dependent build files.

Signed-off-by: Tobias Kohlbau <[email protected]>
  • Loading branch information
@tobiaskohlbau
tobiaskohlbau committed Oct 25, 2024
1 parent 778a4da commit 4958ed2
Show file tree
Hide file tree
Showing 3 changed files with 186 additions and 43 deletions.
6 changes: 5 additions & 1 deletion internal/bazel/testing/mock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ type MockBazel struct {
cqueryResponse map[string]*analysis.CqueryResult
args []string
startupArgs []string
info map[string]string

buildError error
waitError error
Expand All @@ -51,6 +52,9 @@ func (b *MockBazel) SetStartupArgs(args []string) {
b.startupArgs = args
}

func (b *MockBazel) SetInfo(info map[string]string) {
b.info = info
}
func (b *MockBazel) WriteToStderr(v bool) {
b.actions = append(b.actions, []string{"WriteToStderr", fmt.Sprint(v)})
}
Expand All @@ -59,7 +63,7 @@ func (b *MockBazel) WriteToStdout(v bool) {
}
func (b *MockBazel) Info() (map[string]string, error) {
b.actions = append(b.actions, []string{"Info"})
return map[string]string{}, nil
return b.info, nil
}
func (b *MockBazel) AddQueryResponse(query string, res *blaze_query.QueryResult) {
if b.queryResponse == nil {
Expand Down
150 changes: 115 additions & 35 deletions internal/ibazel/ibazel.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ const (
)

const sourceQuery = "kind('source file', deps(set(%s)))"
const buildQuery = "buildfiles(deps(set(%s)))"
const targetQuery = "deps(set(%s))"
const buildQuery = "buildfiles(set(%s))"

type IBazel struct {
debounceDuration time.Duration
Expand Down Expand Up @@ -311,8 +312,21 @@ func (i *IBazel) iteration(command string, commandToRun runnableCommand, targets
case QUERY:
// Query for which files to watch.
log.Logf("Querying for files to watch...")
i.watchFiles(fmt.Sprintf(buildQuery, joinedTargets), i.buildFileWatcher)
i.watchFiles(fmt.Sprintf(sourceQuery, joinedTargets), i.sourceFileWatcher)

toWatchBuildFiles, err := i.queryForBuildFiles(joinedTargets)
if err != nil {
log.Errorf("Error querying for build files: %v", err)
} else {
i.watchFiles(toWatchBuildFiles, i.buildFileWatcher)
}

toWatchSourceFiles, err := i.queryForSourceFiles(joinedTargets)
if err != nil {
log.Errorf("Error querying for source files: %v", err)
} else {
i.watchFiles(toWatchSourceFiles, i.sourceFileWatcher)
}

i.state = RUN
case DEBOUNCE_RUN:
select {
Expand Down Expand Up @@ -480,63 +494,95 @@ func (i *IBazel) getInfo() (map[string]string, error) {
return res, nil
}

func (i *IBazel) queryForSourceFiles(query string) ([]string, error) {
func (i *IBazel) queryForSourceFiles(targets string) ([]string, error) {
b := i.newBazel()

localRepositories, err := i.realLocalRepositoryPaths()
res, err := b.CQuery(i.cQueryArgs(fmt.Sprintf(sourceQuery, targets))...)
if err != nil {
log.Errorf("Bazel cquery failed: %v", err)
return nil, err
}

res, err := b.Query(i.queryArgs(query)...)
labels := make([]string, 0, len(res.Results))
for _, target := range res.Results {
switch *target.Target.Type {
case blaze_query.Target_SOURCE_FILE:
label := target.Target.SourceFile.GetName()
labels = append(labels, label)
default:
log.Errorf("%v\n", target)
}
}

return i.labelsToWatch(labels)
}

func (i *IBazel) queryForBuildFiles(targets string) ([]string, error) {
b := i.newBazel()

targetRes, err := b.CQuery(i.cQueryArgs(fmt.Sprintf(targetQuery, targets))...)
if err != nil {
log.Errorf("Bazel query failed: %v", err)
log.Errorf("Bazel target query failed: %v", err)
return nil, err
}

workspacePath, err := i.workspaceFinder.FindWorkspace()
localRepositories, err := i.realLocalRepositoryPaths()
if err != nil {
log.Errorf("Error finding workspace: %v", err)
return nil, err
}

toWatch := make([]string, 0, len(res.GetTarget()))
for _, target := range res.GetTarget() {
switch *target.Type {
case blaze_query.Target_SOURCE_FILE:
label := target.GetSourceFile().GetName()
buildTargets := make([]string, 0, len(targetRes.Results))
for _, configuredTarget := range targetRes.Results {
target := configuredTarget.GetTarget()
if *target.Type == blaze_query.Target_RULE {
label := target.GetRule().GetName()
if strings.HasPrefix(label, "@") {
repo, target := parseTarget(label)
if realPath, ok := localRepositories[repo]; ok {
label = strings.Replace(target, ":", string(filepath.Separator), 1)
toWatch = append(toWatch, filepath.Join(realPath, label))
break
repo, _ := parseTarget(label)
if _, ok := localRepositories[repo]; !ok {
continue
}
continue
}
if strings.HasPrefix(label, "//external") {
continue
}

label = strings.Replace(strings.TrimPrefix(label, "//"), ":", string(filepath.Separator), 1)
toWatch = append(toWatch, filepath.Join(workspacePath, label))
break
default:
log.Errorf("%v\n", target)
buildTargets = append(buildTargets, label)
}
}

return toWatch, nil
}
f, err := os.CreateTemp("", "query")
if err != nil {
return nil, fmt.Errorf("failed to create query file: %w", err)
}
defer func() {
f.Close()
os.Remove(f.Name())
}()
_, err = f.WriteString(fmt.Sprintf(buildQuery, strings.Join(buildTargets, " ")))
if err != nil {
return nil, fmt.Errorf("failed to write query file: %w", err)
}

func (i *IBazel) watchFiles(query string, watcher common.Watcher) {
toWatch, err := i.queryForSourceFiles(query)
res, err := b.Query(i.queryArgs(fmt.Sprintf("--query_file=%s", f.Name()))...)
if err != nil {
// If the query fails, just keep watching the same files as before
log.Errorf("Error querying for source files: %v", err)
return
log.Errorf("Bazel query failed: %v", err)
return nil, err
}

labels := make([]string, 0, len(res.GetTarget()))
for _, target := range res.GetTarget() {
switch *target.Type {
case blaze_query.Target_SOURCE_FILE:
label := target.GetSourceFile().GetName()
labels = append(labels, label)
default:
log.Errorf("%v\n", target)
}
}

return i.labelsToWatch(labels)
}

func (i *IBazel) watchFiles(toWatch []string, watcher common.Watcher) {
filesWatched := map[string]struct{}{}
uniqueDirectories := map[string]struct{}{}

Expand All @@ -557,18 +603,52 @@ func (i *IBazel) watchFiles(query string, watcher common.Watcher) {
}

watchList := keys(uniqueDirectories)
err = watcher.UpdateAll(watchList)
err := watcher.UpdateAll(watchList)
if err != nil {
log.Errorf("Error(s) updating watch list:\n %v", err)
}

if len(filesWatched) == 0 {
log.Errorf("Didn't find any files to watch from query %s", query)
log.Errorf("Didn't find any files to watch for")
}

i.filesWatched[watcher] = filesWatched
}

func (i *IBazel) labelsToWatch(labels []string) ([]string, error) {
localRepositories, err := i.realLocalRepositoryPaths()
if err != nil {
return nil, err
}

workspacePath, err := i.workspaceFinder.FindWorkspace()
if err != nil {
log.Errorf("Error finding workspace: %v", err)
return nil, err
}

toWatch := make([]string, 0, len(labels))
for _, label := range labels {
if strings.HasPrefix(label, "@") {
repo, target := parseTarget(label)
if realPath, ok := localRepositories[repo]; ok {
label = strings.Replace(target, ":", string(filepath.Separator), 1)
toWatch = append(toWatch, filepath.Join(realPath, label))
break
}
continue
}
if strings.HasPrefix(label, "//external") {
continue
}

label = strings.Replace(strings.TrimPrefix(label, "//"), ":", string(filepath.Separator), 1)
toWatch = append(toWatch, filepath.Join(workspacePath, label))
}

return toWatch, nil
}

func (i *IBazel) queryArgs(args ...string) []string {
queryArgs := append([]string(nil), args...)

Expand Down
73 changes: 66 additions & 7 deletions internal/ibazel/ibazel_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"bytes"
"fmt"
"os"
"path/filepath"
"reflect"
"runtime"
"runtime/debug"
Expand Down Expand Up @@ -188,8 +189,68 @@ func TestIBazelLoop(t *testing.T) {
log.SetTesting(t)

i, mockBazel := newIBazel(t)
mockBazel.AddQueryResponse("buildfiles(deps(set(//my:target)))", &blaze_query.QueryResult{})
mockBazel.AddQueryResponse("kind('source file', deps(set(//my:target)))", &blaze_query.QueryResult{})

ruleType := blaze_query.Target_RULE
sourceFileType := blaze_query.Target_SOURCE_FILE
target := "//my:target"
sourceDir := t.TempDir()

basePath := filepath.Join(sourceDir, "/path/to/")
if err := os.MkdirAll(basePath, 0o700); err != nil {
t.Errorf("failed to create based path: %v", err)
t.FailNow()
}

buildFilePath := filepath.Join(basePath, "BUILD")
sourceFilePath := filepath.Join(basePath, "foo")

if err := os.WriteFile(buildFilePath, []byte(""), 0o700); err != nil {
t.Errorf("failed to create BUILD file: %v", err)
t.FailNow()
}
if err := os.WriteFile(sourceFilePath, []byte(""), 0o700); err != nil {
t.Errorf("failed to create source file: %v", err)
t.FailNow()
}

mockBazel.AddQueryResponse(fmt.Sprintf("buildfiles(set(%s))", target), &blaze_query.QueryResult{
Target: []*blaze_query.Target{{
Type: &sourceFileType,
SourceFile: &blaze_query.SourceFile{
Name: &buildFilePath,
},
}},
})
mockBazel.AddCQueryResponse(fmt.Sprintf("deps(set(%s))", target), &analysispb.CqueryResult{
Results: []*analysispb.ConfiguredTarget{{
Target: &blaze_query.Target{
Type: &ruleType,
Rule: &blaze_query.Rule{
Name: &target,
},
},
}},
})
mockBazel.AddCQueryResponse(fmt.Sprintf("kind('source file', deps(set(%s)))", target), &analysispb.CqueryResult{
Results: []*analysispb.ConfiguredTarget{{
Target: &blaze_query.Target{
Type: &sourceFileType,
SourceFile: &blaze_query.SourceFile{
Name: &sourceFilePath,
},
},
}},
})

outputBase := t.TempDir()
if err := os.Mkdir(filepath.Join(outputBase, "external"), 0o700); err != nil {
t.Errorf("failed to create external directory: %v", err)
t.FailNow()
}
mockBazel.SetInfo(map[string]string{
"output_base": outputBase,
"install_base": t.TempDir(),
})

// Replace the file watching channel with one that has a buffer.
i.buildFileWatcher = &fakeFSNotifyWatcher{
Expand All @@ -211,7 +272,7 @@ func TestIBazelLoop(t *testing.T) {

i.state = QUERY
step := func() {
i.iteration("demo", command, []string{}, "//my:target")
i.iteration("demo", command, []string{}, target)
}
assertRun := func() {
t.Helper()
Expand All @@ -236,14 +297,12 @@ func TestIBazelLoop(t *testing.T) {

assertState(QUERY)
step()
i.filesWatched[i.buildFileWatcher] = map[string]struct{}{"/path/to/BUILD": {}}
i.filesWatched[i.sourceFileWatcher] = map[string]struct{}{"/path/to/foo": {}}
assertState(RUN)
step() // Actually run the command
assertRun()
assertState(WAIT)
// Source file change.
go func() { i.sourceFileWatcher.Events() <- common.Event{Op: common.Write, Name: "/path/to/foo"} }()
go func() { i.sourceFileWatcher.Events() <- common.Event{Op: common.Write, Name: sourceFilePath} }()
step()
assertState(DEBOUNCE_RUN)
step()
Expand All @@ -253,7 +312,7 @@ func TestIBazelLoop(t *testing.T) {
assertRun()
assertState(WAIT)
// Build file change.
i.buildFileWatcher.Events() <- common.Event{Op: common.Write, Name: "/path/to/BUILD"}
i.buildFileWatcher.Events() <- common.Event{Op: common.Write, Name: buildFilePath}
step()
assertState(DEBOUNCE_QUERY)
// Don't send another event in to test the timer
Expand Down

0 comments on commit 4958ed2

Please sign in to comment.