diff --git a/Makefile b/Makefile index 24597829b..2f908cfb8 100644 --- a/Makefile +++ b/Makefile @@ -154,7 +154,7 @@ define assert_clean fi endef -.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto demo-requests deps docs-build docs-serve e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator integration-test-submitqueue-validator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache test-race tidy tidy-bazel tidy-go web-build web-check web-e2e-test web-image-load web-install web-proto help +.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug build-tangofake-linux check-gazelle check-mocks check-tidy clean clean-proto demo-requests deps docs-build docs-serve e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator integration-test-submitqueue-validator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache test-race tidy tidy-bazel tidy-go web-build web-check web-e2e-test web-image-load web-install web-proto help build: ## Build all services and examples @@ -224,6 +224,14 @@ build-stovepipe-linux-debug: ## Build Stovepipe Linux debug binary for Docker/de cp -f bazel-bin/service/stovepipe/server/stovepipe .docker-bin/stovepipe-debug @echo "Stovepipe Linux debug binary ready at .docker-bin/stovepipe-debug" +build-tangofake-linux: ## Build the fake Tango server's Linux binary for Docker + @echo "Building the fake Tango server Linux binary for Docker..." + @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //service/tangofake/server:server + @mkdir -p .docker-bin + @cp -f bazel-bin/service/tangofake/server/server_/server .docker-bin/tangofake 2>/dev/null || \ + cp -f bazel-bin/service/tangofake/server/server .docker-bin/tangofake + @echo "Fake Tango server Linux binary ready at .docker-bin/tangofake" + check-gazelle: ## Check BUILD.bazel files are up to date @echo "Running Gazelle to check BUILD files..." @$(BAZEL) run //:gazelle diff --git a/service/README.md b/service/README.md index e55cfc551..b3a02cb9a 100644 --- a/service/README.md +++ b/service/README.md @@ -7,6 +7,7 @@ Each domain has its own subdirectory with a dedicated README: - [`submitqueue/`](submitqueue/README.md) — the multi-service SubmitQueue domain (Gateway + Orchestrator). - [`stovepipe/`](stovepipe/README.md) — the single-service Stovepipe domain (ingest → process → build → buildsignal → record). - [`runway/`](runway/README.md) — the single-service Runway merge execution service. +- [`tangofake/`](tangofake/README.md) — a fake Tango server for running the Tango conflict analyzer without a build system. ## Services @@ -16,6 +17,7 @@ Each domain has its own subdirectory with a dedicated README: | **SubmitQueue Orchestrator** | 8082 | `submitqueue` | `Ping` (+ consumes start, cancel, validate, merge-conflict-check-signal, batch, dependency-analysis, speculate, build, buildsignal, submitqueue-land, merge-signal, conclude, submitqueue-hook, and paired DLQ topics) | MySQL app + queue | | **Stovepipe** | 8083 | `stovepipe` | `Ping`, `Ingest` (+ consumes process, build, buildsignal, record, stovepipe-hook, and paired DLQ topics) | MySQL storage + queue | | **Runway** | 8086 | `runway` | `Ping` (+ consumes merge-conflict-check & runway-merge topics) | MySQL queue | +| **Fake Tango** | free port, logged | none | Tango's `GetChangedTargets` | a git repository of `TARGET` files | Ports above are the `go run` defaults; under Docker Compose each server listens on `:8080` inside its container and is published on a random ephemeral host port. `make local-submitqueue-ps` is the only status target, and it prints the SubmitQueue mappings. `make local-stovepipe-start` and `make local-runway-start` print their ports when those stacks come up. @@ -37,9 +39,11 @@ service/ │ ├── docker-compose.debug.yml # Debug variant with delve │ ├── server/ # Stovepipe gRPC server + Dockerfile + compose │ └── client/ # Stovepipe ping client -└── runway/ - ├── server/ # Runway gRPC server + Dockerfile + compose - └── client/ # Runway ping client +├── runway/ +│ ├── server/ # Runway gRPC server + Dockerfile + compose +│ └── client/ # Runway ping client +└── tangofake/ + └── server/ # Fake Tango gRPC server + Dockerfile + compose ``` ## Running diff --git a/service/tangofake/BUILD.bazel b/service/tangofake/BUILD.bazel new file mode 100644 index 000000000..a6edfba41 --- /dev/null +++ b/service/tangofake/BUILD.bazel @@ -0,0 +1,37 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["tangofake.go"], + importpath = "github.com/uber/submitqueue/service/tangofake", + visibility = [ + "//service/tangofake:__subpackages__", + "//test:__subpackages__", + ], + deps = [ + "//platform/extension/gitworkspace:go_default_library", + "@com_github_uber_tango//tangopb:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["tangofake_test.go"], + # The pinned git, for building the repositories the tests read. + data = ["@git"], + embed = [":go_default_library"], + env = { + "SUBMITQUEUE_TEST_GIT": "$(location @git//:git)", + }, + deps = [ + "//platform/extension/gitworkspace/local:go_default_library", + "//platform/git/exectest:go_default_library", + "//submitqueue/extension/conflict/tango:go_default_library", + "//submitqueue/extension/conflict/tango/tangoclient:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + "@com_github_uber_tango//tangopb:go_default_library", + "@org_uber_go_yarpc//:go_default_library", + "@org_uber_go_yarpc//transport/grpc:go_default_library", + ], +) diff --git a/service/tangofake/README.md b/service/tangofake/README.md new file mode 100644 index 000000000..b6555fd14 --- /dev/null +++ b/service/tangofake/README.md @@ -0,0 +1,14 @@ +# Fake Tango server + +A runnable stand-in for a Tango server, for running the SubmitQueue Tango conflict analyzer without a build system. It serves Tango's real gRPC protocol and derives the changed targets from `TARGET` files in a git repository, so the analyzer exercises the same client path it uses against a real Tango. The target-file format and how a change maps to targets are described in the package doc of [`tangofake.go`](tangofake.go). + +## Running it + +It runs as a container like the other services here. Build the Linux binary with `make build-tangofake-linux`, then bring it up with [`server/docker-compose.yml`](server/docker-compose.yml), pointing `SQ_TANGOFAKE_REPO_DIR` at a git repository that holds the `TARGET` files. The server listens on port 8080 in the container. + +Outside a container, `bazel run //service/tangofake/server -- -repo ` serves the same repository on a free port, which it logs. + +## Flags + +- `-repo` is the path to the git repository holding the `TARGET` files. It is required, and the repository must contain every commit a change URI names. +- `-address` is the gRPC listen address and defaults to `127.0.0.1:0`. diff --git a/service/tangofake/server/BUILD.bazel b/service/tangofake/server/BUILD.bazel new file mode 100644 index 000000000..eb537c720 --- /dev/null +++ b/service/tangofake/server/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/uber/submitqueue/service/tangofake/server", + visibility = ["//visibility:private"], + deps = [ + "//platform/extension/gitworkspace/local:go_default_library", + "//service/tangofake:go_default_library", + "@com_github_uber_tango//tangopb:go_default_library", + "@org_uber_go_yarpc//:go_default_library", + "@org_uber_go_yarpc//transport/grpc:go_default_library", + ], +) + +go_binary( + name = "server", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/service/tangofake/server/Dockerfile b/service/tangofake/server/Dockerfile new file mode 100644 index 000000000..a30103cd1 --- /dev/null +++ b/service/tangofake/server/Dockerfile @@ -0,0 +1,15 @@ +FROM debian:bookworm-slim + +# git is a runtime dependency: the fake reads targets out of a repository by +# running it. +RUN apt-get update && apt-get install -y ca-certificates git && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /app && chmod 0755 /app +WORKDIR /app + +# Built via: make build-tangofake-linux +COPY .docker-bin/tangofake ./tangofake +RUN chmod 0555 ./tangofake + +EXPOSE 8080 + +CMD ["/app/tangofake"] diff --git a/service/tangofake/server/docker-compose.yml b/service/tangofake/server/docker-compose.yml new file mode 100644 index 000000000..a47b7c1c1 --- /dev/null +++ b/service/tangofake/server/docker-compose.yml @@ -0,0 +1,21 @@ +# Docker Compose for the fake Tango server on its own. +# +# IMPORTANT: Before running compose, build the Linux binary: +# make build-tangofake-linux +# +# Quick start, pointing it at a git repository that holds TARGET files: +# SQ_TANGOFAKE_REPO_DIR=/path/to/repo REPO_ROOT=$PWD \ +# docker compose -f service/tangofake/server/docker-compose.yml up --build + +services: + tangofake-service: + image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-tangofake-service:latest + build: + context: ${REPO_ROOT} + dockerfile: service/tangofake/server/Dockerfile + user: "${SQ_CONTAINER_USER:-0:0}" + command: ["/app/tangofake", "-repo", "/var/tangofake/repo", "-address", "0.0.0.0:8080"] + ports: + - "8080" # Random ephemeral port to avoid conflicts + volumes: + - ${SQ_TANGOFAKE_REPO_DIR:?set SQ_TANGOFAKE_REPO_DIR to the repository holding the TARGET files}:/var/tangofake/repo:ro diff --git a/service/tangofake/server/main.go b/service/tangofake/server/main.go new file mode 100644 index 000000000..0cbbb8527 --- /dev/null +++ b/service/tangofake/server/main.go @@ -0,0 +1,74 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command server serves a Tango server over gRPC that derives changed +// targets from the TARGET files of a git repository (see package tangofake), +// for running the SubmitQueue demo without a build system. +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "os/signal" + "syscall" + + "github.com/uber/tango/tangopb" + "go.uber.org/yarpc" + "go.uber.org/yarpc/transport/grpc" + + "github.com/uber/submitqueue/platform/extension/gitworkspace/local" + "github.com/uber/submitqueue/service/tangofake" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + repo := flag.String("repo", "", "path to the git repository holding the TARGET files") + address := flag.String("address", "127.0.0.1:0", "address to listen on for gRPC; port 0 picks a free port, which is logged") + flag.Parse() + if *repo == "" { + return fmt.Errorf("-repo is required") + } + + listener, err := net.Listen("tcp", *address) + if err != nil { + return fmt.Errorf("failed to listen on %s: %w", *address, err) + } + + transport := grpc.NewTransport() + dispatcher := yarpc.NewDispatcher(yarpc.Config{ + Name: "tango", + Inbounds: yarpc.Inbounds{transport.NewInbound(listener)}, + }) + dispatcher.Register(tangopb.BuildTangoYARPCProcedures(tangofake.New(tangofake.Params{ + Workspaces: local.NewFactory(local.Params{Dir: *repo}), + }))) + if err := dispatcher.Start(); err != nil { + return fmt.Errorf("failed to start the server: %w", err) + } + defer func() { _ = dispatcher.Stop() }() + + log.Printf("fake tango serving %s on %s", *repo, listener.Addr()) + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + <-stop + return nil +} diff --git a/service/tangofake/tangofake.go b/service/tangofake/tangofake.go new file mode 100644 index 000000000..db8111c81 --- /dev/null +++ b/service/tangofake/tangofake.go @@ -0,0 +1,358 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tangofake is a Tango server that answers GetChangedTargets from the +// contents of a git repository instead of a build graph, so a demo or a test can +// exercise the real wire protocol without a build system. +// +// A directory holding a file named TARGET is a target, labelled // +// (// for the repository root). Each non-blank line of the file that does not +// start with # is the label of a target it depends on; a dependency that names +// no target is ignored. The graph a change is read against holds the edges of +// the base and of every change head. A change affects the nearest target above +// each file it touches, at distance 0, and every target that depends on one, at +// its distance along the dependency edges. Adding the TARGET file makes a target +// new, and deleting it makes the target deleted. +// +// A change URI is resolved by the commit SHA in its last path segment, which +// both the git:// and github:// forms end in, so the repository must hold that +// commit. The request's remote and strategy are not consulted. +package tangofake + +import ( + "fmt" + "path" + "slices" + "sort" + "strings" + + "github.com/uber/tango/tangopb" + + "github.com/uber/submitqueue/platform/extension/gitworkspace" +) + +// targetFile is the name of the file that makes its directory a target. +const targetFile = "TARGET" + +// Params are the dependencies of a Server. +type Params struct { + // Workspaces opens the workspace of the repository holding the targets. + Workspaces gitworkspace.Factory +} + +type server struct { + Params +} + +// New returns a Tango server that serves GetChangedTargets from the repository +// in params. The other methods return an error. +func New(params Params) tangopb.TangoYARPCServer { + return &server{Params: params} +} + +// graph maps a target label to the labels it depends on. +type graph map[string][]string + +func (s *server) GetTargetGraph(*tangopb.GetTargetGraphRequest, tangopb.TangoServiceGetTargetGraphYARPCServer) error { + return fmt.Errorf("GetTargetGraph is not implemented by the fake server") +} + +func (s *server) GetChangedTargetGraph(*tangopb.GetChangedTargetGraphRequest, tangopb.TangoServiceGetChangedTargetGraphYARPCServer) error { + return fmt.Errorf("GetChangedTargetGraph is not implemented by the fake server") +} + +func (s *server) GetChangedTargets(request *tangopb.GetChangedTargetsRequest, stream tangopb.TangoServiceGetChangedTargetsYARPCServer) error { + base := request.GetSecondRevision().GetBaseSha() + if base == "" { + return fmt.Errorf("second revision has no base_sha") + } + var heads []string + for _, r := range request.GetSecondRevision().GetRequests() { + heads = append(heads, r.GetUrl()) + } + + workspace, err := s.Workspaces.For(stream.Context(), "") + if err != nil { + return fmt.Errorf("failed to open a workspace: %w", err) + } + defer workspace.Close() + + changed, err := changedTargets(workspace, base, heads) + if err != nil { + return err + } + + ids := make(map[string]int32, len(changed.labels)) + for i, label := range changed.labels { + ids[label] = int32(i + 1) + } + targets := make([]*tangopb.ChangedTarget, 0, len(changed.labels)) + mapping := make(map[int32]string, len(changed.labels)) + for _, label := range changed.labels { + mapping[ids[label]] = label + target := &tangopb.ChangedTarget{ChangeType: changed.kinds[label], Distance: changed.distances[label]} + if node := optimized(ids, label, changed.deps); changed.kinds[label] != tangopb.CHANGE_TYPE_DELETED { + target.NewTarget = node + } + if changed.kinds[label] != tangopb.CHANGE_TYPE_NEW { + target.OldTarget = optimized(ids, label, changed.deps) + } + targets = append(targets, target) + } + + if err := stream.Send(&tangopb.GetChangedTargetsResponse{ + Item: &tangopb.GetChangedTargetsResponse_ChangedTargets{ + ChangedTargets: &tangopb.ChangedTargets{ChangedTargets: targets}, + }, + }); err != nil { + return fmt.Errorf("failed to send changed targets: %w", err) + } + return stream.Send(&tangopb.GetChangedTargetsResponse{ + Item: &tangopb.GetChangedTargetsResponse_Metadata{ + Metadata: &tangopb.Metadata{TargetIdMapping: mapping}, + }, + }) +} + +func optimized(ids map[string]int32, label string, deps graph) *tangopb.OptimizedTarget { + node := &tangopb.OptimizedTarget{Id: ids[label]} + for _, dep := range deps[label] { + if id, ok := ids[dep]; ok { + node.DirectDependencies = append(node.DirectDependencies, id) + } + } + return node +} + +type changeSet struct { + // labels lists the changed targets in sorted order. + labels []string + kinds map[string]tangopb.ChangeType + distances map[string]int32 + deps graph +} + +// changedTargets computes the targets that the changes, each named by a URI +// ending in its head commit, affect on top of base. +func changedTargets(workspace gitworkspace.Workspace, base string, changeURIs []string) (changeSet, error) { + union, err := readGraph(workspace, base) + if err != nil { + return changeSet{}, err + } + + kinds := map[string]tangopb.ChangeType{} + var touched []touchedFile + for _, uri := range changeURIs { + head, err := commitOf(uri) + if err != nil { + return changeSet{}, err + } + headGraph, err := readGraph(workspace, head) + if err != nil { + return changeSet{}, err + } + for label, deps := range headGraph { + if _, existed := union[label]; !existed { + kinds[label] = tangopb.CHANGE_TYPE_NEW + } + union[label] = withDeps(union[label], deps) + } + files, err := changedFiles(workspace, base, head) + if err != nil { + return changeSet{}, err + } + touched = append(touched, files...) + } + + for _, file := range touched { + if path.Base(file.path) == targetFile && file.deleted { + label := labelOf(path.Dir(file.path)) + if _, ok := union[label]; ok && kinds[label] != tangopb.CHANGE_TYPE_NEW { + kinds[label] = tangopb.CHANGE_TYPE_DELETED + } + } + } + for _, file := range touched { + if label, ok := owner(union, file.path); ok { + if _, set := kinds[label]; !set { + kinds[label] = tangopb.CHANGE_TYPE_CHANGED + } + } + } + + distances := map[string]int32{} + var frontier []string + for label := range kinds { + distances[label] = 0 + frontier = append(frontier, label) + } + dependents := map[string][]string{} + for label, deps := range union { + for _, dep := range deps { + dependents[dep] = append(dependents[dep], label) + } + } + for distance := int32(1); len(frontier) > 0; distance++ { + var next []string + for _, label := range frontier { + for _, dependent := range dependents[label] { + if _, seen := distances[dependent]; seen { + continue + } + distances[dependent] = distance + kinds[dependent] = tangopb.CHANGE_TYPE_CHANGED + next = append(next, dependent) + } + } + frontier = next + } + + labels := make([]string, 0, len(distances)) + for label := range distances { + labels = append(labels, label) + } + sort.Strings(labels) + return changeSet{labels: labels, kinds: kinds, distances: distances, deps: union}, nil +} + +// withDeps returns deps plus each of extra it does not already hold, so an edge +// that only the base or only a change has still carries a change to its target. +func withDeps(deps, extra []string) []string { + merged := append([]string(nil), deps...) + for _, dep := range extra { + if !slices.Contains(merged, dep) { + merged = append(merged, dep) + } + } + return merged +} + +type touchedFile struct { + path string + deleted bool +} + +// owner returns the label of the nearest target at or above file's directory. +func owner(targets graph, file string) (string, bool) { + for dir := path.Dir(file); ; dir = path.Dir(dir) { + label := labelOf(dir) + if _, ok := targets[label]; ok { + return label, true + } + if dir == "." { + return "", false + } + } +} + +func labelOf(dir string) string { + if dir == "." { + return "//" + } + return "//" + dir +} + +func commitOf(changeURI string) (string, error) { + sha := changeURI[strings.LastIndex(changeURI, "/")+1:] + if len(sha) != 40 && len(sha) != 64 { + return "", fmt.Errorf("change URI %q does not end in a full commit SHA", changeURI) + } + for _, c := range sha { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return "", fmt.Errorf("change URI %q does not end in a full commit SHA", changeURI) + } + } + return sha, nil +} + +func run(workspace gitworkspace.Workspace, commands []gitworkspace.Command) ([]gitworkspace.Output, error) { + outputs, err := workspace.Exec(commands) + if err != nil { + return nil, fmt.Errorf("failed to run git: %w", err) + } + for _, out := range outputs { + if out.ExitCode != 0 { + return nil, fmt.Errorf("git %s failed with exit code %d: %s", out.Alias, out.ExitCode, strings.TrimSpace(out.Stderr)) + } + } + return outputs, nil +} + +// readGraph returns the targets of the tree at commit. +func readGraph(workspace gitworkspace.Workspace, commit string) (graph, error) { + listing, err := run(workspace, []gitworkspace.Command{ + {Alias: "ls-tree", Bin: "git", Args: []string{"ls-tree", "-r", "--name-only", commit}}, + }) + if err != nil { + return nil, err + } + var commands []gitworkspace.Command + var dirs []string + for _, file := range strings.Split(listing[0].Stdout, "\n") { + if file != "" && path.Base(file) == targetFile { + dirs = append(dirs, path.Dir(file)) + commands = append(commands, gitworkspace.Command{ + Alias: file, Bin: "git", Args: []string{"show", commit + ":" + file}, + }) + } + } + + targets := graph{} + if len(commands) == 0 { + return targets, nil + } + outputs, err := run(workspace, commands) + if err != nil { + return nil, err + } + for i, out := range outputs { + var deps []string + for _, line := range strings.Split(out.Stdout, "\n") { + if line = strings.TrimSpace(line); line != "" && !strings.HasPrefix(line, "#") { + deps = append(deps, line) + } + } + targets[labelOf(dirs[i])] = deps + } + for label, deps := range targets { + known := deps[:0] + for _, dep := range deps { + if _, ok := targets[dep]; ok { + known = append(known, dep) + } + } + targets[label] = known + } + return targets, nil +} + +// changedFiles lists the files head changes relative to where it diverged from +// base. +func changedFiles(workspace gitworkspace.Workspace, base, head string) ([]touchedFile, error) { + outputs, err := run(workspace, []gitworkspace.Command{ + {Alias: "diff", Bin: "git", Args: []string{"diff", "--name-status", "--no-renames", base + "..." + head}}, + }) + if err != nil { + return nil, err + } + var files []touchedFile + for _, line := range strings.Split(outputs[0].Stdout, "\n") { + status, file, ok := strings.Cut(line, "\t") + if !ok { + continue + } + files = append(files, touchedFile{path: file, deleted: status == "D"}) + } + return files, nil +} diff --git a/service/tangofake/tangofake_test.go b/service/tangofake/tangofake_test.go new file mode 100644 index 000000000..9937dec68 --- /dev/null +++ b/service/tangofake/tangofake_test.go @@ -0,0 +1,254 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tangofake + +import ( + "context" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/uber/tango/tangopb" + "go.uber.org/yarpc" + "go.uber.org/yarpc/transport/grpc" + + "github.com/uber/submitqueue/platform/extension/gitworkspace/local" + gitexectest "github.com/uber/submitqueue/platform/git/exectest" + "github.com/uber/submitqueue/submitqueue/extension/conflict/tango" + "github.com/uber/submitqueue/submitqueue/extension/conflict/tango/tangoclient" +) + +type fakeStream struct { + tangopb.TangoServiceGetChangedTargetsYARPCServer + sent []*tangopb.GetChangedTargetsResponse +} + +func (s *fakeStream) Context() context.Context { return context.Background() } + +func (s *fakeStream) Send(response *tangopb.GetChangedTargetsResponse, _ ...yarpc.StreamOption) error { + s.sent = append(s.sent, response) + return nil +} + +type testRepo struct { + t *testing.T + git string + work string +} + +func (r *testRepo) run(args ...string) string { + r.t.Helper() + cmd := exec.Command(r.git, args...) + cmd.Dir = r.work + out, err := cmd.CombinedOutput() + require.NoError(r.t, err, "git %v: %s", args, out) + return strings.TrimSpace(string(out)) +} + +func (r *testRepo) write(file, content string) { + r.t.Helper() + full := filepath.Join(r.work, file) + require.NoError(r.t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(r.t, os.WriteFile(full, []byte(content), 0o644)) +} + +func (r *testRepo) commit(message string) string { + r.t.Helper() + r.run("add", "-A") + r.run("commit", "--no-gpg-sign", "-m", message) + return r.run("rev-parse", "HEAD") +} + +// newTestRepo commits three targets: //app depends on //lib, which depends on +// //base. +func newTestRepo(t *testing.T) (*testRepo, string) { + r := &testRepo{t: t, git: gitexectest.Git(t), work: t.TempDir()} + r.run("init", "-b", "main") + r.run("config", "user.name", "Test") + r.run("config", "user.email", "test@example.invalid") + r.write("base/TARGET", "") + r.write("base/base.txt", "a\n") + r.write("lib/TARGET", "//base\n") + r.write("lib/lib.txt", "a\n") + r.write("app/TARGET", "# comment\n//lib\n//missing\n") + r.write("app/app.txt", "a\n") + r.write("docs/readme.txt", "a\n") + return r, r.commit("seed") +} + +func TestGetChangedTargets(t *testing.T) { + tests := []struct { + name string + change func(r *testRepo) + want map[string]struct { + kind tangopb.ChangeType + distance int32 + } + }{ + { + name: "a change to the lowest target reaches its dependents by distance", + change: func(r *testRepo) { r.write("base/base.txt", "b\n") }, + want: map[string]struct { + kind tangopb.ChangeType + distance int32 + }{ + "//base": {tangopb.CHANGE_TYPE_CHANGED, 0}, + "//lib": {tangopb.CHANGE_TYPE_CHANGED, 1}, + "//app": {tangopb.CHANGE_TYPE_CHANGED, 2}, + }, + }, + { + name: "a change to a leaf target reaches nothing else", + change: func(r *testRepo) { r.write("app/app.txt", "b\n") }, + want: map[string]struct { + kind tangopb.ChangeType + distance int32 + }{"//app": {tangopb.CHANGE_TYPE_CHANGED, 0}}, + }, + { + name: "a file outside every target changes nothing", + change: func(r *testRepo) { r.write("docs/readme.txt", "b\n") }, + }, + { + name: "a new target is new", + change: func(r *testRepo) { + r.write("extra/TARGET", "//lib\n") + r.write("extra/extra.txt", "a\n") + }, + want: map[string]struct { + kind tangopb.ChangeType + distance int32 + }{"//extra": {tangopb.CHANGE_TYPE_NEW, 0}}, + }, + { + name: "a deleted target is deleted and its dependents change", + change: func(r *testRepo) { + require.NoError(r.t, os.RemoveAll(filepath.Join(r.work, "lib"))) + }, + want: map[string]struct { + kind tangopb.ChangeType + distance int32 + }{ + "//lib": {tangopb.CHANGE_TYPE_DELETED, 0}, + "//app": {tangopb.CHANGE_TYPE_CHANGED, 1}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo, base := newTestRepo(t) + tt.change(repo) + head := repo.commit("change") + + stream := &fakeStream{} + err := New(Params{Workspaces: local.NewFactory(local.Params{Dir: repo.work})}).GetChangedTargets( + &tangopb.GetChangedTargetsRequest{SecondRevision: &tangopb.BuildDescription{ + BaseSha: base, + Requests: []*tangopb.Request{{Url: "git://example.invalid/org/repo/refs%2Fheads%2Fmain/" + head}}, + }}, stream) + require.NoError(t, err) + require.Len(t, stream.sent, 2, "targets, then metadata") + + labels := stream.sent[1].GetMetadata().GetTargetIdMapping() + got := map[string]struct { + kind tangopb.ChangeType + distance int32 + }{} + for _, target := range stream.sent[0].GetChangedTargets().GetChangedTargets() { + node := target.GetNewTarget() + if target.GetChangeType() == tangopb.CHANGE_TYPE_DELETED { + node = target.GetOldTarget() + } + got[labels[node.GetId()]] = struct { + kind tangopb.ChangeType + distance int32 + }{target.GetChangeType(), target.GetDistance()} + } + if len(tt.want) == 0 { + assert.Empty(t, got) + return + } + assert.Equal(t, tt.want, got) + }) + } +} + +func TestGetChangedTargets_Rejects(t *testing.T) { + repo, base := newTestRepo(t) + server := New(Params{Workspaces: local.NewFactory(local.Params{Dir: repo.work})}) + + tests := []struct { + name string + request *tangopb.GetChangedTargetsRequest + }{ + {name: "no base", request: &tangopb.GetChangedTargetsRequest{SecondRevision: &tangopb.BuildDescription{}}}, + { + name: "change URI without a commit", + request: &tangopb.GetChangedTargetsRequest{SecondRevision: &tangopb.BuildDescription{ + BaseSha: base, Requests: []*tangopb.Request{{Url: "git://example.invalid/org/repo/refs%2Fheads%2Fmain/main"}}, + }}, + }, + { + name: "commit the repository does not hold", + request: &tangopb.GetChangedTargetsRequest{SecondRevision: &tangopb.BuildDescription{ + BaseSha: base, Requests: []*tangopb.Request{{Url: "git://example.invalid/org/repo/refs%2Fheads%2Fmain/" + strings.Repeat("a", 40)}}, + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Error(t, server.GetChangedTargets(tt.request, &fakeStream{})) + }) + } +} + +func TestServedOverGRPC(t *testing.T) { + repo, base := newTestRepo(t) + repo.write("base/base.txt", "b\n") + head := repo.commit("change") + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + dispatcher := yarpc.NewDispatcher(yarpc.Config{ + Name: "tango", + Inbounds: yarpc.Inbounds{grpc.NewTransport().NewInbound(listener)}, + }) + dispatcher.Register(tangopb.BuildTangoYARPCProcedures(New(Params{Workspaces: local.NewFactory(local.Params{Dir: repo.work})}))) + require.NoError(t, dispatcher.Start()) + t.Cleanup(func() { _ = dispatcher.Stop() }) + + connection, err := tangoclient.Dial(listener.Addr().String()) + require.NoError(t, err) + t.Cleanup(func() { _ = connection.Close() }) + client, err := connection.ChangedTargetsClient("org/repo", "") + require.NoError(t, err) + + got, err := client.ChangedTargets(context.Background(), tango.ChangedTargetsRequest{ + BaseRevision: base, + ChangeURIs: []string{"git://example.invalid/org/repo/refs%2Fheads%2Fmain/" + head}, + }) + require.NoError(t, err) + assert.ElementsMatch(t, []tango.LabelDistance{ + {Label: "//base", Distance: 0}, + {Label: "//lib", Distance: 1}, + {Label: "//app", Distance: 2}, + }, got) +} diff --git a/submitqueue/extension/conflict/tango/tangoclient/BUILD.bazel b/submitqueue/extension/conflict/tango/tangoclient/BUILD.bazel index 638f7720e..d2f2e0eb8 100644 --- a/submitqueue/extension/conflict/tango/tangoclient/BUILD.bazel +++ b/submitqueue/extension/conflict/tango/tangoclient/BUILD.bazel @@ -9,6 +9,7 @@ go_library( importpath = "github.com/uber/submitqueue/submitqueue/extension/conflict/tango/tangoclient", visibility = [ "//service/submitqueue/orchestrator:__subpackages__", + "//service/tangofake:__pkg__", "//submitqueue/extension/conflict/tango:__subpackages__", "//test/integration/submitqueue/extension/conflict/tango:__subpackages__", ], diff --git a/submitqueue/extension/conflict/tango/tangoclient/tangoclient.go b/submitqueue/extension/conflict/tango/tangoclient/tangoclient.go index 7452d4a53..72447b0cc 100644 --- a/submitqueue/extension/conflict/tango/tangoclient/tangoclient.go +++ b/submitqueue/extension/conflict/tango/tangoclient/tangoclient.go @@ -13,8 +13,8 @@ // limitations under the License. // Package tangoclient implements tango.ChangedTargetsClient over the Tango -// service's YARPC client. It is the only package that imports Tango's -// generated types. +// service's YARPC client. It is the only production package that imports +// Tango's generated types. package tangoclient import (