Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion cmd/workspace/up/detach.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ func (cmd *UpCmd) openTask() (*task.Task, error) {
}
t := store.Open(cmd.taskID)
if err := t.HoldWorkerLock(); err != nil {
traceDetachedAdmission("worker lock")
return nil, err
}
verifiedOwner, err := waitForTaskLaunch(t, store)
Expand Down Expand Up @@ -184,6 +185,7 @@ func waitForTaskLaunch(t *task.Task, store *task.Store) (bool, error) {
for {
state, err := readWorkerLaunchState(t, store)
if err != nil {
traceDetachedAdmission("reading launch state")
return progress.verifiedOwner, err
}
ready, err := advanceTaskLaunch(t, store, state, &progress)
Expand All @@ -195,6 +197,7 @@ func waitForTaskLaunch(t *task.Task, store *task.Store) (bool, error) {
}
select {
case <-deadline.C:
traceDetachedAdmission("waiting for launch commit")
return progress.verifiedOwner, errors.New(
"timed out waiting for detached worker launch metadata",
)
Expand Down Expand Up @@ -223,6 +226,7 @@ func advanceTaskLaunch(
owner, err := verifyWorkerTaskOwner(state)
progress.verifiedOwner = owner
if err != nil {
traceDetachedAdmission("verifying owner")
return false, err
}
if !state.LaunchPending {
Expand All @@ -233,17 +237,29 @@ func advanceTaskLaunch(
return false, nil
}
if err := retainWorkerLaunchLease(&progress.leaseRetained); err != nil {
traceDetachedAdmission("retaining launch lease")
return false, err
}
if err := checkWorkerLauncher(t, store); err != nil {
traceDetachedAdmission("checking launcher")
return false, err
}
if state.PID != 0 && !state.WorkerReady {
return false, t.WorkerLaunchReady(os.Getpid())
if err := t.WorkerLaunchReady(os.Getpid()); err != nil {
traceDetachedAdmission("acknowledging worker")
return false, err
}
return false, nil
}
return false, nil
}

func traceDetachedAdmission(stage string) {
if os.Getenv("DEVSY_DETACHED_DIAGNOSTICS") == "1" {
fmt.Fprintln(os.Stderr, "detached worker admission failed:", stage)
}
}

// The worker lock excludes concurrent invocations; only the saved kernel identity
// authorizes this invocation to acknowledge launch or persist startup failures.
func verifyWorkerTaskOwner(state *task.State) (bool, error) {
Expand Down
1 change: 1 addition & 0 deletions e2e/tests/ide/vscode_settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ var _ = ginkgo.Describe("VS Code Settings JSONC Handling", ginkgo.Label("ide"),
err = os.MkdirAll(settingsDir, 0o750)
framework.ExpectNoError(err)

framework.SetEnvForSpec("XDG_CONFIG_HOME", filepath.Join(tmpHome, ".config"))
framework.SetEnvForSpec("HOME", tmpHome)
})

Expand Down
61 changes: 61 additions & 0 deletions e2e/tests/ssh/detached_diagnostics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package ssh

import (
"os"
"path/filepath"
"strings"
"testing"

"github.com/devsy-org/devsy/pkg/config"
"github.com/devsy-org/devsy/pkg/task"
"github.com/devsy-org/devsy/pkg/workspace"
"github.com/stretchr/testify/require"
)

func TestReadDiagnosticStreamsLimitsOutput(t *testing.T) {
home := t.TempDir()
path := filepath.Join(home, "worker.streams")
marker := "detached worker admission failed: reading launch state\n"
content := "private credential\n" + marker + strings.Repeat("x", 64*1024)
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
got, err := readDiagnosticStreams(home, path)
require.NoError(t, err)
require.Equal(t, marker+"[worker stream truncated at 65536 bytes]\n", string(got))
require.NoError(t, os.WriteFile(path, []byte(strings.Repeat("x", 64*1024+1)), 0o600))
got, err = readDiagnosticStreams(home, path)
require.NoError(t, err)
require.Equal(t, "[worker stream truncated at 65536 bytes]\n", string(got))
require.NoError(t, os.WriteFile(path, []byte(strings.Repeat("x", 64*1024)), 0o600))
got, err = readDiagnosticStreams(home, path)
require.NoError(t, err)
require.Empty(t, got)
_, err = readDiagnosticStreams(home, filepath.Join(t.TempDir(), "outside"))
require.Error(t, err)
}

func TestNewDetachedTaskExcludesExistingAndAmbiguousTasks(t *testing.T) {
home := t.TempDir()
t.Setenv(config.EnvHome, home)
config.ResetPathManager()
t.Cleanup(config.ResetPathManager)
source := filepath.Join(home, "project")
store, err := task.NewStore()
require.NoError(t, err)
existing, err := store.Create(
task.CreateOptions{Command: "up", WorkspaceID: workspace.ToID(source)},
)
require.NoError(t, err)
before := map[string]struct{}{existing.ID(): {}}
_, err = store.Create(task.CreateOptions{Command: "up", WorkspaceID: "unrelated"})
require.NoError(t, err)
created, err := store.Create(
task.CreateOptions{Command: "up", WorkspaceID: workspace.ToID(source)},
)
require.NoError(t, err)
got := newDetachedTask(before, source)
require.NotNil(t, got)
require.Equal(t, created.ID(), got.ID)
_, err = store.Create(task.CreateOptions{Command: "up", WorkspaceID: workspace.ToID(source)})
require.NoError(t, err)
require.Nil(t, newDetachedTask(before, source))
}
88 changes: 88 additions & 0 deletions e2e/tests/ssh/detached_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,17 @@ import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"runtime"
"strings"
"time"

"github.com/devsy-org/devsy/e2e/framework"
"github.com/devsy-org/devsy/pkg/config"
"github.com/devsy-org/devsy/pkg/flags/names"
"github.com/devsy-org/devsy/pkg/task"
workspacepkg "github.com/devsy-org/devsy/pkg/workspace"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
)

Expand Down Expand Up @@ -55,9 +60,11 @@ func startDetachedTunnelUp(
}
args = append(args, extraArgs...)
args = append(args, workspace)
before := detachedTaskIDs()

out, err := f.ExecCommandOutput(ctx, args)
if err != nil {
reportDetachedLaunchFailure(before, workspace)
return "", err
}
var env struct {
Expand All @@ -74,6 +81,87 @@ func startDetachedTunnelUp(
return env.ID, nil
}

func detachedTaskIDs() map[string]struct{} {
store, err := task.NewStore()
if err != nil {
return nil
}
states, err := store.List()
if err != nil {
return nil
}
ids := make(map[string]struct{}, len(states))
for _, state := range states {
ids[state.ID] = struct{}{}
}
return ids
}

func reportDetachedLaunchFailure(before map[string]struct{}, source string) {
state := newDetachedTask(before, source)
if state == nil {
return
}
reportDiagnosticState(state)
path, err := config.DefaultPathManager().ProcessStreamsFile(task.WorkerProcessName(state.ID))
if err != nil {
return
}
streams, err := readDiagnosticStreams(filepath.Dir(path), path)
if err != nil {
return
}
ginkgo.GinkgoWriter.Printf("detached worker streams: %s\n", streams)
}

func reportDiagnosticState(state *task.State) {
ginkgo.GinkgoWriter.Printf(
"detached launch state: pidPublished=%t launchPending=%t launchOwned=%t workerReady=%t cleanupComplete=%t\n",
state.PID != 0,
state.LaunchPending,
state.LaunchOwned,
state.WorkerReady,
state.ProcessCleanupComplete,
)
}

func newDetachedTask(before map[string]struct{}, source string) *task.State {
if before == nil {
return nil
}
absolute, err := filepath.Abs(source)
if err != nil {
return nil
}
workspaceID := workspacepkg.ToID(absolute)
store, err := task.NewStore()
if err != nil {
return nil
}
states, err := store.List()
if err != nil {
return nil
}
return selectNewDetachedTask(states, before, workspaceID)
}

func selectNewDetachedTask(
states []*task.State, before map[string]struct{}, workspaceID string,
) *task.State {
var state *task.State
for _, candidate := range states {
if _, existed := before[candidate.ID]; existed || candidate.Command != "up" ||
candidate.WorkspaceID != workspaceID {
continue
}
if state != nil {
return nil
}
state = candidate
}
return state
}

func findDetachedUpTask(
ctx context.Context,
f *framework.Framework,
Expand Down
77 changes: 75 additions & 2 deletions e2e/tests/ssh/detached_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"io"
"os"
"os/exec"
"path/filepath"
Expand All @@ -13,6 +14,7 @@ import (
"github.com/devsy-org/devsy/e2e/framework"
"github.com/devsy-org/devsy/pkg/command"
"github.com/devsy-org/devsy/pkg/config"
"github.com/devsy-org/devsy/pkg/secrets"
"github.com/devsy-org/devsy/pkg/task"
"github.com/devsy-org/devsy/pkg/workspace"
"github.com/onsi/ginkgo/v2"
Expand Down Expand Up @@ -62,10 +64,14 @@ var _ = ginkgo.Describe("detached workspace input validation", ginkgo.Label("tas
home, err := os.MkdirTemp("", "devsy-early-worker-")
framework.ExpectNoError(err)
ginkgo.DeferCleanup(os.RemoveAll, home)
ginkgo.GinkgoT().Setenv(config.EnvHome, home)
config.ResetPathManager()
ginkgo.DeferCleanup(config.ResetPathManager)
cli := detachedValidationCLI{framework: f, home: home}
source := filepath.Join(home, "Backend.API")
framework.ExpectNoError(os.MkdirAll(source, 0o700))
sentinel := filepath.Join(source, "KeepCase.txt")
before := detachedTaskIDs()
framework.ExpectNoError(os.WriteFile(sentinel, []byte("original source"), 0o600))
out, err := cli.command(ctx,
cmdWorkspace,
Expand All @@ -75,7 +81,11 @@ var _ = ginkgo.Describe("detached workspace input validation", ginkgo.Label("tas
"--provider=missing-provider-1471",
source,
)
framework.ExpectNoError(err)
if err != nil {
reportDetachedLaunchFailure(before, source)
}
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "CLI output: %s",
secrets.NewEnvironmentRedactor(detachedValidationEnvironment(home)).Redact(out))
var envelope struct {
Kind string `json:"kind"`
ID string `json:"id"`
Expand Down Expand Up @@ -236,11 +246,73 @@ func (cli detachedValidationCLI) launchWorker(
return worker, nil
})
if err != nil {
store, storeErr := task.NewStoreAt(filepath.Join(cli.home, "state", "tasks"))
if storeErr == nil {
state, stateErr := store.Get(tk.ID())
if stateErr == nil {
reportDiagnosticState(state)
}
}
streamsPath, pathErr := config.DefaultPathManager().
ProcessStreamsFile(task.WorkerProcessName(tk.ID()))
if pathErr == nil {
streams, readErr := readDiagnosticStreams(cli.home, streamsPath)
if readErr == nil {
ginkgo.GinkgoWriter.Printf("worker streams: %s\n", streams)
}
}
return ref, errors.Join(err, tk.FailLaunch(err))
}
return ref, nil
}

func readDiagnosticStreams(home, path string) ([]byte, error) {
root, err := os.OpenRoot(home)
if err != nil {
return nil, err
}
defer func() { _ = root.Close() }()
relative, err := filepath.Rel(home, path)
if err != nil {
return nil, err
}
file, err := root.Open(relative)
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
data, err := io.ReadAll(io.LimitReader(file, 64*1024+1))
if err != nil {
return nil, err
}
return filterDiagnosticStreams(data), nil
}

func filterDiagnosticStreams(data []byte) []byte {
truncated := len(data) > 64*1024
if truncated {
data = data[:64*1024]
}
var result strings.Builder
for line := range strings.SplitSeq(string(data), "\n") {
switch strings.TrimSuffix(line, "\r") {
case "detached worker admission failed: worker lock",
"detached worker admission failed: reading launch state",
"detached worker admission failed: waiting for launch commit",
"detached worker admission failed: verifying owner",
"detached worker admission failed: retaining launch lease",
"detached worker admission failed: checking launcher",
"detached worker admission failed: acknowledging worker":
result.WriteString(strings.TrimSuffix(line, "\r"))
result.WriteByte('\n')
}
}
if truncated {
result.WriteString("[worker stream truncated at 65536 bytes]\n")
}
return []byte(result.String())
}

func (cli detachedValidationCLI) expectFailureEnvelope(ctx context.Context, id, diagnostic string) {
followCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
Expand All @@ -265,5 +337,6 @@ func (cli detachedValidationCLI) expectFailureEnvelope(ctx context.Context, id,
}

func detachedValidationEnvironment(home string) []string {
return append(os.Environ(), "DEVSY_HOME="+home, "DEVSY_TEST_TOKEN=missing-provider-1471")
return append(os.Environ(), "DEVSY_HOME="+home, "DEVSY_TEST_TOKEN=missing-provider-1471",
"DEVSY_DETACHED_DIAGNOSTICS=1")
}
Loading
Loading