Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 56 additions & 1 deletion .github/workflows/pr-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -444,11 +444,58 @@ jobs:
GH_CREDENTIAL_USERNAME: x-access-token
run: '"$CI_TOOL" e2e run --label "${{ matrix.name }}" --runner "${{ matrix.runner }}" --e2e-dir ./e2e'

prepare-microsandbox-provider:
name: Prepare MicroSandbox provider assets
needs: [changes, build-cli]
if: needs.changes.outputs.go == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: read pinned provider version
id: pin
shell: bash
run: |
versions="$(sed -nE 's#.*devsy-provider-microsandbox@(v[0-9]+\.[0-9]+\.[0-9]+)".*#\1#p' e2e/tests/up/microsandbox_session.go)"
if [ "$(printf '%s\n' "$versions" | grep -c .)" != 1 ]; then
echo "::error::expected exactly one pinned microsandbox provider version, got: $versions"
exit 1
fi
echo "version=$versions" >> "$GITHUB_OUTPUT"

- name: download MicroSandbox CI helper
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: devsy-ci-microsandbox-linux
path: ${{ runner.temp }}

- name: cache provider assets
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ runner.temp }}/microsandbox-provider
key: ${{ runner.os }}-${{ runner.arch }}-microsandbox-provider-${{ steps.pin.outputs.version }}

- name: download and verify provider assets
shell: bash
run: |
chmod +x "${RUNNER_TEMP}/devsy-ci-microsandbox"
"${RUNNER_TEMP}/devsy-ci-microsandbox" prepare-provider \
--version "${{ steps.pin.outputs.version }}" \
--output-dir "${RUNNER_TEMP}/microsandbox-provider"

- name: upload provider assets
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: microsandbox-provider-assets
path: ${{ runner.temp }}/microsandbox-provider/
if-no-files-found: error

# E2E tier, group "microsandbox": the MicroSandbox CI helper owns the run and
# its report checks.
integration-tests-microsandbox:
name: ${{ matrix.title }}
needs: [changes, plan, build-cli]
needs: [changes, plan, build-cli, prepare-microsandbox-provider]
if: needs.changes.outputs.go == 'true'
strategy:
fail-fast: false
Expand Down Expand Up @@ -522,6 +569,12 @@ jobs:
shell: bash
run: '"$CI_TOOL" e2e prepare --bin-src "$RUNNER_TEMP/devsy-bin" --e2e-dir ./e2e'

- name: download provider assets
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: microsandbox-provider-assets
path: ${{ runner.temp }}/microsandbox-provider/

- name: cache microsandbox release bundle (Linux)
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
Expand All @@ -540,6 +593,7 @@ jobs:
GH_CREDENTIAL_USERNAME: x-access-token
TEST_TIMEOUT: ${{ matrix.test_timeout }}
DEVSY_REQUIRE_MICROSANDBOX: "true"
DEVSY_E2E_PROVIDER_DOWNLOAD_CACHE: ${{ runner.temp }}/microsandbox-provider/cache
run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" run --label "${{ matrix.name }}" --expected-count "${{ matrix.expected_spec_count }}" --provider-mode "${{ matrix.provider_mode }}"'

- name: upload MicroSandbox test evidence
Expand Down Expand Up @@ -870,6 +924,7 @@ jobs:
- build-cli
- integration-tests-unprivileged
- integration-tests
- prepare-microsandbox-provider
- integration-tests-microsandbox
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
Expand Down
12 changes: 9 additions & 3 deletions e2e/framework/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,13 @@ func (f *Framework) DevsyStop(ctx context.Context, workspace string) error {
func (f *Framework) DevsyProviderAdd(ctx context.Context, args ...string) error {
baseArgs := []string{cmdProvider, "add"}
baseArgs = append(baseArgs, args...)
_, stderr, err := f.ExecCommandCapture(ctx, baseArgs)
if err := seedProviderDownloadCache(); err != nil {
return fmt.Errorf("devsy provider add failed: %w", err)
}
capture := func(ctx context.Context) (string, string, error) {
return f.ExecCommandCapture(ctx, baseArgs)
}
stderr, err := execWithProviderAddRetry(ctx, capture)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if err != nil {
if strings.Contains(stderr, "already exists") {
providerName := providerNameFromAddArgs(args)
Expand All @@ -355,11 +361,11 @@ func (f *Framework) DevsyProviderAdd(ctx context.Context, args ...string) error
)
}

_, stderr, err = f.ExecCommandCapture(ctx, baseArgs)
stderr, err = execWithProviderAddRetry(ctx, capture)
}
}
if err != nil {
return fmt.Errorf("devsy provider add failed: %s", stderr)
return fmt.Errorf("devsy provider add failed: %w: %s", err, stderr)
}
return nil
}
Expand Down
108 changes: 108 additions & 0 deletions e2e/framework/provider_cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package framework

import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"time"

"github.com/devsy-org/devsy/pkg/config"
)

const (
providerDownloadCacheEnv = "DEVSY_E2E_PROVIDER_DOWNLOAD_CACHE"
providerCacheHomePerm = 0o700
providerCacheDirPerm = 0o750
providerCacheFilePerm = 0o755
)

var providerCacheRelativeDir = filepath.Join("cache", "providers")

func seedProviderDownloadCache() error {
source := os.Getenv(providerDownloadCacheEnv)
home := os.Getenv(config.EnvHome)
if source == "" || home == "" {
return nil
}
sourceRoot, err := os.OpenRoot(source)
if err != nil {
return fmt.Errorf("open %s: %w", providerDownloadCacheEnv, err)
}
defer func() { _ = sourceRoot.Close() }()
homeRoot, err := openHomeRoot(home)
if err != nil {
return fmt.Errorf("open %s: %w", config.EnvHome, err)
}
defer func() { _ = homeRoot.Close() }()
if err := homeRoot.MkdirAll(providerCacheRelativeDir, providerCacheDirPerm); err != nil {
return fmt.Errorf("create provider download cache: %w", err)
}
return copyMissingFiles(source, sourceRoot, homeRoot)
}

func openHomeRoot(home string) (*os.Root, error) {
parent, err := os.OpenRoot(filepath.Dir(home))
if err != nil {
return nil, err
}
defer func() { _ = parent.Close() }()
name := filepath.Base(home)
if err := parent.Mkdir(
name,
providerCacheHomePerm,
); err != nil &&
!errors.Is(err, os.ErrExist) {
return nil, err
}
return parent.OpenRoot(name)
}

func copyMissingFiles(source string, sourceRoot, homeRoot *os.Root) error {
entries, err := os.ReadDir(source)
if err != nil {
return fmt.Errorf("read %s: %w", providerDownloadCacheEnv, err)
}
for _, entry := range entries {
if !entry.Type().IsRegular() {
continue
}
destination := filepath.Join(providerCacheRelativeDir, entry.Name())
if err := copyIfMissing(sourceRoot, entry.Name(), homeRoot, destination); err != nil {
return fmt.Errorf("seed provider download cache: %w", err)
}
}
return nil
}

func copyIfMissing(
sourceRoot *os.Root, name string, destinationRoot *os.Root, destination string,
) (err error) {
if _, statErr := destinationRoot.Stat(destination); statErr == nil {
return nil
}
in, err := sourceRoot.Open(name)
if err != nil {
return err
}
defer func() { err = errors.Join(err, in.Close()) }()
partial := fmt.Sprintf("%s.%d.%d.partial", destination, os.Getpid(), time.Now().UnixNano())
out, err := destinationRoot.OpenFile(
partial, os.O_WRONLY|os.O_CREATE|os.O_EXCL, providerCacheFilePerm,
)
if err != nil {
return err
Comment thread
skevetter marked this conversation as resolved.
Resolved
}
_, err = io.Copy(out, in)
if closeErr := out.Close(); err == nil {
err = closeErr
}
if err != nil {
return errors.Join(err, destinationRoot.Remove(partial))
}
if err := destinationRoot.Rename(partial, destination); err != nil {
return errors.Join(err, destinationRoot.Remove(partial))
}
return nil
}
53 changes: 53 additions & 0 deletions e2e/framework/retry.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os/exec"
"regexp"
"strings"
"time"

Expand Down Expand Up @@ -32,6 +33,27 @@ var sshBackoff = wait.Backoff{
Jitter: 0.1,
}

// providerAddBackoff defines retry timing for provider manifest downloads.
// 5 total attempts with waits of ~5s, ~10s, ~20s, ~40s.
var providerAddBackoff = wait.Backoff{
Steps: 5,
Duration: 5 * time.Second,
Factor: 2.0,
Jitter: 0.1,
}

var providerServerErrorPattern = regexp.MustCompile(
`(?i)(received status code|download failed with status|status) 5\d\d`,
)

var retryableProviderNetworkPatterns = []string{
"TLS handshake timeout",
"i/o timeout",
"connection reset by peer",
"unexpected EOF",
"stalled: no progress within timeout",
}
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

// retryableDockerPatterns are stderr substrings indicating a transient Docker
// registry error that is worth retrying.
var retryableDockerPatterns = []string{
Expand Down Expand Up @@ -93,6 +115,22 @@ func isRetryableDockerError(stderr string) bool {
return false
}

// isRetryableProviderDownloadError returns true when a provider download
// failed on a server error or a network error. Client errors such as 404 and
// provider validation failures are never retried.
func isRetryableProviderDownloadError(_ error, stderr string) bool {
if providerServerErrorPattern.MatchString(stderr) {
return true
}
lower := strings.ToLower(stderr)
for _, pattern := range retryableProviderNetworkPatterns {
if strings.Contains(lower, strings.ToLower(pattern)) {
return true
}
}
return false
}

type execFunc func(ctx context.Context) (stdout, stderr string, err error)

// retryPolicy describes one retry loop. label prefixes the retry log line and
Expand Down Expand Up @@ -168,6 +206,21 @@ func execWithDockerRetry(ctx context.Context, fn execFunc) (string, string, erro
return res.stdout, res.stderr, res.err
}

// execWithProviderAddRetry runs fn and retries when the provider manifest
// download failed with a server or network error.
func execWithProviderAddRetry(ctx context.Context, fn execFunc) (string, error) {
res := runWithRetry(ctx, retryPolicy{
backoff: providerAddBackoff,
retryable: isRetryableProviderDownloadError,
label: "[retry] provider add:",
kind: "retryable provider download error",
}, fn)
if res.gaveUp {
return res.stderr, fmt.Errorf("after %d attempts: %w", res.attempts, res.err)
}
return res.stderr, res.err
}

// execWithSSHRetry runs fn and retries if the error indicates a transient SSH
// connection failure. The fn callback must return stdout, stderr, and error so
// that stderr can be inspected for SSH-specific patterns.
Expand Down
Loading
Loading