diff --git a/cli/cmd/codesphere/codesphere_suite_test.go b/cli/cmd/codesphere/codesphere_suite_test.go new file mode 100644 index 000000000..b8f38f853 --- /dev/null +++ b/cli/cmd/codesphere/codesphere_suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package codesphere_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCodesphere(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Codesphere Cmd Suite") +} diff --git a/cli/cmd/codesphere/smoketest_codesphere.go b/cli/cmd/codesphere/smoketest_codesphere.go index 54425365b..a93b78bbb 100644 --- a/cli/cmd/codesphere/smoketest_codesphere.go +++ b/cli/cmd/codesphere/smoketest_codesphere.go @@ -40,14 +40,15 @@ type SmoketestCodesphereCmd struct { Opts *teststeps.SmoketestCodesphereOpts } -func (c *SmoketestCodesphereCmd) RunE(_ *cobra.Command, args []string) error { +// RunE runs the smoke test against the configured Codesphere installation. +func (c *SmoketestCodesphereCmd) RunE(cmd *cobra.Command, _ []string) error { client, err := codesphere.NewClient(c.Opts.BaseURL, c.Opts.Token) if err != nil { return fmt.Errorf("failed to create Codesphere client: %w", err) } c.Opts.Client = client - return c.RunSmoketest() + return c.RunSmoketest(cmd.Context()) } func AddSmoketestCmd(parent *cobra.Command, opts *util.GlobalOptions) { @@ -113,8 +114,11 @@ func AddSmoketestCmd(parent *cobra.Command, opts *util.GlobalOptions) { util.AddCmd(parent, c.cmd) } -func (c *SmoketestCodesphereCmd) RunSmoketest() (err error) { - ctx, cancel := context.WithTimeout(context.Background(), c.Opts.Timeout) +// RunSmoketest runs the selected smoke test steps. The passed context bounds +// the run in addition to the configured timeout, so callers that orchestrate +// several tests (see the test command) can cancel it. +func (c *SmoketestCodesphereCmd) RunSmoketest(ctx context.Context) (err error) { + ctx, cancel := context.WithTimeout(ctx, c.Opts.Timeout) defer cancel() availableStepsMap := make(map[string]teststeps.SmokeTestStep) diff --git a/cli/cmd/codesphere/smoketest_codesphere_test.go b/cli/cmd/codesphere/smoketest_codesphere_test.go index e740356e2..70bb7a7ab 100644 --- a/cli/cmd/codesphere/smoketest_codesphere_test.go +++ b/cli/cmd/codesphere/smoketest_codesphere_test.go @@ -4,6 +4,7 @@ package codesphere_test import ( + "context" "fmt" "strconv" "strings" @@ -124,7 +125,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { It("returns an error indicating no teams are available", func() { mockClient.EXPECT().ListTeams("").Return([]api.Team{}, nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("no teams available"))) }) }) @@ -138,7 +139,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { mockFullTestRun(mockClient, 99, 456, 789) - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(BeNil()) }) }) @@ -152,7 +153,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { mockFullTestRun(mockClient, 21, 456, 789) - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(BeNil()) }) }) @@ -164,7 +165,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { It("returns an error indicating no workspace plans are available", func() { mockClient.EXPECT().ListWorkspacePlans().Return([]api.WorkspacePlan{}, nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("no workspace plans available"))) }) }) @@ -176,13 +177,14 @@ var _ = Describe("SmoketestCodesphereCmd", func() { mockFullTestRun(mockClient, teamIdInt, 42, 789) - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(BeNil()) }) }) It("completes successfully with all steps", func() { mockFullTestRun(mockClient, teamIdInt, planIdInt, 789) - err := c.RunSmoketest() + + err := c.RunSmoketest(context.Background()) Expect(err).To(BeNil()) }) @@ -194,7 +196,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { (*string)(nil), // empty workspace ).Return(0, fmt.Errorf("create failed")).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("failed to create workspace"))) }) @@ -218,7 +220,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { workspaceID, ).Return(nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("failed to set environment variable"))) }) @@ -248,7 +250,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { workspaceId, ).Return(nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("failed to create ci.yml"))) }) @@ -291,7 +293,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { workspaceId, ).Return(nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("failed to sync landscape"))) }) @@ -340,7 +342,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { workspaceId, ).Return(nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("failed to start pipeline"))) }) @@ -394,7 +396,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { workspaceId, ).Return(nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("unexpected state"))) }) @@ -448,7 +450,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { workspaceId, ).Return(nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("unexpected state"))) }) @@ -501,7 +503,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { ).Return(nil).Once() opts.Timeout = 100 * time.Millisecond - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("timed out"))) Expect(err).To(MatchError(ContainSubstring("connection refused"))) }) @@ -558,7 +560,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { ).Return(nil).Once() opts.Timeout = 100 * time.Millisecond - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("timed out"))) }) @@ -614,7 +616,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { workspaceId, ).Return(fmt.Errorf("delete failed")).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(MatchError(ContainSubstring("failed to delete workspace"))) }) @@ -634,7 +636,7 @@ var _ = Describe("SmoketestCodesphereCmd", func() { "smoketest", ).Return(nil).Once() - err := c.RunSmoketest() + err := c.RunSmoketest(context.Background()) Expect(err).To(BeNil()) }) }) diff --git a/cli/cmd/codesphere/status_codesphere.go b/cli/cmd/codesphere/status_codesphere.go new file mode 100644 index 000000000..23e32d9bf --- /dev/null +++ b/cli/cmd/codesphere/status_codesphere.go @@ -0,0 +1,87 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package codesphere + +import ( + "fmt" + "time" + + csio "github.com/codesphere-cloud/cs-go/pkg/io" + "github.com/codesphere-cloud/oms/cli/cmd/util" + "github.com/codesphere-cloud/oms/internal/codesphere" + "github.com/spf13/cobra" +) + +const ( + defaultStatusTimeout = 5 * time.Minute + statusPollInterval = 5 * time.Second +) + +// StatusCodesphereOpts configures the status report of a Codesphere installation. +type StatusCodesphereOpts struct { + BaseURL string + Token string + Wait bool + Timeout time.Duration + Client codesphere.Client +} + +// StatusCodesphereCmd represents the status codesphere command. +type StatusCodesphereCmd struct { + cmd *cobra.Command + Opts *StatusCodesphereOpts +} + +// RunE prints the status report and fails the command if the installation is not ready. +func (c *StatusCodesphereCmd) RunE(cmd *cobra.Command, _ []string) error { + client, err := codesphere.NewClient(c.Opts.BaseURL, c.Opts.Token) + if err != nil { + return fmt.Errorf("failed to create Codesphere client: %w", err) + } + + c.Opts.Client = client + + report := fetchStatus(cmd.Context(), c.Opts) + printStatus(cmd.OutOrStdout(), c.Opts.BaseURL, report) + + if !report.Ready { + return fmt.Errorf("codesphere installation is not ready") + } + + return nil +} + +// AddStatusCmd adds the status codesphere command to the given parent command. +func AddStatusCmd(parent *cobra.Command, _ *util.GlobalOptions) { + c := StatusCodesphereCmd{ + cmd: &cobra.Command{ + Use: "codesphere", + Short: "Check the status of a Codesphere installation", + Long: csio.Long(`Check whether a Codesphere installation is reachable and ready to use, + by querying the Codesphere API.`), + Example: util.FormatExamples("status codesphere", []csio.Example{ + { + Cmd: "--baseurl https://codesphere.example.com/api --token YOUR_TOKEN", + Desc: "Check the status of a Codesphere installation", + }, + { + Cmd: "--baseurl https://codesphere.example.com/api --token YOUR_TOKEN --wait", + Desc: "Block and retry until the Codesphere installation is ready", + }, + }), + }, + Opts: &StatusCodesphereOpts{}, + } + c.cmd.Flags().StringVar(&c.Opts.BaseURL, "baseurl", "", "Base URL of the Codesphere API") + c.cmd.Flags().StringVar(&c.Opts.Token, "token", "", "API token for authentication") + c.cmd.Flags().BoolVar(&c.Opts.Wait, "wait", false, "Block and retry until the installation is ready") + c.cmd.Flags().DurationVar(&c.Opts.Timeout, "timeout", defaultStatusTimeout, "Timeout when waiting for the installation to become ready") + + util.MarkFlagRequired(c.cmd, "baseurl") + util.MarkFlagRequired(c.cmd, "token") + + c.cmd.RunE = c.RunE + + util.AddCmd(parent, c.cmd) +} diff --git a/cli/cmd/codesphere/status_report.go b/cli/cmd/codesphere/status_report.go new file mode 100644 index 000000000..5ad8eb580 --- /dev/null +++ b/cli/cmd/codesphere/status_report.go @@ -0,0 +1,155 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package codesphere + +import ( + "context" + "fmt" + "io" + "net/url" + "strings" + "time" +) + +const ( + ansiReset = "\x1b[0m" + ansiBold = "\x1b[1m" + ansiCyan = "\x1b[36m" + ansiGreen = "\x1b[32m" + ansiRed = "\x1b[31m" +) + +// logo is a small ASCII mark printed next to the status report, neofetch-style. +var logo = []string{ + " ▄▄▄▄▄▄▄▄▄▄▄▄ ", + " ▄█████████████████▄ ", + " ▄███▀▀▀ ▀▀▀███▄ ", + "███ ████", + "██ ▄▄▄▄▄▄▄▄▄ ███", + "██ ███████████ ███", + "██ ███████████ ███", + "██ ▀▀▀▀▀▀▀▀▀ ███", + "████ ████", + " ▀███▄▄▄ ▄▄▄███▀ ", + " ▀██████████████████▀ ", + " ▀▀▀▀▀▀▀▀▀▀▀▀ ", +} + +type statusReport struct { + Ready bool + Latency time.Duration + Teams int + Plans int + Attempts int + Err error +} + +// fetchStatus pings the Codesphere API with a cheap, side-effect-free call +// (ListWorkspacePlans) to determine readiness. With Wait set, it retries on +// failure until the installation becomes ready or opts.Timeout elapses. +func fetchStatus(ctx context.Context, opts *StatusCodesphereOpts) *statusReport { + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + + report := &statusReport{} + for { + report.Attempts++ + + start := time.Now() + plans, err := opts.Client.ListWorkspacePlans() + report.Latency = time.Since(start) + + if err == nil { + report.Ready = true + + report.Plans = len(plans) + if teams, terr := opts.Client.ListTeams(""); terr == nil { + report.Teams = len(teams) + } + + return report + } + + report.Err = err + + if !opts.Wait { + return report + } + + select { + case <-ctx.Done(): + return report + case <-time.After(statusPollInterval): + } + } +} + +// printStatus renders a neofetch-style report: a small ASCII logo alongside +// key/value status lines. +func printStatus(w io.Writer, baseURL string, r *statusReport) { + host := baseURL + if u, err := url.Parse(baseURL); err == nil && u.Host != "" { + host = u.Host + } + + statusColor, statusText := ansiGreen, "Ready" + if !r.Ready { + statusColor, statusText = ansiRed, "Not Ready" + } + + header := fmt.Sprintf("%s%scodesphere%s@%s", ansiBold, ansiCyan, ansiReset, host) + rule := strings.Repeat("-", len("codesphere@")+len(host)) + + lines := []string{ + header, + rule, + fmt.Sprintf("%sStatus%s: %s%s%s", ansiBold, ansiReset, statusColor, statusText, ansiReset), + fmt.Sprintf("%sLatency%s: %s", ansiBold, ansiReset, r.Latency.Round(time.Millisecond)), + } + if r.Ready { + lines = append(lines, + fmt.Sprintf("%sTeams%s: %d", ansiBold, ansiReset, r.Teams), + fmt.Sprintf("%sPlans%s: %d", ansiBold, ansiReset, r.Plans), + ) + } else { + lines = append(lines, fmt.Sprintf("%sError%s: %s", ansiBold, ansiReset, r.Err)) + } + + if r.Attempts > 1 { + lines = append(lines, fmt.Sprintf("%sAttempts%s: %d", ansiBold, ansiReset, r.Attempts)) + } + + rows := len(logo) + if len(lines) > rows { + rows = len(lines) + } + + // Pad the logo to a fixed width so the status lines form a straight column. + logoWidth := 0 + for _, l := range logo { + if n := len([]rune(l)); n > logoWidth { + logoWidth = n + } + } + + _, _ = fmt.Fprintln(w) + + for i := 0; i < rows; i++ { + logoLine := "" + if i < len(logo) { + logoLine = logo[i] + } + + logoLine += strings.Repeat(" ", logoWidth-len([]rune(logoLine))) + + statLine := "" + if i < len(lines) { + statLine = lines[i] + } + + _, _ = fmt.Fprintf(w, " %s%s%s %s\n", ansiCyan, logoLine, ansiReset, statLine) + } + + _, _ = fmt.Fprintln(w) +} diff --git a/cli/cmd/codesphere/test_codesphere.go b/cli/cmd/codesphere/test_codesphere.go new file mode 100644 index 000000000..d4394ac59 --- /dev/null +++ b/cli/cmd/codesphere/test_codesphere.go @@ -0,0 +1,249 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package codesphere + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + csio "github.com/codesphere-cloud/cs-go/pkg/io" + "github.com/codesphere-cloud/oms/cli/cmd/util" + "github.com/codesphere-cloud/oms/internal/codesphere" + "github.com/codesphere-cloud/oms/internal/codesphere/testplan" + "github.com/codesphere-cloud/oms/internal/codesphere/teststeps" + "github.com/spf13/cobra" +) + +const ( + // defaultTestTimeout bounds the whole playlist, not a single test. + defaultTestTimeout = 20 * time.Minute + // DefaultPlaylist is run when neither --playlist nor --tests is given. + DefaultPlaylist = "default" +) + +// Names of the tests that can be part of a playlist. +const ( + StatusTestName = "status" + SmoketestTestName = "smoketest" +) + +// TestCodesphereOpts configures a test run against a Codesphere installation. +type TestCodesphereOpts struct { + BaseURL string + Token string + TeamID string + PlanID string + Profile string + Playlist string + Tests []string + Wait bool + WaitTimeout time.Duration + Timeout time.Duration + FailFast bool + Quiet bool + Client codesphere.Client +} + +// TestCodesphereCmd represents the test codesphere command. +type TestCodesphereCmd struct { + cmd *cobra.Command + Opts *TestCodesphereOpts +} + +// Registry returns the tests that can run against a Codesphere installation, +// together with the playlists that group them. The tests close over opts, so +// the registry has to be built after the flags are parsed and the client is +// set. Building it with zero options is safe as long as no test is run, which +// is what the test list command does. +func Registry(opts *TestCodesphereOpts) *testplan.Registry { + statusTest := &testplan.Func{ + TestName: StatusTestName, + Desc: "Report the state of the installation and verify the API answers", + Fn: func(ctx context.Context, out io.Writer) error { + waitTimeout := opts.WaitTimeout + if waitTimeout <= 0 { + waitTimeout = defaultStatusTimeout + } + + statusOpts := &StatusCodesphereOpts{ + BaseURL: opts.BaseURL, + Token: opts.Token, + Wait: opts.Wait, + Timeout: waitTimeout, + Client: opts.Client, + } + + report := fetchStatus(ctx, statusOpts) + printStatus(out, opts.BaseURL, report) + + if !report.Ready { + if report.Err != nil { + return fmt.Errorf("codesphere installation is not ready: %w", report.Err) + } + + return fmt.Errorf("codesphere installation is not ready") + } + + return nil + }, + } + + smoketest := &testplan.Func{ + TestName: SmoketestTestName, + Desc: "Create a workspace, deploy a sample app in it and clean up afterwards", + Fn: func(ctx context.Context, _ io.Writer) error { + c := SmoketestCodesphereCmd{ + Opts: &teststeps.SmoketestCodesphereOpts{ + BaseURL: opts.BaseURL, + Token: opts.Token, + TeamID: opts.TeamID, + PlanID: opts.PlanID, + Profile: opts.Profile, + Quiet: opts.Quiet, + Timeout: opts.Timeout, + Client: opts.Client, + }, + } + + return c.RunSmoketest(ctx) + }, + } + + registry := testplan.NewRegistry(statusTest, smoketest) + registry.AddPlaylist(testplan.Playlist{ + Name: DefaultPlaylist, + Description: "Verify the installation is up and can run a workspace", + Tests: []string{StatusTestName, SmoketestTestName}, + }) + registry.AddPlaylist(testplan.Playlist{ + Name: "readiness", + Description: "Only check that the installation is reachable and ready", + Tests: []string{StatusTestName}, + }) + + return registry +} + +// selectTests resolves the requested tests. An explicit --tests selection wins +// over --playlist, so a playlist default doesn't have to be unset first. +func (c *TestCodesphereCmd) selectTests() ([]testplan.Test, error) { + registry := Registry(c.Opts) + + if len(c.Opts.Tests) > 0 { + tests, err := registry.Select(c.Opts.Tests) + if err != nil { + return nil, fmt.Errorf("failed to select tests: %w", err) + } + + return tests, nil + } + + tests, err := registry.SelectPlaylist(c.Opts.Playlist) + if err != nil { + return nil, fmt.Errorf("failed to select playlist: %w", err) + } + + return tests, nil +} + +// RunE runs the selected tests and fails the command if any of them failed. +func (c *TestCodesphereCmd) RunE(cmd *cobra.Command, _ []string) error { + tests, err := c.selectTests() + if err != nil { + return err + } + + client, err := codesphere.NewClient(c.Opts.BaseURL, c.Opts.Token) + if err != nil { + return fmt.Errorf("failed to create Codesphere client: %w", err) + } + + c.Opts.Client = client + + ctx, cancel := context.WithTimeout(cmd.Context(), c.Opts.Timeout) + defer cancel() + + out := cmd.OutOrStdout() + runner := &testplan.Runner{ + Out: out, + FailFast: c.Opts.FailFast, + Quiet: c.Opts.Quiet, + } + + results := runner.Run(ctx, tests) + testplan.Summarize(out, results) + + if err := testplan.Err(results); err != nil { + return fmt.Errorf("test run failed: %w", err) + } + + return nil +} + +// AddTestCmd adds the test codesphere command to the given parent command. +func AddTestCmd(parent *cobra.Command, _ *util.GlobalOptions) { + registry := Registry(&TestCodesphereOpts{}) + + c := TestCodesphereCmd{ + cmd: &cobra.Command{ + Use: "codesphere", + Short: "Run a playlist of tests against a Codesphere installation", + Long: csio.Long(`Run a playlist of tests against a Codesphere installation. + + A playlist is an ordered selection of tests, for example a status report + followed by a smoke test. Every test is run even if an earlier one failed, + unless --fail-fast is set, and the results are summarized at the end. + + Run 'oms test list' to see the available tests and playlists.`), + Example: util.FormatExamples("test codesphere", []csio.Example{ + { + Cmd: "--baseurl https://codesphere.example.com/api --token YOUR_TOKEN", + Desc: fmt.Sprintf("Run the %q playlist against a Codesphere installation", DefaultPlaylist), + }, + { + Cmd: "--baseurl https://codesphere.example.com/api --token YOUR_TOKEN --playlist readiness", + Desc: "Run a specific playlist", + }, + { + Cmd: "--baseurl https://codesphere.example.com/api --token YOUR_TOKEN --tests status,smoketest", + Desc: "Run a specific list of tests, in the given order", + }, + { + Cmd: "--baseurl https://codesphere.example.com/api --token YOUR_TOKEN --wait", + Desc: "Wait for the installation to become ready before running the remaining tests", + }, + { + Cmd: "--baseurl https://codesphere.example.com/api --token YOUR_TOKEN --fail-fast", + Desc: "Stop at the first failing test instead of running the whole playlist", + }, + }), + }, + Opts: &TestCodesphereOpts{}, + } + + c.cmd.Flags().StringVar(&c.Opts.BaseURL, "baseurl", "", "Base URL of the Codesphere API") + c.cmd.Flags().StringVar(&c.Opts.Token, "token", "", "API token for authentication") + c.cmd.Flags().StringVar(&c.Opts.TeamID, "team-id", "", "Team ID to run tests in") + c.cmd.Flags().StringVar(&c.Opts.PlanID, "plan-id", "", "Plan ID to use for workspaces created by tests") + c.cmd.Flags().StringVar(&c.Opts.Profile, "profile", defaultProfile, "CI profile to use for landscape and pipeline") + c.cmd.Flags().StringVar(&c.Opts.Playlist, "playlist", DefaultPlaylist, + fmt.Sprintf("Playlist of tests to run (%s)", strings.Join(registry.PlaylistNames(), ","))) + c.cmd.Flags().StringSliceVar(&c.Opts.Tests, "tests", []string{}, + fmt.Sprintf("Comma-separated list of tests to run, in the given order (%s). Takes precedence over --playlist.", strings.Join(registry.TestNames(), ","))) + c.cmd.Flags().BoolVar(&c.Opts.Wait, "wait", false, "Wait for the installation to become ready during the status test") + c.cmd.Flags().DurationVar(&c.Opts.WaitTimeout, "wait-timeout", defaultStatusTimeout, "Timeout when waiting for the installation to become ready") + c.cmd.Flags().DurationVar(&c.Opts.Timeout, "timeout", defaultTestTimeout, "Timeout for the entire test run") + c.cmd.Flags().BoolVar(&c.Opts.FailFast, "fail-fast", false, "Skip the remaining tests after the first failure") + c.cmd.Flags().BoolVarP(&c.Opts.Quiet, "quiet", "q", false, "Suppress progress logging") + + util.MarkFlagRequired(c.cmd, "baseurl") + util.MarkFlagRequired(c.cmd, "token") + + c.cmd.RunE = c.RunE + + util.AddCmd(parent, c.cmd) +} diff --git a/cli/cmd/codesphere/test_codesphere_test.go b/cli/cmd/codesphere/test_codesphere_test.go new file mode 100644 index 000000000..0c88515aa --- /dev/null +++ b/cli/cmd/codesphere/test_codesphere_test.go @@ -0,0 +1,144 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package codesphere_test + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/codesphere-cloud/cs-go/api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/cli/cmd/codesphere" + intcs "github.com/codesphere-cloud/oms/internal/codesphere" + "github.com/codesphere-cloud/oms/internal/codesphere/testplan" +) + +var _ = Describe("TestCodesphereCmd", func() { + var ( + mockClient *intcs.MockClient + opts *codesphere.TestCodesphereOpts + out *bytes.Buffer + runner *testplan.Runner + ) + + BeforeEach(func() { + mockClient = intcs.NewMockClient(GinkgoT()) + out = &bytes.Buffer{} + opts = &codesphere.TestCodesphereOpts{ + BaseURL: "https://test.codesphere.com/api", + Token: "test-token", + TeamID: "123", + PlanID: "456", + Profile: "ci.yml", + Quiet: true, // Suppress log output in tests + Timeout: time.Minute, + WaitTimeout: time.Minute, + Client: mockClient, + } + runner = &testplan.Runner{Out: out, Quiet: true} + }) + + AfterEach(func() { + mockClient.AssertExpectations(GinkgoT()) + }) + + expectHealthyStatus := func() { + mockClient.EXPECT().ListWorkspacePlans().Return([]api.WorkspacePlan{{Id: 456, Title: "small"}}, nil).Once() + mockClient.EXPECT().ListTeams("").Return([]api.Team{{Id: 123, Name: "team"}}, nil).Once() + } + + Describe("Registry", func() { + It("offers the status and smoketest tests", func() { + registry := codesphere.Registry(opts) + + Expect(registry.TestNames()).To(ContainElements( + codesphere.StatusTestName, + codesphere.SmoketestTestName, + )) + }) + + It("runs the status test before the smoketest in the default playlist", func() { + tests, err := codesphere.Registry(opts).SelectPlaylist(codesphere.DefaultPlaylist) + + Expect(err).NotTo(HaveOccurred()) + Expect(tests).To(HaveLen(2)) + Expect(tests[0].Name()).To(Equal(codesphere.StatusTestName)) + Expect(tests[1].Name()).To(Equal(codesphere.SmoketestTestName)) + }) + + It("offers a readiness playlist that only checks the status", func() { + tests, err := codesphere.Registry(opts).SelectPlaylist("readiness") + + Expect(err).NotTo(HaveOccurred()) + Expect(tests).To(HaveLen(1)) + Expect(tests[0].Name()).To(Equal(codesphere.StatusTestName)) + }) + }) + + Describe("status test", func() { + var tests []testplan.Test + + JustBeforeEach(func() { + var err error + + tests, err = codesphere.Registry(opts).Select([]string{codesphere.StatusTestName}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("passes and reports the installation state if the API answers", func() { + expectHealthyStatus() + + results := runner.Run(context.Background(), tests) + + Expect(testplan.Err(results)).To(BeNil()) + Expect(out.String()).To(ContainSubstring("test.codesphere.com")) + Expect(out.String()).To(ContainSubstring("Ready")) + }) + + It("fails if the installation is not reachable", func() { + mockClient.EXPECT().ListWorkspacePlans().Return(nil, fmt.Errorf("connection refused")).Once() + + results := runner.Run(context.Background(), tests) + + Expect(results[0].Status).To(Equal(testplan.StatusFailed)) + Expect(results[0].Err).To(MatchError(ContainSubstring("not ready"))) + Expect(results[0].Err).To(MatchError(ContainSubstring("connection refused"))) + }) + }) + + Describe("default playlist", func() { + It("passes if the installation is ready and the smoketest succeeds", func() { + expectHealthyStatus() + mockFullTestRun(mockClient, 123, 456, 789) + + tests, err := codesphere.Registry(opts).SelectPlaylist(codesphere.DefaultPlaylist) + Expect(err).NotTo(HaveOccurred()) + + results := runner.Run(context.Background(), tests) + + Expect(testplan.Err(results)).To(BeNil()) + Expect(results).To(HaveLen(2)) + }) + + It("skips the smoketest if the status test fails with fail-fast", func() { + mockClient.EXPECT().ListWorkspacePlans().Return(nil, fmt.Errorf("connection refused")).Once() + + runner.FailFast = true + + tests, err := codesphere.Registry(opts).SelectPlaylist(codesphere.DefaultPlaylist) + Expect(err).NotTo(HaveOccurred()) + + results := runner.Run(context.Background(), tests) + + Expect(results[0].Status).To(Equal(testplan.StatusFailed)) + Expect(results[1].Name).To(Equal(codesphere.SmoketestTestName)) + Expect(results[1].Status).To(Equal(testplan.StatusSkipped)) + Expect(testplan.Err(results)).To(MatchError(ContainSubstring("status"))) + }) + }) +}) diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 591b1c8b0..47814a597 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -74,6 +74,10 @@ func GetRootCmd() *cobra.Command { // Smoke test commands AddSmoketestCmd(rootCmd, opts) + // Status and test commands + AddStatusCmd(rootCmd, opts) + AddTestCmd(rootCmd, opts) + // Resource creation commands AddCreateCmd(rootCmd, opts) diff --git a/cli/cmd/status.go b/cli/cmd/status.go new file mode 100644 index 000000000..878760db0 --- /dev/null +++ b/cli/cmd/status.go @@ -0,0 +1,30 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "github.com/codesphere-cloud/cs-go/pkg/io" + "github.com/codesphere-cloud/oms/cli/cmd/codesphere" + "github.com/codesphere-cloud/oms/cli/cmd/util" + "github.com/spf13/cobra" +) + +// StatusCmd represents the status command +type StatusCmd struct { + cmd *cobra.Command +} + +// AddStatusCmd adds the status command and its subcommands to the root command. +func AddStatusCmd(rootCmd *cobra.Command, opts *util.GlobalOptions) { + status := StatusCmd{ + cmd: &cobra.Command{ + Use: "status", + Short: "Check the status of Codesphere components", + Long: io.Long(`Check whether Codesphere installations or components are up and ready.`), + }, + } + util.AddCmd(rootCmd, status.cmd) + + codesphere.AddStatusCmd(status.cmd, opts) +} diff --git a/cli/cmd/test.go b/cli/cmd/test.go new file mode 100644 index 000000000..0e60c6510 --- /dev/null +++ b/cli/cmd/test.go @@ -0,0 +1,63 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "github.com/codesphere-cloud/cs-go/pkg/io" + "github.com/codesphere-cloud/oms/cli/cmd/codesphere" + "github.com/codesphere-cloud/oms/cli/cmd/util" + "github.com/spf13/cobra" +) + +// TestCmd represents the test command +type TestCmd struct { + cmd *cobra.Command +} + +// TestListCmd represents the test list command +type TestListCmd struct { + cmd *cobra.Command +} + +// AddTestCmd adds the test command and its subcommands to the root command. +func AddTestCmd(rootCmd *cobra.Command, opts *util.GlobalOptions) { + test := TestCmd{ + cmd: &cobra.Command{ + Use: "test", + Short: "Run playlists of tests against Codesphere components", + Long: io.Long(`Run playlists of tests against Codesphere components. + + A playlist bundles individual tests, such as a status report or a smoke test, + into a single run with a summarized result.`), + }, + } + util.AddCmd(rootCmd, test.cmd) + + codesphere.AddTestCmd(test.cmd, opts) + AddTestListCmd(test.cmd) +} + +// AddTestListCmd adds the test list command to the given parent command. +func AddTestListCmd(parent *cobra.Command) { + list := TestListCmd{ + cmd: &cobra.Command{ + Use: "list", + Short: "List the available tests and playlists", + Long: io.Long(`List the tests that can be run against a Codesphere installation and the playlists that group them.`), + Example: util.FormatExamples("test list", []io.Example{ + { + Cmd: "", + Desc: "List the available tests and playlists", + }, + }), + }, + } + + list.cmd.RunE = func(cmd *cobra.Command, _ []string) error { + codesphere.Registry(&codesphere.TestCodesphereOpts{}).Describe(cmd.OutOrStdout()) + return nil + } + + util.AddCmd(parent, list.cmd) +} diff --git a/docs/README.md b/docs/README.md index dbbe2f036..7ed3c1788 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,7 +29,9 @@ like downloading new versions. * [oms register](oms_register.md) - Register a new API key * [oms revoke](oms_revoke.md) - Revoke resources available through OMS * [oms smoketest](oms_smoketest.md) - Run smoke tests for Codesphere components +* [oms status](oms_status.md) - Check the status of Codesphere components * [oms template](oms_template.md) - Render OMS configuration templates +* [oms test](oms_test.md) - Run playlists of tests against Codesphere components * [oms update](oms_update.md) - Update OMS related resources * [oms version](oms_version.md) - Print version diff --git a/docs/oms.md b/docs/oms.md index dbbe2f036..7ed3c1788 100644 --- a/docs/oms.md +++ b/docs/oms.md @@ -29,7 +29,9 @@ like downloading new versions. * [oms register](oms_register.md) - Register a new API key * [oms revoke](oms_revoke.md) - Revoke resources available through OMS * [oms smoketest](oms_smoketest.md) - Run smoke tests for Codesphere components +* [oms status](oms_status.md) - Check the status of Codesphere components * [oms template](oms_template.md) - Render OMS configuration templates +* [oms test](oms_test.md) - Run playlists of tests against Codesphere components * [oms update](oms_update.md) - Update OMS related resources * [oms version](oms_version.md) - Print version diff --git a/docs/oms_status.md b/docs/oms_status.md new file mode 100644 index 000000000..6183fb6ce --- /dev/null +++ b/docs/oms_status.md @@ -0,0 +1,19 @@ +## oms status + +Check the status of Codesphere components + +### Synopsis + +Check whether Codesphere installations or components are up and ready. + +### Options + +``` + -h, --help help for status +``` + +### SEE ALSO + +* [oms](oms.md) - Codesphere Operations Management System (OMS) +* [oms status codesphere](oms_status_codesphere.md) - Check the status of a Codesphere installation + diff --git a/docs/oms_status_codesphere.md b/docs/oms_status_codesphere.md new file mode 100644 index 000000000..b21f9c04f --- /dev/null +++ b/docs/oms_status_codesphere.md @@ -0,0 +1,38 @@ +## oms status codesphere + +Check the status of a Codesphere installation + +### Synopsis + +Check whether a Codesphere installation is reachable and ready to use, +by querying the Codesphere API. + +``` +oms status codesphere [flags] +``` + +### Examples + +``` +# Check the status of a Codesphere installation +$ oms status codesphere --baseurl https://codesphere.example.com/api --token YOUR_TOKEN + +# Block and retry until the Codesphere installation is ready +$ oms status codesphere --baseurl https://codesphere.example.com/api --token YOUR_TOKEN --wait + +``` + +### Options + +``` + --baseurl string Base URL of the Codesphere API + -h, --help help for codesphere + --timeout duration Timeout when waiting for the installation to become ready (default 5m0s) + --token string API token for authentication + --wait Block and retry until the installation is ready +``` + +### SEE ALSO + +* [oms status](oms_status.md) - Check the status of Codesphere components + diff --git a/docs/oms_test.md b/docs/oms_test.md new file mode 100644 index 000000000..171cecb7b --- /dev/null +++ b/docs/oms_test.md @@ -0,0 +1,23 @@ +## oms test + +Run playlists of tests against Codesphere components + +### Synopsis + +Run playlists of tests against Codesphere components. + +A playlist bundles individual tests, such as a status report or a smoke test, +into a single run with a summarized result. + +### Options + +``` + -h, --help help for test +``` + +### SEE ALSO + +* [oms](oms.md) - Codesphere Operations Management System (OMS) +* [oms test codesphere](oms_test_codesphere.md) - Run a playlist of tests against a Codesphere installation +* [oms test list](oms_test_list.md) - List the available tests and playlists + diff --git a/docs/oms_test_codesphere.md b/docs/oms_test_codesphere.md new file mode 100644 index 000000000..508ebd4a0 --- /dev/null +++ b/docs/oms_test_codesphere.md @@ -0,0 +1,60 @@ +## oms test codesphere + +Run a playlist of tests against a Codesphere installation + +### Synopsis + +Run a playlist of tests against a Codesphere installation. + +A playlist is an ordered selection of tests, for example a status report +followed by a smoke test. Every test is run even if an earlier one failed, +unless --fail-fast is set, and the results are summarized at the end. + +Run 'oms test list' to see the available tests and playlists. + +``` +oms test codesphere [flags] +``` + +### Examples + +``` +# Run the "default" playlist against a Codesphere installation +$ oms test codesphere --baseurl https://codesphere.example.com/api --token YOUR_TOKEN + +# Run a specific playlist +$ oms test codesphere --baseurl https://codesphere.example.com/api --token YOUR_TOKEN --playlist readiness + +# Run a specific list of tests, in the given order +$ oms test codesphere --baseurl https://codesphere.example.com/api --token YOUR_TOKEN --tests status,smoketest + +# Wait for the installation to become ready before running the remaining tests +$ oms test codesphere --baseurl https://codesphere.example.com/api --token YOUR_TOKEN --wait + +# Stop at the first failing test instead of running the whole playlist +$ oms test codesphere --baseurl https://codesphere.example.com/api --token YOUR_TOKEN --fail-fast + +``` + +### Options + +``` + --baseurl string Base URL of the Codesphere API + --fail-fast Skip the remaining tests after the first failure + -h, --help help for codesphere + --plan-id string Plan ID to use for workspaces created by tests + --playlist string Playlist of tests to run (default,readiness) (default "default") + --profile string CI profile to use for landscape and pipeline (default "ci.yml") + -q, --quiet Suppress progress logging + --team-id string Team ID to run tests in + --tests strings Comma-separated list of tests to run, in the given order (status,smoketest). Takes precedence over --playlist. + --timeout duration Timeout for the entire test run (default 20m0s) + --token string API token for authentication + --wait Wait for the installation to become ready during the status test + --wait-timeout duration Timeout when waiting for the installation to become ready (default 5m0s) +``` + +### SEE ALSO + +* [oms test](oms_test.md) - Run playlists of tests against Codesphere components + diff --git a/docs/oms_test_list.md b/docs/oms_test_list.md new file mode 100644 index 000000000..a3f00c8b9 --- /dev/null +++ b/docs/oms_test_list.md @@ -0,0 +1,30 @@ +## oms test list + +List the available tests and playlists + +### Synopsis + +List the tests that can be run against a Codesphere installation and the playlists that group them. + +``` +oms test list [flags] +``` + +### Examples + +``` +# List the available tests and playlists +$ oms test list + +``` + +### Options + +``` + -h, --help help for list +``` + +### SEE ALSO + +* [oms test](oms_test.md) - Run playlists of tests against Codesphere components + diff --git a/internal/codesphere/testplan/testplan.go b/internal/codesphere/testplan/testplan.go new file mode 100644 index 000000000..fdc1fe42c --- /dev/null +++ b/internal/codesphere/testplan/testplan.go @@ -0,0 +1,347 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package testplan runs ordered playlists of tests against a Codesphere +// installation and reports their results. +// +// A Test is a single, self-contained check (for example a status report or a +// smoke test). A Playlist is a named, ordered selection of those tests, so +// operators can run a well-known set of checks with a single command. +package testplan + +import ( + "context" + "errors" + "fmt" + "io" + "slices" + "strings" + "text/tabwriter" + "time" +) + +const ( + // ANSI color codes + colorGreen = "\033[32m" + colorRed = "\033[31m" + colorYellow = "\033[33m" + colorBold = "\033[1m" + colorReset = "\033[0m" +) + +// Status is the outcome of a single test run. +type Status string + +// The outcomes a test can have. A test that was not run at all, because an +// earlier test failed or the run was cancelled, is skipped. +const ( + StatusPassed Status = "PASS" + StatusFailed Status = "FAIL" + StatusSkipped Status = "SKIP" +) + +func (s Status) colored() string { + switch s { + case StatusPassed: + return colorGreen + string(s) + colorReset + case StatusFailed: + return colorRed + string(s) + colorReset + default: + return colorYellow + string(s) + colorReset + } +} + +// Test is a single, independently runnable check of a Codesphere installation. +type Test interface { + Name() string + Description() string + Run(ctx context.Context, out io.Writer) error +} + +// Func adapts a plain function into a Test. +type Func struct { + TestName string + Desc string + Fn func(ctx context.Context, out io.Writer) error +} + +// Name returns the name the test is selected by. +func (f *Func) Name() string { return f.TestName } + +// Description returns what the test does, as shown in listings and progress logs. +func (f *Func) Description() string { return f.Desc } + +// Run executes the wrapped function. +func (f *Func) Run(ctx context.Context, out io.Writer) error { + return f.Fn(ctx, out) +} + +// Result records the outcome of a single test. +type Result struct { + Name string + Status Status + Duration time.Duration + Err error +} + +// Playlist is a named, ordered selection of tests. +type Playlist struct { + Name string + Description string + Tests []string +} + +// Registry holds the tests that can be run and the playlists that select them. +type Registry struct { + tests []Test + playlists []Playlist +} + +// NewRegistry returns a registry of the given tests, in the order they are +// passed. Tests keep that order unless a playlist specifies a different one. +func NewRegistry(tests ...Test) *Registry { + return &Registry{tests: tests} +} + +// AddPlaylist registers a named selection of tests. +func (r *Registry) AddPlaylist(p Playlist) { + r.playlists = append(r.playlists, p) +} + +// Tests returns all registered tests. +func (r *Registry) Tests() []Test { + return slices.Clone(r.tests) +} + +// Playlists returns all registered playlists. +func (r *Registry) Playlists() []Playlist { + return slices.Clone(r.playlists) +} + +// TestNames returns the names of all registered tests, in registration order. +func (r *Registry) TestNames() []string { + names := make([]string, 0, len(r.tests)) + for _, t := range r.tests { + names = append(names, t.Name()) + } + + return names +} + +// PlaylistNames returns the names of all registered playlists. +func (r *Registry) PlaylistNames() []string { + names := make([]string, 0, len(r.playlists)) + for _, p := range r.playlists { + names = append(names, p.Name) + } + + return names +} + +// Select resolves test names to tests, keeping the requested order. Unknown +// names are reported instead of silently ignored, so a typo doesn't quietly +// shrink the test run. +func (r *Registry) Select(names []string) ([]Test, error) { + if len(names) == 0 { + return nil, errors.New("no tests selected") + } + + byName := make(map[string]Test, len(r.tests)) + for _, t := range r.tests { + byName[t.Name()] = t + } + + selected := make([]Test, 0, len(names)) + + var unknown []string + + for _, name := range names { + test, ok := byName[name] + if !ok { + unknown = append(unknown, name) + continue + } + + if slices.ContainsFunc(selected, func(t Test) bool { return t.Name() == name }) { + continue + } + + selected = append(selected, test) + } + + if len(unknown) > 0 { + return nil, fmt.Errorf("unknown test(s) %s, available tests are %s", + strings.Join(unknown, ","), strings.Join(r.TestNames(), ",")) + } + + return selected, nil +} + +// SelectPlaylist resolves a playlist name to the tests it contains. +func (r *Registry) SelectPlaylist(name string) ([]Test, error) { + idx := slices.IndexFunc(r.playlists, func(p Playlist) bool { return p.Name == name }) + if idx < 0 { + return nil, fmt.Errorf("unknown playlist %q, available playlists are %s", + name, strings.Join(r.PlaylistNames(), ",")) + } + + tests, err := r.Select(r.playlists[idx].Tests) + if err != nil { + return nil, fmt.Errorf("playlist %q: %w", name, err) + } + + return tests, nil +} + +// Describe writes the available tests and playlists in a human readable form. +func (r *Registry) Describe(w io.Writer) { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + + printf(tw, "%sTests%s\n", colorBold, colorReset) + + for _, t := range r.tests { + printf(tw, " %s\t%s\n", t.Name(), t.Description()) + } + + printf(tw, "\n%sPlaylists%s\n", colorBold, colorReset) + + for _, p := range r.playlists { + printf(tw, " %s\t%s\t[%s]\n", p.Name, p.Description, strings.Join(p.Tests, ", ")) + } + + //nolint:errcheck // flushing to the command's output stream, nothing to recover from + tw.Flush() +} + +// Runner executes tests in order and reports what happened. +type Runner struct { + // Out receives both the progress log and the output of the tests themselves. + Out io.Writer + // FailFast skips the remaining tests as soon as one fails. + FailFast bool + // Quiet suppresses the per-test progress log, but not the summary. + Quiet bool +} + +// Run executes the tests in order and returns one result per test. Tests that +// are not run (because of a failure with FailFast, or an expired context) are +// reported as skipped, so the result list always covers the full playlist. +func (r *Runner) Run(ctx context.Context, tests []Test) []Result { + results := make([]Result, 0, len(tests)) + + for i, test := range tests { + if err := ctx.Err(); err != nil { + results = append(results, skipRemaining(tests[i:], fmt.Errorf("test run aborted: %w", err))...) + break + } + + r.logf("\n%s▶ %s%s: %s\n", colorBold, test.Name(), colorReset, test.Description()) + + start := time.Now() + err := test.Run(ctx, r.Out) + result := Result{Name: test.Name(), Duration: time.Since(start), Err: err} + + result.Status = StatusPassed + if err != nil { + result.Status = StatusFailed + } + + results = append(results, result) + + r.logf("%s %s (%s)\n", test.Name(), result.Status.colored(), formatDuration(result.Duration)) + + if err != nil && r.FailFast { + results = append(results, skipRemaining(tests[i+1:], errors.New("skipped after earlier failure"))...) + break + } + } + + return results +} + +func (r *Runner) logf(format string, args ...any) { + if r.Quiet || r.Out == nil { + return + } + + printf(r.Out, format, args...) +} + +func skipRemaining(tests []Test, reason error) []Result { + skipped := make([]Result, 0, len(tests)) + for _, t := range tests { + skipped = append(skipped, Result{Name: t.Name(), Status: StatusSkipped, Err: reason}) + } + + return skipped +} + +// Summarize writes a table of results followed by a one line tally. +func Summarize(w io.Writer, results []Result) { + var ( + passed, failed, skipped int + total time.Duration + ) + for _, res := range results { + total += res.Duration + switch res.Status { + case StatusPassed: + passed++ + case StatusFailed: + failed++ + default: + skipped++ + } + } + + printf(w, "\n%sTest results%s\n", colorBold, colorReset) + + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + + for _, res := range results { + detail := "" + if res.Err != nil { + detail = res.Err.Error() + } + + printf(tw, " %s\t%s\t%s\t%s\n", res.Status.colored(), res.Name, formatDuration(res.Duration), detail) + } + //nolint:errcheck // flushing to the command's output stream, nothing to recover from + tw.Flush() + + printf(w, "\n%d test(s): %d passed, %d failed, %d skipped in %s\n", + len(results), passed, failed, skipped, formatDuration(total)) +} + +// Err aggregates the failures of a test run into a single error, or returns +// nil if nothing failed. +func Err(results []Result) error { + var failed []string + + for _, res := range results { + if res.Status == StatusFailed { + failed = append(failed, res.Name) + } + } + + if len(failed) == 0 { + return nil + } + + return fmt.Errorf("%d of %d test(s) failed: %s", len(failed), len(results), strings.Join(failed, ",")) +} + +// printf writes to the report output. Write errors are ignored: the output is +// the operator's terminal, and there is no fallback to report them on. +func printf(w io.Writer, format string, args ...any) { + //nolint:errcheck // see above + fmt.Fprintf(w, format, args...) +} + +func formatDuration(d time.Duration) string { + if d < time.Second { + return d.Round(time.Millisecond).String() + } + + return d.Round(100 * time.Millisecond).String() +} diff --git a/internal/codesphere/testplan/testplan_suite_test.go b/internal/codesphere/testplan/testplan_suite_test.go new file mode 100644 index 000000000..fbe3f880b --- /dev/null +++ b/internal/codesphere/testplan/testplan_suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package testplan_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestTestplan(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Testplan Suite") +} diff --git a/internal/codesphere/testplan/testplan_test.go b/internal/codesphere/testplan/testplan_test.go new file mode 100644 index 000000000..987988ea2 --- /dev/null +++ b/internal/codesphere/testplan/testplan_test.go @@ -0,0 +1,235 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package testplan_test + +import ( + "bytes" + "context" + "fmt" + "io" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/codesphere/testplan" +) + +// recordingTest records that it ran and returns a fixed error. +type recordingTest struct { + name string + err error + ran *[]string +} + +func (t *recordingTest) Name() string { return t.name } +func (t *recordingTest) Description() string { return t.name + " description" } + +func (t *recordingTest) Run(_ context.Context, out io.Writer) error { + *t.ran = append(*t.ran, t.name) + _, _ = fmt.Fprintf(out, "output of %s\n", t.name) + + return t.err +} + +var _ = Describe("Testplan", func() { + var ( + ran []string + out *bytes.Buffer + passes *recordingTest + fails *recordingTest + second *recordingTest + ) + + newTest := func(name string, err error) *recordingTest { + return &recordingTest{name: name, err: err, ran: &ran} + } + + BeforeEach(func() { + ran = []string{} + out = &bytes.Buffer{} + passes = newTest("passes", nil) + fails = newTest("fails", fmt.Errorf("boom")) + second = newTest("second", nil) + }) + + Describe("Registry", func() { + var registry *testplan.Registry + + BeforeEach(func() { + registry = testplan.NewRegistry(passes, fails, second) + registry.AddPlaylist(testplan.Playlist{ + Name: "default", + Tests: []string{"fails", "passes"}, + }) + }) + + It("lists tests and playlists in registration order", func() { + Expect(registry.TestNames()).To(Equal([]string{"passes", "fails", "second"})) + Expect(registry.PlaylistNames()).To(Equal([]string{"default"})) + }) + + It("selects tests in the requested order", func() { + tests, err := registry.Select([]string{"second", "passes"}) + + Expect(err).NotTo(HaveOccurred()) + Expect(tests).To(HaveLen(2)) + Expect(tests[0].Name()).To(Equal("second")) + Expect(tests[1].Name()).To(Equal("passes")) + }) + + It("ignores duplicates in a selection", func() { + tests, err := registry.Select([]string{"passes", "passes"}) + + Expect(err).NotTo(HaveOccurred()) + Expect(tests).To(HaveLen(1)) + }) + + It("reports unknown test names", func() { + _, err := registry.Select([]string{"passes", "nope"}) + + Expect(err).To(MatchError(ContainSubstring("unknown test(s) nope"))) + Expect(err).To(MatchError(ContainSubstring("passes,fails,second"))) + }) + + It("returns an error for an empty selection", func() { + _, err := registry.Select(nil) + + Expect(err).To(MatchError(ContainSubstring("no tests selected"))) + }) + + It("resolves a playlist to its tests, keeping the playlist order", func() { + tests, err := registry.SelectPlaylist("default") + + Expect(err).NotTo(HaveOccurred()) + Expect(tests[0].Name()).To(Equal("fails")) + Expect(tests[1].Name()).To(Equal("passes")) + }) + + It("reports an unknown playlist", func() { + _, err := registry.SelectPlaylist("nope") + + Expect(err).To(MatchError(ContainSubstring(`unknown playlist "nope"`))) + Expect(err).To(MatchError(ContainSubstring("available playlists are default"))) + }) + + It("reports a playlist that references an unknown test", func() { + registry.AddPlaylist(testplan.Playlist{Name: "broken", Tests: []string{"nope"}}) + + _, err := registry.SelectPlaylist("broken") + + Expect(err).To(MatchError(ContainSubstring(`playlist "broken"`))) + Expect(err).To(MatchError(ContainSubstring("unknown test(s) nope"))) + }) + + It("describes tests and playlists", func() { + registry.Describe(out) + + Expect(out.String()).To(ContainSubstring("passes description")) + Expect(out.String()).To(ContainSubstring("default")) + Expect(out.String()).To(ContainSubstring("[fails, passes]")) + }) + }) + + Describe("Runner", func() { + var runner *testplan.Runner + + BeforeEach(func() { + runner = &testplan.Runner{Out: out} + }) + + It("runs all tests and reports their status", func() { + results := runner.Run(context.Background(), []testplan.Test{passes, fails, second}) + + Expect(ran).To(Equal([]string{"passes", "fails", "second"})) + Expect(results).To(HaveLen(3)) + Expect(results[0].Status).To(Equal(testplan.StatusPassed)) + Expect(results[1].Status).To(Equal(testplan.StatusFailed)) + Expect(results[1].Err).To(MatchError("boom")) + Expect(results[2].Status).To(Equal(testplan.StatusPassed)) + }) + + It("continues after a failure by default", func() { + runner.Run(context.Background(), []testplan.Test{fails, second}) + + Expect(ran).To(Equal([]string{"fails", "second"})) + }) + + It("skips the remaining tests with fail-fast", func() { + runner.FailFast = true + + results := runner.Run(context.Background(), []testplan.Test{fails, second}) + + Expect(ran).To(Equal([]string{"fails"})) + Expect(results).To(HaveLen(2)) + Expect(results[1].Name).To(Equal("second")) + Expect(results[1].Status).To(Equal(testplan.StatusSkipped)) + }) + + It("skips all tests when the context is already done", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + results := runner.Run(ctx, []testplan.Test{passes, second}) + + Expect(ran).To(BeEmpty()) + Expect(results).To(HaveLen(2)) + + for _, res := range results { + Expect(res.Status).To(Equal(testplan.StatusSkipped)) + Expect(res.Err).To(MatchError(ContainSubstring("test run aborted"))) + } + }) + + It("forwards test output and logs progress", func() { + runner.Run(context.Background(), []testplan.Test{passes}) + + Expect(out.String()).To(ContainSubstring("passes description")) + Expect(out.String()).To(ContainSubstring("output of passes")) + Expect(out.String()).To(ContainSubstring("PASS")) + }) + + It("keeps test output but drops progress logging when quiet", func() { + runner.Quiet = true + + runner.Run(context.Background(), []testplan.Test{passes}) + + Expect(out.String()).To(ContainSubstring("output of passes")) + Expect(out.String()).NotTo(ContainSubstring("passes description")) + }) + }) + + Describe("Summarize", func() { + It("lists every result and tallies them", func() { + results := []testplan.Result{ + {Name: "passes", Status: testplan.StatusPassed}, + {Name: "fails", Status: testplan.StatusFailed, Err: fmt.Errorf("boom")}, + {Name: "second", Status: testplan.StatusSkipped}, + } + + testplan.Summarize(out, results) + + Expect(out.String()).To(ContainSubstring("passes")) + Expect(out.String()).To(ContainSubstring("boom")) + Expect(out.String()).To(ContainSubstring("3 test(s): 1 passed, 1 failed, 1 skipped")) + }) + }) + + Describe("Err", func() { + It("returns nil if nothing failed", func() { + Expect(testplan.Err([]testplan.Result{ + {Name: "passes", Status: testplan.StatusPassed}, + {Name: "second", Status: testplan.StatusSkipped}, + })).To(BeNil()) + }) + + It("names the failed tests", func() { + err := testplan.Err([]testplan.Result{ + {Name: "passes", Status: testplan.StatusPassed}, + {Name: "fails", Status: testplan.StatusFailed}, + }) + + Expect(err).To(MatchError("1 of 2 test(s) failed: fails")) + }) + }) +})