diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml new file mode 100644 index 0000000..4aca2ba --- /dev/null +++ b/.github/workflows/pr-build.yml @@ -0,0 +1,25 @@ +name: Verify PR Build + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.26.4" + + - name: Download Go modules + run: go mod download + + - name: Build Conveyor + run: | + go build -o harness cmd/harness/main.go diff --git a/.gitignore b/.gitignore index d7a88a0..8041270 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,8 @@ harness # linux image -images/*-urunc/bzImage + +images/*/bzImage # python cache diff --git a/experiment.yml b/experiment.yml index 5bd8e20..61034a9 100644 --- a/experiment.yml +++ b/experiment.yml @@ -53,6 +53,7 @@ experiments: runtime: urunc snapshotter: overlayfs + network: workloads: default: @@ -60,3 +61,19 @@ experiments: other: - image: docker.io/jimjuniorb/iperf3-urunc:0.3 runtime: urunc + + cpu: + workloads: + default: + image: jimjuniorb/stress-ng-benchmark:0.1 + cpu: 1 + cpuMethod: matrixprod + timeout: 30s + metricsBrief: true + other: + - image: jimjuniorb/stress-ng-urunc:0.7 + cpu: 1 + cpuMethod: matrixprod + timeout: 30s + metricsBrief: true + runtime: urunc diff --git a/images/stress-ng-urunc/bunnyfile b/images/stress-ng-urunc/bunnyfile new file mode 100644 index 0000000..e0279ca --- /dev/null +++ b/images/stress-ng-urunc/bunnyfile @@ -0,0 +1,22 @@ +#syntax=harbor.nbfc.io/nubificus/bunny:latest +version: v0.1 + +platforms: + framework: linux + monitor: qemu + architecture: x86 + +rootfs: + from: jimjuniorb/stress-ng-benchmark:0.1 + type: raw + include: + - from: harbor.nbfc.io/nubificus/urunit:latest + source: /urunit + destination: /urunit + +kernel: + from: local + path: bzImage + + +entrypoint: ["/urunit", "/usr/local/bin/stress-ng-wrapper"] \ No newline at end of file diff --git a/images/stress-ng/Dockerfile b/images/stress-ng/Dockerfile new file mode 100644 index 0000000..7d03a59 --- /dev/null +++ b/images/stress-ng/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + stress-ng \ + && rm -rf /var/lib/apt/lists/* + +COPY entrypoint.sh /usr/local/bin/stress-ng-wrapper + +RUN chmod +x /usr/local/bin/stress-ng-wrapper + +ENTRYPOINT ["/usr/local/bin/stress-ng-wrapper"] \ No newline at end of file diff --git a/images/stress-ng/entrypoint.sh b/images/stress-ng/entrypoint.sh new file mode 100644 index 0000000..cbd0c73 --- /dev/null +++ b/images/stress-ng/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -u + +RESULT_FILE="$(mktemp /tmp/stress-ng-result.XXXXXX.yaml)" + +cleanup() { + rm -f "$RESULT_FILE" +} + +trap cleanup EXIT INT TERM + +stress-ng "$@" --yaml "$RESULT_FILE" >&2 +status=$? + +if [ -s "$RESULT_FILE" ]; then + cat "$RESULT_FILE" +fi + +exit "$status" \ No newline at end of file diff --git a/internal/cli/run.go b/internal/cli/run.go index ae88404..f69d7bb 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -16,6 +16,7 @@ import ( "github.com/urunc-dev/evaluation_suite/internal/orchestrator" "github.com/urunc-dev/evaluation_suite/internal/plan" harnessruntime "github.com/urunc-dev/evaluation_suite/internal/runtime" + runtimeCPU "github.com/urunc-dev/evaluation_suite/internal/runtime/cpu" runtimeHTTPReadiness "github.com/urunc-dev/evaluation_suite/internal/runtime/httpreadiness" runtimeLifecycle "github.com/urunc-dev/evaluation_suite/internal/runtime/lifecycle" runtimeNetwork "github.com/urunc-dev/evaluation_suite/internal/runtime/network" @@ -87,6 +88,9 @@ func NewRunCommand() *cobra.Command { func(trial plan.Trial) (harnessruntime.Adapter, error) { return runtimeHTTPReadiness.NewAdapter(), nil }, + func(trial plan.Trial) (harnessruntime.Adapter, error) { + return runtimeCPU.NewAdapter(), nil + }, ) orch := orchestrator.New(adapterFactories...) diff --git a/internal/doctor/checks.go b/internal/doctor/checks.go index 18a7ac8..7e5d226 100644 --- a/internal/doctor/checks.go +++ b/internal/doctor/checks.go @@ -28,7 +28,7 @@ func Run( checkTool(report, "ctr", true) checkTool(report, "containerd", false) - + checkTool(report, "nerdctl", true) if hasStorageExperiment(m) { checkTool(report, "fio", false) report.Add( @@ -305,8 +305,12 @@ func checkEnvironmentVisibility(ctx context.Context, report *Report) { } func hasStorageExperiment(m *manifest.Manifest) bool { + return hasExperiment(m, "storage") +} + +func hasExperiment(m *manifest.Manifest, expected string) bool { for name := range m.Experiments { - if strings.EqualFold(name, "storage") { + if strings.EqualFold(name, expected) { return true } } diff --git a/internal/manifest/types.go b/internal/manifest/types.go index ae72bce..fb4a3a8 100644 --- a/internal/manifest/types.go +++ b/internal/manifest/types.go @@ -20,11 +20,15 @@ type Workloads struct { } type Workload struct { - Image string `yaml:"image"` - Runtime string `yaml:"runtime,omitempty"` - Ports *Ports `yaml:"ports,omitempty"` - Snapshotter string `yaml:"snapshotter,omitempty"` - Volumes []Volume `yaml:"volumes,omitempty"` + Image string `yaml:"image"` + Runtime string `yaml:"runtime,omitempty"` + Ports *Ports `yaml:"ports,omitempty"` + Snapshotter string `yaml:"snapshotter,omitempty"` + Volumes []Volume `yaml:"volumes,omitempty"` + CPU int `yaml:"cpu,omitempty"` + CPUMethod string `yaml:"cpuMethod,omitempty"` + Timeout string `yaml:"timeout,omitempty"` + MetricsBrief bool `yaml:"metricsBrief,omitempty"` } type Ports struct { diff --git a/internal/manifest/validate.go b/internal/manifest/validate.go index f1f052d..373ade1 100644 --- a/internal/manifest/validate.go +++ b/internal/manifest/validate.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "time" ) func Validate(m *Manifest) error { @@ -67,6 +68,20 @@ func Validate(m *Manifest) error { errs = append(errs, vpath+".type is required") } } + + if name == "cpu" { + if w.CPU <= 0 { + errs = append(errs, path+".cpu must be greater than zero") + } + if w.CPUMethod == "" { + errs = append(errs, path+".cpuMethod is required") + } + if w.Timeout == "" { + errs = append(errs, path+".timeout is required") + } else if duration, err := time.ParseDuration(w.Timeout); err != nil || duration <= 0 { + errs = append(errs, path+".timeout must be a positive Go duration such as 30s") + } + } } validateWorkload("experiments."+name+".workloads.default", exp.Workloads.Default) diff --git a/internal/plan/generate.go b/internal/plan/generate.go index 1235916..b595c22 100644 --- a/internal/plan/generate.go +++ b/internal/plan/generate.go @@ -129,6 +129,10 @@ func buildTrial( Ports: workload.Ports, Volumes: workload.Volumes, Snapshotter: workload.Snapshotter, + CPU: workload.CPU, + CPUMethod: workload.CPUMethod, + Timeout: workload.Timeout, + MetricsBrief: workload.MetricsBrief, } } diff --git a/internal/plan/types.go b/internal/plan/types.go index a9ba116..f3749ae 100644 --- a/internal/plan/types.go +++ b/internal/plan/types.go @@ -16,5 +16,9 @@ type Trial struct { Image string `json:"image"` Ports *manifest.Ports `json:"ports,omitempty"` Volumes []manifest.Volume `json:"volumes,omitempty"` - Snapshotter string `yaml:"snapshotter,omitempty"` + Snapshotter string `json:"snapshotter,omitempty"` + CPU int `json:"cpu,omitempty"` + CPUMethod string `json:"cpuMethod,omitempty"` + Timeout string `json:"timeout,omitempty"` + MetricsBrief bool `json:"metricsBrief,omitempty"` } diff --git a/internal/runtime/cpu/adapter.go b/internal/runtime/cpu/adapter.go new file mode 100644 index 0000000..78cfdf6 --- /dev/null +++ b/internal/runtime/cpu/adapter.go @@ -0,0 +1,193 @@ +package cpu + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" + + harnessruntime "github.com/urunc-dev/evaluation_suite/internal/runtime" +) + +type Adapter struct{} + +func NewAdapter() *Adapter { + return &Adapter{} +} + +func (a *Adapter) ExperimentName() string { + return "cpu" +} + +func (a *Adapter) Prepare(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return noOpStage(ctx, harnessruntime.StagePrepare, tc) +} + +func (a *Adapter) CreateTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return noOpStage(ctx, harnessruntime.StageCreate, tc) +} + +func (a *Adapter) StartTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + startedAt := time.Now() + args := nerdctlArgs(tc) + cmd := exec.CommandContext(ctx, "nerdctl", args...) + + log.Println("Running Command: ", cmd.String()) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdin = os.Stdin + cmd.Stdout = &stdout // The benchmark image writes YAML only to stdout. + cmd.Stderr = &stderr // stress-ng and nerdctl logs remain separate. + + err := cmd.Run() + finishedAt := time.Now() + result := harnessruntime.StageResult{ + Stage: harnessruntime.StageStart, + StartedAt: startedAt, + FinishedAt: finishedAt, + Duration: finishedAt.Sub(startedAt), + Description: fmt.Sprintf( + "Run CPU benchmark: trial=%s runtime=%s handler=%s image=%s", + tc.Trial.ID, + tc.Trial.RuntimeName, + tc.Trial.RuntimeHandler, + tc.Trial.Image, + ), + Data: map[string]any{ + "stderr": stderr.String(), + }, + } + + if err != nil { + return result, fmt.Errorf("run CPU benchmark with nerdctl: %w; stderr: %s", err, stderr.String()) + } + + fmt.Println(stdout.String()) + + benchmarkJSON, err := YAMLToJSON(stdout.String()) + if err != nil { + return result, fmt.Errorf("convert stress-ng YAML output to JSON: %w; stdout: %s; stderr: %s", err, stdout.String(), stderr.String()) + } + + result.Data = map[string]any{ + "benchmark": benchmarkJSON, + "stderr": stderr.String(), + } + return result, nil +} + +func (a *Adapter) WaitReady(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return noOpStage(ctx, harnessruntime.StageWaitReady, tc) +} + +func (a *Adapter) Stop(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return noOpStage(ctx, harnessruntime.StageStop, tc) +} + +func (a *Adapter) DeleteTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return noOpStage(ctx, harnessruntime.StageDelete, tc) +} + +func (a *Adapter) Cleanup(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return noOpStage(ctx, harnessruntime.StageCleanup, tc) +} + +func nerdctlArgs(tc harnessruntime.TrialContext) []string { + args := []string{ + "run", + "-it", + "--rm", + "--runtime=" + tc.Trial.RuntimeHandler, + tc.Trial.Image, + "--cpu", strconv.Itoa(tc.Trial.CPU), + "--cpu-method", tc.Trial.CPUMethod, + "--timeout", tc.Trial.Timeout, + } + if tc.Trial.MetricsBrief { + args = append(args, "--metrics-brief") + } + return args +} + +// YAMLToJSON extracts a YAML document embedded in stdout, converts it to JSON, +// and returns the resulting raw JSON. +// +// It expects the YAML document to begin with a line containing "---". +// The terminating "..." marker is optional. +func YAMLToJSON(stdout string) (json.RawMessage, error) { + yamlDocument, err := extractYAMLDocument(stdout) + if err != nil { + return nil, err + } + + var value any + if err := yaml.Unmarshal([]byte(yamlDocument), &value); err != nil { + return nil, fmt.Errorf("decode stress-ng YAML: %w", err) + } + + jsonData, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("encode stress-ng result as JSON: %w", err) + } + + return json.RawMessage(jsonData), nil +} + +// extractYAMLDocument extracts the content beginning at the YAML document +// marker "---" and ending at the optional document marker "...". +func extractYAMLDocument(output string) (string, error) { + lines := strings.Split(strings.ReplaceAll(output, "\r\n", "\n"), "\n") + + start := -1 + end := len(lines) + + for index, line := range lines { + switch strings.TrimSpace(line) { + case "---": + if start == -1 { + start = index + } + case "...": + if start != -1 { + end = index + 1 + goto extracted + } + } + } + +extracted: + if start == -1 { + return "", errors.New("YAML document marker '---' not found in stress-ng output") + } + + document := strings.TrimSpace(strings.Join(lines[start:end], "\n")) + if document == "" { + return "", errors.New("stress-ng YAML document is empty") + } + + return document, nil +} +func noOpStage(ctx context.Context, stage harnessruntime.Stage, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + startedAt := time.Now() + if err := ctx.Err(); err != nil { + return harnessruntime.StageResult{}, err + } + finishedAt := time.Now() + return harnessruntime.StageResult{ + Stage: stage, + StartedAt: startedAt, + FinishedAt: finishedAt, + Duration: finishedAt.Sub(startedAt), + Description: fmt.Sprintf("No-op for CLI CPU benchmark: trial=%s", tc.Trial.ID), + }, nil +} diff --git a/internal/runtime/cpu/adapter_test.go b/internal/runtime/cpu/adapter_test.go new file mode 100644 index 0000000..647ac40 --- /dev/null +++ b/internal/runtime/cpu/adapter_test.go @@ -0,0 +1,29 @@ +package cpu + +import ( + "reflect" + "testing" + + "github.com/urunc-dev/evaluation_suite/internal/plan" + harnessruntime "github.com/urunc-dev/evaluation_suite/internal/runtime" +) + +func TestNerdctlArgs(t *testing.T) { + tc := harnessruntime.TrialContext{Trial: plan.Trial{ + RuntimeHandler: "io.containerd.runc.v2", + Image: "jimjuniorb/stress-ng-benchmark:0.1", + CPU: 1, + CPUMethod: "matrixprod", + Timeout: "30s", + MetricsBrief: true, + }} + + want := []string{ + "run", "-it", "--rm", "--runtime=io.containerd.runc.v2", + "jimjuniorb/stress-ng-benchmark:0.1", + "--cpu", "1", "--cpu-method", "matrixprod", "--timeout", "30s", "--metrics-brief", + } + if got := nerdctlArgs(tc); !reflect.DeepEqual(got, want) { + t.Fatalf("nerdctlArgs() = %#v, want %#v", got, want) + } +} diff --git a/workloads/fio-kernel/.config b/workloads/fio-kernel/.config index b6dc96c..8dba203 100644 --- a/workloads/fio-kernel/.config +++ b/workloads/fio-kernel/.config @@ -146,6 +146,7 @@ CONFIG_RCU_NEED_SEGCBLIST=y # end of RCU Subsystem # CONFIG_IKCONFIG is not set +# CONFIG_IKHEADERS is not set CONFIG_LOG_BUF_SHIFT=17 CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y @@ -670,6 +671,7 @@ CONFIG_SLAB_MERGE_DEFAULT=y # CONFIG_SLAB_FREELIST_RANDOM is not set # CONFIG_SLAB_FREELIST_HARDENED is not set # CONFIG_SLAB_BUCKETS is not set +# CONFIG_SLUB_STATS is not set # CONFIG_RANDOM_KMALLOC_CACHES is not set # end of Slab allocator options @@ -704,6 +706,7 @@ CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK=y CONFIG_HAVE_SETUP_PER_CPU_AREA=y # CONFIG_CMA is not set CONFIG_GENERIC_EARLY_IOREMAP=y +# CONFIG_IDLE_PAGE_TRACKING is not set CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y CONFIG_ARCH_HAS_CURRENT_STACK_POINTER=y CONFIG_ARCH_HAS_PTE_DEVMAP=y @@ -808,6 +811,7 @@ CONFIG_VIRTIO_VSOCKETS_COMMON=y # CONFIG_NET_NCSI is not set CONFIG_MAX_SKB_FRAGS=17 CONFIG_NET_RX_BUSY_POLL=y +CONFIG_BQL=y # # Network testing @@ -864,6 +868,7 @@ CONFIG_PCIE_BUS_DEFAULT=y # CONFIG_PCIE_BUS_PERFORMANCE is not set # CONFIG_PCIE_BUS_PEER2PEER is not set # CONFIG_VGA_ARB is not set +# CONFIG_HOTPLUG_PCI is not set # # PCI controller drivers @@ -957,6 +962,7 @@ CONFIG_GENERIC_CPU_VULNERABILITIES=y # CONFIG_EDD is not set # CONFIG_FIRMWARE_MEMMAP is not set +# CONFIG_FW_CFG_SYSFS is not set # CONFIG_SYSFB_SIMPLEFB is not set # CONFIG_GOOGLE_FIRMWARE is not set @@ -980,6 +986,7 @@ CONFIG_BLK_DEV=y # CONFIG_BLK_DEV_NULL_BLK is not set # CONFIG_BLK_DEV_FD is not set # CONFIG_BLK_DEV_PCIESSD_MTIP32XX is not set +# CONFIG_ZRAM is not set # CONFIG_BLK_DEV_LOOP is not set # CONFIG_BLK_DEV_DRBD is not set # CONFIG_BLK_DEV_NBD is not set @@ -1394,6 +1401,7 @@ CONFIG_VHOST_NET=y # CONFIG_HYPERV is not set # end of Microsoft Hyper-V guest support +# CONFIG_GREYBUS is not set # CONFIG_COMEDI is not set # CONFIG_STAGING is not set # CONFIG_GOLDFISH is not set @@ -1650,7 +1658,8 @@ CONFIG_PROC_SYSCTL=y CONFIG_PROC_PAGE_MONITOR=y # CONFIG_PROC_CHILDREN is not set CONFIG_PROC_PID_ARCH_STATUS=y -# CONFIG_SYSFS is not set +CONFIG_KERNFS=y +CONFIG_SYSFS=y CONFIG_TMPFS=y # CONFIG_TMPFS_POSIX_ACL is not set # CONFIG_TMPFS_XATTR is not set @@ -1685,6 +1694,7 @@ CONFIG_9P_FS_POSIX_ACL=y CONFIG_PROC_MEM_ALWAYS_FORCE=y # CONFIG_PROC_MEM_FORCE_PTRACE is not set # CONFIG_PROC_MEM_NO_FORCE is not set +# CONFIG_SECURITY is not set # CONFIG_SECURITYFS is not set # CONFIG_HARDENED_USERCOPY is not set # CONFIG_FORTIFY_SOURCE is not set @@ -1958,6 +1968,7 @@ CONFIG_SWIOTLB=y CONFIG_DMA_NEED_SYNC=y # CONFIG_DMA_API_DEBUG is not set CONFIG_FORCE_NR_CPUS=y +CONFIG_DQL=y CONFIG_NLATTR=y # CONFIG_IRQ_POLL is not set CONFIG_DIMLIB=y @@ -1972,6 +1983,8 @@ CONFIG_ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION=y CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE=y CONFIG_ARCH_HAS_COPY_MC=y CONFIG_ARCH_STACKWALK=y +CONFIG_STACKDEPOT=y +CONFIG_STACKDEPOT_MAX_FRAMES=64 CONFIG_SBITMAP=y # CONFIG_LWQ_TEST is not set # end of Library routines @@ -2049,6 +2062,8 @@ CONFIG_HAVE_KCSAN_COMPILER=y # # CONFIG_PAGE_EXTENSION is not set # CONFIG_DEBUG_PAGEALLOC is not set +CONFIG_SLUB_DEBUG=y +# CONFIG_SLUB_DEBUG_ON is not set # CONFIG_PAGE_OWNER is not set # CONFIG_PAGE_TABLE_CHECK is not set # CONFIG_PAGE_POISONING is not set @@ -2075,6 +2090,7 @@ CONFIG_HAVE_ARCH_KASAN_VMALLOC=y CONFIG_CC_HAS_KASAN_GENERIC=y CONFIG_CC_HAS_KASAN_SW_TAGS=y CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y +# CONFIG_KASAN is not set CONFIG_HAVE_ARCH_KFENCE=y # CONFIG_KFENCE is not set CONFIG_HAVE_ARCH_KMSAN=y @@ -2125,7 +2141,7 @@ CONFIG_LOCK_DEBUGGING_SUPPORT=y # CONFIG_NMI_CHECK_CPU is not set # CONFIG_DEBUG_IRQFLAGS is not set -# CONFIG_STACKTRACE is not set +CONFIG_STACKTRACE=y # CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set # CONFIG_DEBUG_KOBJECT is not set @@ -2199,7 +2215,6 @@ CONFIG_IO_DELAY_0X80=y # CONFIG_PUNIT_ATOM_DEBUG is not set # CONFIG_UNWINDER_ORC is not set CONFIG_UNWINDER_FRAME_POINTER=y -# CONFIG_UNWINDER_GUESS is not set # end of x86 Debugging #