From a64d44e49bf77213f085cb5a2283bbf001e833bf Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 25 Jun 2026 11:55:47 -0500 Subject: [PATCH 1/3] Add libvirt/KVM VM scripts for Linux x86_64 testing Add setup and test scripts for running netdevsim DKMS smoke tests and ptp-operator e2e tests inside libvirt/KVM VMs on Linux x86_64 hosts. Scripts: - setup-libvirt-ubuntu.sh: create VM, install DKMS, add SSH config - test-libvirt-ubuntu.sh: full lifecycle including ptp-operator tests Update README with libvirt setup docs and test instructions: ./run-on-vm.sh --dkms --mode oc,bc,dualnicbc,dualnicbcha,dualfollower --- README.md | 54 +++ scripts/setup-libvirt-ubuntu.sh | 214 +++++++++++ scripts/test-libvirt-ubuntu.sh | 660 ++++++++++++++++++++++++++++++++ 3 files changed, 928 insertions(+) create mode 100755 scripts/setup-libvirt-ubuntu.sh create mode 100755 scripts/test-libvirt-ubuntu.sh diff --git a/README.md b/README.md index d0aade7..d17c0b9 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,60 @@ When using netdevsim inside containers or Kubernetes pods: - **Kubernetes pods:** The udev rule creates real `/dev/ptpN` device nodes (not symlinks) so they propagate into containers. +## Local libvirt/KVM VMs (Linux x86_64) + +### Prerequisites + +1. A Linux x86_64 host with KVM support. +2. Install libvirt, QEMU, and cloud-image-utils: + +```bash +sudo apt-get install -y qemu-kvm libvirt-daemon-system virtinst cloud-image-utils +``` + +### Quick Setup (no tests) + +`scripts/setup-libvirt-ubuntu.sh` creates a libvirt VM with the DKMS modules +installed and loaded, adds an SSH config entry, and stops — no tests are run. + +```bash +./scripts/setup-libvirt-ubuntu.sh + +# Then connect: +ssh netdevsim-ubuntu-test +``` + +### Running ptp-operator Tests + +Once the VM is set up, SSH in and run the ptp-operator test suite. Clone the +[ptp-operator](https://github.com/k8snetworkplumbingwg/ptp-operator) repo on +the VM, then run: + +```bash +cd ptp-operator/scripts +./run-on-vm.sh --dkms --mode oc,bc,dualnicbc,dualnicbcha,dualfollower | tee logs.txt +``` + +This builds and deploys the ptp-operator on a Kind cluster inside the VM and +runs all five test scenarios (ordinary clock, boundary clock, dual-NIC boundary +clock, dual-NIC boundary clock HA, dual follower). Results are streamed to +`logs.txt`. + +To run a single scenario: + +```bash +./run-on-vm.sh --dkms --mode bc | tee logs.txt +``` + +### Managing the VM + +```bash +virsh list --all # list VMs +virsh domifaddr netdevsim-ubuntu-test # get VM IP +virsh shutdown netdevsim-ubuntu-test # stop VM +virsh undefine --remove-all-storage netdevsim-ubuntu-test # delete VM +``` + ## Local UTM VMs (macOS) ### Prerequisites diff --git a/scripts/setup-libvirt-ubuntu.sh b/scripts/setup-libvirt-ubuntu.sh new file mode 100755 index 0000000..7726324 --- /dev/null +++ b/scripts/setup-libvirt-ubuntu.sh @@ -0,0 +1,214 @@ +#!/bin/bash +# +# Set up a libvirt/KVM VM with the netdevsim DKMS modules installed and +# loaded, without running any tests. Adds an SSH config entry so you +# can connect with just: ssh +# +# Delegates to test-libvirt-ubuntu.sh --skip-tests for the heavy lifting. +# +# Usage: +# ./scripts/setup-libvirt-ubuntu.sh [options] +# +# Options: +# --release 22.04|24.04 Ubuntu release (default: 24.04) +# --ssh-key PATH SSH private key for VM (auto-detected) +# --vm-name NAME libvirt VM name (default: netdevsim-ubuntu-test) +# --ram MiB RAM in MiB (default: 16384) +# --cpus N CPU cores (default: 4) +# --disk-size SIZE qemu-img resize target (default: 40G) +# --network NAME libvirt network (default: default) +# --cleanup Delete VM and remove SSH config entry +# --shell Drop into an interactive SSH shell after setup +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SSH_CONFIG="${HOME}/.ssh/config" + +# --------------------------------------------------------------------------- +# Parse --vm-name and --cleanup from args (needed before delegation) +# --------------------------------------------------------------------------- +VM_NAME="netdevsim-ubuntu-test" +CLEANUP=false +ARGS=("$@") + +i=0 +while (( i < ${#ARGS[@]} )); do + case "${ARGS[$i]}" in + --vm-name) + (( i+1 < ${#ARGS[@]} )) && VM_NAME="${ARGS[$((i+1))]}" + ;; + --cleanup) + CLEANUP=true + ;; + esac + (( i++ )) || true +done + +MARKER_BEGIN="# --- netdevsim-dkms managed: ${VM_NAME} ---" +MARKER_END="# --- end netdevsim-dkms: ${VM_NAME} ---" + +# --------------------------------------------------------------------------- +# Helper: remove the fenced SSH config block for VM_NAME +# --------------------------------------------------------------------------- +remove_ssh_config_entry() { + [[ -f "$SSH_CONFIG" ]] || return 0 + if grep -qF "$MARKER_BEGIN" "$SSH_CONFIG"; then + local tmp + tmp="$(mktemp)" + awk -v begin="$MARKER_BEGIN" -v end="$MARKER_END" ' + $0 == begin { skip=1; next } + $0 == end { skip=0; next } + !skip + ' "$SSH_CONFIG" > "$tmp" + mv "$tmp" "$SSH_CONFIG" + chmod 600 "$SSH_CONFIG" + echo "==> Removed SSH config entry for '${VM_NAME}'." + fi +} + +# --------------------------------------------------------------------------- +# Cleanup-only mode: remove SSH config + delegate --cleanup +# --------------------------------------------------------------------------- +if [[ "$CLEANUP" == true ]]; then + remove_ssh_config_entry + exec "${SCRIPT_DIR}/test-libvirt-ubuntu.sh" ${ARGS[@]+"${ARGS[@]}"} +fi + +# --------------------------------------------------------------------------- +# Guard: check for existing SSH config entry +# --------------------------------------------------------------------------- +if [[ -f "$SSH_CONFIG" ]] && grep -qF "$MARKER_BEGIN" "$SSH_CONFIG"; then + echo "WARNING: ${SSH_CONFIG} already has an entry for '${VM_NAME}'." + read -r -p "Remove the old entry and continue? [y/N] " answer + case "$answer" in + [yY]|[yY][eE][sS]) + remove_ssh_config_entry + ;; + *) + echo "Aborting. Pick a different name:" + echo " $0 --vm-name ${ARGS[*]:-}" + exit 1 + ;; + esac +fi + +# --------------------------------------------------------------------------- +# Delegate to test-libvirt-ubuntu.sh --skip-tests, capturing output for IP +# --------------------------------------------------------------------------- +LOGFILE="$(mktemp)" +trap 'rm -f "$LOGFILE"' EXIT + +echo "==> Setting up libvirt VM '${VM_NAME}' with DKMS (no tests) ..." + +"${SCRIPT_DIR}/test-libvirt-ubuntu.sh" --skip-tests ${ARGS[@]+"${ARGS[@]}"} 2>&1 | tee "$LOGFILE" +TEST_RC=${PIPESTATUS[0]} + +if [[ $TEST_RC -ne 0 ]]; then + echo "ERROR: test-libvirt-ubuntu.sh exited with code ${TEST_RC}." + exit "$TEST_RC" +fi + +# --------------------------------------------------------------------------- +# Extract VM IP and SSH key from test-libvirt-ubuntu.sh output +# --------------------------------------------------------------------------- +VM_IP="$(grep 'VM IP: ' "$LOGFILE" | awk '{for(i=1;i<=NF;i++) if($(i-1)=="IP:") print $i}' | tail -1 || true)" +SSH_KEY="$(grep 'ssh -i ' "$LOGFILE" | awk '{for(i=1;i<=NF;i++) if($i=="-i") {print $(i+1); exit}}' | tail -1 || true)" + +if [[ -z "$VM_IP" ]]; then + echo "WARNING: Could not extract VM IP from output. Skipping SSH config." + exit 0 +fi + +# --------------------------------------------------------------------------- +# Install test dependencies (but not the test repo itself) +# --------------------------------------------------------------------------- +SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5) +vm_ssh() { ssh "${SSH_OPTS[@]}" "root@${VM_IP}" "$@"; } + +echo "==> Installing test dependencies on VM ..." +vm_ssh bash <<'DEPS' +set -euo pipefail + +apt-get update -qq +apt-get install -y -qq podman pciutils uidmap slirp4netns openvswitch-switch \ + git ethtool linuxptp zsh 2>&1 | tail -5 + +GOARCH=amd64 + +# kind +curl -fsSLo /usr/bin/kind "https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-${GOARCH}" +chmod +x /usr/bin/kind + +# kubectl + oc +curl -fsSLo /tmp/oc.tar.gz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz +tar -C /tmp -xzf /tmp/oc.tar.gz oc kubectl +mv /tmp/oc /tmp/kubectl /usr/local/bin/ + +# Go +GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | head -n 1) +curl -fsSLo /tmp/go.tar.gz "https://go.dev/dl/${GO_VERSION}.linux-${GOARCH}.tar.gz" +rm -rf /usr/local/go +tar -C /usr/local -xzf /tmp/go.tar.gz + +# helm +curl -fsSL "https://get.helm.sh/helm-v3.17.3-linux-${GOARCH}.tar.gz" | tar -C /tmp -xzf - +mv "/tmp/linux-${GOARCH}/helm" /usr/local/bin/helm + +# PATH for root and ubuntu +for RC in /root/.bashrc /home/ubuntu/.bashrc; do + grep -q /usr/local/go/bin "$RC" 2>/dev/null || \ + echo 'export PATH=$PATH:$HOME/go/bin:/usr/local/go/bin' >> "$RC" +done + +# inotify limits (kind needs these) +sysctl -w fs.inotify.max_user_instances=512 +sysctl -w fs.inotify.max_user_watches=524288 + +# ginkgo (test runner) +export PATH=/usr/local/go/bin:/root/go/bin:$PATH +go install github.com/onsi/ginkgo/v2/ginkgo@latest + +# zsh + oh-my-zsh for root +chsh -s /usr/bin/zsh root +sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended +# Carry PATH into .zshrc +grep -q /usr/local/go/bin /root/.zshrc 2>/dev/null || \ + echo 'export PATH=$PATH:$HOME/go/bin:/usr/local/go/bin' >> /root/.zshrc + +echo "--- Verification ---" +podman --version +kind version +kubectl version --client 2>/dev/null | head -1 +go version +helm version --short +ovs-vsctl --version | head -1 +ginkgo version +DEPS +echo "==> Test dependencies installed." + +# --------------------------------------------------------------------------- +# Add SSH config entry +# --------------------------------------------------------------------------- +mkdir -p "$(dirname "$SSH_CONFIG")" +[[ -f "$SSH_CONFIG" ]] || touch "$SSH_CONFIG" +chmod 600 "$SSH_CONFIG" + +# Ensure a trailing newline before our block +[[ -s "$SSH_CONFIG" && "$(tail -c1 "$SSH_CONFIG")" != "" ]] && echo >> "$SSH_CONFIG" + +cat >> "$SSH_CONFIG" < SSH config entry added. Connect with:" +echo " ssh ${VM_NAME}" diff --git a/scripts/test-libvirt-ubuntu.sh b/scripts/test-libvirt-ubuntu.sh new file mode 100755 index 0000000..28db787 --- /dev/null +++ b/scripts/test-libvirt-ubuntu.sh @@ -0,0 +1,660 @@ +#!/bin/bash +# +# Launch an Ubuntu Cloud VM via libvirt/KVM, install the netdevsim DKMS +# modules from the source tree, load them, run smoke tests, then install +# all ptp-operator prerequisites and run the full ptp-operator test suite +# via run-on-vm.sh. +# +# This mirrors what the GitHub Actions CI does (ci.yml), but locally +# on Linux x86_64 via libvirt/KVM so you can iterate before pushing. +# +# Usage: +# ./scripts/test-libvirt-ubuntu.sh [options] +# +# Options: +# --release 22.04|24.04 Ubuntu release (default: 24.04) +# --ssh-key PATH SSH private key for VM (auto-detected) +# --vm-name NAME libvirt VM name (default: netdevsim-ubuntu-test) +# --ram MiB RAM in MiB (default: 16384) +# --cpus N CPU cores (default: 4) +# --disk-size SIZE qemu-img resize target (default: 40G) +# --network NAME libvirt network (default: default) +# --cleanup Delete VM after tests finish +# --skip-tests Only create VM + install DKMS, don't run tests +# --skip-ptp-operator Run smoke tests but skip ptp-operator suite +# --shell Drop into an interactive SSH shell after setup +# +set -euo pipefail + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +UBUNTU_RELEASE="24.04" +VM_NAME="netdevsim-ubuntu-test" +VM_RAM=16384 +VM_CPUS=4 +VM_DISK_SIZE="40G" +LIBVIRT_NET="default" +CLEANUP=false +SKIP_TESTS=false +SKIP_PTP_OPERATOR=false +DROP_SHELL=false +SSH_KEY="" + +DKMS_SRC="$(cd "$(dirname "$0")/.." && pwd)" +DKMS_PKG="netdevsim" +DKMS_VER="6.9.5" + +TMPDIR_BASE="/tmp/netdevsim-dkms-libvirt-ubuntu" +SSH_USER="ubuntu" +IMAGE_CACHE="${HOME}/.cache/netdevsim-dkms" + +# --------------------------------------------------------------------------- +# Image URLs (amd64) — resolved after argument parsing +# --------------------------------------------------------------------------- +image_url_for_release() { + case "$1" in + 22.04) echo "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" ;; + 24.04) echo "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" ;; + *) echo "" ;; + esac +} + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + sed -n '3,29p' "$0" | sed 's/^# \?//' + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --release) UBUNTU_RELEASE="$2"; shift 2 ;; + --ssh-key) SSH_KEY="$2"; shift 2 ;; + --vm-name) VM_NAME="$2"; shift 2 ;; + --ram) VM_RAM="$2"; shift 2 ;; + --cpus) VM_CPUS="$2"; shift 2 ;; + --disk-size) VM_DISK_SIZE="$2"; shift 2 ;; + --network) LIBVIRT_NET="$2"; shift 2 ;; + --cleanup) CLEANUP=true; shift ;; + --skip-tests) SKIP_TESTS=true; shift ;; + --skip-ptp-operator) SKIP_PTP_OPERATOR=true; shift ;; + --shell) DROP_SHELL=true; shift ;; + --help|-h) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +IMAGE_URL="$(image_url_for_release "$UBUNTU_RELEASE")" +[[ -n "$IMAGE_URL" ]] || { echo "Unsupported release: $UBUNTU_RELEASE (use 22.04 or 24.04)"; exit 1; } +IMAGE_FILENAME="$(basename "$IMAGE_URL")" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +log() { echo "==> $*"; } +die() { echo "ERROR: $*" >&2; exit 1; } +ts() { date "+%H:%M:%S"; } + +VM_IP="" + +cleanup_on_exit() { + local rc=$? + if [[ "$CLEANUP" == true ]]; then + log "Cleaning up VM '$VM_NAME' ..." + virsh destroy "$VM_NAME" 2>/dev/null || true + sleep 2 + virsh undefine "$VM_NAME" --remove-all-storage 2>/dev/null || true + fi + rm -rf "$TMPDIR_BASE" + if [[ $rc -eq 0 ]]; then + log "Done — all steps finished successfully." + else + log "Script exited with code $rc." + fi +} +trap cleanup_on_exit EXIT + +vm_ssh() { + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${VM_IP}" "$@" +} + +vm_scp() { + scp "${SSH_OPTS[@]}" "$@" +} + +# --------------------------------------------------------------------------- +# 1. Prerequisites +# --------------------------------------------------------------------------- +log "Checking prerequisites ..." + +if [[ -n "$SSH_KEY" ]]; then + SSH_KEY="$(cd "$(dirname "$SSH_KEY")" && pwd)/$(basename "$SSH_KEY")" + [[ -f "$SSH_KEY" ]] || die "SSH private key not found: $SSH_KEY" +else + for candidate in ~/.ssh/id_ed25519 ~/.ssh/id_rsa ~/.ssh/id_ecdsa; do + if [[ -f "$candidate" ]]; then + SSH_KEY="$candidate" + break + fi + done + [[ -n "$SSH_KEY" ]] || die "No SSH key found. Specify --ssh-key or generate: ssh-keygen -t ed25519" +fi + +SSH_PUBKEY_FILE="${SSH_KEY}.pub" +[[ -f "$SSH_PUBKEY_FILE" ]] || die "Public key not found at ${SSH_PUBKEY_FILE}" +SSH_PUBKEY="$(cat "$SSH_PUBKEY_FILE")" + +SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5) + +command -v virsh >/dev/null 2>&1 || die "virsh not found. Install with: dnf install libvirt-client" +command -v virt-install >/dev/null 2>&1 || die "virt-install not found. Install with: dnf install virt-install" +command -v qemu-img >/dev/null 2>&1 || die "qemu-img not found. Install with: dnf install qemu-img" + +MKISO_CMD="" +if command -v genisoimage >/dev/null 2>&1; then + MKISO_CMD="genisoimage" +elif command -v mkisofs >/dev/null 2>&1; then + MKISO_CMD="mkisofs" +fi +[[ -n "$MKISO_CMD" ]] || die "genisoimage/mkisofs not found. Install with: dnf install genisoimage" + +systemctl is-active --quiet libvirtd 2>/dev/null || { + log " Starting libvirtd ..." + systemctl start libvirtd +} + +virsh net-info "$LIBVIRT_NET" >/dev/null 2>&1 || die "Libvirt network '$LIBVIRT_NET' not found. Create it or specify --network." +virsh net-list --name 2>/dev/null | grep -qw "$LIBVIRT_NET" || { + log " Starting network '$LIBVIRT_NET' ..." + virsh net-start "$LIBVIRT_NET" 2>/dev/null || true +} + +[[ -e /dev/kvm ]] || die "/dev/kvm not available. Ensure KVM is enabled (modprobe kvm_intel or kvm_amd)." + +log " DKMS source: $DKMS_SRC" +log " Ubuntu: $UBUNTU_RELEASE (amd64)" +log " SSH key: $SSH_KEY" +log " ISO tool: $MKISO_CMD" +log " Network: $LIBVIRT_NET" + +# --------------------------------------------------------------------------- +# 2. Remove any existing VM with the same name (before preparing new disk) +# --------------------------------------------------------------------------- +if virsh dominfo "$VM_NAME" >/dev/null 2>&1; then + log "Removing existing VM '$VM_NAME' ..." + virsh destroy "$VM_NAME" 2>/dev/null || true + sleep 2 + virsh undefine "$VM_NAME" --remove-all-storage 2>/dev/null || true + virsh undefine "$VM_NAME" 2>/dev/null || true + sleep 3 +fi + +# --------------------------------------------------------------------------- +# 3. Download Ubuntu Cloud image (cached) +# --------------------------------------------------------------------------- +mkdir -p "$IMAGE_CACHE" +CACHED_IMG="$IMAGE_CACHE/$IMAGE_FILENAME" + +if [[ -f "$CACHED_IMG" ]]; then + log "Using cached image: $CACHED_IMG" +else + log "Downloading Ubuntu ${UBUNTU_RELEASE} Cloud amd64 image ..." + curl -fSL -C - --retry 5 --retry-delay 3 --retry-all-errors \ + -o "${CACHED_IMG}.partial" "$IMAGE_URL" + mv "${CACHED_IMG}.partial" "$CACHED_IMG" + log " Saved to $CACHED_IMG" +fi + +# --------------------------------------------------------------------------- +# 4. Prepare working directory and resize disk +# --------------------------------------------------------------------------- +log "Preparing disk image (resize to $VM_DISK_SIZE) ..." + +mkdir -p "$TMPDIR_BASE" +WORK_QCOW2="$TMPDIR_BASE/${VM_NAME}.qcow2" +qemu-img convert -f qcow2 -O qcow2 "$CACHED_IMG" "$WORK_QCOW2" +qemu-img resize "$WORK_QCOW2" "$VM_DISK_SIZE" + +# --------------------------------------------------------------------------- +# 4. Create cloud-init NoCloud ISO +# --------------------------------------------------------------------------- +log "Creating cloud-init NoCloud ISO ..." + +CIDATA_DIR="$TMPDIR_BASE/cidata" +CIDATA_ISO="$TMPDIR_BASE/cidata.iso" +mkdir -p "$CIDATA_DIR" + +cat > "$CIDATA_DIR/meta-data" < "$CIDATA_DIR/user-data" </dev/null \ + | awk '/virtio/{print $5}' | head -1 || true)" +MAC_LOWER="$(echo "$VM_MAC" | tr '[:upper:]' '[:lower:]')" +log "Waiting for VM IPv4 address (MAC: ${VM_MAC}) ..." +log " (cloud-init must install qemu-guest-agent first — this can take up to 5 min)" +SECONDS=0 +MAX_WAIT=420 +while (( SECONDS < MAX_WAIT )); do + # Primary: match VM's MAC in DHCP leases + if [[ -n "$MAC_LOWER" ]]; then + VM_IP="$(virsh net-dhcp-leases "$LIBVIRT_NET" 2>/dev/null \ + | tr '[:upper:]' '[:lower:]' \ + | grep "$MAC_LOWER" \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' \ + | head -1 || true)" + fi + + # Fallback: domifaddr with guest agent + if [[ -z "$VM_IP" ]]; then + VM_IP="$(virsh domifaddr "$VM_NAME" --source agent 2>/dev/null \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' \ + | grep -v '^127\.' | head -1 || true)" + fi + + if [[ -n "$VM_IP" ]]; then + break + fi + + if (( SECONDS % 30 < 6 )); then + log " ... still waiting (${SECONDS}s elapsed)" + fi + sleep 5 +done + +if [[ -z "$VM_IP" ]]; then + log "Could not discover IP via virsh after ${MAX_WAIT}s." + if [[ -n "$MAC_LOWER" ]]; then + log "Trying ARP scan for MAC ${MAC_LOWER} ..." + VM_IP="$(arp -an 2>/dev/null \ + | tr '[:upper:]' '[:lower:]' \ + | grep "$MAC_LOWER" \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' \ + | head -1 || true)" + fi + [[ -n "$VM_IP" ]] || die "Could not discover VM IP address after ${MAX_WAIT}s." +fi + +log " VM IP: $VM_IP (discovered in ${SECONDS}s)" + +log "Waiting for SSH on $VM_IP ..." +SECONDS=0 +MAX_SSH_WAIT=120 +while (( SECONDS < MAX_SSH_WAIT )); do + if vm_ssh "true" 2>/dev/null; then + break + fi + sleep 5 +done + +if ! vm_ssh "true" 2>/dev/null; then + die "SSH not reachable at ${SSH_USER}@${VM_IP} after ${MAX_SSH_WAIT}s." +fi + +log "SSH is up (took ${SECONDS}s)." + +log "Waiting for cloud-init to finish ..." +vm_ssh "sudo cloud-init status --wait" 2>/dev/null || true + +# --------------------------------------------------------------------------- +# 7. Reboot into HWE kernel (needed for both 22.04 and 24.04) +# --------------------------------------------------------------------------- +if [[ "$UBUNTU_RELEASE" == "22.04" || "$UBUNTU_RELEASE" == "24.04" ]]; then + KERNEL_BEFORE="$(vm_ssh 'uname -r')" + log "Current kernel: $KERNEL_BEFORE" + log "Rebooting into HWE kernel ..." + vm_ssh "sudo reboot" 2>/dev/null || true + sleep 15 + + SECONDS=0 + MAX_REBOOT_WAIT=180 + while (( SECONDS < MAX_REBOOT_WAIT )); do + if vm_ssh "true" 2>/dev/null; then + break + fi + sleep 5 + done + + if ! vm_ssh "true" 2>/dev/null; then + die "SSH not reachable after reboot (waited ${MAX_REBOOT_WAIT}s)." + fi + + KERNEL_AFTER="$(vm_ssh 'uname -r')" + log "Kernel after reboot: $KERNEL_AFTER" + if [[ "$KERNEL_BEFORE" == "$KERNEL_AFTER" ]]; then + log "WARNING: kernel did not change after reboot — HWE may not be installed" + fi +fi + +# --------------------------------------------------------------------------- +# 8. Report kernel version and discover config +# --------------------------------------------------------------------------- +log "VM kernel info:" +vm_ssh "uname -a" +RUNNING_KERNEL="$(vm_ssh 'uname -r')" +log " Running kernel: $RUNNING_KERNEL" + +log "Kernel config for relevant modules:" +vm_ssh "grep -E 'NETDEVSIM|PTP_1588|DPLL|DEVLINK|BPF_SYSCALL|XFRM_OFFLOAD|PSAMPLE|MACSEC|GNSS|OPENVSWITCH' \ + /boot/config-\$(uname -r) 2>/dev/null || echo ' /boot/config not found'" + +log "Module presence (builtin / loadable / absent):" +vm_ssh "echo '--- modules.builtin ---' && \ + grep -E 'netdevsim|ptp|dpll' /lib/modules/\$(uname -r)/modules.builtin 2>/dev/null || echo ' (none builtin)'; \ + echo '--- modinfo ---' && \ + modinfo netdevsim ptp dpll 2>&1 | grep -E '^filename|^description' || echo ' (none loadable)'" + +# --------------------------------------------------------------------------- +# 9. Ensure kernel headers match running kernel +# --------------------------------------------------------------------------- +log "Ensuring kernel headers and extra modules are installed ..." +vm_ssh sudo bash -c "' +set -euo pipefail +apt-get update -qq +apt-get install -y -qq linux-headers-\$(uname -r) linux-modules-extra-\$(uname -r) dkms gcc make 2>&1 | tail -5 +echo \"Headers dir: /lib/modules/\$(uname -r)/build\" +ls /lib/modules/\$(uname -r)/build/Makefile >/dev/null 2>&1 && echo \" OK\" || echo \" MISSING\" +'" + +# --------------------------------------------------------------------------- +# 10. Copy DKMS source tree and install +# --------------------------------------------------------------------------- +log "Copying DKMS source tree to VM ..." + +REMOTE_DKMS="/usr/src/${DKMS_PKG}-${DKMS_VER}" +vm_ssh "sudo mkdir -p ${REMOTE_DKMS}" + +TAR_STAGING="$TMPDIR_BASE/dkms-src.tar.gz" +tar -C "$DKMS_SRC" -czf "$TAR_STAGING" \ + Makefile dkms.conf install-udev-rule.sh 99-nsim-ptp.rules \ + include ptp dpll netdevsim + +vm_scp "$TAR_STAGING" "${SSH_USER}@${VM_IP}:/tmp/dkms-src.tar.gz" +vm_ssh "sudo tar -C ${REMOTE_DKMS} -xzf /tmp/dkms-src.tar.gz" + +log " DKMS add ..." +vm_ssh "sudo dkms add ${DKMS_PKG}/${DKMS_VER}" || { + log " (already registered — continuing)" +} + +log " DKMS build ..." +vm_ssh "sudo dkms build ${DKMS_PKG}/${DKMS_VER}" || { + log "--- make.log ---" + vm_ssh "sudo cat /var/lib/dkms/${DKMS_PKG}/${DKMS_VER}/build/make.log" || true + die "DKMS build failed" +} + +log " DKMS install ..." +vm_ssh "sudo dkms install --force ${DKMS_PKG}/${DKMS_VER}" +vm_ssh "dkms status" + +# --------------------------------------------------------------------------- +# 11. Install udev rule for /dev/ptp* compat device nodes +# --------------------------------------------------------------------------- +log "Installing nsim_ptp udev rule ..." +vm_ssh "sudo cp ${REMOTE_DKMS}/99-nsim-ptp.rules /etc/udev/rules.d/ && \ + sudo udevadm control --reload-rules" + +# --------------------------------------------------------------------------- +# 12. Load modules +# --------------------------------------------------------------------------- +log "Loading modules ..." + +vm_ssh sudo bash -c "' +set -euo pipefail +modprobe gnss && echo \" gnss loaded\" || echo \" gnss skipped\" +modprobe nsim_ptp && echo \" nsim_ptp loaded\" +modprobe nsim_ptp_mock && echo \" nsim_ptp_mock loaded\" +modprobe nsim_dpll && echo \" nsim_dpll loaded\" +modprobe netdevsim pci_bus_nr=0x1f && echo \" netdevsim loaded\" +echo +echo \"--- lsmod ---\" +lsmod | grep -E \"ptp|netdevsim|dpll|gnss\" || true +'" + +# --------------------------------------------------------------------------- +# 13. Smoke tests +# --------------------------------------------------------------------------- +if [[ "$SKIP_TESTS" == true ]]; then + log "Skipping tests (--skip-tests)." +else + log "Running smoke tests ..." + + vm_ssh sudo bash -c "' + set -euo pipefail + set -x + + # Read the pci_bus_nr parameter and find the fake PCI root device + BUS_NR=\$(cat /sys/module/netdevsim/parameters/pci_bus_nr 2>/dev/null || echo 31) + BUS=\$(printf \"%02x\" \"\$BUS_NR\") + FAKE_ROOT=\$(ls /sys/bus/pci/devices/ 2>/dev/null | grep \":\${BUS}:00\\.0\" | head -1) + DOMAIN=\"\${FAKE_ROOT:+\$(echo \"\$FAKE_ROOT\" | cut -d: -f1)}\" + : \${DOMAIN:=0000} + PCI_ADDR=\"\${DOMAIN}:\${BUS}:02.0\" + echo \"Using PCI address: \${PCI_ADDR}\" + + # Create a netdevsim device (id=1, pci_addr, clk_id=1) + echo \"1 \${PCI_ADDR} 1\" > /sys/bus/netdevsim/new_device + sleep 1 + + # Verify the bus device appeared + ls /sys/bus/netdevsim/devices/netdevsim1/ + + # Verify PTP clock (nsim_ptp class + /dev/ptp compat device node) + ls /sys/class/nsim_ptp/ + ls -la /dev/nsim_ptp* /dev/ptp* 2>/dev/null + + # Verify /dev/ptpN has the same major:minor as /dev/nsim_ptpN + PTP_RDEV=\$(stat -c \"%t:%T\" /dev/ptp0 2>/dev/null || true) + NSIM_RDEV=\$(stat -c \"%t:%T\" /dev/nsim_ptp0 2>/dev/null || true) + [ -n \"\$PTP_RDEV\" ] && [ \"\$PTP_RDEV\" = \"\$NSIM_RDEV\" ] \\ + && echo \" /dev/ptp0 matches /dev/nsim_ptp0 (rdev \$PTP_RDEV)\" \\ + || echo \" WARN: /dev/ptp0 rdev mismatch or missing\" + + # Verify ethtool reports a valid PHC + IFACE=\$(ls /sys/bus/pci/devices/\${PCI_ADDR}/net/ 2>/dev/null | head -1) + if [ -n \"\$IFACE\" ]; then + echo \"Interface: \$IFACE\" + ethtool -T \"\$IFACE\" | grep -E \"PTP Hardware Clock|hardware\" + fi + + # Check dmesg for netdevsim messages + dmesg | grep -i netdevsim | tail -10 + + # Cleanup + echo \"1\" > /sys/bus/netdevsim/del_device 2>/dev/null || true + + echo + echo \"=== ALL SMOKE TESTS PASSED ===\" + '" +fi + +# --------------------------------------------------------------------------- +# 14. Install ptp-operator prerequisites +# --------------------------------------------------------------------------- +PTP_REPO="${PTP_REPO:-https://github.com/edcdavid/ptp-operator-upstream.git}" +PTP_BRANCH="${PTP_BRANCH:-netdevsim-dkms}" + +if [[ "$SKIP_TESTS" == true || "$SKIP_PTP_OPERATOR" == true ]]; then + log "Skipping ptp-operator setup." +else + log "Installing ptp-operator prerequisites ..." + + vm_ssh sudo bash <<'PREREQS' +set -euo pipefail +set -x + +apt-get install -y -qq podman pciutils uidmap slirp4netns openvswitch-switch git 2>&1 | tail -3 + +GOARCH=amd64 + +# kind +curl -fsSLo /usr/bin/kind "https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-${GOARCH}" +chmod +x /usr/bin/kind + +# kubectl + oc +curl -fsSLo /tmp/oc.tar.gz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz +tar -C /tmp -xzf /tmp/oc.tar.gz oc kubectl +mv /tmp/oc /tmp/kubectl /usr/local/bin/ + +# Go (latest) +GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | head -n 1) +curl -fsSLo /tmp/go.tar.gz "https://go.dev/dl/${GO_VERSION}.linux-${GOARCH}.tar.gz" +rm -rf /usr/local/go +tar -C /usr/local -xzf /tmp/go.tar.gz + +# helm +curl -fsSL "https://get.helm.sh/helm-v3.17.3-linux-${GOARCH}.tar.gz" | tar -C /tmp -xzf - +mv "/tmp/linux-${GOARCH}/helm" /usr/local/bin/helm + +# PATH for root +grep -q /usr/local/go/bin /root/.bashrc 2>/dev/null || \ + echo 'export PATH=$PATH:$HOME/go/bin:/usr/local/go/bin' >> /root/.bashrc +export PATH=/usr/local/go/bin:/root/go/bin:$PATH + +# inotify limits (kind needs these) +sysctl -w fs.inotify.max_user_instances=512 +sysctl -w fs.inotify.max_user_watches=524288 + +echo "--- Verification ---" +podman --version +kind version +kubectl version --client 2>/dev/null | head -1 +go version +helm version --short +ovs-vsctl --version | head -1 +PREREQS + + # ----------------------------------------------------------------------- + # 15. Clone ptp-operator, install ginkgo, go mod vendor + # ----------------------------------------------------------------------- + log "Cloning ptp-operator (branch: ${PTP_BRANCH}) ..." + vm_ssh "sudo rm -rf /root/ptp-operator" + vm_ssh "sudo git clone --depth 1 --branch '${PTP_BRANCH}' '${PTP_REPO}' /root/ptp-operator" + + log "Installing ginkgo and vendoring Go modules ..." + vm_ssh sudo bash <<'GO_SETUP' +set -euo pipefail +export PATH=/usr/local/go/bin:/root/go/bin:$PATH +cd /root/ptp-operator +go mod tidy 2>&1 | tail -3 +go mod vendor 2>&1 | tail -3 +go install github.com/onsi/ginkgo/v2/ginkgo@latest +GO_SETUP + + # ----------------------------------------------------------------------- + # 16. Run ptp-operator test suite (--dkms flag handles all adjustments) + # ----------------------------------------------------------------------- + log "[$(ts)] Running ptp-operator run-on-vm.sh --dkms ..." + vm_ssh sudo bash -l <<'REMOTE_SCRIPT' +set -euo pipefail +set -x + +export PATH=/usr/local/go/bin:/root/go/bin:$PATH +export GOMAXPROCS=$(nproc) + +VM_IP=$(hostname -I | awk '{print $1}') +echo "VM_IP=${VM_IP}" + +cd /root/ptp-operator/scripts +./run-on-vm.sh --dkms "${VM_IP}" +REMOTE_SCRIPT + log "[$(ts)] ptp-operator tests completed." +fi + +# --------------------------------------------------------------------------- +# 17. Interactive shell or finish +# --------------------------------------------------------------------------- +if [[ "$DROP_SHELL" == true ]]; then + log "Dropping into interactive shell (Ctrl-D to exit) ..." + log " ssh -i ${SSH_KEY} ${SSH_USER}@${VM_IP}" + ssh "${SSH_OPTS[@]}" -t "${SSH_USER}@${VM_IP}" +fi + +if [[ "$CLEANUP" != true ]]; then + echo + log "VM is still running. Connect with:" + log " ssh -i ${SSH_KEY} ${SSH_USER}@${VM_IP}" + log "Destroy later with: virsh destroy ${VM_NAME} && virsh undefine ${VM_NAME} --remove-all-storage" +fi From 1a3e0f78188fbd6b900dadaa8ebdd0182a816354 Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 25 Jun 2026 14:53:02 -0500 Subject: [PATCH 2/3] Allow libvirt scripts to run without root Use LIBVIRT_DEFAULT_URI=qemu:///system so virsh/virt-install connect to the system daemon via polkit instead of requiring root. Only sudo is used for starting libvirtd if it is not already running. The script now checks for libvirt and kvm group membership and gives actionable errors. --- README.md | 7 +++++++ scripts/test-libvirt-ubuntu.sh | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d17c0b9..4d87fff 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,13 @@ When using netdevsim inside containers or Kubernetes pods: sudo apt-get install -y qemu-kvm libvirt-daemon-system virtinst cloud-image-utils ``` +3. Add your user to the `libvirt` and `kvm` groups (no root needed after this): + +```bash +sudo usermod -aG libvirt,kvm $USER +# Log out and back in for group membership to take effect +``` + ### Quick Setup (no tests) `scripts/setup-libvirt-ubuntu.sh` creates a libvirt VM with the DKMS modules diff --git a/scripts/test-libvirt-ubuntu.sh b/scripts/test-libvirt-ubuntu.sh index 28db787..28c9cf2 100755 --- a/scripts/test-libvirt-ubuntu.sh +++ b/scripts/test-libvirt-ubuntu.sh @@ -49,6 +49,8 @@ TMPDIR_BASE="/tmp/netdevsim-dkms-libvirt-ubuntu" SSH_USER="ubuntu" IMAGE_CACHE="${HOME}/.cache/netdevsim-dkms" +export LIBVIRT_DEFAULT_URI="${LIBVIRT_DEFAULT_URI:-qemu:///system}" + # --------------------------------------------------------------------------- # Image URLs (amd64) — resolved after argument parsing # --------------------------------------------------------------------------- @@ -160,10 +162,15 @@ elif command -v mkisofs >/dev/null 2>&1; then fi [[ -n "$MKISO_CMD" ]] || die "genisoimage/mkisofs not found. Install with: dnf install genisoimage" -systemctl is-active --quiet libvirtd 2>/dev/null || { - log " Starting libvirtd ..." - systemctl start libvirtd -} +if [[ $(id -u) -ne 0 ]]; then + id -nG | grep -qw libvirt 2>/dev/null \ + || die "Current user is not in the 'libvirt' group. Fix with: sudo usermod -aG libvirt \$USER (then log out/in)" +fi + +if ! systemctl is-active --quiet libvirtd 2>/dev/null; then + log " libvirtd is not running — starting with sudo ..." + sudo systemctl start libvirtd +fi virsh net-info "$LIBVIRT_NET" >/dev/null 2>&1 || die "Libvirt network '$LIBVIRT_NET' not found. Create it or specify --network." virsh net-list --name 2>/dev/null | grep -qw "$LIBVIRT_NET" || { @@ -172,6 +179,7 @@ virsh net-list --name 2>/dev/null | grep -qw "$LIBVIRT_NET" || { } [[ -e /dev/kvm ]] || die "/dev/kvm not available. Ensure KVM is enabled (modprobe kvm_intel or kvm_amd)." +[[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm not accessible. Add user to 'kvm' group: sudo usermod -aG kvm \$USER" log " DKMS source: $DKMS_SRC" log " Ubuntu: $UBUNTU_RELEASE (amd64)" From d20e3f418b5f05e3ac4d0da1edacf4317727781c Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 25 Jun 2026 15:10:20 -0500 Subject: [PATCH 3/3] Store VM disks persistently and run without root - Store qcow2 disk and cloud-init ISO in ~/.local/share/netdevsim-dkms/ so the VM survives script exit - Use LIBVIRT_DEFAULT_URI=qemu:///system with polkit instead of root - Detect QEMU user (libvirt-qemu/qemu) and check ACL permissions - No sudo calls; script exits with actionable fix commands instead - Update README with group membership and ACL setup instructions --- README.md | 10 +++++++++ scripts/test-libvirt-ubuntu.sh | 37 ++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4d87fff..1e2bbc0 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,16 @@ sudo usermod -aG libvirt,kvm $USER # Log out and back in for group membership to take effect ``` +4. VM disks are stored in `~/.local/share/netdevsim-dkms/`. QEMU needs + search permission (`+x`) on each parent directory. The script grants + this automatically via ACL, but if you hit permission errors, run: + +```bash +# Find the QEMU user on your system (libvirt-qemu on Debian/Ubuntu, qemu on RHEL/Fedora) +QEMU_USER=$(id -nu libvirt-qemu 2>/dev/null || id -nu qemu 2>/dev/null) +setfacl -m "u:${QEMU_USER}:x" ~ ~/.local ~/.local/share +``` + ### Quick Setup (no tests) `scripts/setup-libvirt-ubuntu.sh` creates a libvirt VM with the DKMS modules diff --git a/scripts/test-libvirt-ubuntu.sh b/scripts/test-libvirt-ubuntu.sh index 28c9cf2..b9e15b6 100755 --- a/scripts/test-libvirt-ubuntu.sh +++ b/scripts/test-libvirt-ubuntu.sh @@ -45,6 +45,7 @@ DKMS_SRC="$(cd "$(dirname "$0")/.." && pwd)" DKMS_PKG="netdevsim" DKMS_VER="6.9.5" +VM_DATA_DIR="${HOME}/.local/share/netdevsim-dkms" TMPDIR_BASE="/tmp/netdevsim-dkms-libvirt-ubuntu" SSH_USER="ubuntu" IMAGE_CACHE="${HOME}/.cache/netdevsim-dkms" @@ -108,6 +109,7 @@ cleanup_on_exit() { virsh destroy "$VM_NAME" 2>/dev/null || true sleep 2 virsh undefine "$VM_NAME" --remove-all-storage 2>/dev/null || true + rm -rf "${VM_DATA_DIR:?}/${VM_NAME}" fi rm -rf "$TMPDIR_BASE" if [[ $rc -eq 0 ]]; then @@ -167,10 +169,8 @@ if [[ $(id -u) -ne 0 ]]; then || die "Current user is not in the 'libvirt' group. Fix with: sudo usermod -aG libvirt \$USER (then log out/in)" fi -if ! systemctl is-active --quiet libvirtd 2>/dev/null; then - log " libvirtd is not running — starting with sudo ..." - sudo systemctl start libvirtd -fi +systemctl is-active --quiet libvirtd 2>/dev/null \ + || die "libvirtd is not running. Start it with: sudo systemctl start libvirtd" virsh net-info "$LIBVIRT_NET" >/dev/null 2>&1 || die "Libvirt network '$LIBVIRT_NET' not found. Create it or specify --network." virsh net-list --name 2>/dev/null | grep -qw "$LIBVIRT_NET" || { @@ -179,7 +179,26 @@ virsh net-list --name 2>/dev/null | grep -qw "$LIBVIRT_NET" || { } [[ -e /dev/kvm ]] || die "/dev/kvm not available. Ensure KVM is enabled (modprobe kvm_intel or kvm_amd)." -[[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm not accessible. Add user to 'kvm' group: sudo usermod -aG kvm \$USER" +[[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm not accessible. Add user to 'kvm' group: sudo usermod -aG kvm \$USER (then log out/in)" + +# QEMU runs as a separate user and needs +x on every parent directory +# leading to the disk image. Check and tell the user what to run. +QEMU_USER="" +for u in libvirt-qemu qemu; do + id "$u" &>/dev/null && QEMU_USER="$u" && break +done +if [[ -n "$QEMU_USER" ]]; then + NEED_ACL=() + for d in "$HOME" "$HOME/.local" "$HOME/.local/share"; do + [[ -d "$d" ]] || continue + getfacl -p "$d" 2>/dev/null | grep -q "user:${QEMU_USER}:.*x" && continue + NEED_ACL+=("$d") + done + if (( ${#NEED_ACL[@]} > 0 )); then + die "QEMU user '${QEMU_USER}' lacks search (+x) permission on: ${NEED_ACL[*]} + Fix with: setfacl -m u:${QEMU_USER}:x ${NEED_ACL[*]}" + fi +fi log " DKMS source: $DKMS_SRC" log " Ubuntu: $UBUNTU_RELEASE (amd64)" @@ -220,10 +239,12 @@ fi # --------------------------------------------------------------------------- log "Preparing disk image (resize to $VM_DISK_SIZE) ..." -mkdir -p "$TMPDIR_BASE" -WORK_QCOW2="$TMPDIR_BASE/${VM_NAME}.qcow2" +VM_DIR="${VM_DATA_DIR}/${VM_NAME}" +mkdir -p "$VM_DIR" "$TMPDIR_BASE" +WORK_QCOW2="${VM_DIR}/${VM_NAME}.qcow2" qemu-img convert -f qcow2 -O qcow2 "$CACHED_IMG" "$WORK_QCOW2" qemu-img resize "$WORK_QCOW2" "$VM_DISK_SIZE" +log " Disk: $WORK_QCOW2" # --------------------------------------------------------------------------- # 4. Create cloud-init NoCloud ISO @@ -231,7 +252,7 @@ qemu-img resize "$WORK_QCOW2" "$VM_DISK_SIZE" log "Creating cloud-init NoCloud ISO ..." CIDATA_DIR="$TMPDIR_BASE/cidata" -CIDATA_ISO="$TMPDIR_BASE/cidata.iso" +CIDATA_ISO="${VM_DIR}/cidata.iso" mkdir -p "$CIDATA_DIR" cat > "$CIDATA_DIR/meta-data" <