add ip allocation fix and update readme with qemu 11.1 regression

README.md

@@ -51,6 +51,8 @@ Macpine depends on QEMU >= 7.22.0:
 brew install qemu
 ```
 
+> **Known issue:** QEMU 11.1.1 has a regression that can hang `aarch64` instances on boot when using `vmnet` networking (see [Troubleshooting](https://beringresearch.github.io/macpine/troubleshooting/)). If you hit this, pin QEMU to 10.0.3 until it's fixed upstream.
+
 ## Install from MacPorts
 
 You can also install `macpine` via [MacPorts](https://www.macports.org):

docs/docs/troubleshooting.md

@@ -53,6 +53,21 @@ More information on `chronyd` can be found on [the Arch wiki](https://wiki.archl
     click "Allow" for incoming connection to `qemu` when prompted by macOS as loopback connections (i.e. directly from the host itself)
     will still be allowed.
 
+### Instance hangs on boot with no console output (`qemu` 11.1.1 regression)
+
+On Apple Silicon, `aarch64` instances using `vmnet` networking (`vmnet: true` in `config.yaml`) may fail to boot when using `qemu` 11.1.1: the `qemu-system-aarch64` process pins a CPU core near 100%, produces no serial console output at all, and the instance never acquires a DHCP lease or becomes reachable. This reproduces with a bare `qemu-system-aarch64` invocation (outside of `macpine`), so it is not a `macpine` bug — it appears to be a regression in `qemu` 11.1.1 itself affecting early boot/firmware on the `aarch64` `virt` machine type with HVF acceleration.
+
+Downgrading to `qemu` 10.0.3 resolves the issue. If you have an older `10.0.3` keg still available via Homebrew:
+
+```bash
+brew unlink qemu
+brew link qemu@10.0.3   # or manually symlink the qemu-system-* binaries from
+                         # /opt/homebrew/Cellar/qemu/10.0.3/bin into /opt/homebrew/bin
+brew pin qemu            # prevent `brew upgrade` from reintroducing the regression
+```
+
+If Homebrew has already removed the old keg, you can also build/install `qemu` 10.0.3 from source or an older bottle. Track upstream for a fix before unpinning.
+
 ### Other issues
 
 * If alpine is not able to resize the disk, it will error out with this message: `unable to resize disk: signal: abort trap`. Internally, it runs the command `qemu-img resize <IMAGE_LOCATION> <+SIZE>`. If the `qemu-img resize` command errors out with `dyld[...]: Library not loaded: /opt/homebrew/opt/libunistring/lib/libunistring.2.dylib` then re-installing `gettext` via `brew reinstall gettext` may resolve the issue.

docs/mkdocs.yml

@@ -30,6 +30,7 @@
       - Instance Security: hardening.md
       - Create an Incus container in Macpine: incus_macpine.md
       - Create an LXD container in Macpine: lxd_macpine.md
+    - Troubleshooting: troubleshooting.md
 
  #theme: material
 

qemu/ops.go

@@ -43,47 +43,48 @@ type MachineConfig struct {
 	Tags         []string `yaml:"tags"`
 }
 
-// Exec starts an interactive shell terminal in VM
-func (c *MachineConfig) Exec(cmd string, root bool) (string, error) {
-	if cmd == "" {
-		return "", nil
+// ResolveMachineIP looks up the instance's DHCP-assigned IP address by MAC
+// address and persists it to config.yaml. It is a no-op unless VMNet is
+// enabled and MachineIP is unset ("" or "localhost").
+func (c *MachineConfig) ResolveMachineIP() error {
+	if !c.VMNet || (c.MachineIP != "" && c.MachineIP != "localhost") {
+		return nil
 	}
-	ip := c.MachineIP
 
-	if c.VMNet {
-		if ip == "localhost" || ip == "" {
-			log.Println("getting instance IP address from DHCP leases")
-			for {
-				dhcpLeasesContent, err := os.ReadFile("/var/db/dhcpd_leases")
-				if err != nil {
-					return "", err
-				}
-				lip, _ := c.GetIPAddressByMac(dhcpLeasesContent)
-				//ip = c.GetIPAddressFromMachine()
-				if lip != "" {
-					c.MachineIP = lip
-					break
-				}
-				fmt.Print(".")
-				time.Sleep(4 * time.Second)
-			}
+	log.Println("getting instance IP address from DHCP leases")
+	for {
+		dhcpLeasesContent, err := os.ReadFile("/var/db/dhcpd_leases")
+		if err != nil {
+			return err
+		}
+		lip, _ := c.GetIPAddressByMac(dhcpLeasesContent)
 
-			config, err := yaml.Marshal(&c)
+		if lip != "" {
+			c.MachineIP = lip
+			break
+		}
+		fmt.Print(".")
+		time.Sleep(4 * time.Second)
+	}
 
-			if err != nil {
-				c.Stop()
-				c.CleanPIDFile()
-				return "", err
-			}
+	config, err := yaml.Marshal(&c)
+	if err != nil {
+		return err
+	}
 
-			err = os.WriteFile(filepath.Join(c.Location, "config.yaml"), config, 0644)
-			if err != nil {
-				c.Stop()
-				c.CleanPIDFile()
-				return "", err
-			}
-		}
+	return os.WriteFile(filepath.Join(c.Location, "config.yaml"), config, 0644)
+}
+
+// Exec starts an interactive shell terminal in VM
+func (c *MachineConfig) Exec(cmd string, root bool) (string, error) {
+	if cmd == "" {
+		return "", nil
+	}
 
+	if err := c.ResolveMachineIP(); err != nil {
+		c.Stop()
+		c.CleanPIDFile()
+		return "", err
 	}
 
 	host := c.MachineIP + ":" + c.SSHPort
@@ -275,15 +276,23 @@ func (c *MachineConfig) Stop() error {
 			}
 
 			if err := p.Signal(syscall.SIGKILL); err != nil {
-				return err
+				if errors.Is(err, syscall.EPERM) {
+					return fmt.Errorf("insufficient privileges to stop `%s` (its process is owned by a different user) — try again with sudo", c.Alias)
+				}
+				if !errors.Is(err, syscall.ESRCH) && !errors.Is(err, os.ErrProcessDone) {
+					return err
+				}
+				// process is already gone; fall through and clean up stale files
 			}
 
 			pidFile := filepath.Join(c.Location, "alpine.pid")
 			sockFile := filepath.Join(c.Location, "alpine.sock")
 			qmpFile := filepath.Join(c.Location, "alpine.qmp")
-			os.Remove(pidFile)
-			os.Remove(sockFile)
-			os.Remove(qmpFile)
+			for _, f := range []string{pidFile, sockFile, qmpFile} {
+				if err := os.Remove(f); err != nil && !errors.Is(err, os.ErrNotExist) {
+					log.Printf("warning: failed to remove %s: %v", f, err)
+				}
+			}
 
 			log.Println(c.Alias + " stopped")
 			return nil
@@ -522,6 +531,12 @@ func (c *MachineConfig) Start() error {
 		return err
 	}
 
+	if err := c.ResolveMachineIP(); err != nil {
+		c.Stop()
+		c.CleanPIDFile()
+		return err
+	}
+
 	if c.Mount != "" {
 		basename := filepath.Base(c.Mount)
 		mntcmd := make([]string, 3)
@@ -842,7 +857,11 @@ func (c *MachineConfig) CreateQemuDiskImage(imageName string) error {
 func (c *MachineConfig) CleanPIDFile() {
 	pidFile := filepath.Join(c.Location, "alpine.pid")
 	if err := os.Remove(pidFile); err != nil && !errors.Is(err, os.ErrNotExist) {
-		log.Fatalf("error deleting pidfile at %s. Manually delete it before proceeding.", pidFile)
+		if errors.Is(err, syscall.EPERM) {
+			log.Printf("warning: insufficient privileges to remove pidfile at %s (owned by a different user) — remove it with sudo, or rerun the command with sudo", pidFile)
+			return
+		}
+		log.Printf("warning: error deleting pidfile at %s: %v. Manually delete it before proceeding.", pidFile, err)
 	}
 }
 

utils/utils.go

@@ -486,11 +486,37 @@ func ConvertStringArrayToDhcpDataArray(dataArray [][]string) []DhcpData {
 	return data
 }
 
+// ExpiresAt parses the lease's expiry timestamp, stored by bootpd as a hex
+// Unix epoch time (e.g. "0x6a8e014a").
+func (d DhcpData) ExpiresAt() (time.Time, error) {
+	sec, err := strconv.ParseInt(strings.TrimPrefix(d.Lease, "0x"), 16, 64)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("could not parse lease value %q: %w", d.Lease, err)
+	}
+	return time.Unix(sec, 0), nil
+}
+
+// MatchHwAddress returns the most recently issued, still-valid lease for the
+// given hardware address, or nil if none is found. dhcpd_leases can retain
+// entries for a MAC address long after they've expired (e.g. from a previous
+// boot of the same VM with a static MAC), so expired entries are ignored
+// rather than treated as the instance's current address.
 func MatchHwAddress(data []DhcpData, targetHwAddress string) *DhcpData {
+	var best *DhcpData
+	var bestExpiry time.Time
+
 	for i := range data {
-		if data[i].HwAddress == targetHwAddress {
-			return &data[i]
+		if data[i].HwAddress != targetHwAddress {
+			continue
+		}
+		expiry, err := data[i].ExpiresAt()
+		if err != nil || time.Now().After(expiry) {
+			continue
+		}
+		if best == nil || expiry.After(bestExpiry) {
+			best = &data[i]
+			bestExpiry = expiry
 		}
 	}
-	return nil
+	return best
 }