perf(controller): react to node and endpoint changes instead of polling

README.md

@@ -35,6 +35,8 @@ After the chart is installed, you should be able to create `LoadBalancer` servic
 
 The operator listens to the Kubernetes API for services of type `LoadBalancer` and creates Hetzner load balancers that point to nodes based on `node-ip`.
 
+A balancer is updated when its service changes, when a node is added, removed, relabelled, cordoned or changes readiness or addresses, and, for services with `externalTrafficPolicy: Local` while `ROBOTLB_DYNAMIC_NODE_SELECTOR` is on, when the nodes of their endpoints change; a deleted endpoint slice of such a service rechecks every balancer. Apart from that robotlb checks each balancer every `ROBOTLB_RESYNC_INTERVAL` seconds, 300 by default, which bounds how long a change made to the balancer in Hetzner survives. Each check costs one or two Hetzner API requests per service. When Hetzner refuses a target for a reason other than the rate limit, the service is checked again within 30 seconds instead, so a node it refuses for good, such as one outside the vSwitch subnet, keeps its service on that 30-second cycle.
+
 Target nodes are selected according to the service's `externalTrafficPolicy`:
 
 - `Cluster`, the Kubernetes default: every node of the cluster becomes a target, since kube-proxy forwards the traffic to a node that hosts a pod. Cordoned and not-ready nodes are left out, as they would only take up target slots.
@@ -87,6 +89,8 @@ Options:
           Default load balancer proxy mode. If enabled, the load balancer will act as a proxy for the target servers. The default value is `false`. https://docs.hetzner.com/cloud/load-balancers/faq/#what-does-proxy-protocol-mean-and-should-i-enable-it [env: ROBOTLB_DEFAULT_LB_PROXY_MODE_ENABLED=]
       --ipv6-ingress
           Whether to enable IPv6 ingress for the load balancer. If enabled, the load balancer's IPv6 will be attached to the service as an external IP along with IPv4 [env: ROBOTLB_IPV6_INGRESS=]
+      --resync-interval <RESYNC_INTERVAL>
+          Seconds between reconciliations of a service that nothing changed. Node changes, and endpoint changes of Local services, trigger a reconciliation on their own; this interval bounds how long a change made to a balancer outside robotlb survives. A service whose balancer refused a target is retried within 30 seconds [env: ROBOTLB_RESYNC_INTERVAL=] [default: 300]
       --log-level <LOG_LEVEL>
           [env: ROBOTLB_LOG_LEVEL=] [default: INFO]
   -h, --help

src/config.rs

@@ -67,7 +67,46 @@ pub struct OperatorConfig {
     #[arg(long, env = "ROBOTLB_IPV6_INGRESS", default_value = "false")]
     pub ipv6_ingress: bool,
 
+    /// Seconds between reconciliations of a service that nothing changed. Node changes,
+    /// and endpoint changes of Local services, trigger a reconciliation on their own;
+    /// this interval bounds how long a change made to a balancer outside robotlb survives.
+    /// A service whose balancer refused a target is retried within 30 seconds.
+    #[arg(
+        long,
+        env = "ROBOTLB_RESYNC_INTERVAL",
+        default_value = "300",
+        value_parser = clap::value_parser!(u64).range(1..=31_536_000)
+    )]
+    pub resync_interval: u64,
+
     // Log level of the operator.
     #[arg(long, env = "ROBOTLB_LOG_LEVEL", default_value = "INFO")]
     pub log_level: LevelFilter,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::OperatorConfig;
+    use clap::Parser;
+
+    fn parse(resync: &str) -> Result<OperatorConfig, clap::Error> {
+        OperatorConfig::try_parse_from([
+            "robotlb",
+            "--hcloud-token",
+            "t",
+            "--resync-interval",
+            resync,
+        ])
+    }
+
+    // Zero would requeue every successful reconcile right away, spending Hetzner
+    // API requests on every service all the time. The controller's delay queue
+    // panics on delays past about two years.
+    #[test]
+    fn the_resync_interval_must_be_between_a_second_and_a_year() {
+        assert!(parse("0").is_err());
+        assert!(parse("31536001").is_err());
+        assert_eq!(parse("31536000").unwrap().resync_interval, 31_536_000);
+        assert_eq!(parse("1").unwrap().resync_interval, 1);
+    }
+}

src/lb.rs

@@ -172,14 +172,15 @@ impl LoadBalancer {
 
     /// Reconcile the load balancer to match the desired configuration.
     #[tracing::instrument(skip(self), fields(lb_name=self.name))]
-    pub async fn reconcile(&self) -> RobotLBResult<hcloud::models::LoadBalancer> {
+    /// Returns the balancer, and whether some of its targets could not be added.
+    pub async fn reconcile(&self) -> RobotLBResult<(hcloud::models::LoadBalancer, bool)> {
         let hcloud_balancer = self.get_or_create_hcloud_lb().await?;
         self.reconcile_algorithm(&hcloud_balancer).await?;
         self.reconcile_lb_type(&hcloud_balancer).await?;
         self.reconcile_network(&hcloud_balancer).await?;
         self.reconcile_services(&hcloud_balancer).await?;
-        self.reconcile_targets(&hcloud_balancer).await?;
-        Ok(hcloud_balancer)
+        let targets_missing = self.reconcile_targets(&hcloud_balancer).await?;
+        Ok((hcloud_balancer, targets_missing))
     }
 
     /// Reconcile the services of the load balancer.
@@ -299,7 +300,7 @@ impl LoadBalancer {
     async fn reconcile_targets(
         &self,
         hcloud_balancer: &hcloud::models::LoadBalancer,
-    ) -> RobotLBResult<()> {
+    ) -> RobotLBResult<bool> {
         let max_targets =
             usize::try_from(hcloud_balancer.load_balancer_type.max_targets).unwrap_or(usize::MAX);
         let planned = plan_targets(&self.targets, max_targets);
@@ -386,7 +387,7 @@ impl LoadBalancer {
                 last_error.unwrap_or_else(|| "no reason reported".to_string()),
             )));
         }
-        Ok(())
+        Ok(live < planned.len())
     }
 
     /// Reconcile the load balancer algorithm.