From 29d5ba3f1fb22e5290a6bdfbfa4b5e3800a142f2 Mon Sep 17 00:00:00 2001 From: George Melikov Date: Sun, 11 Oct 2026 21:34:26 +0000 Subject: [PATCH] feat(ovs): add opt-in VXLAN source admission Gate ingress behind [ovs] restrict_tunnel_sources, disabled by default. Learn VTEPs from all EVPN Type 2, 3 and 5 paths on each RIB read. Admit each VTEP once in table 0 and classify its VNI in table 3, preserving the existing actions without multiplying classifiers by sources. Normalize legacy fail-static classifiers behind the gate and restore qualified snapshot sources. Count the final flows sent to OVS. Cover both settings, linear flow growth, multipath next hops, address replacement, withdrawal, and snapshot recovery. --- README.md | 13 + etc/evpn_connector/evpn_connector.cfg.sample | 1 + evpn_connector/bgp/client.py | 8 + evpn_connector/cmd/evpn.py | 1 + evpn_connector/common/conf_opts.py | 8 + evpn_connector/common/constants.py | 1 + evpn_connector/service/evpn.py | 12 +- evpn_connector/service/objects.py | 43 ++- evpn_connector/tests/unit/test_bgp_client.py | 62 ++++ .../tests/unit/test_tunnel_sources.py | 289 ++++++++++++++++++ 10 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 evpn_connector/tests/unit/test_tunnel_sources.py diff --git a/README.md b/README.md index eb88425..419a1fe 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,19 @@ A service for automating the management distributed switch or router based on EV * **Reconciliation Loop Design:** Built on a closed-loop control architecture to ensure fault tolerance and reduce the impact of external factors * **Configurability:** Flexible setup via INI daemon configuration files and JSON clients configuration files +Set `[ovs] restrict_tunnel_sources = True` to accept VXLAN ingress only +from source addresses learned as next hops in EVPN +Type 2, 3 and 5 routes. The connector updates this set automatically and +retains the source admission rules during a control-plane outage with fail-static +enabled. BGP peer addresses, including route reflectors, are not added merely +because they are peers. This relies on trusted EVPN announcements; it does +not authenticate the sender or prevent underlay source address spoofing. +The option defaults to `False`, preserving unrestricted tunnel source +admission for existing installations. When enabled, table 0 admits each VTEP +once and table 3 classifies VNIs without duplicating their rules per VTEP. +Legacy snapshot classifiers are moved behind this source check; qualified +snapshot rules also restore their permitted sources. + ## Documentation The details of the EVPN's operation and evpn-connector daemon workflow are described in the [presentation](https://vkvideo.ru/video-164978780_456239752) (in Russian only). diff --git a/etc/evpn_connector/evpn_connector.cfg.sample b/etc/evpn_connector/evpn_connector.cfg.sample index a5fbf27..a3ddd00 100644 --- a/etc/evpn_connector/evpn_connector.cfg.sample +++ b/etc/evpn_connector/evpn_connector.cfg.sample @@ -29,6 +29,7 @@ source_ip = "10.10.10.2" as_number = 1 [ovs] +restrict_tunnel_sources = False switch_name = "evpn" vxlan_udp_port = 4789 enable_sudo = True diff --git a/evpn_connector/bgp/client.py b/evpn_connector/bgp/client.py index 84e9d4b..82a2395 100644 --- a/evpn_connector/bgp/client.py +++ b/evpn_connector/bgp/client.py @@ -287,6 +287,7 @@ def __init__(self, gobgp_channel, grpc_timeout_sec, router_mac_type5): self._router_mac_type5 = router_mac_type5 self.gobgp_channel = gobgp_channel self.timeout_seconds = grpc_timeout_sec + self.vtep_addresses = set() self.bgp_origin = Any() self.bgp_origin.Pack(attribute_pb2.OriginAttribute(origin=BGP_ORIGIN)) @@ -592,6 +593,7 @@ def _get_paths_from_gobgp(self, prefix_type): def _get_paths(self): """Get paths from GoBGP""" + self.vtep_addresses.clear() return ( self._get_obj(constants.EVPN_MACADV_TYPE), self._get_obj(constants.EVPN_MULTICAST_TYPE), @@ -604,6 +606,12 @@ def _get_obj(self, prefix_type): req = self._get_paths_from_gobgp(prefix_type=prefix_type) start_time = time.time() for line in req: + for path in line.destination.paths: + for attribute in path.pattrs: + if attribute.type_url == MPREACHNLRI_TYPE_NAME: + self.vtep_addresses.update( + unpack_protobuf_message(attribute).next_hops + ) simplify_func = self.simplify_map[prefix_type] func_res = simplify_func(line.destination) if type(func_res) is list: diff --git a/evpn_connector/cmd/evpn.py b/evpn_connector/cmd/evpn.py index a082676..688cc56 100644 --- a/evpn_connector/cmd/evpn.py +++ b/evpn_connector/cmd/evpn.py @@ -94,6 +94,7 @@ def main(): policy_enabled=CONF.gobgp.policy_enabled, fail_static=CONF.gobgp.fail_static, fail_static_min_peers=CONF.gobgp.fail_static_min_peers, + restrict_tunnel_sources=CONF.ovs.restrict_tunnel_sources, configs_dir=CONF.daemon.configs_dir, router_mac_type5=CONF.gobgp.router_mac_type5, anycast_status_file=CONF.anycast.anycast_status_file, diff --git a/evpn_connector/common/conf_opts.py b/evpn_connector/common/conf_opts.py index fdfc41e..b599cf1 100644 --- a/evpn_connector/common/conf_opts.py +++ b/evpn_connector/common/conf_opts.py @@ -110,6 +110,14 @@ ] ovs_opts = [ + cfg.BoolOpt( + name="restrict_tunnel_sources", + default=False, + help=( + "Accept VXLAN ingress only from next-hop addresses learned " + "in EVPN Type 2, 3 and 5 routes. Requires trusted EVPN routes." + ), + ), cfg.StrOpt( name="switch_name", required=True, help="OpenvSwitch switch name" ), diff --git a/evpn_connector/common/constants.py b/evpn_connector/common/constants.py index da75e11..7c38078 100644 --- a/evpn_connector/common/constants.py +++ b/evpn_connector/common/constants.py @@ -44,6 +44,7 @@ INPUT_FILTER_TABLE_NUM = 0 OUTPUT_TABLE_NUM = 1 ECMP_TABLE_NUM = 2 +TUNNEL_CLASSIFY_TABLE_NUM = 3 # Priority for flows NORMAL_PRIO = 10 LOW_PRIO = 9 diff --git a/evpn_connector/service/evpn.py b/evpn_connector/service/evpn.py index f47a172..6b9ac0d 100644 --- a/evpn_connector/service/evpn.py +++ b/evpn_connector/service/evpn.py @@ -28,7 +28,6 @@ from evpn_connector.common.services import softirq from evpn_connector.service import objects as evpnobj - LOG = logging.getLogger(__name__) COMMON_NAME = "local-vni-" # Steps a session must outlive when nothing reports End-of-RIB. @@ -81,6 +80,7 @@ def __init__( policy_enabled=True, fail_static=False, fail_static_min_peers=0, + restrict_tunnel_sources=False, ): super(EvpnConnectorService, self).__init__( step_period=step_period, @@ -108,6 +108,7 @@ def __init__( # converged, retained if one is later lost. self.fail_static = fail_static self.fail_static_min_peers = fail_static_min_peers + self.restrict_tunnel_sources = restrict_tunnel_sources self._peer_settle_sec = PEER_SETTLE_STEPS * step_period self._last_good_flows = set() self._peers_healthy = True @@ -439,6 +440,10 @@ def _apply_fail_static(self, target_flows, metrics, peers_were_healthy): return target_flows flows_to_sync = target_flows.union(self._last_good_flows) + if self.restrict_tunnel_sources: + flows_to_sync = evpnobj.restrict_tunnel_sources( + flows_to_sync, set() + ) metrics["fail_static_active"] = 1 metrics["fail_static_retained_cnt"] = len(flows_to_sync) - len( target_flows @@ -680,7 +685,6 @@ def _step(self): len(uni_flows), len(bum_flows), ) - metrics["ovs_flow_cnt"] = len(uni_flows) + len(bum_flows) duration_metrics["prep_ovs_time"] = time.time() - start_time LOG.info( "Prepare ovs flows done for %0.4f sec", @@ -688,10 +692,14 @@ def _step(self): ) uni_flows = uni_flows.union(bum_flows) + if self.restrict_tunnel_sources: + sources = self.gobgp_client.vtep_addresses - {self.source_ip} + uni_flows = evpnobj.restrict_tunnel_sources(uni_flows, sources) flows_to_sync = self._apply_fail_static( uni_flows, metrics, peers_were_healthy ) + metrics["ovs_flow_cnt"] = len(flows_to_sync) start_time = time.time() LOG.debug("Sync flows in ovs") diff --git a/evpn_connector/service/objects.py b/evpn_connector/service/objects.py index c985b2c..bc7454c 100644 --- a/evpn_connector/service/objects.py +++ b/evpn_connector/service/objects.py @@ -24,7 +24,6 @@ from evpn_connector.common import constants from evpn_connector.ovs import common as ovs_cm - LOG = logging.getLogger(__name__) # Whether the fabric carries a sender's group: switch-wide, set at start-up. @@ -151,7 +150,6 @@ def _ovs_filter_match( if self.port_type == constants.EVPN_EDGE_TYPE_VLAN: res += "in_port=%d,dl_vlan=%d" % (ofport, self.tag) elif self.port_type == constants.EVPN_EDGE_TYPE_VXLAN: - # TODO(aleks.popov): May be add remote ip filter? res += "in_port=%d,tun_id=%d" % (ofport, self.vni) # Type: flat else: @@ -220,6 +218,47 @@ def __lt__(self, other): return NotImplemented +def restrict_tunnel_sources(flows, sources): + table = constants.TUNNEL_CLASSIFY_TABLE_NUM + restricted = {OvsDropFlow(table)} + sources = set(sources) + tunnel_port = "in_port=%d" % constants.VXLAN_PORT_OFPORT + gate_action = "resubmit(,%d)" % table + for flow in flows: + fields = flow.match.split(",") + if "table=0" not in fields or tunnel_port not in fields: + restricted.add(flow) + continue + source_fields = { + field + for field in fields + if field.startswith(("tun_src=", "tun_ipv6_src=")) + } + sources.update(field.split("=", 1)[1] for field in source_fields) + if flow.action.split("=", 1)[1] == gate_action: + continue + restricted.add( + OvsFlow( + match=",".join( + "table=%d" % table if field == "table=0" else field + for field in fields + if field not in source_fields + ), + action=flow.action, + ) + ) + for source in sources: + field = "tun_ipv6_src" if ":" in source else "tun_src" + restricted.add( + OvsFlow( + match="table=0,priority=%d,%s,%s=%s" + % (constants.NORMAL_PRIO, tunnel_port, field, source), + action="actions=" + gate_action, + ) + ) + return restricted + + class OvsDropFlow(OvsFlow): def __init__(self, table): super(OvsDropFlow, self).__init__( diff --git a/evpn_connector/tests/unit/test_bgp_client.py b/evpn_connector/tests/unit/test_bgp_client.py index 69241bb..8a424ec 100644 --- a/evpn_connector/tests/unit/test_bgp_client.py +++ b/evpn_connector/tests/unit/test_bgp_client.py @@ -15,9 +15,11 @@ # under the License. import mock +import pytest from evpn_connector.bgp import client from evpn_connector.bgp.generated import gobgp_pb2 +from evpn_connector.common import constants from evpn_connector.service import objects @@ -231,3 +233,63 @@ def test_an_unset_uptime_is_not_trusted(self): peer = self._peer(uptime=0) with mock.patch("time.time", return_value=1005): assert self._count([peer], settle_sec=10) == (1, 0) + + +class TestVtepSources(object): + @pytest.mark.parametrize( + "route_type", + [ + constants.EVPN_MACADV_TYPE, + constants.EVPN_MULTICAST_TYPE, + constants.EVPN_PREFIX_TYPE, + ], + ) + def test_all_paths_supply_vtep_sources(self, route_type): + bgp = client.BGPClient("127.0.0.1:50051", 1, "02:00:00:00:00:01") + common = dict( + vni=10000, + as_number=65001, + next_hop="10.10.10.2", + rt=objects.RouteTarget([(65001, 10000)]), + ) + if route_type == constants.EVPN_MULTICAST_TYPE: + route = objects.VirtNet(**common) + prepare = bgp.prepare_multicast_path + else: + common.update( + mac="02:00:00:00:00:02", ofport=65278, port_type="vxlan", tag=0 + ) + if route_type == constants.EVPN_MACADV_TYPE: + route = objects.ClientEdge(ip="10.100.0.1", **common) + prepare = bgp.prepare_unicast_path + else: + route = objects.ClientEdgePrefix( + prefix="10.100.0.1", + prefix_len=32, + router_mac="02:00:00:00:00:03", + **common, + ) + prepare = bgp.prepare_prefix_path + first = prepare(route) + route.next_hop = "10.10.10.3" + second = prepare(route) + destination = gobgp_pb2.Destination(paths=[first, second]) + response = gobgp_pb2.ListPathResponse(destination=destination) + bgp._get_paths_from_gobgp = mock.Mock( + side_effect=( + lambda prefix_type: ( + [response] if prefix_type == route_type else [] + ) + ) + ) + bgp.get_paths(local_next_hop="10.10.10.1") + assert bgp.vtep_addresses == {"10.10.10.2", "10.10.10.3"} + route.next_hop = "10.10.10.4" + destination.ClearField("paths") + destination.paths.add().CopyFrom(prepare(route)) + response.destination.CopyFrom(destination) + bgp.get_paths(local_next_hop="10.10.10.1") + assert bgp.vtep_addresses == {"10.10.10.4"} + bgp._get_paths_from_gobgp = mock.Mock(return_value=[]) + bgp.get_paths(local_next_hop="10.10.10.1") + assert not bgp.vtep_addresses diff --git a/evpn_connector/tests/unit/test_tunnel_sources.py b/evpn_connector/tests/unit/test_tunnel_sources.py new file mode 100644 index 0000000..1aea06b --- /dev/null +++ b/evpn_connector/tests/unit/test_tunnel_sources.py @@ -0,0 +1,289 @@ +# Copyright 2024 VK Cloud. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import pytest + +from evpn_connector.common import conf_opts +from evpn_connector.common import constants +from evpn_connector.service import evpn +from evpn_connector.service import objects + + +@pytest.fixture +def service(tmp_path): + return evpn.EvpnConnectorService( + source_ip="10.10.10.1", + as_number=65001, + configs_dir=str(tmp_path), + gobgp_client=mock.MagicMock(), + ovs_client=mock.MagicMock(), + sender=mock.MagicMock(), + vxlan_udp_port=4789, + router_mac_type5="02:00:00:00:00:01", + anycast_status_file=str(tmp_path / "anycast"), + anycast_check_ofport=65277, + anycast_check_mac="02:00:00:00:00:02", + policy_enabled=False, + fail_static=True, + ) + + +def prefix(source, address="10.100.0.1"): + return objects.ClientEdgePrefix( + mac="02:00:00:00:00:01", + prefix=address, + prefix_len=32, + router_mac="02:00:00:00:00:02", + vni=10000, + as_number=65001, + next_hop=source, + rt=objects.RouteTarget([(65001, 10000)]), + ofport=10, + port_type="flat", + tag=0, + ) + + +def ingress(flows): + return { + flow + for flow in flows + if "table=0" in flow.match.split(",") + and ("in_port=%d" % constants.VXLAN_PORT_OFPORT) + in flow.match.split(",") + } + + +@pytest.mark.parametrize("restrict", [False, True]) +def test_step_restricts_sources_from_every_evpn_route_type(service, restrict): + assert service.restrict_tunnel_sources is False + service.restrict_tunnel_sources = restrict + local = prefix(service.source_ip) + remote = prefix("10.10.10.2", "10.100.0.2") + edge = objects.ClientEdge( + mac="02:00:00:00:00:03", + ip="", + as_number=65001, + vni=10001, + next_hop="10.10.10.3", + rt=objects.RouteTarget([(65001, 10001)]), + ofport=65278, + port_type="vxlan", + tag=0, + ) + vnet = objects.VirtNet( + vni=10002, + next_hop="10.10.10.4", + as_number=65001, + rt=objects.RouteTarget([(65001, 10002)]), + ) + service.read_client_configs = mock.Mock(return_value=(set(), [local])) + service._vrfs = {objects.VRF(10000, local.rt)} + service.gobgp_client.get_paths.return_value = ( + set(), + set(), + {edge}, + {vnet}, + {local}, + {remote}, + ) + service.gobgp_client.count_peers.return_value = (1, 1) + service.gobgp_client.vtep_addresses = { + service.source_ip, + "10.10.10.2", + "10.10.10.3", + "10.10.10.4", + } + service._step() + flows = service.ovs_client.sync_flows.call_args[0][0] + tunnel = ingress(flows) + assert tunnel + if not restrict: + assert all("tun_src=" not in f.match for f in tunnel) + return + assert { + next( + field + for field in flow.match.split(",") + if field.startswith("tun_src=") + ) + for flow in tunnel + } == { + "tun_src=10.10.10.2", + "tun_src=10.10.10.3", + "tun_src=10.10.10.4", + } + assert service._last_good_flows == flows + service._sender.send_counter.assert_any_call("ovs_flow_cnt", len(flows)) + + +def test_sources_withdraw_and_survive_rr_failure_and_restart( + service, tmp_path +): + service.restrict_tunnel_sources = True + local = prefix(service.source_ip) + remote = prefix("10.10.10.2", "10.100.0.2") + service.read_client_configs = mock.Mock(return_value=(set(), [local])) + service._vrfs = {objects.VRF(10000, local.rt)} + service.gobgp_client.count_peers.return_value = (1, 1) + service.gobgp_client.get_paths.return_value = ( + set(), + set(), + set(), + set(), + {local}, + {remote}, + ) + service.gobgp_client.vtep_addresses = {service.source_ip, remote.next_hop} + service._step() + saved = service.ovs_client.sync_flows.call_args[0][0] + assert ingress(saved) + assert all("tun_src=10.10.10.2" in f.match for f in ingress(saved)) + flow_file = tmp_path / "flows" + flow_file.write_text("\n".join(f.to_string() for f in sorted(saved))) + service.ovs_client.tmp_flow_file_path = str(flow_file) + service._last_good_flows = service._read_synced_flows() + service.gobgp_client.count_peers.return_value = (1, 0) + service.gobgp_client.get_paths.return_value = ( + set(), + set(), + set(), + set(), + {local}, + set(), + ) + service.gobgp_client.vtep_addresses = {service.source_ip} + service._step() + held = service.ovs_client.sync_flows.call_args[0][0] + assert ingress(held) == ingress(saved) + service.gobgp_client.count_peers.return_value = (1, 1) + service._step() + withdrawn = service.ovs_client.sync_flows.call_args[0][0] + assert not ingress(withdrawn) + assert any("in_port=10" in f.match for f in withdrawn) + + +def test_source_restriction_is_disabled_by_default(service): + option = next( + option + for option in conf_opts.ovs_opts + if option.name == "restrict_tunnel_sources" + ) + assert option.default is False + assert service.restrict_tunnel_sources is False + + +def test_source_rules_grow_with_vteps_instead_of_classifiers(): + classifiers = { + objects.OvsFlow( + "table=0,priority=10,in_port=65278,tun_id=%d" % vni, + "actions=set_field:%d->reg2,resubmit(,1)" % vni, + ) + for vni in range(10000, 10100) + } + sources = {"10.10.10.%d" % n for n in range(2, 22)} + result = objects.restrict_tunnel_sources(classifiers, sources) + assert len(ingress(result)) == len(sources) + classified = {f for f in result if "table=3" in f.match.split(",")} + assert len(classified) == len(classifiers) + 1 + assert len(result) == len(classifiers) + len(sources) + 1 + assert objects.restrict_tunnel_sources(result, set()) == result + + +@pytest.mark.parametrize("restrict", [False, True]) +def test_legacy_snapshot_cannot_bypass_enabled_filter( + service, tmp_path, restrict +): + service.restrict_tunnel_sources = restrict + broad = objects.OvsFlow( + "table=0,priority=10,in_port=65278,tun_id=10000", + "action=resubmit(,1)", + ) + qualified = objects.OvsFlow( + broad.match + ",tun_src=10.10.10.2", broad.action + ) + local = objects.OvsFlow("table=0,in_port=10", "action=resubmit(,1)") + saved = {broad, qualified, local} + path = tmp_path / "flows" + path.write_text("\n".join(flow.to_string() for flow in saved)) + service.ovs_client.tmp_flow_file_path = str(path) + service._last_good_flows = service._read_synced_flows() + service.gobgp_client.count_peers.return_value = (1, 0) + result = service._apply_fail_static({local}, {}, False) + expected = ( + objects.restrict_tunnel_sources(saved, set()) if restrict else saved + ) + assert result == expected + if restrict: + assert {f.to_string() for f in ingress(result)} == { + "table=0,priority=10,in_port=65278,tun_src=10.10.10.2 " + "actions=resubmit(,3)" + } + + +@pytest.mark.parametrize( + "match", + [ + "table=0,priority=10,in_port=65278,tun_id=10000", + "table=0,priority=11,in_port=65278,tun_id=10000,ip,nw_src=10.100.0.1", + "table=0,priority=10,in_port=65278,tun_id=10000,arp", + ], +) +def test_every_tunnel_classifier_keeps_its_actions(match): + action = ( + "actions=move:NXM_NX_TUN_GBP_ID[]->NXM_NX_PKT_MARK[0..15],resubmit(,1)" + ) + original = objects.OvsFlow(match, action) + local = objects.OvsFlow( + "table=0,priority=10,in_port=10", "actions=output:11" + ) + recirculated = objects.OvsFlow( + "table=1,priority=10,in_port=65278", "actions=output:10" + ) + flows = {original, local, recirculated, objects.OvsDropFlow(0)} + result = objects.restrict_tunnel_sources( + flows, {"10.10.10.2", "2001:db8::2"} + ) + assert local in result and recirculated in result + assert original not in result + assert {f.to_string() for f in ingress(result)} == { + "table=0,priority=10,in_port=65278,tun_src=10.10.10.2 " + "actions=resubmit(,3)", + "table=0,priority=10,in_port=65278,tun_ipv6_src=2001:db8::2 " + "actions=resubmit(,3)", + } + classified = objects.OvsFlow(match.replace("table=0", "table=3"), action) + assert classified in result + assert next(f for f in result if f == classified).action == action + empty = objects.restrict_tunnel_sources(flows, set()) + assert not ingress(empty) + assert empty == flows - {original} | {classified, objects.OvsDropFlow(3)} + + +def test_unqualified_legacy_snapshot_cannot_admit_sources(service): + service.restrict_tunnel_sources = True + service.gobgp_client.count_peers.return_value = (1, 0) + service._last_good_flows = { + objects.OvsFlow( + "table=0,priority=10,in_port=65278,tun_id=10000", + "actions=resubmit(,1)", + ) + } + result = service._apply_fail_static({objects.OvsDropFlow(0)}, {}, False) + assert not ingress(result) + assert objects.OvsDropFlow(0) in result + assert objects.OvsDropFlow(3) in result