diff --git a/README.md b/README.md index eb88425..419a1fe 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,19 @@ A service for automating the management distributed switch or router based on EV * **Reconciliation Loop Design:** Built on a closed-loop control architecture to ensure fault tolerance and reduce the impact of external factors * **Configurability:** Flexible setup via INI daemon configuration files and JSON clients configuration files +Set `[ovs] restrict_tunnel_sources = True` to accept VXLAN ingress only +from source addresses learned as next hops in EVPN +Type 2, 3 and 5 routes. The connector updates this set automatically and +retains the source admission rules during a control-plane outage with fail-static +enabled. BGP peer addresses, including route reflectors, are not added merely +because they are peers. This relies on trusted EVPN announcements; it does +not authenticate the sender or prevent underlay source address spoofing. +The option defaults to `False`, preserving unrestricted tunnel source +admission for existing installations. When enabled, table 0 admits each VTEP +once and table 3 classifies VNIs without duplicating their rules per VTEP. +Legacy snapshot classifiers are moved behind this source check; qualified +snapshot rules also restore their permitted sources. + ## Documentation The details of the EVPN's operation and evpn-connector daemon workflow are described in the [presentation](https://vkvideo.ru/video-164978780_456239752) (in Russian only). diff --git a/etc/evpn_connector/evpn_connector.cfg.sample b/etc/evpn_connector/evpn_connector.cfg.sample index a5fbf27..a3ddd00 100644 --- a/etc/evpn_connector/evpn_connector.cfg.sample +++ b/etc/evpn_connector/evpn_connector.cfg.sample @@ -29,6 +29,7 @@ source_ip = "10.10.10.2" as_number = 1 [ovs] +restrict_tunnel_sources = False switch_name = "evpn" vxlan_udp_port = 4789 enable_sudo = True diff --git a/evpn_connector/bgp/client.py b/evpn_connector/bgp/client.py index 84e9d4b..82a2395 100644 --- a/evpn_connector/bgp/client.py +++ b/evpn_connector/bgp/client.py @@ -287,6 +287,7 @@ def __init__(self, gobgp_channel, grpc_timeout_sec, router_mac_type5): self._router_mac_type5 = router_mac_type5 self.gobgp_channel = gobgp_channel self.timeout_seconds = grpc_timeout_sec + self.vtep_addresses = set() self.bgp_origin = Any() self.bgp_origin.Pack(attribute_pb2.OriginAttribute(origin=BGP_ORIGIN)) @@ -592,6 +593,7 @@ def _get_paths_from_gobgp(self, prefix_type): def _get_paths(self): """Get paths from GoBGP""" + self.vtep_addresses.clear() return ( self._get_obj(constants.EVPN_MACADV_TYPE), self._get_obj(constants.EVPN_MULTICAST_TYPE), @@ -604,6 +606,12 @@ def _get_obj(self, prefix_type): req = self._get_paths_from_gobgp(prefix_type=prefix_type) start_time = time.time() for line in req: + for path in line.destination.paths: + for attribute in path.pattrs: + if attribute.type_url == MPREACHNLRI_TYPE_NAME: + self.vtep_addresses.update( + unpack_protobuf_message(attribute).next_hops + ) simplify_func = self.simplify_map[prefix_type] func_res = simplify_func(line.destination) if type(func_res) is list: diff --git a/evpn_connector/cmd/evpn.py b/evpn_connector/cmd/evpn.py index a082676..688cc56 100644 --- a/evpn_connector/cmd/evpn.py +++ b/evpn_connector/cmd/evpn.py @@ -94,6 +94,7 @@ def main(): policy_enabled=CONF.gobgp.policy_enabled, fail_static=CONF.gobgp.fail_static, fail_static_min_peers=CONF.gobgp.fail_static_min_peers, + restrict_tunnel_sources=CONF.ovs.restrict_tunnel_sources, configs_dir=CONF.daemon.configs_dir, router_mac_type5=CONF.gobgp.router_mac_type5, anycast_status_file=CONF.anycast.anycast_status_file, diff --git a/evpn_connector/common/conf_opts.py b/evpn_connector/common/conf_opts.py index fdfc41e..b599cf1 100644 --- a/evpn_connector/common/conf_opts.py +++ b/evpn_connector/common/conf_opts.py @@ -110,6 +110,14 @@ ] ovs_opts = [ + cfg.BoolOpt( + name="restrict_tunnel_sources", + default=False, + help=( + "Accept VXLAN ingress only from next-hop addresses learned " + "in EVPN Type 2, 3 and 5 routes. Requires trusted EVPN routes." + ), + ), cfg.StrOpt( name="switch_name", required=True, help="OpenvSwitch switch name" ), diff --git a/evpn_connector/common/constants.py b/evpn_connector/common/constants.py index da75e11..7c38078 100644 --- a/evpn_connector/common/constants.py +++ b/evpn_connector/common/constants.py @@ -44,6 +44,7 @@ INPUT_FILTER_TABLE_NUM = 0 OUTPUT_TABLE_NUM = 1 ECMP_TABLE_NUM = 2 +TUNNEL_CLASSIFY_TABLE_NUM = 3 # Priority for flows NORMAL_PRIO = 10 LOW_PRIO = 9 diff --git a/evpn_connector/service/evpn.py b/evpn_connector/service/evpn.py index f47a172..6b9ac0d 100644 --- a/evpn_connector/service/evpn.py +++ b/evpn_connector/service/evpn.py @@ -28,7 +28,6 @@ from evpn_connector.common.services import softirq from evpn_connector.service import objects as evpnobj - LOG = logging.getLogger(__name__) COMMON_NAME = "local-vni-" # Steps a session must outlive when nothing reports End-of-RIB. @@ -81,6 +80,7 @@ def __init__( policy_enabled=True, fail_static=False, fail_static_min_peers=0, + restrict_tunnel_sources=False, ): super(EvpnConnectorService, self).__init__( step_period=step_period, @@ -108,6 +108,7 @@ def __init__( # converged, retained if one is later lost. self.fail_static = fail_static self.fail_static_min_peers = fail_static_min_peers + self.restrict_tunnel_sources = restrict_tunnel_sources self._peer_settle_sec = PEER_SETTLE_STEPS * step_period self._last_good_flows = set() self._peers_healthy = True @@ -439,6 +440,10 @@ def _apply_fail_static(self, target_flows, metrics, peers_were_healthy): return target_flows flows_to_sync = target_flows.union(self._last_good_flows) + if self.restrict_tunnel_sources: + flows_to_sync = evpnobj.restrict_tunnel_sources( + flows_to_sync, set() + ) metrics["fail_static_active"] = 1 metrics["fail_static_retained_cnt"] = len(flows_to_sync) - len( target_flows @@ -680,7 +685,6 @@ def _step(self): len(uni_flows), len(bum_flows), ) - metrics["ovs_flow_cnt"] = len(uni_flows) + len(bum_flows) duration_metrics["prep_ovs_time"] = time.time() - start_time LOG.info( "Prepare ovs flows done for %0.4f sec", @@ -688,10 +692,14 @@ def _step(self): ) uni_flows = uni_flows.union(bum_flows) + if self.restrict_tunnel_sources: + sources = self.gobgp_client.vtep_addresses - {self.source_ip} + uni_flows = evpnobj.restrict_tunnel_sources(uni_flows, sources) flows_to_sync = self._apply_fail_static( uni_flows, metrics, peers_were_healthy ) + metrics["ovs_flow_cnt"] = len(flows_to_sync) start_time = time.time() LOG.debug("Sync flows in ovs") diff --git a/evpn_connector/service/objects.py b/evpn_connector/service/objects.py index c985b2c..bc7454c 100644 --- a/evpn_connector/service/objects.py +++ b/evpn_connector/service/objects.py @@ -24,7 +24,6 @@ from evpn_connector.common import constants from evpn_connector.ovs import common as ovs_cm - LOG = logging.getLogger(__name__) # Whether the fabric carries a sender's group: switch-wide, set at start-up. @@ -151,7 +150,6 @@ def _ovs_filter_match( if self.port_type == constants.EVPN_EDGE_TYPE_VLAN: res += "in_port=%d,dl_vlan=%d" % (ofport, self.tag) elif self.port_type == constants.EVPN_EDGE_TYPE_VXLAN: - # TODO(aleks.popov): May be add remote ip filter? res += "in_port=%d,tun_id=%d" % (ofport, self.vni) # Type: flat else: @@ -220,6 +218,47 @@ def __lt__(self, other): return NotImplemented +def restrict_tunnel_sources(flows, sources): + table = constants.TUNNEL_CLASSIFY_TABLE_NUM + restricted = {OvsDropFlow(table)} + sources = set(sources) + tunnel_port = "in_port=%d" % constants.VXLAN_PORT_OFPORT + gate_action = "resubmit(,%d)" % table + for flow in flows: + fields = flow.match.split(",") + if "table=0" not in fields or tunnel_port not in fields: + restricted.add(flow) + continue + source_fields = { + field + for field in fields + if field.startswith(("tun_src=", "tun_ipv6_src=")) + } + sources.update(field.split("=", 1)[1] for field in source_fields) + if flow.action.split("=", 1)[1] == gate_action: + continue + restricted.add( + OvsFlow( + match=",".join( + "table=%d" % table if field == "table=0" else field + for field in fields + if field not in source_fields + ), + action=flow.action, + ) + ) + for source in sources: + field = "tun_ipv6_src" if ":" in source else "tun_src" + restricted.add( + OvsFlow( + match="table=0,priority=%d,%s,%s=%s" + % (constants.NORMAL_PRIO, tunnel_port, field, source), + action="actions=" + gate_action, + ) + ) + return restricted + + class OvsDropFlow(OvsFlow): def __init__(self, table): super(OvsDropFlow, self).__init__( diff --git a/evpn_connector/tests/unit/test_bgp_client.py b/evpn_connector/tests/unit/test_bgp_client.py index 69241bb..8a424ec 100644 --- a/evpn_connector/tests/unit/test_bgp_client.py +++ b/evpn_connector/tests/unit/test_bgp_client.py @@ -15,9 +15,11 @@ # under the License. import mock +import pytest from evpn_connector.bgp import client from evpn_connector.bgp.generated import gobgp_pb2 +from evpn_connector.common import constants from evpn_connector.service import objects @@ -231,3 +233,63 @@ def test_an_unset_uptime_is_not_trusted(self): peer = self._peer(uptime=0) with mock.patch("time.time", return_value=1005): assert self._count([peer], settle_sec=10) == (1, 0) + + +class TestVtepSources(object): + @pytest.mark.parametrize( + "route_type", + [ + constants.EVPN_MACADV_TYPE, + constants.EVPN_MULTICAST_TYPE, + constants.EVPN_PREFIX_TYPE, + ], + ) + def test_all_paths_supply_vtep_sources(self, route_type): + bgp = client.BGPClient("127.0.0.1:50051", 1, "02:00:00:00:00:01") + common = dict( + vni=10000, + as_number=65001, + next_hop="10.10.10.2", + rt=objects.RouteTarget([(65001, 10000)]), + ) + if route_type == constants.EVPN_MULTICAST_TYPE: + route = objects.VirtNet(**common) + prepare = bgp.prepare_multicast_path + else: + common.update( + mac="02:00:00:00:00:02", ofport=65278, port_type="vxlan", tag=0 + ) + if route_type == constants.EVPN_MACADV_TYPE: + route = objects.ClientEdge(ip="10.100.0.1", **common) + prepare = bgp.prepare_unicast_path + else: + route = objects.ClientEdgePrefix( + prefix="10.100.0.1", + prefix_len=32, + router_mac="02:00:00:00:00:03", + **common, + ) + prepare = bgp.prepare_prefix_path + first = prepare(route) + route.next_hop = "10.10.10.3" + second = prepare(route) + destination = gobgp_pb2.Destination(paths=[first, second]) + response = gobgp_pb2.ListPathResponse(destination=destination) + bgp._get_paths_from_gobgp = mock.Mock( + side_effect=( + lambda prefix_type: ( + [response] if prefix_type == route_type else [] + ) + ) + ) + bgp.get_paths(local_next_hop="10.10.10.1") + assert bgp.vtep_addresses == {"10.10.10.2", "10.10.10.3"} + route.next_hop = "10.10.10.4" + destination.ClearField("paths") + destination.paths.add().CopyFrom(prepare(route)) + response.destination.CopyFrom(destination) + bgp.get_paths(local_next_hop="10.10.10.1") + assert bgp.vtep_addresses == {"10.10.10.4"} + bgp._get_paths_from_gobgp = mock.Mock(return_value=[]) + bgp.get_paths(local_next_hop="10.10.10.1") + assert not bgp.vtep_addresses diff --git a/evpn_connector/tests/unit/test_tunnel_sources.py b/evpn_connector/tests/unit/test_tunnel_sources.py new file mode 100644 index 0000000..1aea06b --- /dev/null +++ b/evpn_connector/tests/unit/test_tunnel_sources.py @@ -0,0 +1,289 @@ +# Copyright 2024 VK Cloud. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import pytest + +from evpn_connector.common import conf_opts +from evpn_connector.common import constants +from evpn_connector.service import evpn +from evpn_connector.service import objects + + +@pytest.fixture +def service(tmp_path): + return evpn.EvpnConnectorService( + source_ip="10.10.10.1", + as_number=65001, + configs_dir=str(tmp_path), + gobgp_client=mock.MagicMock(), + ovs_client=mock.MagicMock(), + sender=mock.MagicMock(), + vxlan_udp_port=4789, + router_mac_type5="02:00:00:00:00:01", + anycast_status_file=str(tmp_path / "anycast"), + anycast_check_ofport=65277, + anycast_check_mac="02:00:00:00:00:02", + policy_enabled=False, + fail_static=True, + ) + + +def prefix(source, address="10.100.0.1"): + return objects.ClientEdgePrefix( + mac="02:00:00:00:00:01", + prefix=address, + prefix_len=32, + router_mac="02:00:00:00:00:02", + vni=10000, + as_number=65001, + next_hop=source, + rt=objects.RouteTarget([(65001, 10000)]), + ofport=10, + port_type="flat", + tag=0, + ) + + +def ingress(flows): + return { + flow + for flow in flows + if "table=0" in flow.match.split(",") + and ("in_port=%d" % constants.VXLAN_PORT_OFPORT) + in flow.match.split(",") + } + + +@pytest.mark.parametrize("restrict", [False, True]) +def test_step_restricts_sources_from_every_evpn_route_type(service, restrict): + assert service.restrict_tunnel_sources is False + service.restrict_tunnel_sources = restrict + local = prefix(service.source_ip) + remote = prefix("10.10.10.2", "10.100.0.2") + edge = objects.ClientEdge( + mac="02:00:00:00:00:03", + ip="", + as_number=65001, + vni=10001, + next_hop="10.10.10.3", + rt=objects.RouteTarget([(65001, 10001)]), + ofport=65278, + port_type="vxlan", + tag=0, + ) + vnet = objects.VirtNet( + vni=10002, + next_hop="10.10.10.4", + as_number=65001, + rt=objects.RouteTarget([(65001, 10002)]), + ) + service.read_client_configs = mock.Mock(return_value=(set(), [local])) + service._vrfs = {objects.VRF(10000, local.rt)} + service.gobgp_client.get_paths.return_value = ( + set(), + set(), + {edge}, + {vnet}, + {local}, + {remote}, + ) + service.gobgp_client.count_peers.return_value = (1, 1) + service.gobgp_client.vtep_addresses = { + service.source_ip, + "10.10.10.2", + "10.10.10.3", + "10.10.10.4", + } + service._step() + flows = service.ovs_client.sync_flows.call_args[0][0] + tunnel = ingress(flows) + assert tunnel + if not restrict: + assert all("tun_src=" not in f.match for f in tunnel) + return + assert { + next( + field + for field in flow.match.split(",") + if field.startswith("tun_src=") + ) + for flow in tunnel + } == { + "tun_src=10.10.10.2", + "tun_src=10.10.10.3", + "tun_src=10.10.10.4", + } + assert service._last_good_flows == flows + service._sender.send_counter.assert_any_call("ovs_flow_cnt", len(flows)) + + +def test_sources_withdraw_and_survive_rr_failure_and_restart( + service, tmp_path +): + service.restrict_tunnel_sources = True + local = prefix(service.source_ip) + remote = prefix("10.10.10.2", "10.100.0.2") + service.read_client_configs = mock.Mock(return_value=(set(), [local])) + service._vrfs = {objects.VRF(10000, local.rt)} + service.gobgp_client.count_peers.return_value = (1, 1) + service.gobgp_client.get_paths.return_value = ( + set(), + set(), + set(), + set(), + {local}, + {remote}, + ) + service.gobgp_client.vtep_addresses = {service.source_ip, remote.next_hop} + service._step() + saved = service.ovs_client.sync_flows.call_args[0][0] + assert ingress(saved) + assert all("tun_src=10.10.10.2" in f.match for f in ingress(saved)) + flow_file = tmp_path / "flows" + flow_file.write_text("\n".join(f.to_string() for f in sorted(saved))) + service.ovs_client.tmp_flow_file_path = str(flow_file) + service._last_good_flows = service._read_synced_flows() + service.gobgp_client.count_peers.return_value = (1, 0) + service.gobgp_client.get_paths.return_value = ( + set(), + set(), + set(), + set(), + {local}, + set(), + ) + service.gobgp_client.vtep_addresses = {service.source_ip} + service._step() + held = service.ovs_client.sync_flows.call_args[0][0] + assert ingress(held) == ingress(saved) + service.gobgp_client.count_peers.return_value = (1, 1) + service._step() + withdrawn = service.ovs_client.sync_flows.call_args[0][0] + assert not ingress(withdrawn) + assert any("in_port=10" in f.match for f in withdrawn) + + +def test_source_restriction_is_disabled_by_default(service): + option = next( + option + for option in conf_opts.ovs_opts + if option.name == "restrict_tunnel_sources" + ) + assert option.default is False + assert service.restrict_tunnel_sources is False + + +def test_source_rules_grow_with_vteps_instead_of_classifiers(): + classifiers = { + objects.OvsFlow( + "table=0,priority=10,in_port=65278,tun_id=%d" % vni, + "actions=set_field:%d->reg2,resubmit(,1)" % vni, + ) + for vni in range(10000, 10100) + } + sources = {"10.10.10.%d" % n for n in range(2, 22)} + result = objects.restrict_tunnel_sources(classifiers, sources) + assert len(ingress(result)) == len(sources) + classified = {f for f in result if "table=3" in f.match.split(",")} + assert len(classified) == len(classifiers) + 1 + assert len(result) == len(classifiers) + len(sources) + 1 + assert objects.restrict_tunnel_sources(result, set()) == result + + +@pytest.mark.parametrize("restrict", [False, True]) +def test_legacy_snapshot_cannot_bypass_enabled_filter( + service, tmp_path, restrict +): + service.restrict_tunnel_sources = restrict + broad = objects.OvsFlow( + "table=0,priority=10,in_port=65278,tun_id=10000", + "action=resubmit(,1)", + ) + qualified = objects.OvsFlow( + broad.match + ",tun_src=10.10.10.2", broad.action + ) + local = objects.OvsFlow("table=0,in_port=10", "action=resubmit(,1)") + saved = {broad, qualified, local} + path = tmp_path / "flows" + path.write_text("\n".join(flow.to_string() for flow in saved)) + service.ovs_client.tmp_flow_file_path = str(path) + service._last_good_flows = service._read_synced_flows() + service.gobgp_client.count_peers.return_value = (1, 0) + result = service._apply_fail_static({local}, {}, False) + expected = ( + objects.restrict_tunnel_sources(saved, set()) if restrict else saved + ) + assert result == expected + if restrict: + assert {f.to_string() for f in ingress(result)} == { + "table=0,priority=10,in_port=65278,tun_src=10.10.10.2 " + "actions=resubmit(,3)" + } + + +@pytest.mark.parametrize( + "match", + [ + "table=0,priority=10,in_port=65278,tun_id=10000", + "table=0,priority=11,in_port=65278,tun_id=10000,ip,nw_src=10.100.0.1", + "table=0,priority=10,in_port=65278,tun_id=10000,arp", + ], +) +def test_every_tunnel_classifier_keeps_its_actions(match): + action = ( + "actions=move:NXM_NX_TUN_GBP_ID[]->NXM_NX_PKT_MARK[0..15],resubmit(,1)" + ) + original = objects.OvsFlow(match, action) + local = objects.OvsFlow( + "table=0,priority=10,in_port=10", "actions=output:11" + ) + recirculated = objects.OvsFlow( + "table=1,priority=10,in_port=65278", "actions=output:10" + ) + flows = {original, local, recirculated, objects.OvsDropFlow(0)} + result = objects.restrict_tunnel_sources( + flows, {"10.10.10.2", "2001:db8::2"} + ) + assert local in result and recirculated in result + assert original not in result + assert {f.to_string() for f in ingress(result)} == { + "table=0,priority=10,in_port=65278,tun_src=10.10.10.2 " + "actions=resubmit(,3)", + "table=0,priority=10,in_port=65278,tun_ipv6_src=2001:db8::2 " + "actions=resubmit(,3)", + } + classified = objects.OvsFlow(match.replace("table=0", "table=3"), action) + assert classified in result + assert next(f for f in result if f == classified).action == action + empty = objects.restrict_tunnel_sources(flows, set()) + assert not ingress(empty) + assert empty == flows - {original} | {classified, objects.OvsDropFlow(3)} + + +def test_unqualified_legacy_snapshot_cannot_admit_sources(service): + service.restrict_tunnel_sources = True + service.gobgp_client.count_peers.return_value = (1, 0) + service._last_good_flows = { + objects.OvsFlow( + "table=0,priority=10,in_port=65278,tun_id=10000", + "actions=resubmit(,1)", + ) + } + result = service._apply_fail_static({objects.OvsDropFlow(0)}, {}, False) + assert not ingress(result) + assert objects.OvsDropFlow(0) in result + assert objects.OvsDropFlow(3) in result