vxlan: use remote next-hop gateway for pod CIDR tunnel routes

Commit 47563a4 changed VXLAN pod-CIDR routes from `Gw: nodeIP` to `Gw: nil`
to avoid local-IP adj-sourced side effects and UNRESOLVED encap DPO issues.

`Gw: nil` is valid for IPIP (P2P tunnel semantics), but VXLAN is non-P2P.
VXLAN tunnels in VPP lack `VNET_HW_INTERFACE_CLASS_FLAG_P2P` (unlike IPIP).
With `Gw:nil` on a non-P2P interface, VPP creates an attached/glean FIB
entry (`FIB_PATH_TYPE_ATTACHED`) and attempts NDP for each destination
directly on the VXLAN tunnel which triggers unresolved neighbor resolution
behavior on the tunnel path.

Fixed the code to use `cn.NextHop` (remote node IP / tunnel destination)
as the gateway. This creates `FIB_PATH_TYPE_ATTACHED_NEXT_HOP`, which
resolves via NDP on the uplink (tunnel is unnumbered) and does not
conflict with the encap DPO since the remote IP has no local receive route.

Signed-off-by: Aritra Basu <aritrbas@cisco.com>

Author: Aritra Basu

calico-vpp-agent/connectivity/vxlan.go

@@ -211,7 +211,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error {
 			})
 			if err != nil {
 				// TODO : delete tunnel
-				return errors.Wrapf(err, "Error adding route to %s in ipip tunnel %d for pods", cn.NextHop.String(), swIfIndex)
+				return errors.Wrapf(err, "Error adding route to %s in vxlan tunnel %d for pods", cn.NextHop.String(), swIfIndex)
 			}
 		}
 
@@ -261,7 +261,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error {
 		Dst: &cn.Dst,
 		Paths: []types.RoutePath{{
 			SwIfIndex: tunnel.SwIfIndex,
-			Gw:        nil,
+			Gw:        cn.NextHop,
 		}},
 		Table: table,
 	}
@@ -286,7 +286,7 @@ func (p *VXLanProvider) DelConnectivity(cn *common.NodeConnectivity) error {
 			Dst: &cn.Dst,
 			Paths: []types.RoutePath{{
 				SwIfIndex: tunnel.SwIfIndex,
-				Gw:        nil,
+				Gw:        cn.NextHop,
 			}},
 		}
 	} else {
@@ -296,7 +296,7 @@ func (p *VXLanProvider) DelConnectivity(cn *common.NodeConnectivity) error {
 			Dst: &cn.Dst,
 			Paths: []types.RoutePath{{
 				SwIfIndex: tunnel.SwIfIndex,
-				Gw:        nil,
+				Gw:        cn.NextHop,
 			}},
 			Table: vrfIndex,
 		}