UDP not getting picked up

I am using C# UDP to get data from multiple clients. My code is working fine for most of the clients. But on a few the udp is not getting sent to my program. The 2 clients that are not working are coming in over a VPN, but I have 1 other on the same VPN which works.

I am using WireShark and I can see the UDP message making it to my server from those IPs but the message never triggers my ReceiveData function.

Any ideas will be great.

ipEndPoint = new IPEndPoint(IPAddress.Any, 4612);
UdpClient udpClient = new UdpClient(ipEndPoint);
serverListeners.Add(udpClient);
udpClient.BeginReceive(new AsyncCallback(ReceiveData), udpClient);

private void ReceiveData(IAsyncResult ar)
    {

        //this is never written out for the missing ips
        LogController.Current.WriteLog(LoggingController.LogMessageType.Debug, "GOT UPD", -1);

        byte[] data = null;
        long ip = 0;
        int port = 0;
        bool hasData = true;
        UdpClient u = (UdpClient)ar.AsyncState;

        try
        {

            data = u.EndReceive(ar, ref clientEndPoint);
            LogController.Current.WriteLog(LoggingController.LogMessageType.Debug, "GOT FROM: " + clientEndPoint.Address.ToString(), -1);


        }
        catch(Exception doh)
        {
            LogController.Current.WriteError(doh, -1, "receive upd");
        }

        u.BeginReceive(new AsyncCallback(ReceiveData), u);