private readonly SortedList<Distance, List<BearingPoint>> pointsByX;
private readonly SortedList<Distance, List<BearingPoint>> pointsByY;
 
 
...
 
Point targetPoint;
var bestMatch = 
   (from pt in 
    (from kvp in pointsByX
     where kvp.Key > minX && kvp.Key < maxX
     select kvp.Value).Union(
     from kvp in pointsByY
     where kvp.Key > minY && kvp.Key < maxY
     select kvp.Value).Distinct()
    order pt by pt.Value.DistanceTo(targetPoint)
    select pt.Value).FirstOrDefault();