diff --git a/common/dialer/default.go b/common/dialer/default.go
index f8546fa6..9fbd1d8e 100644
--- a/common/dialer/default.go
+++ b/common/dialer/default.go
@@ -137,10 +137,12 @@ func (d *DefaultDialer) DialContext(ctx context.Context, network string, address
 }
 
 func (d *DefaultDialer) ListenPacket(ctx context.Context, destination M.Socksaddr) (net.PacketConn, error) {
-	if !destination.IsIPv6() {
-		return trackPacketConn(d.udpListener.ListenPacket(ctx, N.NetworkUDP, d.udpAddr4))
-	} else {
+	if destination.IsIPv6() {
 		return trackPacketConn(d.udpListener.ListenPacket(ctx, N.NetworkUDP, d.udpAddr6))
+	} else if destination.IsIPv4() && !destination.Addr.IsUnspecified() {
+		return trackPacketConn(d.udpListener.ListenPacket(ctx, N.NetworkUDP+"4", d.udpAddr4))
+	} else {
+		return trackPacketConn(d.udpListener.ListenPacket(ctx, N.NetworkUDP, d.udpAddr4))
 	}
 }