diff --git a/include/linux/netfilter_ipv4/ipt_policy.h b/include/linux/netfilter_ipv4/ipt_policy.h
index 7fd1bec453f1..a3f6eff39d33 100644
--- a/include/linux/netfilter_ipv4/ipt_policy.h
+++ b/include/linux/netfilter_ipv4/ipt_policy.h
@@ -27,16 +27,22 @@ struct ipt_policy_spec
 			reqid:1;
 };
 
+union ipt_policy_addr
+{
+	struct in_addr	a4;
+	struct in6_addr	a6;
+};
+
 struct ipt_policy_elem
 {
-	u_int32_t	saddr;
-	u_int32_t	smask;
-	u_int32_t	daddr;
-	u_int32_t	dmask;
-	u_int32_t	spi;
-	u_int32_t	reqid;
-	u_int8_t	proto;
-	u_int8_t	mode;
+	union ipt_policy_addr	saddr;
+	union ipt_policy_addr	smask;
+	union ipt_policy_addr	daddr;
+	union ipt_policy_addr	dmask;
+	u_int32_t		spi;
+	u_int32_t		reqid;
+	u_int8_t		proto;
+	u_int8_t		mode;
 
 	struct ipt_policy_spec	match;
 	struct ipt_policy_spec	invert;
diff --git a/include/linux/netfilter_ipv6/ip6t_policy.h b/include/linux/netfilter_ipv6/ip6t_policy.h
index 5a93afcd2ff1..671bd818300f 100644
--- a/include/linux/netfilter_ipv6/ip6t_policy.h
+++ b/include/linux/netfilter_ipv6/ip6t_policy.h
@@ -27,16 +27,22 @@ struct ip6t_policy_spec
 			reqid:1;
 };
 
+union ip6t_policy_addr
+{
+	struct in_addr	a4;
+	struct in6_addr	a6;
+};
+
 struct ip6t_policy_elem
 {
-	struct in6_addr	saddr;
-	struct in6_addr	smask;
-	struct in6_addr	daddr;
-	struct in6_addr	dmask;
-	u_int32_t	spi;
-	u_int32_t	reqid;
-	u_int8_t	proto;
-	u_int8_t	mode;
+	union ip6t_policy_addr	saddr;
+	union ip6t_policy_addr	smask;
+	union ip6t_policy_addr	daddr;
+	union ip6t_policy_addr	dmask;
+	u_int32_t		spi;
+	u_int32_t		reqid;
+	u_int8_t		proto;
+	u_int8_t		mode;
 
 	struct ip6t_policy_spec	match;
 	struct ip6t_policy_spec	invert;
diff --git a/net/ipv4/netfilter/ipt_policy.c b/net/ipv4/netfilter/ipt_policy.c
index a48949a3a750..5a7a265280f9 100644
--- a/net/ipv4/netfilter/ipt_policy.c
+++ b/net/ipv4/netfilter/ipt_policy.c
@@ -26,10 +26,13 @@ MODULE_LICENSE("GPL");
 static inline int
 match_xfrm_state(struct xfrm_state *x, const struct ipt_policy_elem *e)
 {
-#define MATCH(x,y)	(!e->match.x || ((e->x == (y)) ^ e->invert.x))
+#define MATCH_ADDR(x,y,z)	(!e->match.x ||				     \
+		                 ((e->x.a4.s_addr == (e->y.a4.s_addr & (z))) \
+				  ^ e->invert.x))
+#define MATCH(x,y)		(!e->match.x || ((e->x == (y)) ^ e->invert.x))
 
-	return MATCH(saddr, x->props.saddr.a4 & e->smask) &&
-	       MATCH(daddr, x->id.daddr.a4 & e->dmask) &&
+	return MATCH_ADDR(saddr, smask, x->props.saddr.a4) &&
+	       MATCH_ADDR(daddr, dmask, x->id.daddr.a4) &&
 	       MATCH(proto, x->id.proto) &&
 	       MATCH(mode, x->props.mode) &&
 	       MATCH(spi, x->id.spi) &&
diff --git a/net/ipv6/netfilter/ip6t_policy.c b/net/ipv6/netfilter/ip6t_policy.c
index 1d0f48276123..3d39ec924041 100644
--- a/net/ipv6/netfilter/ip6t_policy.c
+++ b/net/ipv6/netfilter/ip6t_policy.c
@@ -26,8 +26,8 @@ MODULE_LICENSE("GPL");
 static inline int
 match_xfrm_state(struct xfrm_state *x, const struct ip6t_policy_elem *e)
 {
-#define MATCH_ADDR(x,y,z)	(!e->match.x ||				 \
-				 ((!ip6_masked_addrcmp(&e->x, &e->y, z)) \
+#define MATCH_ADDR(x,y,z)	(!e->match.x ||				       \
+				 ((!ip6_masked_addrcmp(&e->x.a6, &e->y.a6, z)) \
 				  ^ e->invert.x))
 #define MATCH(x,y)		(!e->match.x || ((e->x == (y)) ^ e->invert.x))