diff options
| -rw-r--r-- | include/net/rps.h | 24 | ||||
| -rw-r--r-- | net/ipv4/inet_hashtables.c | 6 | ||||
| -rw-r--r-- | net/ipv4/udp.c | 2 | ||||
| -rw-r--r-- | net/sctp/socket.c | 2 |
4 files changed, 31 insertions, 3 deletions
diff --git a/include/net/rps.h b/include/net/rps.h index 507f4aa5d39b..d8ab3a08bcc4 100644 --- a/include/net/rps.h +++ b/include/net/rps.h @@ -123,6 +123,30 @@ static inline void sock_rps_record_flow(const struct sock *sk) #endif } +static inline void sock_rps_delete_flow(const struct sock *sk) +{ +#ifdef CONFIG_RPS + struct rps_sock_flow_table *table; + u32 hash, index; + + if (!static_branch_unlikely(&rfs_needed)) + return; + + hash = READ_ONCE(sk->sk_rxhash); + if (!hash) + return; + + rcu_read_lock(); + table = rcu_dereference(net_hotdata.rps_sock_flow_table); + if (table) { + index = hash & table->mask; + if (READ_ONCE(table->ents[index]) != RPS_NO_CPU) + WRITE_ONCE(table->ents[index], RPS_NO_CPU); + } + rcu_read_unlock(); +#endif +} + static inline u32 rps_input_queue_tail_incr(struct softnet_data *sd) { #ifdef CONFIG_RPS diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index da85cc30e382..77a0b52b2eab 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -23,11 +23,12 @@ #if IS_ENABLED(CONFIG_IPV6) #include <net/inet6_hashtables.h> #endif -#include <net/secure_seq.h> #include <net/hotdata.h> #include <net/ip.h> -#include <net/tcp.h> +#include <net/rps.h> +#include <net/secure_seq.h> #include <net/sock_reuseport.h> +#include <net/tcp.h> u32 inet_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport, const __be32 faddr, @@ -790,6 +791,7 @@ void inet_unhash(struct sock *sk) if (sk_unhashed(sk)) return; + sock_rps_delete_flow(sk); if (sk->sk_state == TCP_LISTEN) { struct inet_listen_hashbucket *ilb2; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 358b49caa7b9..dde52b8050b8 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -120,6 +120,7 @@ #if IS_ENABLED(CONFIG_IPV6) #include <net/ipv6_stubs.h> #endif +#include <net/rps.h> struct udp_table udp_table __read_mostly; @@ -2200,6 +2201,7 @@ void udp_lib_unhash(struct sock *sk) struct udp_table *udptable = udp_get_table_prot(sk); struct udp_hslot *hslot, *hslot2; + sock_rps_delete_flow(sk); hslot = udp_hashslot(udptable, sock_net(sk), udp_sk(sk)->udp_port_hash); hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 53725ee7ba06..85a9dfeff4d6 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -8321,7 +8321,7 @@ static int sctp_hash(struct sock *sk) static void sctp_unhash(struct sock *sk) { - /* STUB */ + sock_rps_delete_flow(sk); } /* Check if port is acceptable. Possibly find first available port. |