xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rhashtable: Add rht_ptr_rcu and improve rht_ptr

Browse Source 
This patch moves common code between rht_ptr and rht_ptr_exclusive
into __rht_ptr.  It also adds a new helper rht_ptr_rcu exclusively
for the RCU case.  This way rht_ptr becomes a lock-only construct
so we can use the lighter rcu_dereference_protected primitive.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Herbert Xu
2019-05-28 15:02:31 +08:00
committed by David S. Miller
parent af64935213
commit 279758f800
1 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
include/linux/rhashtable.h
View File

@@ -352,37 +352,38 @@ static inline void rht_unlock(struct bucket_table *tbl,
static inline struct rhash_head __rcu *__rht_ptr( static inline struct rhash_head __rcu *__rht_ptr(
struct rhash_lock_head *const *bkt) struct rhash_lock_head *const *bkt)
{ {
return (struct rhash_head __rcu *)((unsigned long)*bkt & ~BIT(0)); return (struct rhash_head __rcu *)
((unsigned long)*bkt & ~BIT(0) ?:
(unsigned long)RHT_NULLS_MARKER(bkt));
} }
/* /*
* Where 'bkt' is a bucket and might be locked: * Where 'bkt' is a bucket and might be locked:
* rht_ptr() dereferences that pointer and clears the lock bit. * rht_ptr_rcu() dereferences that pointer and clears the lock bit.
* rht_ptr() dereferences in a context where the bucket is locked.
* rht_ptr_exclusive() dereferences in a context where exclusive * rht_ptr_exclusive() dereferences in a context where exclusive
* access is guaranteed, such as when destroying the table. * access is guaranteed, such as when destroying the table.
*/ */
static inline struct rhash_head *rht_ptr_rcu(
struct rhash_lock_head *const *bkt)
{
struct rhash_head __rcu *p = __rht_ptr(bkt);
return rcu_dereference(p);
}
static inline struct rhash_head *rht_ptr( static inline struct rhash_head *rht_ptr(
struct rhash_lock_head *const *bkt, struct rhash_lock_head *const *bkt,
struct bucket_table *tbl, struct bucket_table *tbl,
unsigned int hash) unsigned int hash)
{ {
struct rhash_head __rcu *p = __rht_ptr(bkt); return rht_dereference_bucket(__rht_ptr(bkt), tbl, hash);
if (!p)
return RHT_NULLS_MARKER(bkt);
return rht_dereference_bucket_rcu(p, tbl, hash);
} }
static inline struct rhash_head *rht_ptr_exclusive( static inline struct rhash_head *rht_ptr_exclusive(
struct rhash_lock_head *const *bkt) struct rhash_lock_head *const *bkt)
{ {
struct rhash_head __rcu *p = __rht_ptr(bkt); return rcu_dereference_protected(__rht_ptr(bkt), 1);
if (!p)
return RHT_NULLS_MARKER(bkt);
return rcu_dereference_protected(p, 1);
} }
static inline void rht_assign_locked(struct rhash_lock_head **bkt, static inline void rht_assign_locked(struct rhash_lock_head **bkt,
@@ -509,7 +510,7 @@ static inline void rht_assign_unlock(struct bucket_table *tbl,
*/ */
#define rht_for_each_rcu(pos, tbl, hash) \ #define rht_for_each_rcu(pos, tbl, hash) \
for (({barrier(); }), \ for (({barrier(); }), \
pos = rht_ptr(rht_bucket(tbl, hash), tbl, hash); \ pos = rht_ptr_rcu(rht_bucket(tbl, hash)); \
!rht_is_a_nulls(pos); \ !rht_is_a_nulls(pos); \
pos = rcu_dereference_raw(pos->next)) pos = rcu_dereference_raw(pos->next))
@@ -546,8 +547,7 @@ static inline void rht_assign_unlock(struct bucket_table *tbl,
*/ */
#define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member) \ #define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member) \
rht_for_each_entry_rcu_from(tpos, pos, \ rht_for_each_entry_rcu_from(tpos, pos, \
rht_ptr(rht_bucket(tbl, hash), \ rht_ptr_rcu(rht_bucket(tbl, hash)), \
tbl, hash), \
tbl, hash, member) tbl, hash, member)
/** /**
@@ -603,7 +603,7 @@ restart:
hash = rht_key_hashfn(ht, tbl, key, params); hash = rht_key_hashfn(ht, tbl, key, params);
bkt = rht_bucket(tbl, hash); bkt = rht_bucket(tbl, hash);
do { do {
rht_for_each_rcu_from(he, rht_ptr(bkt, tbl, hash), tbl, hash) { rht_for_each_rcu_from(he, rht_ptr_rcu(bkt), tbl, hash) {
if (params.obj_cmpfn ? if (params.obj_cmpfn ?
params.obj_cmpfn(&arg, rht_obj(ht, he)) : params.obj_cmpfn(&arg, rht_obj(ht, he)) :
rhashtable_compare(&arg, rht_obj(ht, he))) rhashtable_compare(&arg, rht_obj(ht, he)))