Merge branch 'xarray' of git://git.infradead.org/users/willy/linux-dax
Pull XArray conversion from Matthew Wilcox: "The XArray provides an improved interface to the radix tree data structure, providing locking as part of the API, specifying GFP flags at allocation time, eliminating preloading, less re-walking the tree, more efficient iterations and not exposing RCU-protected pointers to its users. This patch set 1. Introduces the XArray implementation 2. Converts the pagecache to use it 3. Converts memremap to use it The page cache is the most complex and important user of the radix tree, so converting it was most important. Converting the memremap code removes the only other user of the multiorder code, which allows us to remove the radix tree code that supported it. I have 40+ followup patches to convert many other users of the radix tree over to the XArray, but I'd like to get this part in first. The other conversions haven't been in linux-next and aren't suitable for applying yet, but you can see them in the xarray-conv branch if you're interested" * 'xarray' of git://git.infradead.org/users/willy/linux-dax: (90 commits) radix tree: Remove multiorder support radix tree test: Convert multiorder tests to XArray radix tree tests: Convert item_delete_rcu to XArray radix tree tests: Convert item_kill_tree to XArray radix tree tests: Move item_insert_order radix tree test suite: Remove multiorder benchmarking radix tree test suite: Remove __item_insert memremap: Convert to XArray xarray: Add range store functionality xarray: Move multiorder_check to in-kernel tests xarray: Move multiorder_shrink to kernel tests xarray: Move multiorder account test in-kernel radix tree test suite: Convert iteration test to XArray radix tree test suite: Convert tag_tagged_items to XArray radix tree: Remove radix_tree_clear_tags radix tree: Remove radix_tree_maybe_preload_order radix tree: Remove split/join code radix tree: Remove radix_tree_update_node_t page cache: Finish XArray conversion dax: Convert page fault handlers to XArray ...
This commit is contained in:
Linus Torvalds
@@ -403,24 +403,40 @@ int pagecache_write_end(struct file *, struct address_space *mapping,
|
||||
loff_t pos, unsigned len, unsigned copied,
|
||||
struct page *page, void *fsdata);
|
||||
|
||||
/**
|
||||
* struct address_space - Contents of a cacheable, mappable object.
|
||||
* @host: Owner, either the inode or the block_device.
|
||||
* @i_pages: Cached pages.
|
||||
* @gfp_mask: Memory allocation flags to use for allocating pages.
|
||||
* @i_mmap_writable: Number of VM_SHARED mappings.
|
||||
* @i_mmap: Tree of private and shared mappings.
|
||||
* @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable.
|
||||
* @nrpages: Number of page entries, protected by the i_pages lock.
|
||||
* @nrexceptional: Shadow or DAX entries, protected by the i_pages lock.
|
||||
* @writeback_index: Writeback starts here.
|
||||
* @a_ops: Methods.
|
||||
* @flags: Error bits and flags (AS_*).
|
||||
* @wb_err: The most recent error which has occurred.
|
||||
* @private_lock: For use by the owner of the address_space.
|
||||
* @private_list: For use by the owner of the address_space.
|
||||
* @private_data: For use by the owner of the address_space.
|
||||
*/
|
||||
struct address_space {
|
||||
struct inode *host; /* owner: inode, block_device */
|
||||
struct radix_tree_root i_pages; /* cached pages */
|
||||
atomic_t i_mmap_writable;/* count VM_SHARED mappings */
|
||||
struct rb_root_cached i_mmap; /* tree of private and shared mappings */
|
||||
struct rw_semaphore i_mmap_rwsem; /* protect tree, count, list */
|
||||
/* Protected by the i_pages lock */
|
||||
unsigned long nrpages; /* number of total pages */
|
||||
/* number of shadow or DAX exceptional entries */
|
||||
struct inode *host;
|
||||
struct xarray i_pages;
|
||||
gfp_t gfp_mask;
|
||||
atomic_t i_mmap_writable;
|
||||
struct rb_root_cached i_mmap;
|
||||
struct rw_semaphore i_mmap_rwsem;
|
||||
unsigned long nrpages;
|
||||
unsigned long nrexceptional;
|
||||
pgoff_t writeback_index;/* writeback starts here */
|
||||
const struct address_space_operations *a_ops; /* methods */
|
||||
unsigned long flags; /* error bits */
|
||||
spinlock_t private_lock; /* for use by the address_space */
|
||||
gfp_t gfp_mask; /* implicit gfp mask for allocations */
|
||||
struct list_head private_list; /* for use by the address_space */
|
||||
void *private_data; /* ditto */
|
||||
pgoff_t writeback_index;
|
||||
const struct address_space_operations *a_ops;
|
||||
unsigned long flags;
|
||||
errseq_t wb_err;
|
||||
spinlock_t private_lock;
|
||||
struct list_head private_list;
|
||||
void *private_data;
|
||||
} __attribute__((aligned(sizeof(long)))) __randomize_layout;
|
||||
/*
|
||||
* On most architectures that alignment is already the case; but
|
||||
@@ -467,15 +483,18 @@ struct block_device {
|
||||
struct mutex bd_fsfreeze_mutex;
|
||||
} __randomize_layout;
|
||||
|
||||
/*
|
||||
* Radix-tree tags, for tagging dirty and writeback pages within the pagecache
|
||||
* radix trees
|
||||
*/
|
||||
#define PAGECACHE_TAG_DIRTY 0
|
||||
#define PAGECACHE_TAG_WRITEBACK 1
|
||||
#define PAGECACHE_TAG_TOWRITE 2
|
||||
/* XArray tags, for tagging dirty and writeback pages in the pagecache. */
|
||||
#define PAGECACHE_TAG_DIRTY XA_MARK_0
|
||||
#define PAGECACHE_TAG_WRITEBACK XA_MARK_1
|
||||
#define PAGECACHE_TAG_TOWRITE XA_MARK_2
|
||||
|
||||
int mapping_tagged(struct address_space *mapping, int tag);
|
||||
/*
|
||||
* Returns true if any of the pages in the mapping are marked with the tag.
|
||||
*/
|
||||
static inline bool mapping_tagged(struct address_space *mapping, xa_mark_t tag)
|
||||
{
|
||||
return xa_marked(&mapping->i_pages, tag);
|
||||
}
|
||||
|
||||
static inline void i_mmap_lock_write(struct address_space *mapping)
|
||||
{
|
||||
|
||||
@@ -214,8 +214,7 @@ static inline void idr_preload_end(void)
|
||||
++id, (entry) = idr_get_next((idr), &(id)))
|
||||
|
||||
/*
|
||||
* IDA - IDR based id allocator, use when translation from id to
|
||||
* pointer isn't necessary.
|
||||
* IDA - ID Allocator, use when translation from id to pointer isn't necessary.
|
||||
*/
|
||||
#define IDA_CHUNK_SIZE 128 /* 128 bytes per chunk */
|
||||
#define IDA_BITMAP_LONGS (IDA_CHUNK_SIZE / sizeof(long))
|
||||
@@ -225,14 +224,14 @@ struct ida_bitmap {
|
||||
unsigned long bitmap[IDA_BITMAP_LONGS];
|
||||
};
|
||||
|
||||
DECLARE_PER_CPU(struct ida_bitmap *, ida_bitmap);
|
||||
|
||||
struct ida {
|
||||
struct radix_tree_root ida_rt;
|
||||
struct xarray xa;
|
||||
};
|
||||
|
||||
#define IDA_INIT_FLAGS (XA_FLAGS_LOCK_IRQ | XA_FLAGS_ALLOC)
|
||||
|
||||
#define IDA_INIT(name) { \
|
||||
.ida_rt = RADIX_TREE_INIT(name, IDR_RT_MARKER | GFP_NOWAIT), \
|
||||
.xa = XARRAY_INIT(name, IDA_INIT_FLAGS) \
|
||||
}
|
||||
#define DEFINE_IDA(name) struct ida name = IDA_INIT(name)
|
||||
|
||||
@@ -292,7 +291,7 @@ static inline int ida_alloc_max(struct ida *ida, unsigned int max, gfp_t gfp)
|
||||
|
||||
static inline void ida_init(struct ida *ida)
|
||||
{
|
||||
INIT_RADIX_TREE(&ida->ida_rt, IDR_RT_MARKER | GFP_NOWAIT);
|
||||
xa_init_flags(&ida->xa, IDA_INIT_FLAGS);
|
||||
}
|
||||
|
||||
#define ida_simple_get(ida, start, end, gfp) \
|
||||
@@ -301,9 +300,6 @@ static inline void ida_init(struct ida *ida)
|
||||
|
||||
static inline bool ida_is_empty(const struct ida *ida)
|
||||
{
|
||||
return radix_tree_empty(&ida->ida_rt);
|
||||
return xa_empty(&ida->xa);
|
||||
}
|
||||
|
||||
/* in lib/radix-tree.c */
|
||||
int ida_pre_get(struct ida *ida, gfp_t gfp_mask);
|
||||
#endif /* __IDR_H__ */
|
||||
|
||||
@@ -241,9 +241,9 @@ static inline gfp_t readahead_gfp_mask(struct address_space *x)
|
||||
|
||||
typedef int filler_t(void *, struct page *);
|
||||
|
||||
pgoff_t page_cache_next_hole(struct address_space *mapping,
|
||||
pgoff_t page_cache_next_miss(struct address_space *mapping,
|
||||
pgoff_t index, unsigned long max_scan);
|
||||
pgoff_t page_cache_prev_hole(struct address_space *mapping,
|
||||
pgoff_t page_cache_prev_miss(struct address_space *mapping,
|
||||
pgoff_t index, unsigned long max_scan);
|
||||
|
||||
#define FGP_ACCESSED 0x00000001
|
||||
@@ -363,17 +363,17 @@ static inline unsigned find_get_pages(struct address_space *mapping,
|
||||
unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start,
|
||||
unsigned int nr_pages, struct page **pages);
|
||||
unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
|
||||
pgoff_t end, int tag, unsigned int nr_pages,
|
||||
pgoff_t end, xa_mark_t tag, unsigned int nr_pages,
|
||||
struct page **pages);
|
||||
static inline unsigned find_get_pages_tag(struct address_space *mapping,
|
||||
pgoff_t *index, int tag, unsigned int nr_pages,
|
||||
pgoff_t *index, xa_mark_t tag, unsigned int nr_pages,
|
||||
struct page **pages)
|
||||
{
|
||||
return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag,
|
||||
nr_pages, pages);
|
||||
}
|
||||
unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
|
||||
int tag, unsigned int nr_entries,
|
||||
xa_mark_t tag, unsigned int nr_entries,
|
||||
struct page **entries, pgoff_t *indices);
|
||||
|
||||
struct page *grab_cache_page_write_begin(struct address_space *mapping,
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
#ifndef _LINUX_PAGEVEC_H
|
||||
#define _LINUX_PAGEVEC_H
|
||||
|
||||
#include <linux/xarray.h>
|
||||
|
||||
/* 15 pointers + header align the pagevec structure to a power of two */
|
||||
#define PAGEVEC_SIZE 15
|
||||
|
||||
@@ -40,12 +42,12 @@ static inline unsigned pagevec_lookup(struct pagevec *pvec,
|
||||
|
||||
unsigned pagevec_lookup_range_tag(struct pagevec *pvec,
|
||||
struct address_space *mapping, pgoff_t *index, pgoff_t end,
|
||||
int tag);
|
||||
xa_mark_t tag);
|
||||
unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec,
|
||||
struct address_space *mapping, pgoff_t *index, pgoff_t end,
|
||||
int tag, unsigned max_pages);
|
||||
xa_mark_t tag, unsigned max_pages);
|
||||
static inline unsigned pagevec_lookup_tag(struct pagevec *pvec,
|
||||
struct address_space *mapping, pgoff_t *index, int tag)
|
||||
struct address_space *mapping, pgoff_t *index, xa_mark_t tag)
|
||||
{
|
||||
return pagevec_lookup_range_tag(pvec, mapping, index, (pgoff_t)-1, tag);
|
||||
}
|
||||
|
||||
@@ -28,34 +28,30 @@
|
||||
#include <linux/rcupdate.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/xarray.h>
|
||||
|
||||
/* Keep unconverted code working */
|
||||
#define radix_tree_root xarray
|
||||
#define radix_tree_node xa_node
|
||||
|
||||
/*
|
||||
* The bottom two bits of the slot determine how the remaining bits in the
|
||||
* slot are interpreted:
|
||||
*
|
||||
* 00 - data pointer
|
||||
* 01 - internal entry
|
||||
* 10 - exceptional entry
|
||||
* 11 - this bit combination is currently unused/reserved
|
||||
* 10 - internal entry
|
||||
* x1 - value entry
|
||||
*
|
||||
* The internal entry may be a pointer to the next level in the tree, a
|
||||
* sibling entry, or an indicator that the entry in this slot has been moved
|
||||
* to another location in the tree and the lookup should be restarted. While
|
||||
* NULL fits the 'data pointer' pattern, it means that there is no entry in
|
||||
* the tree for this index (no matter what level of the tree it is found at).
|
||||
* This means that you cannot store NULL in the tree as a value for the index.
|
||||
* This means that storing a NULL entry in the tree is the same as deleting
|
||||
* the entry from the tree.
|
||||
*/
|
||||
#define RADIX_TREE_ENTRY_MASK 3UL
|
||||
#define RADIX_TREE_INTERNAL_NODE 1UL
|
||||
|
||||
/*
|
||||
* Most users of the radix tree store pointers but shmem/tmpfs stores swap
|
||||
* entries in the same tree. They are marked as exceptional entries to
|
||||
* distinguish them from pointers to struct page.
|
||||
* EXCEPTIONAL_ENTRY tests the bit, EXCEPTIONAL_SHIFT shifts content past it.
|
||||
*/
|
||||
#define RADIX_TREE_EXCEPTIONAL_ENTRY 2
|
||||
#define RADIX_TREE_EXCEPTIONAL_SHIFT 2
|
||||
#define RADIX_TREE_INTERNAL_NODE 2UL
|
||||
|
||||
static inline bool radix_tree_is_internal_node(void *ptr)
|
||||
{
|
||||
@@ -65,75 +61,32 @@ static inline bool radix_tree_is_internal_node(void *ptr)
|
||||
|
||||
/*** radix-tree API starts here ***/
|
||||
|
||||
#define RADIX_TREE_MAX_TAGS 3
|
||||
|
||||
#ifndef RADIX_TREE_MAP_SHIFT
|
||||
#define RADIX_TREE_MAP_SHIFT (CONFIG_BASE_SMALL ? 4 : 6)
|
||||
#endif
|
||||
|
||||
#define RADIX_TREE_MAP_SHIFT XA_CHUNK_SHIFT
|
||||
#define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT)
|
||||
#define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE-1)
|
||||
|
||||
#define RADIX_TREE_TAG_LONGS \
|
||||
((RADIX_TREE_MAP_SIZE + BITS_PER_LONG - 1) / BITS_PER_LONG)
|
||||
#define RADIX_TREE_MAX_TAGS XA_MAX_MARKS
|
||||
#define RADIX_TREE_TAG_LONGS XA_MARK_LONGS
|
||||
|
||||
#define RADIX_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long))
|
||||
#define RADIX_TREE_MAX_PATH (DIV_ROUND_UP(RADIX_TREE_INDEX_BITS, \
|
||||
RADIX_TREE_MAP_SHIFT))
|
||||
|
||||
/*
|
||||
* @count is the count of every non-NULL element in the ->slots array
|
||||
* whether that is an exceptional entry, a retry entry, a user pointer,
|
||||
* a sibling entry or a pointer to the next level of the tree.
|
||||
* @exceptional is the count of every element in ->slots which is
|
||||
* either radix_tree_exceptional_entry() or is a sibling entry for an
|
||||
* exceptional entry.
|
||||
*/
|
||||
struct radix_tree_node {
|
||||
unsigned char shift; /* Bits remaining in each slot */
|
||||
unsigned char offset; /* Slot offset in parent */
|
||||
unsigned char count; /* Total entry count */
|
||||
unsigned char exceptional; /* Exceptional entry count */
|
||||
struct radix_tree_node *parent; /* Used when ascending tree */
|
||||
struct radix_tree_root *root; /* The tree we belong to */
|
||||
union {
|
||||
struct list_head private_list; /* For tree user */
|
||||
struct rcu_head rcu_head; /* Used when freeing node */
|
||||
};
|
||||
void __rcu *slots[RADIX_TREE_MAP_SIZE];
|
||||
unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS];
|
||||
};
|
||||
|
||||
/* The IDR tag is stored in the low bits of the GFP flags */
|
||||
/* The IDR tag is stored in the low bits of xa_flags */
|
||||
#define ROOT_IS_IDR ((__force gfp_t)4)
|
||||
/* The top bits of gfp_mask are used to store the root tags */
|
||||
/* The top bits of xa_flags are used to store the root tags */
|
||||
#define ROOT_TAG_SHIFT (__GFP_BITS_SHIFT)
|
||||
|
||||
struct radix_tree_root {
|
||||
spinlock_t xa_lock;
|
||||
gfp_t gfp_mask;
|
||||
struct radix_tree_node __rcu *rnode;
|
||||
};
|
||||
|
||||
#define RADIX_TREE_INIT(name, mask) { \
|
||||
.xa_lock = __SPIN_LOCK_UNLOCKED(name.xa_lock), \
|
||||
.gfp_mask = (mask), \
|
||||
.rnode = NULL, \
|
||||
}
|
||||
#define RADIX_TREE_INIT(name, mask) XARRAY_INIT(name, mask)
|
||||
|
||||
#define RADIX_TREE(name, mask) \
|
||||
struct radix_tree_root name = RADIX_TREE_INIT(name, mask)
|
||||
|
||||
#define INIT_RADIX_TREE(root, mask) \
|
||||
do { \
|
||||
spin_lock_init(&(root)->xa_lock); \
|
||||
(root)->gfp_mask = (mask); \
|
||||
(root)->rnode = NULL; \
|
||||
} while (0)
|
||||
#define INIT_RADIX_TREE(root, mask) xa_init_flags(root, mask)
|
||||
|
||||
static inline bool radix_tree_empty(const struct radix_tree_root *root)
|
||||
{
|
||||
return root->rnode == NULL;
|
||||
return root->xa_head == NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -143,7 +96,6 @@ static inline bool radix_tree_empty(const struct radix_tree_root *root)
|
||||
* @next_index: one beyond the last index for this chunk
|
||||
* @tags: bit-mask for tag-iterating
|
||||
* @node: node that contains current slot
|
||||
* @shift: shift for the node that holds our slots
|
||||
*
|
||||
* This radix tree iterator works in terms of "chunks" of slots. A chunk is a
|
||||
* subinterval of slots contained within one radix tree leaf node. It is
|
||||
@@ -157,20 +109,8 @@ struct radix_tree_iter {
|
||||
unsigned long next_index;
|
||||
unsigned long tags;
|
||||
struct radix_tree_node *node;
|
||||
#ifdef CONFIG_RADIX_TREE_MULTIORDER
|
||||
unsigned int shift;
|
||||
#endif
|
||||
};
|
||||
|
||||
static inline unsigned int iter_shift(const struct radix_tree_iter *iter)
|
||||
{
|
||||
#ifdef CONFIG_RADIX_TREE_MULTIORDER
|
||||
return iter->shift;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Radix-tree synchronization
|
||||
*
|
||||
@@ -194,12 +134,11 @@ static inline unsigned int iter_shift(const struct radix_tree_iter *iter)
|
||||
* radix_tree_lookup_slot
|
||||
* radix_tree_tag_get
|
||||
* radix_tree_gang_lookup
|
||||
* radix_tree_gang_lookup_slot
|
||||
* radix_tree_gang_lookup_tag
|
||||
* radix_tree_gang_lookup_tag_slot
|
||||
* radix_tree_tagged
|
||||
*
|
||||
* The first 8 functions are able to be called locklessly, using RCU. The
|
||||
* The first 7 functions are able to be called locklessly, using RCU. The
|
||||
* caller must ensure calls to these functions are made within rcu_read_lock()
|
||||
* regions. Other readers (lock-free or otherwise) and modifications may be
|
||||
* running concurrently.
|
||||
@@ -268,17 +207,6 @@ static inline int radix_tree_deref_retry(void *arg)
|
||||
return unlikely(radix_tree_is_internal_node(arg));
|
||||
}
|
||||
|
||||
/**
|
||||
* radix_tree_exceptional_entry - radix_tree_deref_slot gave exceptional entry?
|
||||
* @arg: value returned by radix_tree_deref_slot
|
||||
* Returns: 0 if well-aligned pointer, non-0 if exceptional entry.
|
||||
*/
|
||||
static inline int radix_tree_exceptional_entry(void *arg)
|
||||
{
|
||||
/* Not unlikely because radix_tree_exception often tested first */
|
||||
return (unsigned long)arg & RADIX_TREE_EXCEPTIONAL_ENTRY;
|
||||
}
|
||||
|
||||
/**
|
||||
* radix_tree_exception - radix_tree_deref_slot returned either exception?
|
||||
* @arg: value returned by radix_tree_deref_slot
|
||||
@@ -289,47 +217,28 @@ static inline int radix_tree_exception(void *arg)
|
||||
return unlikely((unsigned long)arg & RADIX_TREE_ENTRY_MASK);
|
||||
}
|
||||
|
||||
int __radix_tree_create(struct radix_tree_root *, unsigned long index,
|
||||
unsigned order, struct radix_tree_node **nodep,
|
||||
void __rcu ***slotp);
|
||||
int __radix_tree_insert(struct radix_tree_root *, unsigned long index,
|
||||
unsigned order, void *);
|
||||
static inline int radix_tree_insert(struct radix_tree_root *root,
|
||||
unsigned long index, void *entry)
|
||||
{
|
||||
return __radix_tree_insert(root, index, 0, entry);
|
||||
}
|
||||
int radix_tree_insert(struct radix_tree_root *, unsigned long index,
|
||||
void *);
|
||||
void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index,
|
||||
struct radix_tree_node **nodep, void __rcu ***slotp);
|
||||
void *radix_tree_lookup(const struct radix_tree_root *, unsigned long);
|
||||
void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *,
|
||||
unsigned long index);
|
||||
typedef void (*radix_tree_update_node_t)(struct radix_tree_node *);
|
||||
void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *,
|
||||
void __rcu **slot, void *entry,
|
||||
radix_tree_update_node_t update_node);
|
||||
void __rcu **slot, void *entry);
|
||||
void radix_tree_iter_replace(struct radix_tree_root *,
|
||||
const struct radix_tree_iter *, void __rcu **slot, void *entry);
|
||||
void radix_tree_replace_slot(struct radix_tree_root *,
|
||||
void __rcu **slot, void *entry);
|
||||
void __radix_tree_delete_node(struct radix_tree_root *,
|
||||
struct radix_tree_node *,
|
||||
radix_tree_update_node_t update_node);
|
||||
void radix_tree_iter_delete(struct radix_tree_root *,
|
||||
struct radix_tree_iter *iter, void __rcu **slot);
|
||||
void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *);
|
||||
void *radix_tree_delete(struct radix_tree_root *, unsigned long);
|
||||
void radix_tree_clear_tags(struct radix_tree_root *, struct radix_tree_node *,
|
||||
void __rcu **slot);
|
||||
unsigned int radix_tree_gang_lookup(const struct radix_tree_root *,
|
||||
void **results, unsigned long first_index,
|
||||
unsigned int max_items);
|
||||
unsigned int radix_tree_gang_lookup_slot(const struct radix_tree_root *,
|
||||
void __rcu ***results, unsigned long *indices,
|
||||
unsigned long first_index, unsigned int max_items);
|
||||
int radix_tree_preload(gfp_t gfp_mask);
|
||||
int radix_tree_maybe_preload(gfp_t gfp_mask);
|
||||
int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order);
|
||||
void radix_tree_init(void);
|
||||
void *radix_tree_tag_set(struct radix_tree_root *,
|
||||
unsigned long index, unsigned int tag);
|
||||
@@ -337,8 +246,6 @@ void *radix_tree_tag_clear(struct radix_tree_root *,
|
||||
unsigned long index, unsigned int tag);
|
||||
int radix_tree_tag_get(const struct radix_tree_root *,
|
||||
unsigned long index, unsigned int tag);
|
||||
void radix_tree_iter_tag_set(struct radix_tree_root *,
|
||||
const struct radix_tree_iter *iter, unsigned int tag);
|
||||
void radix_tree_iter_tag_clear(struct radix_tree_root *,
|
||||
const struct radix_tree_iter *iter, unsigned int tag);
|
||||
unsigned int radix_tree_gang_lookup_tag(const struct radix_tree_root *,
|
||||
@@ -354,12 +261,6 @@ static inline void radix_tree_preload_end(void)
|
||||
preempt_enable();
|
||||
}
|
||||
|
||||
int radix_tree_split_preload(unsigned old_order, unsigned new_order, gfp_t);
|
||||
int radix_tree_split(struct radix_tree_root *, unsigned long index,
|
||||
unsigned new_order);
|
||||
int radix_tree_join(struct radix_tree_root *, unsigned long index,
|
||||
unsigned new_order, void *);
|
||||
|
||||
void __rcu **idr_get_free(struct radix_tree_root *root,
|
||||
struct radix_tree_iter *iter, gfp_t gfp,
|
||||
unsigned long max);
|
||||
@@ -465,7 +366,7 @@ void __rcu **radix_tree_iter_retry(struct radix_tree_iter *iter)
|
||||
static inline unsigned long
|
||||
__radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots)
|
||||
{
|
||||
return iter->index + (slots << iter_shift(iter));
|
||||
return iter->index + slots;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -490,21 +391,9 @@ void __rcu **__must_check radix_tree_iter_resume(void __rcu **slot,
|
||||
static __always_inline long
|
||||
radix_tree_chunk_size(struct radix_tree_iter *iter)
|
||||
{
|
||||
return (iter->next_index - iter->index) >> iter_shift(iter);
|
||||
return iter->next_index - iter->index;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_RADIX_TREE_MULTIORDER
|
||||
void __rcu **__radix_tree_next_slot(void __rcu **slot,
|
||||
struct radix_tree_iter *iter, unsigned flags);
|
||||
#else
|
||||
/* Can't happen without sibling entries, but the compiler can't tell that */
|
||||
static inline void __rcu **__radix_tree_next_slot(void __rcu **slot,
|
||||
struct radix_tree_iter *iter, unsigned flags)
|
||||
{
|
||||
return slot;
|
||||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* radix_tree_next_slot - find next slot in chunk
|
||||
*
|
||||
@@ -563,8 +452,6 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,
|
||||
return NULL;
|
||||
|
||||
found:
|
||||
if (unlikely(radix_tree_is_internal_node(rcu_dereference_raw(*slot))))
|
||||
return __radix_tree_next_slot(slot, iter, flags);
|
||||
return slot;
|
||||
}
|
||||
|
||||
@@ -583,23 +470,6 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,
|
||||
slot || (slot = radix_tree_next_chunk(root, iter, 0)) ; \
|
||||
slot = radix_tree_next_slot(slot, iter, 0))
|
||||
|
||||
/**
|
||||
* radix_tree_for_each_contig - iterate over contiguous slots
|
||||
*
|
||||
* @slot: the void** variable for pointer to slot
|
||||
* @root: the struct radix_tree_root pointer
|
||||
* @iter: the struct radix_tree_iter pointer
|
||||
* @start: iteration starting index
|
||||
*
|
||||
* @slot points to radix tree slot, @iter->index contains its index.
|
||||
*/
|
||||
#define radix_tree_for_each_contig(slot, root, iter, start) \
|
||||
for (slot = radix_tree_iter_init(iter, start) ; \
|
||||
slot || (slot = radix_tree_next_chunk(root, iter, \
|
||||
RADIX_TREE_ITER_CONTIG)) ; \
|
||||
slot = radix_tree_next_slot(slot, iter, \
|
||||
RADIX_TREE_ITER_CONTIG))
|
||||
|
||||
/**
|
||||
* radix_tree_for_each_tagged - iterate over tagged slots
|
||||
*
|
||||
|
||||
@@ -300,17 +300,12 @@ void *workingset_eviction(struct address_space *mapping, struct page *page);
|
||||
void workingset_refault(struct page *page, void *shadow);
|
||||
void workingset_activation(struct page *page);
|
||||
|
||||
/* Do not use directly, use workingset_lookup_update */
|
||||
void workingset_update_node(struct radix_tree_node *node);
|
||||
|
||||
/* Returns workingset_update_node() if the mapping has shadow entries. */
|
||||
#define workingset_lookup_update(mapping) \
|
||||
({ \
|
||||
radix_tree_update_node_t __helper = workingset_update_node; \
|
||||
if (dax_mapping(mapping) || shmem_mapping(mapping)) \
|
||||
__helper = NULL; \
|
||||
__helper; \
|
||||
})
|
||||
/* Only track the nodes of mappings with shadow entries */
|
||||
void workingset_update_node(struct xa_node *node);
|
||||
#define mapping_set_update(xas, mapping) do { \
|
||||
if (!dax_mapping(mapping) && !shmem_mapping(mapping)) \
|
||||
xas_set_update(xas, workingset_update_node); \
|
||||
} while (0)
|
||||
|
||||
/* linux/mm/page_alloc.c */
|
||||
extern unsigned long totalram_pages;
|
||||
@@ -409,7 +404,7 @@ extern void show_swap_cache_info(void);
|
||||
extern int add_to_swap(struct page *page);
|
||||
extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t);
|
||||
extern int __add_to_swap_cache(struct page *page, swp_entry_t entry);
|
||||
extern void __delete_from_swap_cache(struct page *);
|
||||
extern void __delete_from_swap_cache(struct page *, swp_entry_t entry);
|
||||
extern void delete_from_swap_cache(struct page *);
|
||||
extern void free_page_and_swap_cache(struct page *);
|
||||
extern void free_pages_and_swap_cache(struct page **, int);
|
||||
@@ -563,7 +558,8 @@ static inline int add_to_swap_cache(struct page *page, swp_entry_t entry,
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline void __delete_from_swap_cache(struct page *page)
|
||||
static inline void __delete_from_swap_cache(struct page *page,
|
||||
swp_entry_t entry)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -18,9 +18,8 @@
|
||||
*
|
||||
* swp_entry_t's are *never* stored anywhere in their arch-dependent format.
|
||||
*/
|
||||
#define SWP_TYPE_SHIFT(e) ((sizeof(e.val) * 8) - \
|
||||
(MAX_SWAPFILES_SHIFT + RADIX_TREE_EXCEPTIONAL_SHIFT))
|
||||
#define SWP_OFFSET_MASK(e) ((1UL << SWP_TYPE_SHIFT(e)) - 1)
|
||||
#define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT)
|
||||
#define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1)
|
||||
|
||||
/*
|
||||
* Store a type+offset into a swp_entry_t in an arch-independent format
|
||||
@@ -29,8 +28,7 @@ static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset)
|
||||
{
|
||||
swp_entry_t ret;
|
||||
|
||||
ret.val = (type << SWP_TYPE_SHIFT(ret)) |
|
||||
(offset & SWP_OFFSET_MASK(ret));
|
||||
ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -40,7 +38,7 @@ static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset)
|
||||
*/
|
||||
static inline unsigned swp_type(swp_entry_t entry)
|
||||
{
|
||||
return (entry.val >> SWP_TYPE_SHIFT(entry));
|
||||
return (entry.val >> SWP_TYPE_SHIFT);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -49,7 +47,7 @@ static inline unsigned swp_type(swp_entry_t entry)
|
||||
*/
|
||||
static inline pgoff_t swp_offset(swp_entry_t entry)
|
||||
{
|
||||
return entry.val & SWP_OFFSET_MASK(entry);
|
||||
return entry.val & SWP_OFFSET_MASK;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MMU
|
||||
@@ -90,16 +88,13 @@ static inline swp_entry_t radix_to_swp_entry(void *arg)
|
||||
{
|
||||
swp_entry_t entry;
|
||||
|
||||
entry.val = (unsigned long)arg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
|
||||
entry.val = xa_to_value(arg);
|
||||
return entry;
|
||||
}
|
||||
|
||||
static inline void *swp_to_radix_entry(swp_entry_t entry)
|
||||
{
|
||||
unsigned long value;
|
||||
|
||||
value = entry.val << RADIX_TREE_EXCEPTIONAL_SHIFT;
|
||||
return (void *)(value | RADIX_TREE_EXCEPTIONAL_ENTRY);
|
||||
return xa_mk_value(entry.val);
|
||||
}
|
||||
|
||||
#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user