return __is_page_sharing_candidate(stable_node, 0);
}
-static struct stable_node *stable_node_dup(struct stable_node **_stable_node,
- struct page **tree_page,
- struct rb_root *root,
- bool prune_stale_stable_nodes)
+struct page *stable_node_dup(struct stable_node **_stable_node_dup,
+ struct stable_node **_stable_node,
+ struct rb_root *root,
+ bool prune_stale_stable_nodes)
{
struct stable_node *dup, *found = NULL, *stable_node = *_stable_node;
struct hlist_node *hlist_safe;
- struct page *_tree_page;
+ struct page *_tree_page, *tree_page = NULL;
int nr = 0;
int found_rmap_hlist_len;
if (!found ||
dup->rmap_hlist_len > found_rmap_hlist_len) {
if (found)
- put_page(*tree_page);
+ put_page(tree_page);
found = dup;
found_rmap_hlist_len = found->rmap_hlist_len;
- *tree_page = _tree_page;
+ tree_page = _tree_page;
+ /* skip put_page for found dup */
if (!prune_stale_stable_nodes)
break;
- /* skip put_page */
continue;
}
}
}
}
- return found;
+ *_stable_node_dup = found;
+ return tree_page;
}
static struct stable_node *stable_node_dup_any(struct stable_node *stable_node,
typeof(*stable_node), hlist_dup);
}
-static struct stable_node *__stable_node_chain(struct stable_node **_stable_node,
- struct page **tree_page,
- struct rb_root *root,
- bool prune_stale_stable_nodes)
+/*
+ * Like for get_ksm_page, this function can free the *_stable_node and
+ * *_stable_node_dup if the returned tree_page is NULL.
+ *
+ * It can also free and overwrite *_stable_node with the found
+ * stable_node_dup if the chain is collapsed (in which case
+ * *_stable_node will be equal to *_stable_node_dup like if the chain
+ * never existed). It's up to the caller to verify tree_page is not
+ * NULL before dereferencing *_stable_node or *_stable_node_dup.
+ *
+ * *_stable_node_dup is really a second output parameter of this
+ * function and will be overwritten in all cases, the caller doesn't
+ * need to initialize it.
+ */
+static struct page *__stable_node_chain(struct stable_node **_stable_node_dup,
+ struct stable_node **_stable_node,
+ struct rb_root *root,
+ bool prune_stale_stable_nodes)
{
struct stable_node *stable_node = *_stable_node;
if (!is_stable_node_chain(stable_node)) {
if (is_page_sharing_candidate(stable_node)) {
- *tree_page = get_ksm_page(stable_node, false);
- return stable_node;
+ *_stable_node_dup = stable_node;
+ return get_ksm_page(stable_node, false);
}
+ /*
+ * _stable_node_dup set to NULL means the stable_node
+ * reached the ksm_max_page_sharing limit.
+ */
+ *_stable_node_dup = NULL;
return NULL;
}
- return stable_node_dup(_stable_node, tree_page, root,
+ return stable_node_dup(_stable_node_dup, _stable_node, root,
prune_stale_stable_nodes);
}
-static __always_inline struct stable_node *chain_prune(struct stable_node **s_n,
- struct page **t_p,
- struct rb_root *root)
+static __always_inline struct page *chain_prune(struct stable_node **s_n_d,
+ struct stable_node **s_n,
+ struct rb_root *root)
{
- return __stable_node_chain(s_n, t_p, root, true);
+ return __stable_node_chain(s_n_d, s_n, root, true);
}
-static __always_inline struct stable_node *chain(struct stable_node *s_n,
- struct page **t_p,
- struct rb_root *root)
+static __always_inline struct page *chain(struct stable_node **s_n_d,
+ struct stable_node *s_n,
+ struct rb_root *root)
{
- return __stable_node_chain(&s_n, t_p, root, false);
+ struct stable_node *old_stable_node = s_n;
+ struct page *tree_page;
+
+ tree_page = __stable_node_chain(s_n_d, &s_n, root, false);
+ /* not pruning dups so s_n cannot have changed */
+ VM_BUG_ON(s_n != old_stable_node);
+ return tree_page;
}
/*
cond_resched();
stable_node = rb_entry(*new, struct stable_node, node);
stable_node_any = NULL;
- stable_node_dup = chain_prune(&stable_node, &tree_page, root);
+ tree_page = chain_prune(&stable_node_dup, &stable_node, root);
/*
* NOTE: stable_node may have been freed by
* chain_prune() if the returned stable_node_dup is
cond_resched();
stable_node = rb_entry(*new, struct stable_node, node);
stable_node_any = NULL;
- stable_node_dup = chain(stable_node, &tree_page, root);
+ tree_page = chain(&stable_node_dup, stable_node, root);
if (!stable_node_dup) {
/*
* Either all stable_node dups were full in