Page MenuHomeFreeBSD
Files F115982062

D47680.id146884.diff
No OneTemporary
Actions

D47680.id146884.diff
View Options

Index: sys/kern/subr_pctrie.c
===================================================================
--- sys/kern/subr_pctrie.c
+++ sys/kern/subr_pctrie.c
@@ -539,14 +539,27 @@
enum pctrie_access access)
{
struct pctrie_node *node;
- int slot;
+ int slot, top;
+
+ top = it->top;
+ if (top != 0) {
+ node = it->path[top - 1];
+ KASSERT(!powerof2(node->pn_popmap),
+ ("%s: freed node in iter path", __func__));
+ if (!pctrie_keybarr(node, index, &slot)) {
+ node = pctrie_node_load(
+ &node->pn_child[slot], smr, access);
+ if (pctrie_isleaf(node))
+ return (node);
+ }
+ }
/*
* Climb the search path to find the lowest node from which to start the
* search for a value matching 'index'.
*/
- while (it->top != 0) {
- node = it->path[it->top - 1];
+ while (top != 0) {
+ node = it->path[top - 1];
KASSERT(!powerof2(node->pn_popmap),
("%s: freed node in iter path", __func__));
if (!pctrie_keybarr(node, index, &slot)) {
@@ -554,18 +567,19 @@
&node->pn_child[slot], smr, access);
break;
}
- --it->top;
+ --top;
}
- if (it->top == 0)
+ if (top == 0)
node = pctrie_root_load(it->ptree, smr, access);
/* Seek a node that matches index. */
while (!pctrie_isleaf(node) && !pctrie_keybarr(node, index, &slot)) {
- KASSERT(it->top < nitems(it->path),
+ KASSERT(top < nitems(it->path),
("%s: path overflow in trie %p", __func__, it->ptree));
- it->path[it->top++] = node;
+ it->path[top++] = node;
node = pctrie_node_load(&node->pn_child[slot], smr, access);
}
+ it->top = top;
return (node);
}
@@ -781,6 +795,43 @@
return (pctrie_lookup_ge_node(node, index + 1));
}
+static struct pctrie_node *
+_pctrie_iter_lookup_gt_node(struct pctrie_iter *it, uint64_t index)
+{
+ struct pctrie_node *node;
+ int slot, top;
+
+ top = it->top;
+ if (top != 0) {
+ node = it->path[top - 1];
+ slot = pctrie_slot(node, index);
+ if ((node->pn_popmap & (-2 << slot)) != 0) {
+ /* Return least child with a descendant > index. */
+ slot = ffs(node->pn_popmap & (-2 << slot)) - 1;
+ node = pctrie_node_load(&node->pn_child[slot], NULL,
+ PCTRIE_LOCKED);
+ return (node);
+ }
+ --top;
+ }
+ while (top != 0) {
+ node = it->path[top - 1];
+ slot = pctrie_slot(node, index);
+ if ((node->pn_popmap & (-2 << slot)) != 0) {
+ /* Return least child with a descendant > index. */
+ slot = ffs(node->pn_popmap & (-2 << slot)) - 1;
+ node = pctrie_node_load(&node->pn_child[slot], NULL,
+ PCTRIE_LOCKED);
+ it->top = top;
+ return (node);
+ }
+ --top;
+ }
+ it->top = 0;
+ return (NULL);
+
+}
+
/*
* Find first leaf >= index, and fill iter with the path to the parent of that
* leaf. Return NULL if there is no such leaf less than limit.
@@ -801,21 +852,11 @@
*/
if (node == PCTRIE_NULL || *pctrie_toval(node) < index) {
/* Climb the path to find a node with a descendant > index. */
- while (it->top != 0) {
- node = it->path[it->top - 1];
- slot = pctrie_slot(node, index) + 1;
- if ((node->pn_popmap >> slot) != 0)
- break;
- --it->top;
- }
- if (it->top == 0)
+ node = _pctrie_iter_lookup_gt_node(it, index);
+ if (node == NULL)
return (NULL);
-
- /* Step to the least child with a descendant > index. */
- slot += ffs(node->pn_popmap >> slot) - 1;
- node = pctrie_node_load(&node->pn_child[slot], NULL,
- PCTRIE_LOCKED);
}
+
/* Descend to the least leaf of the subtrie. */
while (!pctrie_isleaf(node)) {
if (it->limit != 0 && node->pn_owner >= it->limit)
@@ -938,6 +979,44 @@
return (pctrie_lookup_le_node(node, index - 1));
}
+static struct pctrie_node *
+_pctrie_iter_lookup_lt_node(struct pctrie_iter *it, uint64_t index)
+{
+ struct pctrie_node *node;
+ int slot, top;
+
+ top = it->top;
+ if (top != 0) {
+ node = it->path[top - 1];
+ slot = pctrie_slot(node, index);
+ if ((node->pn_popmap & ((1 << slot) - 1)) != 0) {
+ /* Return greatest child with a descendant < index. */
+ slot = ilog2(node->pn_popmap & ((1 << slot) - 1));
+ node = pctrie_node_load(&node->pn_child[slot], NULL,
+ PCTRIE_LOCKED);
+ return (node);
+ }
+ --top;
+ }
+
+ while (top != 0) {
+ node = it->path[top - 1];
+ slot = pctrie_slot(node, index);
+ if ((node->pn_popmap & ((1 << slot) - 1)) != 0) {
+ /* Return greatest child with a descendant < index. */
+ slot = ilog2(node->pn_popmap & ((1 << slot) - 1));
+ node = pctrie_node_load(&node->pn_child[slot], NULL,
+ PCTRIE_LOCKED);
+ it->top = top;
+ return (node);
+ }
+ --top;
+ }
+ it->top = 0;
+ return (NULL);
+
+}
+
/*
* Find first leaf <= index, and fill iter with the path to the parent of that
* leaf. Return NULL if there is no such leaf greater than limit.
@@ -958,21 +1037,11 @@
*/
if (node == PCTRIE_NULL || *pctrie_toval(node) > index) {
/* Climb the path to find a node with a descendant < index. */
- while (it->top != 0) {
- node = it->path[it->top - 1];
- slot = pctrie_slot(node, index);
- if ((node->pn_popmap & ((1 << slot) - 1)) != 0)
- break;
- --it->top;
- }
- if (it->top == 0)
+ node = _pctrie_iter_lookup_lt_node(it, index);
+ if (node == NULL)
return (NULL);
-
- /* Step to the greatest child with a descendant < index. */
- slot = ilog2(node->pn_popmap & ((1 << slot) - 1));
- node = pctrie_node_load(&node->pn_child[slot], NULL,
- PCTRIE_LOCKED);
}
+
/* Descend to the greatest leaf of the subtrie. */
while (!pctrie_isleaf(node)) {
if (it->limit != 0 && it->limit >=
Index: sys/vm/vm_kern.c
===================================================================
--- sys/vm/vm_kern.c
+++ sys/vm/vm_kern.c
@@ -648,8 +648,8 @@
pmap_remove(kernel_pmap, addr, addr + size);
offset = addr - VM_MIN_KERNEL_ADDRESS;
end = offset + size;
- VM_OBJECT_WLOCK(object);
vm_page_iter_init(&pages, object);
+ VM_OBJECT_WLOCK(object);
m = vm_page_iter_lookup(&pages, atop(offset));
domain = vm_page_domain(m);
if (__predict_true((m->oflags & VPO_KMEM_EXEC) == 0))
@@ -660,7 +660,7 @@
m = vm_page_iter_lookup(&pages, atop(offset))) {
vm_page_xbusy_claim(m);
vm_page_unwire_noq(m);
- vm_page_iter_free(&pages);
+ vm_page_iter_free(m, &pages);
}
VM_OBJECT_WUNLOCK(object);
Index: sys/vm/vm_object.c
===================================================================
--- sys/vm/vm_object.c
+++ sys/vm/vm_object.c
@@ -1702,7 +1702,6 @@
*/
vm_page_iter_init(&pages, backing_object);
for (p = vm_page_iter_lookup_ge(&pages, 0); p != NULL; p = next) {
- next = TAILQ_NEXT(p, listq);
new_pindex = p->pindex - backing_offset_index;
/*
@@ -1997,7 +1996,6 @@
vm_object_pip_add(object, 1);
vm_page_iter_limit_init(&pages, object, end);
again:
- pctrie_iter_reset(&pages);
for (p = vm_page_iter_lookup_ge(&pages, start); p != NULL;
p = vm_radix_iter_step(&pages)) {
/*
@@ -2026,6 +2024,7 @@
if (vm_page_tryxbusy(p) == 0) {
if (vm_page_busy_sleep(p, "vmopar", 0))
VM_OBJECT_WLOCK(object);
+ pctrie_iter_reset(&pages);
goto again;
}
if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p)) {
@@ -2060,7 +2059,7 @@
if ((options & OBJPR_NOTMAPPED) == 0 &&
object->ref_count != 0 && !vm_page_try_remove_all(p))
goto wired;
- vm_page_iter_free(&pages);
+ vm_page_iter_free(p, &pages);
}
vm_object_pip_wakeup(object);
Index: sys/vm/vm_page.h
===================================================================
--- sys/vm/vm_page.h
+++ sys/vm/vm_page.h
@@ -602,7 +602,7 @@
void vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m,
vm_pindex_t pindex, const char *wmesg, int allocflags);
void vm_page_free(vm_page_t m);
-void vm_page_iter_free(struct pctrie_iter *);
+void vm_page_iter_free(vm_page_t m, struct pctrie_iter *);
void vm_page_free_zero(vm_page_t m);
void vm_page_activate (vm_page_t);
Index: sys/vm/vm_page.c
===================================================================
--- sys/vm/vm_page.c
+++ sys/vm/vm_page.c
@@ -1716,11 +1716,8 @@
* Free the current page, as identified by iterator.
*/
void
-vm_page_iter_free(struct pctrie_iter *pages)
+vm_page_iter_free(vm_page_t m, struct pctrie_iter *pages)
{
- vm_page_t m;
-
- m = vm_radix_iter_page(pages);
vm_radix_iter_remove(pages);
vm_page_free_object_prep(m);
vm_page_xunbusy(m);

File Metadata

Mime Type
text/plain
Expires
Fri, May 2, 4:46 AM (11 h, 48 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
17898517
Default Alt Text
D47680.id146884.diff (8 KB)

Event Timeline