Page MenuHomeFreeBSD
Files F108308595

D42509.id130024.diff
No OneTemporary
Actions

D42509.id130024.diff
View Options

Index: sys/vm/vm_phys.c
===================================================================
--- sys/vm/vm_phys.c
+++ sys/vm/vm_phys.c
@@ -1360,108 +1360,75 @@
}
/*
- * Find a run of contiguous physical pages from the specified page list.
+ * Find a run of contiguous physical pages, meeting alignment requirements, from
+ * a list of max-sized page blocks, where we need at least two consecutive
+ * blocks to satisfy the (large) page request.
*/
static vm_page_t
-vm_phys_find_freelist_contig(struct vm_freelist *fl, int oind, u_long npages,
+vm_phys_find_freelist_contig(struct vm_freelist *fl, u_long npages,
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
{
struct vm_phys_seg *seg;
- vm_paddr_t frag, lbound, pa, page_size, pa_end, pa_pre, size;
- vm_page_t m, m_listed, m_ret;
- int order;
+ vm_page_t m, m_iter, m_ret;
+ vm_paddr_t max_size, size;
+ int max_order;
- KASSERT(npages > 0, ("npages is 0"));
- KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
- KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
- /* Search for a run satisfying the specified conditions. */
- page_size = PAGE_SIZE;
+ max_order = VM_NFREEORDER - 1;
size = npages << PAGE_SHIFT;
- frag = (npages & ~(~0UL << oind)) << PAGE_SHIFT;
- TAILQ_FOREACH(m_listed, &fl[oind].pl, listq) {
- /*
- * Determine if the address range starting at pa is
- * too low.
- */
- pa = VM_PAGE_TO_PHYS(m_listed);
- if (pa < low)
- continue;
+ max_size = (vm_paddr_t)1 << (PAGE_SHIFT + max_order);
+ KASSERT(size > max_size, ("size is too small"));
+ /*
+ * In order to avoid examining any free max-sized page block more than
+ * twice, identify the ones that are first in a physically-contiguous
+ * sequence of such blocks, and only for those walk the sequence to
+ * check if there are enough free blocks starting at a properly aligned
+ * block. Thus, no block is checked for free-ness more than twice.
+ */
+ TAILQ_FOREACH(m, &fl[max_order].pl, listq) {
/*
- * If this is not the first free oind-block in this range, bail
- * out. We have seen the first free block already, or will see
- * it before failing to find an appropriate range.
+ * Skip m unless it is first in a sequence of free max page
+ * blocks >= low in its segment.
*/
- seg = &vm_phys_segs[m_listed->segind];
- lbound = low > seg->start ? low : seg->start;
- pa_pre = pa - (page_size << oind);
- m = &seg->first_page[atop(pa_pre - seg->start)];
- if (pa != 0 && pa_pre >= lbound && m->order == oind)
+ seg = &vm_phys_segs[m->segind];
+ if (VM_PAGE_TO_PHYS(m) < MAX(low, seg->start))
+ continue;
+ if (VM_PAGE_TO_PHYS(m) >= max_size &&
+ VM_PAGE_TO_PHYS(m) - max_size >= MAX(low, seg->start) &&
+ max_order == m[-1 << max_order].order)
continue;
-
- if (!vm_addr_align_ok(pa, alignment))
- /* Advance to satisfy alignment condition. */
- pa = roundup2(pa, alignment);
- else if (frag != 0 && lbound + frag <= pa) {
- /*
- * Back up to the first aligned free block in this
- * range, without moving below lbound.
- */
- pa_end = pa;
- for (order = oind - 1; order >= 0; order--) {
- pa_pre = pa_end - (page_size << order);
- if (!vm_addr_align_ok(pa_pre, alignment))
- break;
- m = &seg->first_page[atop(pa_pre - seg->start)];
- if (pa_pre >= lbound && m->order == order)
- pa_end = pa_pre;
- }
- /*
- * If the extra small blocks are enough to complete the
- * fragment, use them. Otherwise, look to allocate the
- * fragment at the other end.
- */
- if (pa_end + frag <= pa)
- pa = pa_end;
- }
-
- /* Advance as necessary to satisfy boundary conditions. */
- if (!vm_addr_bound_ok(pa, size, boundary))
- pa = roundup2(pa + 1, boundary);
- pa_end = pa + size;
/*
- * Determine if the address range is valid (without overflow in
- * pa_end calculation), and fits within the segment.
+ * Advance m_ret from m to the first of the sequence, if any,
+ * that satisfies alignment conditions and might leave enough
+ * space.
*/
- if (pa_end < pa || seg->end < pa_end)
- continue;
-
- m_ret = &seg->first_page[atop(pa - seg->start)];
+ m_ret = m;
+ while (!vm_addr_ok(VM_PAGE_TO_PHYS(m_ret),
+ size, alignment, boundary) &&
+ VM_PAGE_TO_PHYS(m_ret) + size <= MIN(high, seg->end) &&
+ max_order == m_ret[1 << max_order].order)
+ m_ret += 1 << max_order;
/*
- * Determine whether there are enough free oind-blocks here to
- * satisfy the allocation request.
+ * Skip m unless some block m_ret in the sequence is properly
+ * aligned, and begins a sequence of enough pages less than
+ * high, and in the same segment.
*/
- pa = VM_PAGE_TO_PHYS(m_listed);
- do {
- pa += page_size << oind;
- if (pa >= pa_end)
- return (m_ret);
- m = &seg->first_page[atop(pa - seg->start)];
- } while (oind == m->order);
+ if (VM_PAGE_TO_PHYS(m_ret) + size > MIN(high, seg->end))
+ continue;
/*
- * Determine if an additional series of free blocks of
- * diminishing size can help to satisfy the allocation request.
+ * Skip m unless the blocks to allocate starting at m_ret are
+ * all free.
*/
- while (m->order < oind &&
- pa + 2 * (page_size << m->order) > pa_end) {
- pa += page_size << m->order;
- if (pa >= pa_end)
- return (m_ret);
- m = &seg->first_page[atop(pa - seg->start)];
+ for (m_iter = m_ret;
+ m_iter < m_ret + npages && max_order == m_iter->order;
+ m_iter += 1 << max_order) {
}
+ if (m_iter < m_ret + npages)
+ continue;
+ return (m_ret);
}
return (NULL);
}
@@ -1508,11 +1475,10 @@
}
if (order < VM_NFREEORDER)
return (NULL);
- /* Search for a long-enough sequence of small blocks. */
- oind = VM_NFREEORDER - 1;
+ /* Search for a long-enough sequence of max-order blocks. */
for (pind = 0; pind < VM_NFREEPOOL; pind++) {
fl = (*queues)[pind];
- m_ret = vm_phys_find_freelist_contig(fl, oind, npages,
+ m_ret = vm_phys_find_freelist_contig(fl, npages,
low, high, alignment, boundary);
if (m_ret != NULL)
return (m_ret);
@@ -1593,6 +1559,18 @@
/* Return excess pages to the free lists. */
fl = (*queues)[VM_FREEPOOL_DEFAULT];
vm_phys_enq_range(&m_run[npages], m - &m_run[npages], fl, 0);
+
+ /* Return page verified to satisfy conditions of request. */
+ pa_start = VM_PAGE_TO_PHYS(m_run);
+ KASSERT(low <= pa_start,
+ ("memory allocated below minimum requested range"));
+ KASSERT(pa_start + ptoa(npages) <= high,
+ ("memory allocated above maximum requested range"));
+ seg = &vm_phys_segs[m_run->segind];
+ KASSERT(seg->domain == domain,
+ ("memory not allocated from specified domain"));
+ KASSERT(vm_addr_ok(pa_start, ptoa(npages), alignment, boundary),
+ ("memory alignment/boundary constraints not satisfied"));
return (m_run);
}

File Metadata

Mime Type
text/plain
Expires
Fri, Jan 24, 6:24 PM (12 h, 29 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16095449
Default Alt Text
D42509.id130024.diff (6 KB)

Event Timeline