Page MenuHomeFreeBSD
Files F102743113

D29046.diff
No OneTemporary
Actions

D29046.diff
View Options

Index: sys/kern/kern_cpuset.c
===================================================================
--- sys/kern/kern_cpuset.c
+++ sys/kern/kern_cpuset.c
@@ -119,6 +119,7 @@
*/
LIST_HEAD(domainlist, domainset);
+struct domainset __read_mostly domainset_firsttouch;
struct domainset __read_mostly domainset_fixed[MAXMEMDOM];
struct domainset __read_mostly domainset_prefer[MAXMEMDOM];
struct domainset __read_mostly domainset_roundrobin;
@@ -130,7 +131,7 @@
static struct domainlist cpuset_domains;
static struct unrhdr *cpuset_unr;
static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel;
-static struct domainset domainset0, domainset2;
+static struct domainset *domainset0, domainset2;
/* Return the size of cpuset_t at the kernel level */
SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD,
@@ -568,7 +569,7 @@
if (domain->ds_policy == DOMAINSET_POLICY_PREFER &&
!DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask))
return (NULL);
- if (!DOMAINSET_SUBSET(&domainset0.ds_mask, &domain->ds_mask))
+ if (!DOMAINSET_SUBSET(&domainset0->ds_mask, &domain->ds_mask))
return (NULL);
ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO);
domainset_copy(domain, ndomain);
@@ -1532,6 +1533,12 @@
struct domainset *dset;
int i;
+ dset = &domainset_firsttouch;
+ DOMAINSET_COPY(&all_domains, &dset->ds_mask);
+ dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH;
+ dset->ds_prefer = -1;
+ _domainset_create(dset, NULL);
+
dset = &domainset_roundrobin;
DOMAINSET_COPY(&all_domains, &dset->ds_mask);
dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN;
@@ -1563,15 +1570,14 @@
mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE);
- dset = &domainset0;
+ domainset0 = &domainset_firsttouch;
+ curthread->td_domain.dr_policy = domainset0;
+
+ dset = &domainset2;
DOMAINSET_COPY(&all_domains, &dset->ds_mask);
- dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH;
+ dset->ds_policy = DOMAINSET_POLICY_INTERLEAVE;
dset->ds_prefer = -1;
- curthread->td_domain.dr_policy = _domainset_create(dset, NULL);
-
- domainset_copy(dset, &domainset2);
- domainset2.ds_policy = DOMAINSET_POLICY_INTERLEAVE;
- kernel_object->domain.dr_policy = _domainset_create(&domainset2, NULL);
+ kernel_object->domain.dr_policy = _domainset_create(dset, NULL);
/* Remove empty domains from the global policies. */
LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp)
@@ -1613,7 +1619,7 @@
LIST_INSERT_HEAD(&cpuset_ids, set, cs_link);
refcount_init(&set->cs_ref, 1);
set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY;
- set->cs_domain = &domainset0;
+ set->cs_domain = DOMAINSET_FT();
cpuset_zero = set;
cpuset_root = &set->cs_mask;
Index: sys/sys/domainset.h
===================================================================
--- sys/sys/domainset.h
+++ sys/sys/domainset.h
@@ -95,6 +95,8 @@
domainid_t ds_order[MAXMEMDOM]; /* nth domain table. */
};
+extern struct domainset domainset_firsttouch;
+#define DOMAINSET_FT() (&domainset_firsttouch)
extern struct domainset domainset_fixed[MAXMEMDOM], domainset_prefer[MAXMEMDOM];
#define DOMAINSET_FIXED(domain) (&domainset_fixed[(domain)])
#define DOMAINSET_PREF(domain) (&domainset_prefer[(domain)])
Index: sys/vm/uma_core.c
===================================================================
--- sys/vm/uma_core.c
+++ sys/vm/uma_core.c
@@ -2217,14 +2217,6 @@
keg->uk_reserve = 0;
keg->uk_flags = arg->flags;
- /*
- * We use a global round-robin policy by default. Zones with
- * UMA_ZONE_FIRSTTOUCH set will use first-touch instead, in which
- * case the iterator is never run.
- */
- keg->uk_dr.dr_policy = DOMAINSET_RR();
- keg->uk_dr.dr_iter = 0;
-
/*
* The primary zone is passed to us at keg-creation time.
*/
@@ -2244,19 +2236,32 @@
keg_layout(keg);
/*
- * Use a first-touch NUMA policy for kegs that pmap_extract() will
+ * Use a first-touch NUMA policy for kegs that pmap_kextract() will
* work on. Use round-robin for everything else.
*
* Zones may override the default by specifying either.
*/
#ifdef NUMA
+ KASSERT((keg->uk_flags & (UMA_ZONE_ROUNDROBIN | UMA_ZONE_FIRSTTOUCH)) !=
+ (UMA_ZONE_ROUNDROBIN | UMA_ZONE_FIRSTTOUCH),
+ ("%s: invalid keg NUMA flags %x", __func__, keg->uk_flags));
+
if ((keg->uk_flags &
- (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0)
+ (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) {
keg->uk_flags |= UMA_ZONE_FIRSTTOUCH;
- else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0)
+ keg->uk_dr.dr_policy = DOMAINSET_FT();
+ keg->uk_dr.dr_iter = 0;
+ } else {
keg->uk_flags |= UMA_ZONE_ROUNDROBIN;
+ keg->uk_dr.dr_policy = DOMAINSET_RR();
+ keg->uk_dr.dr_iter = 0;
+ }
#endif
+ /*
+ * Zones use a first-touch policy by default.
+ */
+
/*
* If we haven't booted yet we need allocations to go through the
* startup cache until the vm is ready.
@@ -3643,32 +3648,26 @@
return (slab);
}
+/*
+ * Fetch a cached slab containing one or more free items, or attempt to allocate
+ * a new slab. Returns with the per-domain keg lock held if successful.
+ */
static uma_slab_t
-keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
+keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, int flags)
{
struct vm_domainset_iter di;
uma_slab_t slab;
- int aflags, domain;
+ int domain;
bool rr;
-restart:
/*
* Use the keg's policy if upper layers haven't already specified a
* domain (as happens with first-touch zones).
*
- * To avoid races we run the iterator with the keg lock held, but that
- * means that we cannot allow the vm_domainset layer to sleep. Thus,
- * clear M_WAITOK and handle low memory conditions locally.
+ * XXXMJ keg iterator update is racy
*/
rr = rdomain == UMA_ANYDOMAIN;
- if (rr) {
- aflags = (flags & ~M_WAITOK) | M_NOWAIT;
- vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain,
- &aflags);
- } else {
- aflags = flags;
- domain = rdomain;
- }
+ vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, &flags);
for (;;) {
slab = keg_fetch_free_slab(keg, domain, rr, flags);
@@ -3681,29 +3680,18 @@
if (flags & M_NOVM)
break;
- slab = keg_alloc_slab(keg, zone, domain, flags, aflags);
+ slab = keg_alloc_slab(keg, zone, domain, flags, flags);
if (slab != NULL)
return (slab);
- if (!rr && (flags & M_WAITOK) == 0)
+ if (vm_domainset_iter_policy(&di, &domain) != 0)
break;
- if (rr && vm_domainset_iter_policy(&di, &domain) != 0) {
- if ((flags & M_WAITOK) != 0) {
- vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0);
- goto restart;
- }
- break;
- }
}
/*
- * We might not have been able to get a slab but another cpu
- * could have while we were unlocked. Check again before we
- * fail.
+ * We might not have been able to get a slab but another cpu could have
+ * while we were unlocked. Check again before we fail.
*/
- if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL)
- return (slab);
-
- return (NULL);
+ return (keg_fetch_free_slab(keg, rr ? domain : rdomain, true, flags));
}
static void *

File Metadata

Mime Type
text/plain
Expires
Sun, Nov 17, 2:49 PM (21 h, 22 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
14680498
Default Alt Text
D29046.diff (6 KB)

Event Timeline