Page MenuHomeFreeBSD
Files F108978437

D24755.diff
No OneTemporary
Actions

D24755.diff
View Options

Index: sys/kern/subr_smr.c
===================================================================
--- sys/kern/subr_smr.c
+++ sys/kern/subr_smr.c
@@ -146,6 +146,14 @@
static uma_zone_t smr_shared_zone;
static uma_zone_t smr_zone;
+enum {
+ BOOT_COLD = 1,
+ BOOT_ZONE = 2,
+ BOOT_PCPU = 3,
+} booted = BOOT_COLD;
+
+static smr_shared_t smr_init_list;
+
#ifndef INVARIANTS
#define SMR_SEQ_INIT 1 /* All valid sequence numbers are odd. */
#define SMR_SEQ_INCR 2
@@ -582,30 +590,51 @@
return (success);
}
+static void
+smr_init_cpu(smr_shared_t s, int cpu, int limit, int flags)
+{
+ smr_t c;
+
+ c = zpcpu_get_cpu(s->s_cpu, cpu);
+ c->c_seq = SMR_SEQ_INVALID;
+ c->c_shared = s;
+ c->c_deferred = 0;
+ c->c_limit = limit;
+ c->c_flags = flags;
+}
+
smr_t
smr_create(const char *name, int limit, int flags)
{
- smr_t smr, c;
+ smr_t smr;
smr_shared_t s;
int i;
+ KASSERT(booted >= BOOT_ZONE,
+ ("%s: SMR is not yet initialized", __func__));
+
s = uma_zalloc(smr_shared_zone, M_WAITOK);
smr = uma_zalloc_pcpu(smr_zone, M_WAITOK);
s->s_name = name;
s->s_rd_seq = s->s_wr.seq = SMR_SEQ_INIT;
s->s_wr.ticks = ticks;
+ s->s_cpu = smr;
- /* Initialize all CPUS, not just those running. */
- for (i = 0; i <= mp_maxid; i++) {
- c = zpcpu_get_cpu(smr, i);
- c->c_seq = SMR_SEQ_INVALID;
- c->c_shared = s;
- c->c_deferred = 0;
- c->c_limit = limit;
- c->c_flags = flags;
+ smr_init_cpu(s, 0, limit, flags);
+ if (booted < BOOT_PCPU) {
+ /*
+ * Per-CPU memory for APs might not be accessible at this point.
+ * Defer initialization of the rest of the CPU sections.
+ */
+ s->s_next = smr_init_list;
+ smr_init_list = s;
+ } else {
+ /* Initialize all CPUs, not just those running. */
+ for (i = 1; i <= mp_maxid; i++)
+ smr_init_cpu(s, i, limit, flags);
+ atomic_thread_fence_seq_cst();
}
- atomic_thread_fence_seq_cst();
return (smr);
}
@@ -622,12 +651,33 @@
/*
* Initialize the UMA slab zone.
*/
-void
-smr_init(void)
+static void
+smr_init(void *arg __unused)
{
smr_shared_zone = uma_zcreate("SMR SHARED", sizeof(struct smr_shared),
NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, 0);
smr_zone = uma_zcreate("SMR CPU", sizeof(struct smr),
NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, UMA_ZONE_PCPU);
+ booted = BOOT_ZONE;
+}
+SYSINIT(smr_init, SI_SUB_SMR, SI_ORDER_FIRST, smr_init, NULL);
+
+static void
+smr_init2(void *arg __unused)
+{
+ smr_shared_t s;
+ smr_t c0;
+ int i;
+
+ /*
+ * Complete initialization of per-CPU sections, started in smr_create().
+ */
+ for (s = smr_init_list; s != NULL; s = s->s_next) {
+ c0 = zpcpu_get_cpu(s->s_cpu, 0);
+ for (i = 1; i <= mp_maxid; i++)
+ smr_init_cpu(s, i, c0->c_limit, c0->c_flags);
+ }
+ booted = BOOT_PCPU;
}
+SYSINIT(smr_init2, SI_SUB_CPU, SI_ORDER_ANY, smr_init2, NULL);
Index: sys/sys/kernel.h
===================================================================
--- sys/sys/kernel.h
+++ sys/sys/kernel.h
@@ -92,6 +92,7 @@
SI_SUB_TUNABLES = 0x0700000, /* establish tunable values */
SI_SUB_COPYRIGHT = 0x0800001, /* first use of console*/
SI_SUB_VM = 0x1000000, /* virtual memory system init */
+ SI_SUB_SMR = 0x1010000, /* SMR is initialized */
SI_SUB_COUNTER = 0x1100000, /* counter(9) is initialized */
SI_SUB_KMEM = 0x1800000, /* kernel memory*/
SI_SUB_HYPERVISOR = 0x1A40000, /*
Index: sys/sys/smr.h
===================================================================
--- sys/sys/smr.h
+++ sys/sys/smr.h
@@ -65,9 +65,11 @@
uint64_t _pair;
};
struct smr_shared {
- const char *s_name; /* Name for debugging/reporting. */
union s_wr s_wr; /* Write sequence */
smr_seq_t s_rd_seq; /* Minimum observed read sequence. */
+ smr_t s_cpu; /* Per-CPU sections. */
+ struct smr_shared *s_next; /* Deferred init list entry. */
+ const char *s_name; /* Name for debugging/reporting. */
};
typedef struct smr_shared *smr_shared_t;
@@ -255,7 +257,4 @@
smr_wait(smr, smr_advance(smr));
}
-/* Only at startup. */
-void smr_init(void);
-
#endif /* _SYS_SMR_H_ */
Index: sys/vm/uma_core.c
===================================================================
--- sys/vm/uma_core.c
+++ sys/vm/uma_core.c
@@ -2228,6 +2228,9 @@
keg->uk_flags |= UMA_ZONE_ROUNDROBIN;
#endif
+ KASSERT(booted >= BOOT_KVA || (keg->uk_flags & UMA_ZONE_PCPU) == 0,
+ ("%s: early pcpu keg %s", __func__, keg->uk_name));
+
/*
* If we haven't booted yet we need allocations to go through the
* startup cache until the vm is ready.
@@ -2307,10 +2310,7 @@
if (keg->uk_allocf == startup_alloc) {
/* Switch to the real allocator. */
- if (keg->uk_flags & UMA_ZONE_PCPU)
- keg->uk_allocf = pcpu_page_alloc;
- else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 &&
- keg->uk_ppera > 1)
+ if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1)
keg->uk_allocf = contig_alloc;
else
keg->uk_allocf = page_alloc;
@@ -2869,7 +2869,6 @@
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
bucket_init();
- smr_init();
}
#ifndef UMA_MD_SMALL_ALLOC
@@ -3312,7 +3311,7 @@
#ifdef UMA_ZALLOC_DEBUG
void *item;
- KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0,
+ KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0 || !smp_started,
("uma_zalloc_arg: called with non-SMR zone.\n"));
if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN)
return (item);
@@ -4655,7 +4654,8 @@
uma_zone_set_smr(uma_zone_t zone, smr_t smr)
{
- ZONE_ASSERT_COLD(zone);
+ if (smp_started)
+ ZONE_ASSERT_COLD(zone);
KASSERT(smr != NULL, ("Got NULL smr"));
KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0,
Index: sys/vm/vm_radix.c
===================================================================
--- sys/vm/vm_radix.c
+++ sys/vm/vm_radix.c
@@ -375,10 +375,20 @@
vm_radix_node_zone = uma_zcreate("RADIX NODE",
sizeof(struct vm_radix_node), NULL, NULL, NULL, NULL,
- VM_RADIX_PAD, UMA_ZONE_VM | UMA_ZONE_SMR | UMA_ZONE_ZINIT);
- vm_radix_smr = uma_zone_get_smr(vm_radix_node_zone);
+ VM_RADIX_PAD, UMA_ZONE_VM | UMA_ZONE_ZINIT);
}
+/*
+ * Attach SMR structures to the radix node zone.
+ */
+static void
+vm_radix_zinit2(void *arg __unused)
+{
+ vm_radix_smr = smr_create("RADIX NODE", 0, 0);
+ uma_zone_set_smr(vm_radix_node_zone, vm_radix_smr);
+}
+SYSINIT(vm_radix_zinit2, SI_SUB_SMR, SI_ORDER_ANY, vm_radix_zinit2, NULL);
+
/*
* Inserts the key-value pair into the trie.
* Panics if the key already exists.

File Metadata

Mime Type
text/plain
Expires
Fri, Jan 31, 5:31 AM (11 h, 52 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16355292
Default Alt Text
D24755.diff (6 KB)

Event Timeline