Page Menu
Home
FreeBSD
Search
Configure Global Search
Log In
Files
F97674570
D34568.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Size
10 KB
Referenced Files
None
Subscribers
None
D34568.diff
View Options
diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S
--- a/sys/arm64/arm64/locore.S
+++ b/sys/arm64/arm64/locore.S
@@ -38,7 +38,6 @@
#include <machine/vmparam.h>
#define VIRT_BITS 48
-#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
.globl kernbase
.set kernbase, KERNBASE
@@ -378,7 +377,6 @@
* - The Kernel L0 table (TTBR1)
* - The identity (PA = VA) L1 table
* - The identity (PA = VA) L0 table (TTBR0)
- * - The DMAP L1 tables
*/
LENTRY(create_pagetables)
/* Save the Link register */
@@ -476,13 +474,6 @@
mov x10, #1
bl link_l0_pagetable
- /* Link the DMAP tables */
- ldr x8, =DMAP_MIN_ADDRESS
- adrp x9, pagetable_dmap
- add x9, x9, :lo12:pagetable_dmap
- mov x10, #DMAP_TABLES
- bl link_l0_pagetable
-
/*
* Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG.
* They are only needed early on, so the VA = PA map is uncached.
@@ -771,6 +762,7 @@
* L0 bootstrap for user
* L0 for user
*/
+ .globl pagetable_l0_ttbr1
pagetable:
.space PAGE_SIZE
pagetable_l1_ttbr1:
@@ -786,9 +778,6 @@
pagetable_l0_ttbr0:
.space PAGE_SIZE
- .globl pagetable_dmap
-pagetable_dmap:
- .space PAGE_SIZE * DMAP_TABLES
pagetable_end:
el2_pagetable:
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -290,8 +290,7 @@
CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS);
CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS);
-#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
-extern pt_entry_t pagetable_dmap[];
+extern pt_entry_t pagetable_l0_ttbr1[];
#define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1))
static vm_paddr_t physmap[PHYSMAP_SIZE];
@@ -808,104 +807,248 @@
return (pa_page | (va & PAR_LOW_MASK));
}
-static vm_offset_t
-pmap_bootstrap_dmap_l2(vm_offset_t *va, vm_paddr_t *pa, u_int *prev_l1_slot,
- pt_entry_t **l2p, int i, vm_offset_t freemempos)
+/* State of the bootstrapped DMAP page tables */
+struct dmap_bootstrap_state {
+ vm_offset_t va;
+ vm_paddr_t pa;
+ pt_entry_t *l1;
+ pt_entry_t *l2;
+ pt_entry_t *l3;
+ u_int l0_slot;
+ u_int l1_slot;
+ u_int l2_slot;
+ vm_offset_t freemempos;
+};
+
+static void
+pmap_bootstrap_dmap_l0_table(struct dmap_bootstrap_state *state)
+{
+ vm_paddr_t l1_pa;
+ u_int l0_slot;
+
+ /* Link the level 0 table to a level 1 table */
+ l0_slot = pmap_l0_index(state->va);
+ if (l0_slot != state->l0_slot) {
+ MPASS(state->l0_slot < l0_slot ||
+ state->l0_slot == L0_ENTRIES);
+
+ /* Create a new L0 table entry */
+ state->l0_slot = l0_slot;
+ state->l1 = (pt_entry_t *)state->freemempos;
+ memset(state->l1, 0, PAGE_SIZE);
+ state->freemempos += PAGE_SIZE;
+
+ /* Reset lower levels */
+ state->l2 = NULL;
+ state->l3 = NULL;
+ state->l1_slot = Ln_ENTRIES;
+ state->l2_slot = Ln_ENTRIES;
+
+ l1_pa = pmap_early_vtophys((vm_offset_t)state->l1);
+ MPASS((l1_pa & Ln_TABLE_MASK) == 0);
+ MPASS(pagetable_l0_ttbr1[l0_slot] == 0);
+ pmap_store(&pagetable_l0_ttbr1[l0_slot], l1_pa |
+ TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0 | L0_TABLE);
+ }
+ KASSERT(state->l1 != NULL, ("%s: NULL l1", __func__));
+}
+
+static void
+pmap_bootstrap_dmap_l1_table(struct dmap_bootstrap_state *state)
{
- pt_entry_t *l2;
vm_paddr_t l2_pa;
- u_int l1_slot, l2_slot;
- bool first;
+ u_int l1_slot;
+
+ /* Make sure there is a valid L0 -> L1 table */
+ pmap_bootstrap_dmap_l0_table(state);
+
+ /* Link the level 1 table to a level 2 table */
+ l1_slot = pmap_l1_index(state->va);
+ if (l1_slot != state->l1_slot) {
+ MPASS(state->l1_slot < l1_slot ||
+ state->l1_slot == Ln_ENTRIES);
+
+ /* Create a new L1 table entry */
+ state->l1_slot = l1_slot;
+ state->l2 = (pt_entry_t *)state->freemempos;
+ memset(state->l2, 0, PAGE_SIZE);
+ state->freemempos += PAGE_SIZE;
+
+ /* Reset lower levels */
+ state->l3 = NULL;
+ state->l2_slot = Ln_ENTRIES;
+
+ l2_pa = pmap_early_vtophys((vm_offset_t)state->l2);
+ MPASS((l2_pa & Ln_TABLE_MASK) == 0);
+ MPASS(state->l1[l1_slot] == 0);
+ pmap_store(&state->l1[l1_slot], l2_pa | TATTR_PXN_TABLE |
+ L1_TABLE);
+ }
+ KASSERT(state->l2 != NULL, ("%s: NULL l2", __func__));
+}
- l2 = *l2p;
- l1_slot = ((*va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
- if (l1_slot != *prev_l1_slot) {
- *prev_l1_slot = l1_slot;
- l2 = (pt_entry_t *)freemempos;
- l2_pa = pmap_early_vtophys((vm_offset_t)l2);
- freemempos += PAGE_SIZE;
+static void
+pmap_bootstrap_dmap_l2_table(struct dmap_bootstrap_state *state)
+{
+ vm_paddr_t l3_pa;
+ u_int l2_slot;
+
+ /* Make sure there is a valid L1 -> L2 table */
+ pmap_bootstrap_dmap_l1_table(state);
- pmap_store(&pagetable_dmap[l1_slot],
- (l2_pa & ~Ln_TABLE_MASK) |
- TATTR_PXN_TABLE | L1_TABLE);
+ /* Link the level 2 table to a level 3 table */
+ l2_slot = pmap_l2_index(state->va);
+ if (l2_slot != state->l2_slot) {
+ MPASS(state->l2_slot < l2_slot ||
+ state->l2_slot == Ln_ENTRIES);
- memset(l2, 0, PAGE_SIZE);
+ /* Create a new L2 table entry */
+ state->l2_slot = l2_slot;
+ state->l3 = (pt_entry_t *)state->freemempos;
+ memset(state->l3, 0, PAGE_SIZE);
+ state->freemempos += PAGE_SIZE;
+
+ l3_pa = pmap_early_vtophys((vm_offset_t)state->l3);
+ MPASS((l3_pa & Ln_TABLE_MASK) == 0);
+ MPASS(state->l2[l2_slot] == 0);
+ pmap_store(&state->l2[l2_slot], l3_pa | TATTR_PXN_TABLE |
+ L2_TABLE);
}
- KASSERT(l2 != NULL,
- ("pmap_bootstrap_dmap_l2: NULL l2 map"));
- for (first = true; *va < DMAP_MAX_ADDRESS && *pa < physmap[i + 1];
- *pa += L2_SIZE, *va += L2_SIZE) {
+ KASSERT(state->l3 != NULL, ("%s: NULL l3", __func__));
+}
+
+static void
+pmap_bootstrap_dmap_l2_block(struct dmap_bootstrap_state *state, int i)
+{
+ u_int l2_slot;
+ bool first;
+
+ if ((physmap[i + 1] - state->pa) < L2_SIZE)
+ return;
+
+ /* Make sure there is a valid L1 table */
+ pmap_bootstrap_dmap_l1_table(state);
+
+ MPASS((state->va & L2_OFFSET) == 0);
+ for (first = true;
+ state->va < DMAP_MAX_ADDRESS &&
+ (physmap[i + 1] - state->pa) >= L2_SIZE;
+ state->va += L2_SIZE, state->pa += L2_SIZE) {
/*
* Stop if we are about to walk off the end of what the
* current L1 slot can address.
*/
- if (!first && (*pa & L1_OFFSET) == 0)
+ if (!first && (state->pa & L1_OFFSET) == 0)
break;
first = false;
- l2_slot = pmap_l2_index(*va);
- pmap_store(&l2[l2_slot],
- (*pa & ~L2_OFFSET) | ATTR_DEFAULT |
- ATTR_S1_XN |
- ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+ l2_slot = pmap_l2_index(state->va);
+ MPASS((state->pa & L2_OFFSET) == 0);
+ MPASS(state->l2[l2_slot] == 0);
+ pmap_store(&state->l2[l2_slot], state->pa | ATTR_DEFAULT |
+ ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
L2_BLOCK);
}
- MPASS(*va == (*pa - dmap_phys_base + DMAP_MIN_ADDRESS));
- *l2p = l2;
+ MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
+}
- return (freemempos);
+static void
+pmap_bootstrap_dmap_l3_page(struct dmap_bootstrap_state *state, int i)
+{
+ u_int l3_slot;
+ bool first;
+
+ if ((physmap[i + 1] - state->pa) < L3_SIZE)
+ return;
+
+ /* Make sure there is a valid L2 table */
+ pmap_bootstrap_dmap_l2_table(state);
+
+ MPASS((state->va & L3_OFFSET) == 0);
+ for (first = true;
+ state->va < DMAP_MAX_ADDRESS &&
+ (physmap[i + 1] - state->pa) >= L3_SIZE;
+ state->va += L3_SIZE, state->pa += L3_SIZE) {
+ /*
+ * Stop if we are about to walk off the end of what the
+ * current L2 slot can address.
+ */
+ if (!first && (state->pa & L2_OFFSET) == 0)
+ break;
+
+ first = false;
+ l3_slot = pmap_l3_index(state->va);
+ MPASS((state->pa & L3_OFFSET) == 0);
+ MPASS(state->l3[l3_slot] == 0);
+ pmap_store(&state->l3[l3_slot], state->pa | ATTR_DEFAULT |
+ ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+ L3_PAGE);
+ }
+ MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
}
static vm_offset_t
pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa,
vm_offset_t freemempos)
{
- pt_entry_t *l2;
- vm_offset_t va;
- vm_paddr_t pa;
- u_int l1_slot, prev_l1_slot;
+ struct dmap_bootstrap_state state;
int i;
dmap_phys_base = min_pa & ~L1_OFFSET;
dmap_phys_max = 0;
dmap_max_addr = 0;
- l2 = NULL;
- prev_l1_slot = -1;
+
+ state.l1 = state.l2 = state.l3 = NULL;
+ state.l0_slot = L0_ENTRIES;
+ state.l1_slot = Ln_ENTRIES;
+ state.l2_slot = Ln_ENTRIES;
+ state.freemempos = freemempos;
for (i = 0; i < (physmap_idx * 2); i += 2) {
- pa = physmap[i] & ~L2_OFFSET;
- va = pa - dmap_phys_base + DMAP_MIN_ADDRESS;
+ state.pa = physmap[i] & ~L3_OFFSET;
+ state.va = state.pa - dmap_phys_base + DMAP_MIN_ADDRESS;
- /* Create L2 mappings at the start of the region */
- if ((pa & L1_OFFSET) != 0) {
- freemempos = pmap_bootstrap_dmap_l2(&va, &pa,
- &prev_l1_slot, &l2, i, freemempos);
- }
+ /* Create L3 mappings at the start of the region */
+ if ((state.pa & L2_OFFSET) != 0)
+ pmap_bootstrap_dmap_l3_page(&state, i);
+ MPASS(state.pa <= physmap[i + 1]);
- for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1] &&
- (physmap[i + 1] - pa) >= L1_SIZE;
- pa += L1_SIZE, va += L1_SIZE) {
- l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
- pmap_store(&pagetable_dmap[l1_slot],
- (pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_S1_XN |
- ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L1_BLOCK);
- }
+ /* Create L2 mappings at the start of the region */
+ if ((state.pa & L1_OFFSET) != 0)
+ pmap_bootstrap_dmap_l2_block(&state, i);
+ MPASS(state.pa <= physmap[i + 1]);
+
+ /* Create the main L1 block mappings */
+ for (; state.va < DMAP_MAX_ADDRESS &&
+ (physmap[i + 1] - state.pa) >= L1_SIZE;
+ state.va += L1_SIZE, state.pa += L1_SIZE) {
+ /* Make sure there is a valid L1 table */
+ pmap_bootstrap_dmap_l0_table(&state);
+ MPASS((state.pa & L1_OFFSET) == 0);
+ pmap_store(&state.l1[pmap_l1_index(state.va)],
+ state.pa | ATTR_DEFAULT | ATTR_S1_XN |
+ ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+ L1_BLOCK);
+ }
+ MPASS(state.pa <= physmap[i + 1]);
/* Create L2 mappings at the end of the region */
- if (pa < physmap[i + 1]) {
- freemempos = pmap_bootstrap_dmap_l2(&va, &pa,
- &prev_l1_slot, &l2, i, freemempos);
- }
+ pmap_bootstrap_dmap_l2_block(&state, i);
+ MPASS(state.pa <= physmap[i + 1]);
+
+ /* Create L3 mappings at the end of the region */
+ pmap_bootstrap_dmap_l3_page(&state, i);
+ MPASS(state.pa == physmap[i + 1]);
- if (pa > dmap_phys_max) {
- dmap_phys_max = pa;
- dmap_max_addr = va;
+ if (state.pa > dmap_phys_max) {
+ dmap_phys_max = state.pa;
+ dmap_max_addr = state.va;
}
}
cpu_tlb_flushID();
- return (freemempos);
+ return (state.freemempos);
}
static vm_offset_t
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Tue, Oct 1, 5:22 PM (21 h, 48 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
13267325
Default Alt Text
D34568.diff (10 KB)
Attached To
Mode
D34568: Move the arm64 DMAP creation to C
Attached
Detach File
Event Timeline
Log In to Comment