Page MenuHomeFreeBSD
Files F102831359

D45191.id138805.diff
No OneTemporary
Actions

D45191.id138805.diff
View Options

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -1336,6 +1336,14 @@
pmap_pcid_enabled = 0;
}
+ TUNABLE_INT_FETCH("vm.pmap.invlpgb_works", &invlpgb_works);
+ if ((amd_extended_feature_extensions & AMDFEID_INVLPGB) != 0 &&
+ invlpgb_works) {
+ invlpgb_maxcnt = cpu_procinfo3 & AMDID_INVLPGB_MAXCNT;
+ } else {
+ invlpgb_works = 0;
+ }
+
/*
* Now we can do small core initialization, after the PCID
* CPU features and user knobs are evaluated.
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -694,6 +694,12 @@
void
smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ invlpgb(INVLPGB_GLOB, 0, 0);
+ tlbsync();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, 0, 0, curcpu_cb, invl_op_tlb);
#ifdef COUNT_XINVLTLB_HITS
ipi_global++;
@@ -703,6 +709,12 @@
void
smp_masked_invlpg(vm_offset_t addr, pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | addr, 0, 0);
+ tlbsync();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, addr, 0, curcpu_cb, invl_op_pg);
#ifdef COUNT_XINVLTLB_HITS
ipi_page++;
@@ -713,6 +725,36 @@
smp_masked_invlpg_range(vm_offset_t addr1, vm_offset_t addr2, pmap_t pmap,
smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ vm_offset_t va;
+ uint64_t cnt, total;
+
+ total = atop(addr2 - addr1);
+ for (va = addr1; total > 0;) {
+ if ((va & PDRMASK) != 0 || total < NPDEPG) {
+ cnt = atop(NBPDR - (va & PDRMASK));
+ if (cnt > total)
+ cnt = total;
+ if (cnt > invlpgb_maxcnt + 1)
+ cnt = invlpgb_maxcnt + 1;
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
+ cnt - 1);
+ va += ptoa(cnt);
+ total -= cnt;
+ } else {
+ cnt = total / NPTEPG;
+ if (cnt > invlpgb_maxcnt + 1)
+ cnt = invlpgb_maxcnt + 1;
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
+ INVLPGB_2M_CNT | (cnt - 1));
+ va += cnt << PDRSHIFT;
+ total -= cnt * NPTEPG;
+ }
+ }
+ tlbsync();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, addr1, addr2, curcpu_cb,
invl_op_pgrng);
#ifdef COUNT_XINVLTLB_HITS
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -550,6 +550,10 @@
int invpcid_works = 0;
SYSCTL_INT(_vm_pmap, OID_AUTO, invpcid_works, CTLFLAG_RD, &invpcid_works, 0,
"Is the invpcid instruction available ?");
+int invlpgb_works = 1;
+SYSCTL_INT(_vm_pmap, OID_AUTO, invlpgb_works, CTLFLAG_RD, &invlpgb_works, 0,
+ "Is the invlpgb instruction available?");
+int invlpgb_maxcnt;
int pmap_pcid_invlpg_workaround = 0;
SYSCTL_INT(_vm_pmap, OID_AUTO, pcid_invlpg_workaround,
CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -529,6 +529,29 @@
: : "r" (d), "r" ((u_long)type) : "memory");
}
+#define INVLPGB_VA 0x0001
+#define INVLPGB_PCID 0x0002
+#define INVLPGB_ASID 0x0004
+#define INVLPGB_GLOB 0x0008
+#define INVLPGB_FIN 0x0010
+#define INVLPGB_NEST 0x0020
+
+#define INVLPGB_DESCR(asid, pcid) (((pcid) << 16) | (asid))
+
+#define INVLPGB_2M_CNT (1u << 31)
+
+static __inline void
+invlpgb(uint64_t rax, uint32_t edx, uint32_t ecx)
+{
+ __asm __volatile("invlpgb" : : "a" (rax), "d" (edx), "c" (ecx));
+}
+
+static __inline void
+tlbsync(void)
+{
+ __asm __volatile("tlbsync");
+}
+
static __inline u_short
rfs(void)
{
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -424,6 +424,8 @@
extern vm_paddr_t dmaplimit;
extern int pmap_pcid_enabled;
extern int invpcid_works;
+extern int invlpgb_works;
+extern int invlpgb_maxcnt;
extern int pmap_pcid_invlpg_workaround;
extern int pmap_pcid_invlpg_workaround_uena;

File Metadata

Mime Type
text/plain
Expires
Mon, Nov 18, 6:06 PM (19 h, 37 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
14702211
Default Alt Text
D45191.id138805.diff (3 KB)

Event Timeline