Page MenuHomeFreeBSD
Files F108310039

D45191.id140230.diff
No OneTemporary
Actions

D45191.id140230.diff
View Options

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -1487,6 +1487,12 @@
finishidentcpu(); /* Final stage of CPU initialization */
+ invlpgb_works = (amd_extended_feature_extensions &
+ AMDFEID_INVLPGB) != 0;
+ TUNABLE_INT_FETCH("vm.pmap.invlpgb_works", &invlpgb_works);
+ if (invlpgb_works)
+ invlpgb_maxcnt = cpu_procinfo3 & AMDID_INVLPGB_MAXCNT;
+
/*
* Initialize the clock before the console so that console
* initialization can use DELAY().
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -679,6 +679,20 @@
void
smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ invlpgb(INVLPGB_GLOB, 0, 0);
+
+ /*
+ * TLBSYNC syncs only against INVLPGB executed on the
+ * same CPU. Since current thread is pinned by
+ * caller, we do not need to enter critical section to
+ * prevent migration.
+ */
+ tlbsync();
+ sched_unpin();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, 0, 0, curcpu_cb, invl_op_tlb);
#ifdef COUNT_XINVLTLB_HITS
ipi_global++;
@@ -688,6 +702,13 @@
void
smp_masked_invlpg(vm_offset_t addr, pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | trunc_page(addr), 0, 0);
+ tlbsync();
+ sched_unpin();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, addr, 0, curcpu_cb, invl_op_pg);
#ifdef COUNT_XINVLTLB_HITS
ipi_page++;
@@ -698,6 +719,37 @@
smp_masked_invlpg_range(vm_offset_t addr1, vm_offset_t addr2, pmap_t pmap,
smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ vm_offset_t va;
+ uint64_t cnt, total;
+
+ total = atop(addr2 - addr1);
+ for (va = trunc_page(addr1); total > 0;) {
+ if ((va & PDRMASK) != 0 || total < NPDEPG) {
+ cnt = atop(NBPDR - (va & PDRMASK));
+ if (cnt > total)
+ cnt = total;
+ if (cnt > invlpgb_maxcnt + 1)
+ cnt = invlpgb_maxcnt + 1;
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
+ cnt - 1);
+ va += ptoa(cnt);
+ total -= cnt;
+ } else {
+ cnt = total / NPTEPG;
+ if (cnt > invlpgb_maxcnt + 1)
+ cnt = invlpgb_maxcnt + 1;
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
+ INVLPGB_2M_CNT | (cnt - 1));
+ va += cnt << PDRSHIFT;
+ total -= cnt * NPTEPG;
+ }
+ }
+ tlbsync();
+ sched_unpin();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, addr1, addr2, curcpu_cb,
invl_op_pgrng);
#ifdef COUNT_XINVLTLB_HITS
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -550,6 +550,10 @@
int invpcid_works = 0;
SYSCTL_INT(_vm_pmap, OID_AUTO, invpcid_works, CTLFLAG_RD, &invpcid_works, 0,
"Is the invpcid instruction available ?");
+int invlpgb_works;
+SYSCTL_INT(_vm_pmap, OID_AUTO, invlpgb_works, CTLFLAG_RD, &invlpgb_works, 0,
+ "Is the invlpgb instruction available?");
+int invlpgb_maxcnt;
int pmap_pcid_invlpg_workaround = 0;
SYSCTL_INT(_vm_pmap, OID_AUTO, pcid_invlpg_workaround,
CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -525,6 +525,29 @@
: : "r" (d), "r" ((u_long)type) : "memory");
}
+#define INVLPGB_VA 0x0001
+#define INVLPGB_PCID 0x0002
+#define INVLPGB_ASID 0x0004
+#define INVLPGB_GLOB 0x0008
+#define INVLPGB_FIN 0x0010
+#define INVLPGB_NEST 0x0020
+
+#define INVLPGB_DESCR(asid, pcid) (((pcid) << 16) | (asid))
+
+#define INVLPGB_2M_CNT (1u << 31)
+
+static __inline void
+invlpgb(uint64_t rax, uint32_t edx, uint32_t ecx)
+{
+ __asm __volatile("invlpgb" : : "a" (rax), "d" (edx), "c" (ecx));
+}
+
+static __inline void
+tlbsync(void)
+{
+ __asm __volatile("tlbsync");
+}
+
static __inline u_short
rfs(void)
{
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -424,6 +424,8 @@
extern vm_paddr_t dmaplimit;
extern int pmap_pcid_enabled;
extern int invpcid_works;
+extern int invlpgb_works;
+extern int invlpgb_maxcnt;
extern int pmap_pcid_invlpg_workaround;
extern int pmap_pcid_invlpg_workaround_uena;

File Metadata

Mime Type
text/plain
Expires
Fri, Jan 24, 6:35 PM (20 h, 17 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16096341
Default Alt Text
D45191.id140230.diff (4 KB)

Event Timeline