Page MenuHomeFreeBSD
Files F102665043

D44483.id140007.diff
No OneTemporary
Actions

D44483.id140007.diff
View Options

diff --git a/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/sdt/tst.sdtargs.d b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/sdt/tst.sdtargs.d
--- a/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/sdt/tst.sdtargs.d
+++ b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/common/sdt/tst.sdtargs.d
@@ -27,7 +27,7 @@
#pragma ident "%Z%%M% %I% %E% SMI"
/*
- * ASSERTION: Verify that argN (1..7) variables are properly remapped.
+ * ASSERTION: Verify that argN (1..6) variables are properly remapped.
*/
BEGIN
@@ -44,13 +44,12 @@
}
test:::sdttest
-/arg0 != 1 || arg1 != 2 || arg2 != 3 || arg3 != 4 || arg4 != 5 || arg5 != 6 ||
- arg6 != 7/
+/arg0 != 1 || arg1 != 2 || arg2 != 3 || arg3 != 4 || arg4 != 5 || arg5 != 6/
{
printf("sdt arg mismatch\n\n");
- printf("args are : %d, %d, %d, %d, %d, %d, %d\n", arg0, arg1, arg2,
- arg3, arg4, arg5, arg6);
- printf("should be : 1, 2, 3, 4, 5, 6, 7\n");
+ printf("args are : %d, %d, %d, %d, %d, %d\n", arg0, arg1, arg2,
+ arg3, arg4, arg5);
+ printf("should be : 1, 2, 3, 4, 5, 6\n");
exit(1);
}
diff --git a/sys/amd64/include/sdt_machdep.h b/sys/amd64/include/sdt_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/amd64/include/sdt_machdep.h
@@ -0,0 +1,12 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#ifndef _SYS_SDT_MACHDEP_H_
+#define _SYS_SDT_MACHDEP_H_
+
+#define _SDT_ASM_PATCH_INSTR "nop; nop; nop; nop; nop"
+
+#endif
diff --git a/sys/arm/arm/sdt_machdep.c b/sys/arm/arm/sdt_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/arm/arm/sdt_machdep.c
@@ -0,0 +1,63 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#include <sys/systm.h>
+#include <sys/sdt.h>
+
+#include <machine/cpu.h>
+
+/*
+ * Return true if we can overwrite a nop at "patchpoint" with a jump to the
+ * target address.
+ */
+bool
+sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
+{
+ int32_t offset;
+
+ if (patchpoint == target ||
+ (patchpoint & (INSN_SIZE - 1)) != 0 ||
+ (target & (INSN_SIZE - 1)) != 0 ||
+ patchpoint + 2 * INSN_SIZE < patchpoint)
+ return (false);
+ offset = target - (patchpoint + 2 * INSN_SIZE);
+ if (offset < -(1 << 24) || offset > (1 >> 24))
+ return (false);
+ return (true);
+}
+
+/*
+ * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
+ * the target address.
+ */
+void
+sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
+{
+ uint32_t instr;
+
+ KASSERT(sdt_tracepoint_valid(patchpoint, target),
+ ("%s: invalid tracepoint %#x -> %#x",
+ __func__, patchpoint, target));
+
+ instr =
+ (((target - (patchpoint + 2 * INSN_SIZE)) >> 2) & ((1 << 24) - 1)) |
+ 0xea000000;
+ memcpy((void *)patchpoint, &instr, sizeof(instr));
+ icache_sync(patchpoint, sizeof(instr));
+}
+
+/*
+ * Overwrite the patchpoint with a nop instruction.
+ */
+void
+sdt_tracepoint_restore(uintptr_t patchpoint)
+{
+ uint32_t instr;
+
+ instr = 0xe320f000u;
+ memcpy((void *)patchpoint, &instr, sizeof(instr));
+ icache_sync(patchpoint, sizeof(instr));
+}
diff --git a/sys/arm/include/sdt_machdep.h b/sys/arm/include/sdt_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/arm/include/sdt_machdep.h
@@ -0,0 +1,12 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#ifndef _SYS_SDT_MACHDEP_H_
+#define _SYS_SDT_MACHDEP_H_
+
+#define _SDT_ASM_PATCH_INSTR "nop"
+
+#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/arm64/arm64/sdt_machdep.c b/sys/arm64/arm64/sdt_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/arm64/sdt_machdep.c
@@ -0,0 +1,77 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#include <sys/systm.h>
+#include <sys/sdt.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+#include <machine/vmparam.h>
+
+/*
+ * Return true if we can overwrite a nop at "patchpoint" with a jump to the
+ * target address.
+ */
+bool
+sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
+{
+ void *addr;
+ int64_t offset;
+
+ if (!arm64_get_writable_addr((void *)patchpoint, &addr))
+ return (false);
+
+ if (patchpoint == target ||
+ (patchpoint & (INSN_SIZE - 1)) != 0 ||
+ (target & (INSN_SIZE - 1)) != 0)
+ return (false);
+ offset = target - patchpoint;
+ if (offset < -(1 << 26) || offset > (1 << 26))
+ return (false);
+ return (true);
+}
+
+/*
+ * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to the
+ * target address.
+ */
+void
+sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
+{
+ void *addr;
+ uint32_t instr;
+
+ KASSERT(sdt_tracepoint_valid(patchpoint, target),
+ ("%s: invalid tracepoint %#lx -> %#lx",
+ __func__, patchpoint, target));
+
+ if (!arm64_get_writable_addr((void *)patchpoint, &addr))
+ panic("%s: Unable to write new instruction", __func__);
+
+ instr = (((target - patchpoint) >> 2) & 0x3fffffful) | 0x14000000;
+ memcpy(addr, &instr, sizeof(instr));
+ cpu_icache_sync_range((void *)patchpoint, INSN_SIZE);
+}
+
+/*
+ * Overwrite the patchpoint with a nop instruction.
+ */
+void
+sdt_tracepoint_restore(uintptr_t patchpoint)
+{
+ void *addr;
+ uint32_t instr;
+
+ if (!arm64_get_writable_addr((void *)patchpoint, &addr))
+ panic("%s: Unable to write new instruction", __func__);
+
+ instr = 0xd503201f;
+ memcpy(addr, &instr, sizeof(instr));
+ cpu_icache_sync_range((void *)patchpoint, INSN_SIZE);
+}
diff --git a/sys/arm64/include/sdt_machdep.h b/sys/arm64/include/sdt_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/include/sdt_machdep.h
@@ -0,0 +1,12 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#ifndef _SYS_SDT_MACHDEP_H_
+#define _SYS_SDT_MACHDEP_H_
+
+#define _SDT_ASM_PATCH_INSTR "nop"
+
+#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/cddl/dev/dtrace/dtrace_test.c b/sys/cddl/dev/dtrace/dtrace_test.c
--- a/sys/cddl/dev/dtrace/dtrace_test.c
+++ b/sys/cddl/dev/dtrace/dtrace_test.c
@@ -37,8 +37,8 @@
SDT_PROVIDER_DEFINE(test);
-SDT_PROBE_DEFINE7(test, , , sdttest, "int", "int", "int", "int", "int",
- "int", "int");
+SDT_PROBE_DEFINE6(test, , , sdttest, "int", "int", "int", "int", "int",
+ "int");
/*
* These are variables that the DTrace test suite references in the
@@ -68,7 +68,7 @@
else if (val == 0)
return (0);
- SDT_PROBE7(test, , , sdttest, 1, 2, 3, 4, 5, 6, 7);
+ SDT_PROBE6(test, , , sdttest, 1, 2, 3, 4, 5, 6);
return (error);
}
diff --git a/sys/cddl/dev/sdt/sdt.c b/sys/cddl/dev/sdt/sdt.c
--- a/sys/cddl/dev/sdt/sdt.c
+++ b/sys/cddl/dev/sdt/sdt.c
@@ -19,7 +19,7 @@
* CDDL HEADER END
*
* Portions Copyright 2006-2008 John Birrell jb@freebsd.org
- *
+ * Copyright 2024 Mark Johnston <markj@FreeBSD.org>
*/
/*
@@ -187,7 +187,7 @@
if (dtrace_probe_lookup(prov->id, mod, func, name) != DTRACE_IDNONE)
return;
- (void)dtrace_probe_create(prov->id, mod, func, name, 1, probe);
+ (void)dtrace_probe_create(prov->id, mod, func, name, 0, probe);
}
/*
@@ -200,10 +200,62 @@
{
}
+struct sdt_enable_cb_arg {
+ struct sdt_probe *probe;
+ int cpu;
+ int arrived;
+ int done;
+ bool enable;
+};
+
+static void
+sdt_probe_update_cb(void *_arg)
+{
+ struct sdt_enable_cb_arg *arg;
+ struct sdt_tracepoint *tp;
+
+ arg = _arg;
+ if (arg->cpu != curcpu) {
+ atomic_add_rel_int(&arg->arrived, 1);
+ while (atomic_load_acq_int(&arg->done) == 0)
+ cpu_spinwait();
+ return;
+ } else {
+ while (atomic_load_acq_int(&arg->arrived) != mp_ncpus - 1)
+ cpu_spinwait();
+ }
+
+ STAILQ_FOREACH(tp, &arg->probe->tracepoint_list, tracepoint_entry) {
+ if (arg->enable)
+ sdt_tracepoint_patch(tp->patchpoint, tp->target);
+ else
+ sdt_tracepoint_restore(tp->patchpoint);
+ }
+
+ atomic_store_rel_int(&arg->done, 1);
+}
+
+static void
+sdt_probe_update(struct sdt_probe *probe, bool enable)
+{
+ struct sdt_enable_cb_arg cbarg;
+
+ sched_pin();
+ cbarg.probe = probe;
+ cbarg.cpu = curcpu;
+ atomic_store_rel_int(&cbarg.arrived, 0);
+ atomic_store_rel_int(&cbarg.done, 0);
+ cbarg.enable = enable;
+ smp_rendezvous(NULL, sdt_probe_update_cb, NULL, &cbarg);
+ sched_unpin();
+}
+
static void
sdt_enable(void *arg __unused, dtrace_id_t id, void *parg)
{
- struct sdt_probe *probe = parg;
+ struct sdt_probe *probe;
+
+ probe = parg;
probe->id = id;
probe->sdtp_lf->nenabled++;
@@ -215,15 +267,20 @@
sdt_probes_enabled_count++;
if (sdt_probes_enabled_count == 1)
sdt_probes_enabled = true;
+
+ sdt_probe_update(probe, true);
}
static void
sdt_disable(void *arg __unused, dtrace_id_t id, void *parg)
{
- struct sdt_probe *probe = parg;
+ struct sdt_probe *probe;
+ probe = parg;
KASSERT(probe->sdtp_lf->nenabled > 0, ("no probes enabled"));
+ sdt_probe_update(probe, false);
+
sdt_probes_enabled_count--;
if (sdt_probes_enabled_count == 0)
sdt_probes_enabled = false;
@@ -284,26 +341,47 @@
static void
sdt_kld_load_probes(struct linker_file *lf)
{
- struct sdt_probe **probe, **p_begin, **p_end;
- struct sdt_argtype **argtype, **a_begin, **a_end;
+ struct sdt_probe **p_begin, **p_end;
+ struct sdt_argtype **a_begin, **a_end;
+ struct sdt_tracepoint *tp_begin, *tp_end;
if (linker_file_lookup_set(lf, "sdt_probes_set", &p_begin, &p_end,
NULL) == 0) {
- for (probe = p_begin; probe < p_end; probe++) {
+ for (struct sdt_probe **probe = p_begin; probe < p_end;
+ probe++) {
(*probe)->sdtp_lf = lf;
sdt_create_probe(*probe);
TAILQ_INIT(&(*probe)->argtype_list);
+ STAILQ_INIT(&(*probe)->tracepoint_list);
}
}
if (linker_file_lookup_set(lf, "sdt_argtypes_set", &a_begin, &a_end,
NULL) == 0) {
- for (argtype = a_begin; argtype < a_end; argtype++) {
+ for (struct sdt_argtype **argtype = a_begin; argtype < a_end;
+ argtype++) {
(*argtype)->probe->n_args++;
TAILQ_INSERT_TAIL(&(*argtype)->probe->argtype_list,
*argtype, argtype_entry);
}
}
+
+ if (linker_file_lookup_set(lf, __XSTRING(_SDT_TRACEPOINT_SET),
+ &tp_begin, &tp_end, NULL) == 0) {
+ for (struct sdt_tracepoint *tp = tp_begin; tp < tp_end; tp++) {
+ if (!sdt_tracepoint_valid(tp->patchpoint, tp->target)) {
+ printf(
+ "invalid tracepoint %#jx->%#jx for %s:%s:%s:%s\n",
+ (uintmax_t)tp->patchpoint,
+ (uintmax_t)tp->target,
+ tp->probe->prov->name, tp->probe->mod,
+ tp->probe->func, tp->probe->name);
+ continue;
+ }
+ STAILQ_INSERT_TAIL(&tp->probe->tracepoint_list, tp,
+ tracepoint_entry);
+ }
+ }
}
/*
@@ -378,6 +456,7 @@
TAILQ_INIT(&sdt_prov_list);
sdt_probe_func = dtrace_probe;
+ sdt_probe6_func = (sdt_probe6_func_t)dtrace_probe;
sdt_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, sdt_kld_load, NULL,
EVENTHANDLER_PRI_ANY);
@@ -403,6 +482,7 @@
EVENTHANDLER_DEREGISTER(kld_unload_try, sdt_kld_unload_try_tag);
sdt_probe_func = sdt_probe_stub;
+ sdt_probe6_func = (sdt_probe6_func_t)sdt_probe_stub;
TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) {
ret = dtrace_unregister(prov->id);
@@ -419,7 +499,6 @@
static int
sdt_modevent(module_t mod __unused, int type, void *data __unused)
{
-
switch (type) {
case MOD_LOAD:
case MOD_UNLOAD:
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -58,6 +58,7 @@
arm/arm/pmu_fdt.c optional fdt pmu | fdt hwpmc
arm/arm/ptrace_machdep.c standard
arm/arm/sc_machdep.c optional sc
+arm/arm/sdt_machdep.c optional kdtrace_hooks
arm/arm/setcpsr.S standard
arm/arm/setstack.S standard
arm/arm/stack_machdep.c optional ddb | stack
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -68,6 +68,7 @@
compile-with "${NORMAL_C:N-mbranch-protection*} -mbranch-protection=bti"
arm64/arm64/pmap.c standard
arm64/arm64/ptrace_machdep.c standard
+arm64/arm64/sdt_machdep.c optional kdtrace_hooks
arm64/arm64/sigtramp.S standard
arm64/arm64/stack_machdep.c optional ddb | stack
arm64/arm64/strcmp.S standard
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -336,6 +336,7 @@
powerpc/powerpc/platform_if.m standard
powerpc/powerpc/ptrace_machdep.c standard
powerpc/powerpc/sc_machdep.c optional sc
+powerpc/powerpc/sdt_machdep.c optional powerpc64 kdtrace_hooks
powerpc/powerpc/setjmp.S standard
powerpc/powerpc/sigcode32.S optional powerpc | powerpcspe | compat_freebsd32
powerpc/powerpc/sigcode64.S optional powerpc64 | powerpc64le
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv
--- a/sys/conf/files.riscv
+++ b/sys/conf/files.riscv
@@ -61,6 +61,7 @@
riscv/riscv/sigtramp.S standard
riscv/riscv/sbi.c standard
riscv/riscv/sbi_ipi.c optional smp
+riscv/riscv/sdt_machdep.c optional kdtrace_hooks
riscv/riscv/stack_machdep.c optional ddb | stack
riscv/riscv/support.S standard
riscv/riscv/swtch.S standard
diff --git a/sys/conf/files.x86 b/sys/conf/files.x86
--- a/sys/conf/files.x86
+++ b/sys/conf/files.x86
@@ -377,6 +377,7 @@
x86/x86/mp_x86.c optional smp
x86/x86/nexus.c standard
x86/x86/pvclock.c optional kvm_clock | xenhvm
+x86/x86/sdt_machdep.c optional kdtrace_hooks
x86/x86/stack_machdep.c optional ddb | stack
x86/x86/tsc.c standard
x86/x86/ucode.c standard
diff --git a/sys/i386/include/sdt_machdep.h b/sys/i386/include/sdt_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/i386/include/sdt_machdep.h
@@ -0,0 +1,19 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#ifndef _SYS_SDT_MACHDEP_H_
+#define _SYS_SDT_MACHDEP_H_
+
+#define _SDT_ASM_PATCH_INSTR "nop; nop; nop; nop; nop"
+
+/*
+ * Work around an apparent clang bug or limitation which prevents the use of the
+ * "i" (immediate) constraint with the probe structure.
+ */
+#define _SDT_ASM_PROBE_CONSTRAINT "Ws"
+#define _SDT_ASM_PROBE_OPERAND "p"
+
+#endif
diff --git a/sys/kern/kern_sdt.c b/sys/kern/kern_sdt.c
--- a/sys/kern/kern_sdt.c
+++ b/sys/kern/kern_sdt.c
@@ -37,6 +37,7 @@
* dtrace_probe() when it loads.
*/
sdt_probe_func_t sdt_probe_func = sdt_probe_stub;
+sdt_probe6_func_t sdt_probe6_func = (sdt_probe6_func_t)sdt_probe_stub;
volatile bool __read_frequently sdt_probes_enabled;
/*
@@ -45,10 +46,24 @@
* to enable it.
*/
void
-sdt_probe_stub(uint32_t id, uintptr_t arg0, uintptr_t arg1,
- uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
+sdt_probe_stub(uint32_t id __unused, uintptr_t arg0 __unused,
+ uintptr_t arg1 __unused, uintptr_t arg2 __unused, uintptr_t arg3 __unused,
+ uintptr_t arg4 __unused)
{
-
printf("sdt_probe_stub: unexpectedly called\n");
kdb_backtrace();
}
+
+void
+sdt_probe(uint32_t id, uintptr_t arg0, uintptr_t arg1,
+ uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
+{
+ sdt_probe_func(id, arg0, arg1, arg2, arg3, arg4);
+}
+
+void
+sdt_probe6(uint32_t id, uintptr_t arg0, uintptr_t arg1,
+ uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5)
+{
+ sdt_probe6_func(id, arg0, arg1, arg2, arg3, arg4, arg5);
+}
diff --git a/sys/modules/dtrace/Makefile b/sys/modules/dtrace/Makefile
--- a/sys/modules/dtrace/Makefile
+++ b/sys/modules/dtrace/Makefile
@@ -10,7 +10,6 @@
fbt \
profile \
prototype \
- sdt \
systrace
.if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386"
@@ -31,5 +30,8 @@
${MACHINE_ARCH} == "powerpc64"
SUBDIR+= systrace_freebsd32
.endif
+.if ${MACHINE_CPUARCH} != "powerpc" || ${MACHINE_ARCH} == "powerpc64"
+SUBDIR+= sdt
+.endif
.include <bsd.subdir.mk>
diff --git a/sys/powerpc/include/sdt_machdep.h b/sys/powerpc/include/sdt_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/powerpc/include/sdt_machdep.h
@@ -0,0 +1,12 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#ifndef _SYS_SDT_MACHDEP_H_
+#define _SYS_SDT_MACHDEP_H_
+
+#define _SDT_ASM_PATCH_INSTR "nop"
+
+#endif
diff --git a/sys/powerpc/powerpc/sdt_machdep.c b/sys/powerpc/powerpc/sdt_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/powerpc/powerpc/sdt_machdep.c
@@ -0,0 +1,59 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#include <sys/systm.h>
+#include <sys/sdt.h>
+
+#include <machine/md_var.h>
+
+/*
+ * Return true if we can overwrite a nop at "patchpoint" with a jump to the
+ * target address.
+ */
+bool
+sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
+{
+ int64_t offset;
+
+ if (patchpoint == target ||
+ (patchpoint & 3) != 0 || (target & 3) != 0)
+ return (false);
+ offset = target - patchpoint;
+ if (offset < -(1 << 26) || offset > (1 << 26))
+ return (false);
+ return (true);
+}
+
+/*
+ * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
+ * the target address.
+ */
+void
+sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
+{
+ uint32_t instr;
+
+ KASSERT(sdt_tracepoint_valid(patchpoint, target),
+ ("%s: invalid tracepoint %#lx -> %#lx",
+ __func__, patchpoint, target));
+
+ instr = ((target - patchpoint) & 0x7fffffful) | 0x48000000;
+ memcpy((void *)patchpoint, &instr, sizeof(instr));
+ __syncicache((void *)patchpoint, sizeof(instr));
+}
+
+/*
+ * Overwrite the patchpoint with a nop instruction.
+ */
+void
+sdt_tracepoint_restore(uintptr_t patchpoint)
+{
+ uint32_t instr;
+
+ instr = 0x60000000;
+ memcpy((void *)patchpoint, &instr, sizeof(instr));
+ __syncicache((void *)patchpoint, sizeof(instr));
+}
diff --git a/sys/riscv/include/sdt_machdep.h b/sys/riscv/include/sdt_machdep.h
new file mode 100644
--- /dev/null
+++ b/sys/riscv/include/sdt_machdep.h
@@ -0,0 +1,12 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#ifndef _SYS_SDT_MACHDEP_H_
+#define _SYS_SDT_MACHDEP_H_
+
+#define _SDT_ASM_PATCH_INSTR ".option push; .option norvc; nop; .option pop"
+
+#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/riscv/riscv/sdt_machdep.c b/sys/riscv/riscv/sdt_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/riscv/riscv/sdt_machdep.c
@@ -0,0 +1,68 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#include <sys/systm.h>
+#include <sys/sdt.h>
+
+#include <machine/encoding.h>
+
+/*
+ * Return true if we can overwrite a nop at "patchpoint" with a jump to the
+ * target address.
+ */
+bool
+sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
+{
+ int64_t offset;
+
+ if (patchpoint == target ||
+ (patchpoint & (INSN_C_SIZE - 1)) != 0 ||
+ (target & (INSN_C_SIZE - 1)) != 0)
+ return (false);
+ offset = target - patchpoint;
+ if (offset < -(1 << 19) || offset > (1 << 19))
+ return (false);
+ return (true);
+}
+
+/*
+ * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
+ * the target address.
+ */
+void
+sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
+{
+ int32_t imm;
+ uint32_t instr;
+
+ KASSERT(sdt_tracepoint_valid(patchpoint, target),
+ ("%s: invalid tracepoint %#lx -> %#lx",
+ __func__, patchpoint, target));
+
+ imm = target - patchpoint;
+ imm = (imm & 0x100000) |
+ ((imm & 0x7fe) << 8) |
+ ((imm & 0x800) >> 2) |
+ ((imm & 0xff000) >> 12);
+ instr = (imm << 12) | MATCH_JAL;
+
+ memcpy((void *)patchpoint, &instr, sizeof(instr));
+ fence_i();
+}
+
+/*
+ * Overwrite the patchpoint with a nop instruction.
+ */
+void
+sdt_tracepoint_restore(uintptr_t patchpoint)
+{
+ uint32_t instr;
+
+ instr = 0x13; /* uncompressed nop */
+
+ memcpy((void *)patchpoint, &instr, sizeof(instr));
+ fence_i();
+}
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -77,8 +77,8 @@
#else /* _KERNEL */
-#include <sys/cdefs.h>
#include <sys/linker_set.h>
+#include <machine/sdt_machdep.h>
extern volatile bool sdt_probes_enabled;
@@ -102,8 +102,6 @@
#define SDT_PROBE_DEFINE5(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4)
#define SDT_PROBE_DEFINE6(prov, mod, func, name, arg0, arg1, arg2, \
arg3, arg4, arg5)
-#define SDT_PROBE_DEFINE7(prov, mod, func, name, arg0, arg1, arg2, \
- arg3, arg4, arg5, arg6)
#define SDT_PROBE0(prov, mod, func, name)
#define SDT_PROBE1(prov, mod, func, name, arg0)
@@ -112,8 +110,6 @@
#define SDT_PROBE4(prov, mod, func, name, arg0, arg1, arg2, arg3)
#define SDT_PROBE5(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4)
#define SDT_PROBE6(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4, arg5)
-#define SDT_PROBE7(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4, arg5, \
- arg6)
#define MIB_SDT_PROBE1(...)
#define MIB_SDT_PROBE2(...)
@@ -130,9 +126,6 @@
arg1, xarg1, arg2, xarg2, arg3, xarg3, arg4, xarg4)
#define SDT_PROBE_DEFINE6_XLATE(prov, mod, func, name, arg0, xarg0, \
arg1, xarg1, arg2, xarg2, arg3, xarg3, arg4, xarg4, arg5, xarg5)
-#define SDT_PROBE_DEFINE7_XLATE(prov, mod, func, name, arg0, xarg0, \
- arg1, xarg1, arg2, xarg2, arg3, xarg3, arg4, xarg4, arg5, xarg5, arg6, \
- xarg6)
#define DTRACE_PROBE(name)
#define DTRACE_PROBE1(name, type0, arg0)
@@ -144,6 +137,18 @@
#else
+void sdt_probe(uint32_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t,
+ uintptr_t);
+void sdt_probe6(uint32_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t,
+ uintptr_t, uintptr_t);
+
+#define _SDT_TRACEPOINT_SET sdt_tracepoint_set
+#define _SDT_TRACEPOINT_SECTION "set_sdt_tracepoint_set"
+
+bool sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target);
+void sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target);
+void sdt_tracepoint_restore(uintptr_t patchpoint);
+
#define __sdt_used
SET_DECLARE(sdt_providers_set, struct sdt_provider);
@@ -181,14 +186,58 @@
#define SDT_PROBES_ENABLED() __predict_false(sdt_probes_enabled)
-#define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) do { \
- if (SDT_PROBES_ENABLED()) { \
- if (__predict_false(_SDT_PROBE_NAME(prov, mod, func, name)->id)) \
- (*sdt_probe_func)(_SDT_PROBE_NAME(prov, mod, func, name)->id, \
- (uintptr_t) arg0, (uintptr_t) arg1, (uintptr_t) arg2, \
- (uintptr_t) arg3, (uintptr_t) arg4); \
- } \
+#ifdef _ILP32
+#define _SDT_ASM_WORD ".long"
+#else
+#define _SDT_ASM_WORD ".quad"
+#endif
+
+#ifndef _SDT_ASM_PROBE_CONSTRAINT
+#define _SDT_ASM_PROBE_CONSTRAINT "i"
+#endif
+#ifndef _SDT_ASM_PROBE_OPERAND
+#define _SDT_ASM_PROBE_OPERAND "c"
+#endif
+
+/*
+ * The asm below generates records corresponding to the structure's layout, so
+ * the two must be kept in sync.
+ */
+struct sdt_tracepoint {
+ struct sdt_probe *probe;
+ uintptr_t patchpoint;
+ uintptr_t target;
+ STAILQ_ENTRY(sdt_tracepoint) tracepoint_entry;
+};
+
+#define __SDT_PROBE(prov, mod, func, name, uniq, f, ...) do { \
+ __WEAK(__CONCAT(__start_set_, _SDT_TRACEPOINT_SET)); \
+ __WEAK(__CONCAT(__stop_set_, _SDT_TRACEPOINT_SET)); \
+ asm goto( \
+ "0:\n" \
+ _SDT_ASM_PATCH_INSTR "\n" \
+ ".pushsection " _SDT_TRACEPOINT_SECTION ", \"aw\"\n" \
+ _SDT_ASM_WORD " %" _SDT_ASM_PROBE_OPERAND "0\n" \
+ _SDT_ASM_WORD " 0b\n" \
+ _SDT_ASM_WORD " %l1\n" \
+ _SDT_ASM_WORD " 0\n" \
+ ".popsection\n" \
+ : \
+ : _SDT_ASM_PROBE_CONSTRAINT (_SDT_PROBE_NAME(prov, mod, \
+ func, name)) \
+ : \
+ : __sdt_probe##uniq); \
+ if (0) { \
+__sdt_probe##uniq:; \
+ f(_SDT_PROBE_NAME(prov, mod, func, name)->id, __VA_ARGS__); \
+ } \
} while (0)
+#define _SDT_PROBE(prov, mod, func, name, uniq, f, ...) \
+ __SDT_PROBE(prov, mod, func, name, uniq, f, __VA_ARGS__)
+#define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) \
+ _SDT_PROBE(prov, mod, func, name, __COUNTER__, sdt_probe, \
+ (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, \
+ (uintptr_t)arg3, (uintptr_t)arg4)
#define SDT_PROBE_ARGTYPE(_prov, _mod, _func, _name, _num, _type, _xtype) \
static struct sdt_argtype \
@@ -246,17 +295,6 @@
SDT_PROBE_ARGTYPE(prov, mod, func, name, 4, arg4, NULL); \
SDT_PROBE_ARGTYPE(prov, mod, func, name, 5, arg5, NULL)
-#define SDT_PROBE_DEFINE7(prov, mod, func, name, arg0, arg1, arg2, arg3,\
- arg4, arg5, arg6) \
- SDT_PROBE_DEFINE(prov, mod, func, name); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 0, arg0, NULL); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 1, arg1, NULL); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 2, arg2, NULL); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 3, arg3, NULL); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 4, arg4, NULL); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 5, arg5, NULL); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 6, arg6, NULL)
-
#define SDT_PROBE_DEFINE0_XLATE(prov, mod, func, name) \
SDT_PROBE_DEFINE(prov, mod, func, name)
@@ -304,18 +342,6 @@
SDT_PROBE_ARGTYPE(prov, mod, func, name, 4, arg4, xarg4); \
SDT_PROBE_ARGTYPE(prov, mod, func, name, 5, arg5, xarg5)
-#define SDT_PROBE_DEFINE7_XLATE(prov, mod, func, name, arg0, xarg0, \
- arg1, xarg1, arg2, xarg2, arg3, xarg3, arg4, xarg4, arg5, xarg5, arg6, \
- xarg6) \
- SDT_PROBE_DEFINE(prov, mod, func, name); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 0, arg0, xarg0); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 1, arg1, xarg1); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 2, arg2, xarg2); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 3, arg3, xarg3); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 4, arg4, xarg4); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 5, arg5, xarg5); \
- SDT_PROBE_ARGTYPE(prov, mod, func, name, 6, arg6, xarg6)
-
#define SDT_PROBE0(prov, mod, func, name) \
SDT_PROBE(prov, mod, func, name, 0, 0, 0, 0, 0)
#define SDT_PROBE1(prov, mod, func, name, arg0) \
@@ -328,27 +354,10 @@
SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, 0)
#define SDT_PROBE5(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) \
SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4)
-#define SDT_PROBE6(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4, arg5) \
- do { \
- if (_SDT_PROBE_NAME(prov, mod, func, name)->id) \
- (*(void (*)(uint32_t, uintptr_t, uintptr_t, uintptr_t, \
- uintptr_t, uintptr_t, uintptr_t))sdt_probe_func)( \
- _SDT_PROBE_NAME(prov, mod, func, name)->id, \
- (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, \
- (uintptr_t)arg3, (uintptr_t)arg4, (uintptr_t)arg5);\
- } while (0)
-#define SDT_PROBE7(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4, arg5, \
- arg6) \
- do { \
- if (_SDT_PROBE_NAME(prov, mod, func, name)->id) \
- (*(void (*)(uint32_t, uintptr_t, uintptr_t, uintptr_t, \
- uintptr_t, uintptr_t, uintptr_t, uintptr_t)) \
- sdt_probe_func)( \
- _SDT_PROBE_NAME(prov, mod, func, name)->id, \
- (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, \
- (uintptr_t)arg3, (uintptr_t)arg4, (uintptr_t)arg5, \
- (uintptr_t)arg6); \
- } while (0)
+#define SDT_PROBE6(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4, arg5) \
+ _SDT_PROBE(prov, mod, func, name, __COUNTER__, sdt_probe6, \
+ (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, \
+ (uintptr_t)arg3, (uintptr_t)arg4, (uintptr_t)arg5)
#ifndef KDTRACE_NO_MIB_SDT
#define MIB_SDT_PROBE1(...) SDT_PROBE1(mib, __VA_ARGS__)
@@ -411,11 +420,14 @@
*/
typedef void (*sdt_probe_func_t)(uint32_t, uintptr_t arg0, uintptr_t arg1,
uintptr_t arg2, uintptr_t arg3, uintptr_t arg4);
+typedef void (*sdt_probe6_func_t)(uint32_t, uintptr_t arg0, uintptr_t arg1,
+ uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5);
/*
* The 'sdt' provider will set it to dtrace_probe when it loads.
*/
-extern sdt_probe_func_t sdt_probe_func;
+extern sdt_probe_func_t sdt_probe_func;
+extern sdt_probe6_func_t sdt_probe6_func;
struct sdt_probe;
struct sdt_provider;
@@ -436,6 +448,7 @@
TAILQ_ENTRY(sdt_probe)
probe_entry; /* SDT probe list entry. */
TAILQ_HEAD(, sdt_argtype) argtype_list;
+ STAILQ_HEAD(, sdt_tracepoint) tracepoint_list;
const char *mod;
const char *func;
const char *name;
diff --git a/sys/x86/x86/sdt_machdep.c b/sys/x86/x86/sdt_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/x86/x86/sdt_machdep.c
@@ -0,0 +1,84 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
+ */
+
+#include <sys/systm.h>
+#include <sys/sdt.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/md_var.h>
+#include <machine/vmparam.h>
+
+#define SDT_PATCH_SIZE 5
+
+/*
+ * Return true if we can overwrite a nop at "patchpoint" with a jump to the
+ * target address.
+ */
+bool
+sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
+{
+#ifdef __amd64__
+ if (patchpoint < KERNSTART || target < KERNSTART)
+ return (false);
+#endif
+ if (patchpoint == target ||
+ patchpoint + SDT_PATCH_SIZE < patchpoint)
+ return (false);
+#ifdef __amd64__
+ int64_t offset = target - (patchpoint + SDT_PATCH_SIZE);
+ if (offset < -(1l << 31) || offset > (1l << 31))
+ return (false);
+#endif
+ return (true);
+}
+
+/*
+ * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
+ * the target address.
+ */
+void
+sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
+{
+ uint8_t instr[SDT_PATCH_SIZE];
+ int32_t disp;
+ bool old_wp;
+
+ KASSERT(sdt_tracepoint_valid(patchpoint, target),
+ ("%s: invalid tracepoint %#jx -> %#jx",
+ __func__, (uintmax_t)patchpoint, (uintmax_t)target));
+
+ instr[0] = 0xe9;
+ disp = target - (patchpoint + SDT_PATCH_SIZE);
+ memcpy(&instr[1], &disp, sizeof(disp));
+
+ old_wp = disable_wp();
+ memcpy((void *)patchpoint, instr, sizeof(instr));
+ restore_wp(old_wp);
+}
+
+/*
+ * Overwrite the patchpoint with a nop instruction.
+ */
+void
+sdt_tracepoint_restore(uintptr_t patchpoint)
+{
+ uint8_t instr[SDT_PATCH_SIZE];
+ bool old_wp;
+
+#ifdef __amd64__
+ KASSERT(patchpoint >= KERNSTART,
+ ("%s: invalid patchpoint %#lx", __func__, patchpoint));
+#endif
+
+ for (int i = 0; i < SDT_PATCH_SIZE; i++)
+ instr[i] = 0x90;
+
+ old_wp = disable_wp();
+ memcpy((void *)patchpoint, instr, sizeof(instr));
+ restore_wp(old_wp);
+}

File Metadata

Mime Type
text/plain
Expires
Sat, Nov 16, 1:45 PM (21 h, 10 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
14659433
Default Alt Text
D44483.id140007.diff (29 KB)

Event Timeline