mirror of
https://github.com/torvalds/linux.git
synced 2025-11-30 23:16:01 +07:00
378b770819
For now, migrate_enable and migrate_disable are global, which makes them
become hotspots in some case. Take BPF for example, the function calling
to migrate_enable and migrate_disable in BPF trampoline can introduce
significant overhead, and following is the 'perf top' of FENTRY's
benchmark (./tools/testing/selftests/bpf/bench trig-fentry):
54.63% bpf_prog_2dcccf652aac1793_bench_trigger_fentry [k]
bpf_prog_2dcccf652aac1793_bench_trigger_fentry
10.43% [kernel] [k] migrate_enable
10.07% bpf_trampoline_6442517037 [k] bpf_trampoline_6442517037
8.06% [kernel] [k] __bpf_prog_exit_recur
4.11% libc.so.6 [.] syscall
2.15% [kernel] [k] entry_SYSCALL_64
1.48% [kernel] [k] memchr_inv
1.32% [kernel] [k] fput
1.16% [kernel] [k] _copy_to_user
0.73% [kernel] [k] bpf_prog_test_run_raw_tp
So in this commit, we make migrate_enable/migrate_disable inline to obtain
better performance. The struct rq is defined internally in
kernel/sched/sched.h, and the field "nr_pinned" is accessed in
migrate_enable/migrate_disable, which makes it hard to make them inline.
Alexei Starovoitov suggests to generate the offset of "nr_pinned" in [1],
so we can define the migrate_enable/migrate_disable in
include/linux/sched.h and access "this_rq()->nr_pinned" with
"(void *)this_rq() + RQ_nr_pinned".
The offset of "nr_pinned" is generated in include/generated/rq-offsets.h
by kernel/sched/rq-offsets.c.
Generally speaking, we move the definition of migrate_enable and
migrate_disable to include/linux/sched.h from kernel/sched/core.c. The
calling to __set_cpus_allowed_ptr() is leaved in ___migrate_enable().
The "struct rq" is not available in include/linux/sched.h, so we can't
access the "runqueues" with this_cpu_ptr(), as the compilation will fail
in this_cpu_ptr() -> raw_cpu_ptr() -> __verify_pcpu_ptr():
typeof((ptr) + 0)
So we introduce the this_rq_raw() and access the runqueues with
arch_raw_cpu_ptr/PERCPU_PTR directly.
The variable "runqueues" is not visible in the kernel modules, and export
it is not a good idea. As Peter Zijlstra advised in [2], we define and
export migrate_enable/migrate_disable in kernel/sched/core.c too, and use
them for the modules.
Before this patch, the performance of BPF FENTRY is:
fentry : 113.030 ± 0.149M/s
fentry : 112.501 ± 0.187M/s
fentry : 112.828 ± 0.267M/s
fentry : 115.287 ± 0.241M/s
After this patch, the performance of BPF FENTRY increases to:
fentry : 143.644 ± 0.670M/s
fentry : 149.764 ± 0.362M/s
fentry : 149.642 ± 0.156M/s
fentry : 145.263 ± 0.221M/s
Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/bpf/CAADnVQ+5sEDKHdsJY5ZsfGDO_1SEhhQWHrt2SMBG5SYyQ+jt7w@mail.gmail.com/ [1]
Link: https://lore.kernel.org/all/20250819123214.GH4067720@noisy.programming.kicks-ass.net/ [2]
112 lines
2.8 KiB
Makefile
112 lines
2.8 KiB
Makefile
# SPDX-License-Identifier: GPL-2.0
|
|
#
|
|
# Kbuild for top-level directory of the kernel
|
|
|
|
# Prepare global headers and check sanity before descending into sub-directories
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Generate bounds.h
|
|
|
|
bounds-file := include/generated/bounds.h
|
|
|
|
targets := kernel/bounds.s
|
|
|
|
$(bounds-file): kernel/bounds.s FORCE
|
|
$(call filechk,offsets,__LINUX_BOUNDS_H__)
|
|
|
|
# Generate timeconst.h
|
|
|
|
timeconst-file := include/generated/timeconst.h
|
|
|
|
filechk_gentimeconst = echo $(CONFIG_HZ) | bc -q $<
|
|
|
|
$(timeconst-file): kernel/time/timeconst.bc FORCE
|
|
$(call filechk,gentimeconst)
|
|
|
|
# Generate asm-offsets.h
|
|
|
|
offsets-file := include/generated/asm-offsets.h
|
|
|
|
targets += arch/$(SRCARCH)/kernel/asm-offsets.s
|
|
|
|
arch/$(SRCARCH)/kernel/asm-offsets.s: $(timeconst-file) $(bounds-file)
|
|
|
|
$(offsets-file): arch/$(SRCARCH)/kernel/asm-offsets.s FORCE
|
|
$(call filechk,offsets,__ASM_OFFSETS_H__)
|
|
|
|
# Generate rq-offsets.h
|
|
|
|
rq-offsets-file := include/generated/rq-offsets.h
|
|
|
|
targets += kernel/sched/rq-offsets.s
|
|
|
|
kernel/sched/rq-offsets.s: $(offsets-file)
|
|
|
|
$(rq-offsets-file): kernel/sched/rq-offsets.s FORCE
|
|
$(call filechk,offsets,__RQ_OFFSETS_H__)
|
|
|
|
# Check for missing system calls
|
|
|
|
quiet_cmd_syscalls = CALL $<
|
|
cmd_syscalls = $(CONFIG_SHELL) $< $(CC) $(c_flags) $(missing_syscalls_flags)
|
|
|
|
PHONY += missing-syscalls
|
|
missing-syscalls: scripts/checksyscalls.sh $(rq-offsets-file)
|
|
$(call cmd,syscalls)
|
|
|
|
# Check the manual modification of atomic headers
|
|
|
|
quiet_cmd_check_sha1 = CHKSHA1 $<
|
|
cmd_check_sha1 = \
|
|
if ! command -v sha1sum >/dev/null; then \
|
|
echo "warning: cannot check the header due to sha1sum missing"; \
|
|
exit 0; \
|
|
fi; \
|
|
if [ "$$(sed -n '$$s:// ::p' $<)" != \
|
|
"$$(sed '$$d' $< | sha1sum | sed 's/ .*//')" ]; then \
|
|
echo "error: $< has been modified." >&2; \
|
|
exit 1; \
|
|
fi; \
|
|
touch $@
|
|
|
|
atomic-checks += $(addprefix $(obj)/.checked-, \
|
|
atomic-arch-fallback.h \
|
|
atomic-instrumented.h \
|
|
atomic-long.h)
|
|
|
|
targets += $(atomic-checks)
|
|
$(atomic-checks): $(obj)/.checked-%: include/linux/atomic/% FORCE
|
|
$(call if_changed,check_sha1)
|
|
|
|
# A phony target that depends on all the preparation targets
|
|
|
|
PHONY += prepare
|
|
prepare: $(offsets-file) missing-syscalls $(atomic-checks)
|
|
@:
|
|
|
|
# Ordinary directory descending
|
|
# ---------------------------------------------------------------------------
|
|
|
|
obj-y += init/
|
|
obj-y += usr/
|
|
obj-y += arch/$(SRCARCH)/
|
|
obj-y += $(ARCH_CORE)
|
|
obj-y += kernel/
|
|
obj-y += certs/
|
|
obj-y += mm/
|
|
obj-y += fs/
|
|
obj-y += ipc/
|
|
obj-y += security/
|
|
obj-y += crypto/
|
|
obj-$(CONFIG_BLOCK) += block/
|
|
obj-$(CONFIG_IO_URING) += io_uring/
|
|
obj-$(CONFIG_RUST) += rust/
|
|
obj-y += $(ARCH_LIB)
|
|
obj-y += drivers/
|
|
obj-y += sound/
|
|
obj-$(CONFIG_SAMPLES) += samples/
|
|
obj-$(CONFIG_NET) += net/
|
|
obj-y += virt/
|
|
obj-y += $(ARCH_DRIVERS)
|
|
obj-$(CONFIG_DRM_HEADER_TEST) += include/
|