From 1bbbfffdb5d0f291c8fc1787a8c9857ecb4b2f93 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 2 Apr 2020 17:22:36 -0700 Subject: [PATCH 06/24] kbuild: lto: fix recordmcount With LTO, LLVM bitcode won't be compiled into native code until modpost_link. This change postpones calls to recordmcount until after this step. In order to exclude specific functions from inspection, we add a new code section .text..nomcount, which we tell recordmcount to ignore, and a __nomcount attribute for moving functions to this section. Signed-off-by: Sami Tolvanen --- Makefile | 2 +- arch/Kconfig | 2 +- include/asm-generic/vmlinux.lds.h | 1 + include/linux/compiler-clang.h | 4 ++++ include/linux/compiler_types.h | 4 ++++ kernel/trace/ftrace.c | 1 + scripts/Makefile.build | 9 +++++++++ scripts/Makefile.modfinal | 18 ++++++++++++++++-- scripts/link-vmlinux.sh | 29 +++++++++++++++++++++++++++++ scripts/recordmcount.c | 3 ++- 10 files changed, 68 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b59ce19ef248..19e2bf26c52c 100644 --- a/Makefile +++ b/Makefile @@ -862,7 +862,7 @@ KBUILD_AFLAGS += $(CC_FLAGS_USING) ifdef CONFIG_DYNAMIC_FTRACE ifdef CONFIG_HAVE_C_RECORDMCOUNT BUILD_C_RECORDMCOUNT := y - export BUILD_C_RECORDMCOUNT + export BUILD_C_RECORDMCOUNT RECORDMCOUNT_WARN endif endif endif diff --git a/arch/Kconfig b/arch/Kconfig index d9c74e3b4cf8..694bfdc2073f 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -599,7 +599,7 @@ config LTO_CLANG depends on $(success,$(NM) --help | head -n 1 | grep -qi llvm) depends on $(success,$(AR) --help | head -n 1 | grep -qi llvm) depends on ARCH_SUPPORTS_LTO_CLANG - depends on !FTRACE_MCOUNT_RECORD + depends on !FTRACE_MCOUNT_RECORD || HAVE_C_RECORDMCOUNT depends on !KASAN select LTO help diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 7f62f7eef94c..7c9a686cf633 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -566,6 +566,7 @@ *(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \ NOINSTR_TEXT \ *(.text..refcount) \ + *(.text..nomcount) \ *(.ref.text) \ MEM_KEEP(init.text*) \ MEM_KEEP(exit.text*) \ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 5e55302e3bf6..21423d6f6dc4 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -63,3 +63,7 @@ #if __has_feature(shadow_call_stack) # define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) #endif + +#if defined(CONFIG_LTO_CLANG) && defined(CONFIG_FTRACE_MCOUNT_RECORD) +#define __nomcount __attribute__((__section__(".text..nomcount"))) +#endif diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 01dd58c74d80..23c7cf44aa92 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -231,6 +231,10 @@ struct ftrace_likely_data { # define __noscs #endif +#ifndef __nomcount +# define __nomcount +#endif + #ifndef asm_volatile_goto #define asm_volatile_goto(x...) asm goto(x) #endif diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 1903b80db6eb..8e3ddb8123d9 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6062,6 +6062,7 @@ static int ftrace_cmp_ips(const void *a, const void *b) return 0; } +__nomcount static int ftrace_process_locs(struct module *mod, unsigned long *start, unsigned long *end) diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 5c0bbb6ddfcf..64e99f4baa5b 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -187,6 +187,9 @@ endif ifdef CONFIG_FTRACE_MCOUNT_RECORD ifndef CC_USING_RECORD_MCOUNT +ifndef CC_USING_PATCHABLE_FUNCTION_ENTRY +# With LTO, we postpone recordmcount until we compile a native binary +ifndef CONFIG_LTO_CLANG # compiler will not generate __mcount_loc use recordmcount or recordmcount.pl ifdef BUILD_C_RECORDMCOUNT ifeq ("$(origin RECORDMCOUNT_WARN)", "command line") @@ -200,6 +203,8 @@ sub_cmd_record_mcount = \ if [ $(@) != "scripts/mod/empty.o" ]; then \ $(objtree)/scripts/recordmcount $(RECORDMCOUNT_FLAGS) "$(@)"; \ fi; +endif # CONFIG_LTO_CLANG + recordmcount_source := $(srctree)/scripts/recordmcount.c \ $(srctree)/scripts/recordmcount.h else @@ -209,11 +214,15 @@ sub_cmd_record_mcount = perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ "$(OBJDUMP)" "$(OBJCOPY)" "$(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS)" \ "$(LD) $(KBUILD_LDFLAGS)" "$(NM)" "$(RM)" "$(MV)" \ "$(if $(part-of-module),1,0)" "$(@)"; + recordmcount_source := $(srctree)/scripts/recordmcount.pl endif # BUILD_C_RECORDMCOUNT +ifndef CONFIG_LTO_CLANG cmd_record_mcount = $(if $(findstring $(strip $(CC_FLAGS_FTRACE)),$(_c_flags)), \ $(sub_cmd_record_mcount)) +endif # CONFIG_LTO_CLANG endif # CC_USING_RECORD_MCOUNT +endif # CC_USING_PATCHABLE_FUNCTION_ENTRY endif # CONFIG_FTRACE_MCOUNT_RECORD ifdef CONFIG_STACK_VALIDATION diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index 1005b147abd0..d168f0cfe67c 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -34,10 +34,24 @@ ifdef CONFIG_LTO_CLANG # With CONFIG_LTO_CLANG, reuse the object file we compiled for modpost to # avoid a second slow LTO link prelink-ext := .lto -endif + +# ELF processing was skipped earlier because we didn't have native code, +# so let's now process the prelinked binary before we link the module. + +ifdef CONFIG_FTRACE_MCOUNT_RECORD +ifndef CC_USING_RECORD_MCOUNT +ifndef CC_USING_PATCHABLE_FUNCTION_ENTRY +cmd_ld_ko_o += $(objtree)/scripts/recordmcount $(RECORDMCOUNT_FLAGS) \ + $(@:.ko=$(prelink-ext).o); + +endif # CC_USING_PATCHABLE_FUNCTION_ENTRY +endif # CC_USING_RECORD_MCOUNT +endif # CONFIG_FTRACE_MCOUNT_RECORD + +endif # CONFIG_LTO_CLANG quiet_cmd_ld_ko_o = LD [M] $@ - cmd_ld_ko_o = \ + cmd_ld_ko_o += \ $(LD) -r $(KBUILD_LDFLAGS) \ $(KBUILD_LDFLAGS_MODULE) $(LDFLAGS_MODULE) \ $(addprefix -T , $(KBUILD_LDS_MODULE)) \ diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index 712769355a5b..c5715608fe63 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -108,6 +108,29 @@ objtool_link() fi } +# If CONFIG_LTO_CLANG is selected, we postpone running recordmcount until +# we have compiled LLVM IR to an object file. +recordmcount() +{ + if [ "${CONFIG_LTO_CLANG} ${CONFIG_FTRACE_MCOUNT_RECORD}" != "y y" ]; then + return + fi + + if [ -n "${CC_USING_RECORD_MCOUNT}" ]; then + return + fi + if [ -n "${CC_USING_PATCHABLE_FUNCTION_ENTRY}" ]; then + return + fi + + local flags="" + + [ -n "${RECORDMCOUNT_WARN}" ] && flags="-w" + + info MCOUNT $* + ${objtree}/scripts/recordmcount ${flags} $* +} + # Link of vmlinux # ${1} - output file # ${2}, ${3}, ... - optional extra .o files @@ -316,6 +339,12 @@ objtool_link vmlinux.o # modpost vmlinux.o to check for section mismatches ${MAKE} -f "${srctree}/scripts/Makefile.modpost" MODPOST_VMLINUX=1 +if [ -n "${CONFIG_LTO_CLANG}" ]; then + # If we postponed ELF processing steps due to LTO, process + # vmlinux.o instead. + recordmcount vmlinux.o +fi + info MODINFO modules.builtin.modinfo ${OBJCOPY} -j .modinfo -O binary vmlinux.o modules.builtin.modinfo info GEN modules.builtin diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 7225107a9aaf..9e9f10b4d649 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -404,7 +404,8 @@ static uint32_t (*w2)(uint16_t); /* Names of the sections that could contain calls to mcount. */ static int is_mcounted_section_name(char const *const txtname) { - return strncmp(".text", txtname, 5) == 0 || + return (strncmp(".text", txtname, 5) == 0 && + strcmp(".text..nomcount", txtname) != 0) || strcmp(".init.text", txtname) == 0 || strcmp(".ref.text", txtname) == 0 || strcmp(".sched.text", txtname) == 0 || -- 2.28.0