1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
|
From 1bbbfffdb5d0f291c8fc1787a8c9857ecb4b2f93 Mon Sep 17 00:00:00 2001
From: Sami Tolvanen <samitolvanen@google.com>
Date: Thu, 2 Apr 2020 17:22:36 -0700
Subject: [PATCH 06/24] kbuild: lto: fix recordmcount
With LTO, LLVM bitcode won't be compiled into native code until
modpost_link. This change postpones calls to recordmcount until after
this step.
In order to exclude specific functions from inspection, we add a new
code section .text..nomcount, which we tell recordmcount to ignore, and
a __nomcount attribute for moving functions to this section.
Signed-off-by: Sami Tolvanen <samitolvanen@google.com>
---
Makefile | 2 +-
arch/Kconfig | 2 +-
include/asm-generic/vmlinux.lds.h | 1 +
include/linux/compiler-clang.h | 4 ++++
include/linux/compiler_types.h | 4 ++++
kernel/trace/ftrace.c | 1 +
scripts/Makefile.build | 9 +++++++++
scripts/Makefile.modfinal | 18 ++++++++++++++++--
scripts/link-vmlinux.sh | 29 +++++++++++++++++++++++++++++
scripts/recordmcount.c | 3 ++-
10 files changed, 68 insertions(+), 5 deletions(-)
diff --git a/Makefile b/Makefile
index b59ce19ef248..19e2bf26c52c 100644
--- a/Makefile
+++ b/Makefile
@@ -862,7 +862,7 @@ KBUILD_AFLAGS += $(CC_FLAGS_USING)
ifdef CONFIG_DYNAMIC_FTRACE
ifdef CONFIG_HAVE_C_RECORDMCOUNT
BUILD_C_RECORDMCOUNT := y
- export BUILD_C_RECORDMCOUNT
+ export BUILD_C_RECORDMCOUNT RECORDMCOUNT_WARN
endif
endif
endif
diff --git a/arch/Kconfig b/arch/Kconfig
index d9c74e3b4cf8..694bfdc2073f 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -599,7 +599,7 @@ config LTO_CLANG
depends on $(success,$(NM) --help | head -n 1 | grep -qi llvm)
depends on $(success,$(AR) --help | head -n 1 | grep -qi llvm)
depends on ARCH_SUPPORTS_LTO_CLANG
- depends on !FTRACE_MCOUNT_RECORD
+ depends on !FTRACE_MCOUNT_RECORD || HAVE_C_RECORDMCOUNT
depends on !KASAN
select LTO
help
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 7f62f7eef94c..7c9a686cf633 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -566,6 +566,7 @@
*(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \
NOINSTR_TEXT \
*(.text..refcount) \
+ *(.text..nomcount) \
*(.ref.text) \
MEM_KEEP(init.text*) \
MEM_KEEP(exit.text*) \
diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
index 5e55302e3bf6..21423d6f6dc4 100644
--- a/include/linux/compiler-clang.h
+++ b/include/linux/compiler-clang.h
@@ -63,3 +63,7 @@
#if __has_feature(shadow_call_stack)
# define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
#endif
+
+#if defined(CONFIG_LTO_CLANG) && defined(CONFIG_FTRACE_MCOUNT_RECORD)
+#define __nomcount __attribute__((__section__(".text..nomcount")))
+#endif
diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index 01dd58c74d80..23c7cf44aa92 100644
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -231,6 +231,10 @@ struct ftrace_likely_data {
# define __noscs
#endif
+#ifndef __nomcount
+# define __nomcount
+#endif
+
#ifndef asm_volatile_goto
#define asm_volatile_goto(x...) asm goto(x)
#endif
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 1903b80db6eb..8e3ddb8123d9 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -6062,6 +6062,7 @@ static int ftrace_cmp_ips(const void *a, const void *b)
return 0;
}
+__nomcount
static int ftrace_process_locs(struct module *mod,
unsigned long *start,
unsigned long *end)
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index 5c0bbb6ddfcf..64e99f4baa5b 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -187,6 +187,9 @@ endif
ifdef CONFIG_FTRACE_MCOUNT_RECORD
ifndef CC_USING_RECORD_MCOUNT
+ifndef CC_USING_PATCHABLE_FUNCTION_ENTRY
+# With LTO, we postpone recordmcount until we compile a native binary
+ifndef CONFIG_LTO_CLANG
# compiler will not generate __mcount_loc use recordmcount or recordmcount.pl
ifdef BUILD_C_RECORDMCOUNT
ifeq ("$(origin RECORDMCOUNT_WARN)", "command line")
@@ -200,6 +203,8 @@ sub_cmd_record_mcount = \
if [ $(@) != "scripts/mod/empty.o" ]; then \
$(objtree)/scripts/recordmcount $(RECORDMCOUNT_FLAGS) "$(@)"; \
fi;
+endif # CONFIG_LTO_CLANG
+
recordmcount_source := $(srctree)/scripts/recordmcount.c \
$(srctree)/scripts/recordmcount.h
else
@@ -209,11 +214,15 @@ sub_cmd_record_mcount = perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \
"$(OBJDUMP)" "$(OBJCOPY)" "$(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS)" \
"$(LD) $(KBUILD_LDFLAGS)" "$(NM)" "$(RM)" "$(MV)" \
"$(if $(part-of-module),1,0)" "$(@)";
+
recordmcount_source := $(srctree)/scripts/recordmcount.pl
endif # BUILD_C_RECORDMCOUNT
+ifndef CONFIG_LTO_CLANG
cmd_record_mcount = $(if $(findstring $(strip $(CC_FLAGS_FTRACE)),$(_c_flags)), \
$(sub_cmd_record_mcount))
+endif # CONFIG_LTO_CLANG
endif # CC_USING_RECORD_MCOUNT
+endif # CC_USING_PATCHABLE_FUNCTION_ENTRY
endif # CONFIG_FTRACE_MCOUNT_RECORD
ifdef CONFIG_STACK_VALIDATION
diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal
index 1005b147abd0..d168f0cfe67c 100644
--- a/scripts/Makefile.modfinal
+++ b/scripts/Makefile.modfinal
@@ -34,10 +34,24 @@ ifdef CONFIG_LTO_CLANG
# With CONFIG_LTO_CLANG, reuse the object file we compiled for modpost to
# avoid a second slow LTO link
prelink-ext := .lto
-endif
+
+# ELF processing was skipped earlier because we didn't have native code,
+# so let's now process the prelinked binary before we link the module.
+
+ifdef CONFIG_FTRACE_MCOUNT_RECORD
+ifndef CC_USING_RECORD_MCOUNT
+ifndef CC_USING_PATCHABLE_FUNCTION_ENTRY
+cmd_ld_ko_o += $(objtree)/scripts/recordmcount $(RECORDMCOUNT_FLAGS) \
+ $(@:.ko=$(prelink-ext).o);
+
+endif # CC_USING_PATCHABLE_FUNCTION_ENTRY
+endif # CC_USING_RECORD_MCOUNT
+endif # CONFIG_FTRACE_MCOUNT_RECORD
+
+endif # CONFIG_LTO_CLANG
quiet_cmd_ld_ko_o = LD [M] $@
- cmd_ld_ko_o = \
+ cmd_ld_ko_o += \
$(LD) -r $(KBUILD_LDFLAGS) \
$(KBUILD_LDFLAGS_MODULE) $(LDFLAGS_MODULE) \
$(addprefix -T , $(KBUILD_LDS_MODULE)) \
diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh
index 712769355a5b..c5715608fe63 100755
--- a/scripts/link-vmlinux.sh
+++ b/scripts/link-vmlinux.sh
@@ -108,6 +108,29 @@ objtool_link()
fi
}
+# If CONFIG_LTO_CLANG is selected, we postpone running recordmcount until
+# we have compiled LLVM IR to an object file.
+recordmcount()
+{
+ if [ "${CONFIG_LTO_CLANG} ${CONFIG_FTRACE_MCOUNT_RECORD}" != "y y" ]; then
+ return
+ fi
+
+ if [ -n "${CC_USING_RECORD_MCOUNT}" ]; then
+ return
+ fi
+ if [ -n "${CC_USING_PATCHABLE_FUNCTION_ENTRY}" ]; then
+ return
+ fi
+
+ local flags=""
+
+ [ -n "${RECORDMCOUNT_WARN}" ] && flags="-w"
+
+ info MCOUNT $*
+ ${objtree}/scripts/recordmcount ${flags} $*
+}
+
# Link of vmlinux
# ${1} - output file
# ${2}, ${3}, ... - optional extra .o files
@@ -316,6 +339,12 @@ objtool_link vmlinux.o
# modpost vmlinux.o to check for section mismatches
${MAKE} -f "${srctree}/scripts/Makefile.modpost" MODPOST_VMLINUX=1
+if [ -n "${CONFIG_LTO_CLANG}" ]; then
+ # If we postponed ELF processing steps due to LTO, process
+ # vmlinux.o instead.
+ recordmcount vmlinux.o
+fi
+
info MODINFO modules.builtin.modinfo
${OBJCOPY} -j .modinfo -O binary vmlinux.o modules.builtin.modinfo
info GEN modules.builtin
diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c
index 7225107a9aaf..9e9f10b4d649 100644
--- a/scripts/recordmcount.c
+++ b/scripts/recordmcount.c
@@ -404,7 +404,8 @@ static uint32_t (*w2)(uint16_t);
/* Names of the sections that could contain calls to mcount. */
static int is_mcounted_section_name(char const *const txtname)
{
- return strncmp(".text", txtname, 5) == 0 ||
+ return (strncmp(".text", txtname, 5) == 0 &&
+ strcmp(".text..nomcount", txtname) != 0) ||
strcmp(".init.text", txtname) == 0 ||
strcmp(".ref.text", txtname) == 0 ||
strcmp(".sched.text", txtname) == 0 ||
--
2.28.0
|