Skip to content

Commit 09cea61

Browse files
Kefeng Wangtorvalds
Kefeng Wang
authored andcommitted
arm64: support page mapping percpu first chunk allocator
Percpu embedded first chunk allocator is the firstly option, but it could fails on ARM64, eg, percpu: max_distance=0x5fcfdc640000 too large for vmalloc space 0x781fefff0000 percpu: max_distance=0x600000540000 too large for vmalloc space 0x7dffb7ff0000 percpu: max_distance=0x5fff9adb0000 too large for vmalloc space 0x5dffb7ff0000 then we could get WARNING: CPU: 15 PID: 461 at vmalloc.c:3087 pcpu_get_vm_areas+0x488/0x838 and the system could not boot successfully. Let's implement page mapping percpu first chunk allocator as a fallback to the embedding allocator to increase the robustness of the system. Link: https://lkml.kernel.org/r/20210910053354.26721-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> Cc: Andrey Konovalov <andreyknvl@gmail.com> Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com> Cc: Dmitry Vyukov <dvyukov@google.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Marco Elver <elver@google.com> Cc: Will Deacon <will@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent 0eb6843 commit 09cea61

File tree

2 files changed

+76
-10
lines changed

2 files changed

+76
-10
lines changed

arch/arm64/Kconfig

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1042,6 +1042,10 @@ config NEED_PER_CPU_EMBED_FIRST_CHUNK
10421042
def_bool y
10431043
depends on NUMA
10441044

1045+
config NEED_PER_CPU_PAGE_FIRST_CHUNK
1046+
def_bool y
1047+
depends on NUMA
1048+
10451049
source "kernel/Kconfig.hz"
10461050

10471051
config ARCH_SPARSEMEM_ENABLE

drivers/base/arch_numa.c

Lines changed: 72 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <linux/of.h>
1515

1616
#include <asm/sections.h>
17+
#include <asm/pgalloc.h>
1718

1819
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
1920
EXPORT_SYMBOL(node_data);
@@ -168,22 +169,83 @@ static void __init pcpu_fc_free(void *ptr, size_t size)
168169
memblock_free_early(__pa(ptr), size);
169170
}
170171

172+
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
173+
static void __init pcpu_populate_pte(unsigned long addr)
174+
{
175+
pgd_t *pgd = pgd_offset_k(addr);
176+
p4d_t *p4d;
177+
pud_t *pud;
178+
pmd_t *pmd;
179+
180+
p4d = p4d_offset(pgd, addr);
181+
if (p4d_none(*p4d)) {
182+
pud_t *new;
183+
184+
new = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
185+
if (!new)
186+
goto err_alloc;
187+
p4d_populate(&init_mm, p4d, new);
188+
}
189+
190+
pud = pud_offset(p4d, addr);
191+
if (pud_none(*pud)) {
192+
pmd_t *new;
193+
194+
new = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
195+
if (!new)
196+
goto err_alloc;
197+
pud_populate(&init_mm, pud, new);
198+
}
199+
200+
pmd = pmd_offset(pud, addr);
201+
if (!pmd_present(*pmd)) {
202+
pte_t *new;
203+
204+
new = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
205+
if (!new)
206+
goto err_alloc;
207+
pmd_populate_kernel(&init_mm, pmd, new);
208+
}
209+
210+
return;
211+
212+
err_alloc:
213+
panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",
214+
__func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
215+
}
216+
#endif
217+
171218
void __init setup_per_cpu_areas(void)
172219
{
173220
unsigned long delta;
174221
unsigned int cpu;
175-
int rc;
222+
int rc = -EINVAL;
223+
224+
if (pcpu_chosen_fc != PCPU_FC_PAGE) {
225+
/*
226+
* Always reserve area for module percpu variables. That's
227+
* what the legacy allocator did.
228+
*/
229+
rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
230+
PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
231+
pcpu_cpu_distance,
232+
pcpu_fc_alloc, pcpu_fc_free);
233+
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
234+
if (rc < 0)
235+
pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n",
236+
pcpu_fc_names[pcpu_chosen_fc], rc);
237+
#endif
238+
}
176239

177-
/*
178-
* Always reserve area for module percpu variables. That's
179-
* what the legacy allocator did.
180-
*/
181-
rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
182-
PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
183-
pcpu_cpu_distance,
184-
pcpu_fc_alloc, pcpu_fc_free);
240+
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
241+
if (rc < 0)
242+
rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE,
243+
pcpu_fc_alloc,
244+
pcpu_fc_free,
245+
pcpu_populate_pte);
246+
#endif
185247
if (rc < 0)
186-
panic("Failed to initialize percpu areas.");
248+
panic("Failed to initialize percpu areas (err=%d).", rc);
187249

188250
delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
189251
for_each_possible_cpu(cpu)

0 commit comments

Comments
 (0)