1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Sultan Alsawaf <sultan@kerneltoast.com>
Date: Sun, 19 Apr 2020 19:59:18 -0700
Subject: [PATCH] ZEN: mm: Stop kswapd early when nothing's waiting for it to
free pages
Contains:
- mm: Stop kswapd early when nothing's waiting for it to free pages
Keeping kswapd running when all the failed allocations that invoked it
are satisfied incurs a high overhead due to unnecessary page eviction
and writeback, as well as spurious VM pressure events to various
registered shrinkers. When kswapd doesn't need to work to make an
allocation succeed anymore, stop it prematurely to save resources.
Signed-off-by: Sultan Alsawaf <sultan@kerneltoast.com>
- mm: Don't stop kswapd on a per-node basis when there are no waiters
The page allocator wakes all kswapds in an allocation context's allowed
nodemask in the slow path, so it doesn't make sense to have the kswapd-
waiter count per each NUMA node. Instead, it should be a global counter
to stop all kswapds when there are no failed allocation requests.
Signed-off-by: Sultan Alsawaf <sultan@kerneltoast.com>
---
mm/internal.h | 1 +
mm/page_alloc.c | 17 ++++++++++++++---
mm/vmscan.c | 3 ++-
3 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/mm/internal.h b/mm/internal.h
index c0f8fbe0445b5f1704c41e2a4f2b664456b9768f..cd447101241b1e7245c84cf25c2fba4d56c1df72 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -361,6 +361,7 @@ extern void prep_compound_page(struct page *page, unsigned int order);
extern void post_alloc_hook(struct page *page, unsigned int order,
gfp_t gfp_flags);
extern int user_min_free_kbytes;
+extern atomic_long_t kswapd_waiters;
extern void free_unref_page(struct page *page, unsigned int order);
extern void free_unref_page_list(struct list_head *list);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d6fc4723c4e9137a3bf8193fcbad1bb23f041c43..d86f3da635751e00a5ca5f9c386415523d2ff40a 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -122,6 +122,8 @@ typedef int __bitwise fpi_t;
*/
#define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2))
+atomic_long_t kswapd_waiters = ATOMIC_LONG_INIT(0);
+
/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
static DEFINE_MUTEX(pcp_batch_high_lock);
#define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)
@@ -4921,6 +4923,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
int no_progress_loops;
unsigned int cpuset_mems_cookie;
int reserve_flags;
+ bool woke_kswapd = false;
/*
* We also sanity check to catch abuse of atomic reserves being used by
@@ -4967,8 +4970,13 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
goto nopage;
}
- if (alloc_flags & ALLOC_KSWAPD)
+ if (alloc_flags & ALLOC_KSWAPD) {
+ if (!woke_kswapd) {
+ atomic_long_inc(&kswapd_waiters);
+ woke_kswapd = true;
+ }
wake_all_kswapds(order, gfp_mask, ac);
+ }
/*
* The adjusted alloc_flags might result in immediate success, so try
@@ -5173,9 +5181,12 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
goto retry;
}
fail:
- warn_alloc(gfp_mask, ac->nodemask,
- "page allocation failure: order:%u", order);
got_pg:
+ if (woke_kswapd)
+ atomic_long_dec(&kswapd_waiters);
+ if (!page)
+ warn_alloc(gfp_mask, ac->nodemask,
+ "page allocation failure: order:%u", order);
return page;
}
diff --git a/mm/vmscan.c b/mm/vmscan.c
index f7d9a683e3a7d38fe1ffd0265b9f7d7acad938b4..c1936a256ed1e858febfe8f090f889f4a6238ff0 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4228,7 +4228,8 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
__fs_reclaim_release(_THIS_IP_);
ret = try_to_freeze();
__fs_reclaim_acquire(_THIS_IP_);
- if (ret || kthread_should_stop())
+ if (ret || kthread_should_stop() ||
+ !atomic_long_read(&kswapd_waiters))
break;
/*
|