1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
|
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Sultan Alsawaf <sultan@kerneltoast.com>
Date: Fri, 16 Jul 2021 23:35:47 -0700
Subject: [PATCH] ZEN: mm: Increment kswapd_waiters for throttled direct
reclaimers
Throttled direct reclaimers will wake up kswapd and wait for kswapd to
satisfy their page allocation request, even when the failed allocation
lacks the __GFP_KSWAPD_RECLAIM flag in its gfp mask. As a result, kswapd
may think that there are no waiters and thus exit prematurely, causing
throttled direct reclaimers lacking __GFP_KSWAPD_RECLAIM to stall on
waiting for kswapd to wake them up. Incrementing the kswapd_waiters
counter when such direct reclaimers become throttled fixes the problem.
Signed-off-by: Sultan Alsawaf <sultan@kerneltoast.com>
---
mm/vmscan.c | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 3d8cbc64af8c..478df26db0e9 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3658,7 +3658,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
return 0;
}
-static bool allow_direct_reclaim(pg_data_t *pgdat)
+static bool allow_direct_reclaim(pg_data_t *pgdat, bool using_kswapd)
{
struct zone *zone;
unsigned long pfmemalloc_reserve = 0;
@@ -3687,6 +3687,10 @@ static bool allow_direct_reclaim(pg_data_t *pgdat)
wmark_ok = free_pages > pfmemalloc_reserve / 2;
+ /* The throttled direct reclaimer is now a kswapd waiter */
+ if (unlikely(!using_kswapd && !wmark_ok))
+ atomic_long_inc(&kswapd_waiters);
+
/* kswapd must be awake if processes are being throttled */
if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
@@ -3752,7 +3756,7 @@ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
/* Throttle based on the first usable node */
pgdat = zone->zone_pgdat;
- if (allow_direct_reclaim(pgdat))
+ if (allow_direct_reclaim(pgdat, gfp_mask & __GFP_KSWAPD_RECLAIM))
goto out;
break;
}
@@ -3774,11 +3778,14 @@ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
*/
if (!(gfp_mask & __GFP_FS))
wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
- allow_direct_reclaim(pgdat), HZ);
+ allow_direct_reclaim(pgdat, true), HZ);
else
/* Throttle until kswapd wakes the process */
wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
- allow_direct_reclaim(pgdat));
+ allow_direct_reclaim(pgdat, true));
+
+ if (unlikely(!(gfp_mask & __GFP_KSWAPD_RECLAIM)))
+ atomic_long_dec(&kswapd_waiters);
if (fatal_signal_pending(current))
return true;
@@ -4268,7 +4275,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
* able to safely make forward progress. Wake them
*/
if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
- allow_direct_reclaim(pgdat))
+ allow_direct_reclaim(pgdat, true))
wake_up_all(&pgdat->pfmemalloc_wait);
/* Check if kswapd should be suspending */
--
2.35.1
|