blob: 6d0863db3e41fcc90eced432166e6f675c4fb7c7 [file] [log] [blame]
Thomas Gleixnerc942fdd2019-05-27 08:55:06 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Balbir Singh8cdea7c2008-02-07 00:13:50 -08002/* memcontrol.c - Memory Controller
3 *
4 * Copyright IBM Corporation, 2007
5 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08007 * Copyright 2007 OpenVZ SWsoft Inc
8 * Author: Pavel Emelianov <xemul@openvz.org>
9 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080010 * Memory thresholds
11 * Copyright (C) 2009 Nokia Corporation
12 * Author: Kirill A. Shutemov
13 *
Glauber Costa7ae1e1d2012-12-18 14:21:56 -080014 * Kernel Memory Controller
15 * Copyright (C) 2012 Parallels Inc. and Google Inc.
16 * Authors: Glauber Costa and Suleiman Souhlal
17 *
Johannes Weiner1575e682015-04-14 15:44:51 -070018 * Native page reclaim
19 * Charge lifetime sanitation
20 * Lockless page tracking & accounting
21 * Unified hierarchy configuration model
22 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
Alex Shi6168d0d2020-12-15 12:34:29 -080023 *
24 * Per memcg lru locking
25 * Copyright (C) 2020 Alibaba, Inc, Alex Shi
Balbir Singh8cdea7c2008-02-07 00:13:50 -080026 */
27
Johannes Weiner3e32cb22014-12-10 15:42:31 -080028#include <linux/page_counter.h>
Balbir Singh8cdea7c2008-02-07 00:13:50 -080029#include <linux/memcontrol.h>
30#include <linux/cgroup.h>
Christoph Hellwiga5201102019-08-28 16:19:53 +020031#include <linux/pagewalk.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010032#include <linux/sched/mm.h>
Hugh Dickins3a4f8a02017-02-24 14:59:36 -080033#include <linux/shmem_fs.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080034#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080035#include <linux/pagemap.h>
Chris Down1ff9e6e2019-03-05 15:48:09 -080036#include <linux/vm_event_item.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080037#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080038#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080039#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080040#include <linux/bit_spinlock.h>
41#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070042#include <linux/limits.h>
Paul Gortmakerb9e15ba2011-05-26 16:00:52 -040043#include <linux/export.h>
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -080044#include <linux/mutex.h>
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -070045#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070046#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080047#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080048#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080049#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080050#include <linux/eventfd.h>
Tejun Heo79bd9812013-11-22 18:20:42 -050051#include <linux/poll.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080052#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080053#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080054#include <linux/seq_file.h>
Anton Vorontsov70ddf632013-04-29 15:08:31 -070055#include <linux/vmpressure.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070056#include <linux/mm_inline.h>
Johannes Weiner5d1ea482014-12-10 15:44:55 -080057#include <linux/swap_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080058#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070059#include <linux/oom.h>
Johannes Weiner0056f4e2013-10-31 16:34:14 -070060#include <linux/lockdep.h>
Tejun Heo79bd9812013-11-22 18:20:42 -050061#include <linux/file.h>
Tejun Heob23afb92015-11-05 18:46:11 -080062#include <linux/tracehook.h>
Chris Down0e4b01d2019-09-23 15:34:55 -070063#include <linux/psi.h>
Johannes Weinerc8713d02019-07-11 20:55:59 -070064#include <linux/seq_buf.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080065#include "internal.h"
Glauber Costad1a4c0b2011-12-11 21:47:04 +000066#include <net/sock.h>
Michal Hocko4bd2c1e2012-10-08 16:33:10 -070067#include <net/ip.h>
Qiang Huangf35c3a82013-11-12 15:08:22 -080068#include "slab.h"
Balbir Singh8cdea7c2008-02-07 00:13:50 -080069
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080070#include <linux/uaccess.h>
Balbir Singh8697d332008-02-07 00:13:59 -080071
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070072#include <trace/events/vmscan.h>
73
Tejun Heo073219e2014-02-08 10:36:58 -050074struct cgroup_subsys memory_cgrp_subsys __read_mostly;
75EXPORT_SYMBOL(memory_cgrp_subsys);
David Rientjes68ae5642012-12-12 13:51:57 -080076
Johannes Weiner7d828602016-01-14 15:20:56 -080077struct mem_cgroup *root_mem_cgroup __read_mostly;
78
Roman Gushchin37d59852020-10-17 16:13:50 -070079/* Active memory cgroup to use from an interrupt context */
80DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
81
Johannes Weinerf7e1cb62016-01-14 15:21:29 -080082/* Socket memory accounting disabled? */
83static bool cgroup_memory_nosocket;
84
Vladimir Davydov04823c82016-01-20 15:02:38 -080085/* Kernel memory accounting disabled? */
86static bool cgroup_memory_nokmem;
87
Johannes Weiner21afa382015-02-11 15:26:36 -080088/* Whether the swap controller is active */
Andrew Mortonc255a452012-07-31 16:43:02 -070089#ifdef CONFIG_MEMCG_SWAP
Johannes Weinereccb52e2020-06-03 16:02:11 -070090bool cgroup_memory_noswap __read_mostly;
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080091#else
Johannes Weinereccb52e2020-06-03 16:02:11 -070092#define cgroup_memory_noswap 1
Johannes Weiner2d1c4982020-06-03 16:02:14 -070093#endif
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080094
Tejun Heo97b27822019-08-26 09:06:56 -070095#ifdef CONFIG_CGROUP_WRITEBACK
96static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
97#endif
98
Johannes Weiner7941d212016-01-14 15:21:23 -080099/* Whether legacy memory+swap accounting is active */
100static bool do_memsw_account(void)
101{
Johannes Weinereccb52e2020-06-03 16:02:11 -0700102 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
Johannes Weiner7941d212016-01-14 15:21:23 -0800103}
104
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700105#define THRESHOLDS_EVENTS_TARGET 128
106#define SOFTLIMIT_EVENTS_TARGET 1024
Johannes Weinere9f89742011-03-23 16:42:37 -0700107
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700108/*
109 * Cgroups above their limits are maintained in a RB-Tree, independent of
110 * their hierarchy representation
111 */
112
Mel Gormanef8f2322016-07-28 15:46:05 -0700113struct mem_cgroup_tree_per_node {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700114 struct rb_root rb_root;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700115 struct rb_node *rb_rightmost;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700116 spinlock_t lock;
117};
118
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700119struct mem_cgroup_tree {
120 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
121};
122
123static struct mem_cgroup_tree soft_limit_tree __read_mostly;
124
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700125/* for OOM */
126struct mem_cgroup_eventfd_list {
127 struct list_head list;
128 struct eventfd_ctx *eventfd;
129};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800130
Tejun Heo79bd9812013-11-22 18:20:42 -0500131/*
132 * cgroup_event represents events which userspace want to receive.
133 */
Tejun Heo3bc942f2013-11-22 18:20:44 -0500134struct mem_cgroup_event {
Tejun Heo79bd9812013-11-22 18:20:42 -0500135 /*
Tejun Heo59b6f872013-11-22 18:20:43 -0500136 * memcg which the event belongs to.
Tejun Heo79bd9812013-11-22 18:20:42 -0500137 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500138 struct mem_cgroup *memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -0500139 /*
Tejun Heo79bd9812013-11-22 18:20:42 -0500140 * eventfd to signal userspace about the event.
141 */
142 struct eventfd_ctx *eventfd;
143 /*
144 * Each of these stored in a list by the cgroup.
145 */
146 struct list_head list;
147 /*
Tejun Heofba94802013-11-22 18:20:43 -0500148 * register_event() callback will be used to add new userspace
149 * waiter for changes related to this event. Use eventfd_signal()
150 * on eventfd to send notification to userspace.
151 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500152 int (*register_event)(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -0500153 struct eventfd_ctx *eventfd, const char *args);
Tejun Heofba94802013-11-22 18:20:43 -0500154 /*
155 * unregister_event() callback will be called when userspace closes
156 * the eventfd or on cgroup removing. This callback must be set,
157 * if you want provide notification functionality.
158 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500159 void (*unregister_event)(struct mem_cgroup *memcg,
Tejun Heofba94802013-11-22 18:20:43 -0500160 struct eventfd_ctx *eventfd);
161 /*
Tejun Heo79bd9812013-11-22 18:20:42 -0500162 * All fields below needed to unregister event when
163 * userspace closes eventfd.
164 */
165 poll_table pt;
166 wait_queue_head_t *wqh;
Ingo Molnarac6424b2017-06-20 12:06:13 +0200167 wait_queue_entry_t wait;
Tejun Heo79bd9812013-11-22 18:20:42 -0500168 struct work_struct remove;
169};
170
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700171static void mem_cgroup_threshold(struct mem_cgroup *memcg);
172static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800173
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800174/* Stuffs for move charges at task migration. */
175/*
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800176 * Types of charges to be moved.
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800177 */
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800178#define MOVE_ANON 0x1U
179#define MOVE_FILE 0x2U
180#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800181
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800182/* "mc" and its members are protected by cgroup_mutex */
183static struct move_charge_struct {
Daisuke Nishimurab1dd6932010-11-24 12:57:06 -0800184 spinlock_t lock; /* for from, to */
Tejun Heo264a0ae2016-04-21 19:09:02 -0400185 struct mm_struct *mm;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800186 struct mem_cgroup *from;
187 struct mem_cgroup *to;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800188 unsigned long flags;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800189 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800190 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800191 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800192 struct task_struct *moving_task; /* a task moving charges */
193 wait_queue_head_t waitq; /* a waitq for other context */
194} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700195 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800196 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
197};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800198
Balbir Singh4e416952009-09-23 15:56:39 -0700199/*
200 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
201 * limit reclaim to prevent infinite loops, if they ever occur.
202 */
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700203#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700204#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
Balbir Singh4e416952009-09-23 15:56:39 -0700205
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800206/* for encoding cft->private value on file */
Glauber Costa86ae53e2012-12-18 14:21:45 -0800207enum res_type {
208 _MEM,
209 _MEMSWAP,
210 _OOM_TYPE,
Glauber Costa510fc4e2012-12-18 14:21:47 -0800211 _KMEM,
Vladimir Davydovd55f90b2016-01-20 15:02:44 -0800212 _TCP,
Glauber Costa86ae53e2012-12-18 14:21:45 -0800213};
214
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700215#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
216#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800217#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700218/* Used for OOM nofiier */
219#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800220
Kirill Tkhaib05706f2018-08-17 15:47:33 -0700221/*
222 * Iteration constructs for visiting all cgroups (under a tree). If
223 * loops are exited prematurely (break), mem_cgroup_iter_break() must
224 * be used for reference counting.
225 */
226#define for_each_mem_cgroup_tree(iter, root) \
227 for (iter = mem_cgroup_iter(root, NULL, NULL); \
228 iter != NULL; \
229 iter = mem_cgroup_iter(root, iter, NULL))
230
231#define for_each_mem_cgroup(iter) \
232 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
233 iter != NULL; \
234 iter = mem_cgroup_iter(NULL, iter, NULL))
235
Tetsuo Handa7775fac2019-03-05 15:46:47 -0800236static inline bool should_force_charge(void)
237{
238 return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
239 (current->flags & PF_EXITING);
240}
241
Anton Vorontsov70ddf632013-04-29 15:08:31 -0700242/* Some nice accessors for the vmpressure. */
243struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
244{
245 if (!memcg)
246 memcg = root_mem_cgroup;
247 return &memcg->vmpressure;
248}
249
250struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
251{
252 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
253}
254
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700255#ifdef CONFIG_MEMCG_KMEM
Roman Gushchinbf4f0592020-08-06 23:20:49 -0700256extern spinlock_t css_set_lock;
257
Roman Gushchinc1a660d2021-02-24 12:03:58 -0800258static int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
259 unsigned int nr_pages);
260static void __memcg_kmem_uncharge(struct mem_cgroup *memcg,
261 unsigned int nr_pages);
262
Roman Gushchinbf4f0592020-08-06 23:20:49 -0700263static void obj_cgroup_release(struct percpu_ref *ref)
264{
265 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
266 struct mem_cgroup *memcg;
267 unsigned int nr_bytes;
268 unsigned int nr_pages;
269 unsigned long flags;
270
271 /*
272 * At this point all allocated objects are freed, and
273 * objcg->nr_charged_bytes can't have an arbitrary byte value.
274 * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
275 *
276 * The following sequence can lead to it:
277 * 1) CPU0: objcg == stock->cached_objcg
278 * 2) CPU1: we do a small allocation (e.g. 92 bytes),
279 * PAGE_SIZE bytes are charged
280 * 3) CPU1: a process from another memcg is allocating something,
281 * the stock if flushed,
282 * objcg->nr_charged_bytes = PAGE_SIZE - 92
283 * 5) CPU0: we do release this object,
284 * 92 bytes are added to stock->nr_bytes
285 * 6) CPU0: stock is flushed,
286 * 92 bytes are added to objcg->nr_charged_bytes
287 *
288 * In the result, nr_charged_bytes == PAGE_SIZE.
289 * This page will be uncharged in obj_cgroup_release().
290 */
291 nr_bytes = atomic_read(&objcg->nr_charged_bytes);
292 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
293 nr_pages = nr_bytes >> PAGE_SHIFT;
294
295 spin_lock_irqsave(&css_set_lock, flags);
296 memcg = obj_cgroup_memcg(objcg);
297 if (nr_pages)
298 __memcg_kmem_uncharge(memcg, nr_pages);
299 list_del(&objcg->list);
300 mem_cgroup_put(memcg);
301 spin_unlock_irqrestore(&css_set_lock, flags);
302
303 percpu_ref_exit(ref);
304 kfree_rcu(objcg, rcu);
305}
306
307static struct obj_cgroup *obj_cgroup_alloc(void)
308{
309 struct obj_cgroup *objcg;
310 int ret;
311
312 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
313 if (!objcg)
314 return NULL;
315
316 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
317 GFP_KERNEL);
318 if (ret) {
319 kfree(objcg);
320 return NULL;
321 }
322 INIT_LIST_HEAD(&objcg->list);
323 return objcg;
324}
325
326static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
327 struct mem_cgroup *parent)
328{
329 struct obj_cgroup *objcg, *iter;
330
331 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
332
333 spin_lock_irq(&css_set_lock);
334
335 /* Move active objcg to the parent's list */
336 xchg(&objcg->memcg, parent);
337 css_get(&parent->css);
338 list_add(&objcg->list, &parent->objcg_list);
339
340 /* Move already reparented objcgs to the parent's list */
341 list_for_each_entry(iter, &memcg->objcg_list, list) {
342 css_get(&parent->css);
343 xchg(&iter->memcg, parent);
344 css_put(&memcg->css);
345 }
346 list_splice(&memcg->objcg_list, &parent->objcg_list);
347
348 spin_unlock_irq(&css_set_lock);
349
350 percpu_ref_kill(&objcg->refcnt);
351}
352
Glauber Costa55007d82012-12-18 14:22:38 -0800353/*
Roman Gushchin98556092020-08-06 23:21:10 -0700354 * This will be used as a shrinker list's index.
Li Zefanb8627832013-09-23 16:56:47 +0800355 * The main reason for not using cgroup id for this:
356 * this works better in sparse environments, where we have a lot of memcgs,
357 * but only a few kmem-limited. Or also, if we have, for instance, 200
358 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
359 * 200 entry array for that.
Glauber Costa55007d82012-12-18 14:22:38 -0800360 *
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -0800361 * The current size of the caches array is stored in memcg_nr_cache_ids. It
362 * will double each time we have to increase it.
Glauber Costa55007d82012-12-18 14:22:38 -0800363 */
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -0800364static DEFINE_IDA(memcg_cache_ida);
365int memcg_nr_cache_ids;
Glauber Costa749c5412012-12-18 14:23:01 -0800366
Vladimir Davydov05257a12015-02-12 14:59:01 -0800367/* Protects memcg_nr_cache_ids */
368static DECLARE_RWSEM(memcg_cache_ids_sem);
369
370void memcg_get_cache_ids(void)
371{
372 down_read(&memcg_cache_ids_sem);
373}
374
375void memcg_put_cache_ids(void)
376{
377 up_read(&memcg_cache_ids_sem);
378}
379
Glauber Costa55007d82012-12-18 14:22:38 -0800380/*
381 * MIN_SIZE is different than 1, because we would like to avoid going through
382 * the alloc/free process all the time. In a small machine, 4 kmem-limited
383 * cgroups is a reasonable guess. In the future, it could be a parameter or
384 * tunable, but that is strictly not necessary.
385 *
Li Zefanb8627832013-09-23 16:56:47 +0800386 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
Glauber Costa55007d82012-12-18 14:22:38 -0800387 * this constant directly from cgroup, but it is understandable that this is
388 * better kept as an internal representation in cgroup.c. In any case, the
Li Zefanb8627832013-09-23 16:56:47 +0800389 * cgrp_id space is not getting any smaller, and we don't have to necessarily
Glauber Costa55007d82012-12-18 14:22:38 -0800390 * increase ours as well if it increases.
391 */
392#define MEMCG_CACHES_MIN_SIZE 4
Li Zefanb8627832013-09-23 16:56:47 +0800393#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
Glauber Costa55007d82012-12-18 14:22:38 -0800394
Glauber Costad7f25f82012-12-18 14:22:40 -0800395/*
396 * A lot of the calls to the cache allocation functions are expected to be
Roman Gushchin272911a2020-08-06 23:21:17 -0700397 * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are
Glauber Costad7f25f82012-12-18 14:22:40 -0800398 * conditional to this static branch, we'll have to allow modules that does
399 * kmem_cache_alloc and the such to see this symbol as well
400 */
Johannes Weineref129472016-01-14 15:21:34 -0800401DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
Glauber Costad7f25f82012-12-18 14:22:40 -0800402EXPORT_SYMBOL(memcg_kmem_enabled_key);
Yang Shi0a432dc2019-09-23 15:38:12 -0700403#endif
Tejun Heo17cc4df2017-02-22 15:41:36 -0800404
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700405static int memcg_shrinker_map_size;
406static DEFINE_MUTEX(memcg_shrinker_map_mutex);
407
408static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
409{
410 kvfree(container_of(head, struct memcg_shrinker_map, rcu));
411}
412
413static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
414 int size, int old_size)
415{
416 struct memcg_shrinker_map *new, *old;
Johannes Weinera3747b52021-04-29 22:56:14 -0700417 struct mem_cgroup_per_node *pn;
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700418 int nid;
419
420 lockdep_assert_held(&memcg_shrinker_map_mutex);
421
422 for_each_node(nid) {
Johannes Weinera3747b52021-04-29 22:56:14 -0700423 pn = memcg->nodeinfo[nid];
424 old = rcu_dereference_protected(pn->shrinker_map, true);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700425 /* Not yet online memcg */
426 if (!old)
427 return 0;
428
Kirill Tkhai86daf942020-04-01 21:06:33 -0700429 new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700430 if (!new)
431 return -ENOMEM;
432
433 /* Set all old bits, clear all new bits */
434 memset(new->map, (int)0xff, old_size);
435 memset((void *)new->map + old_size, 0, size - old_size);
436
Johannes Weinera3747b52021-04-29 22:56:14 -0700437 rcu_assign_pointer(pn->shrinker_map, new);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700438 call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
439 }
440
441 return 0;
442}
443
444static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
445{
446 struct mem_cgroup_per_node *pn;
447 struct memcg_shrinker_map *map;
448 int nid;
449
450 if (mem_cgroup_is_root(memcg))
451 return;
452
453 for_each_node(nid) {
Johannes Weinera3747b52021-04-29 22:56:14 -0700454 pn = memcg->nodeinfo[nid];
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700455 map = rcu_dereference_protected(pn->shrinker_map, true);
Yang Li8a260162021-02-24 12:04:05 -0800456 kvfree(map);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700457 rcu_assign_pointer(pn->shrinker_map, NULL);
458 }
459}
460
461static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
462{
463 struct memcg_shrinker_map *map;
464 int nid, size, ret = 0;
465
466 if (mem_cgroup_is_root(memcg))
467 return 0;
468
469 mutex_lock(&memcg_shrinker_map_mutex);
470 size = memcg_shrinker_map_size;
471 for_each_node(nid) {
Kirill Tkhai86daf942020-04-01 21:06:33 -0700472 map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700473 if (!map) {
474 memcg_free_shrinker_maps(memcg);
475 ret = -ENOMEM;
476 break;
477 }
478 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
479 }
480 mutex_unlock(&memcg_shrinker_map_mutex);
481
482 return ret;
483}
484
485int memcg_expand_shrinker_maps(int new_id)
486{
487 int size, old_size, ret = 0;
488 struct mem_cgroup *memcg;
489
490 size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
491 old_size = memcg_shrinker_map_size;
492 if (size <= old_size)
493 return 0;
494
495 mutex_lock(&memcg_shrinker_map_mutex);
496 if (!root_mem_cgroup)
497 goto unlock;
498
499 for_each_mem_cgroup(memcg) {
500 if (mem_cgroup_is_root(memcg))
501 continue;
502 ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
Vasily Averin75866af2020-02-20 20:04:18 -0800503 if (ret) {
504 mem_cgroup_iter_break(NULL, memcg);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700505 goto unlock;
Vasily Averin75866af2020-02-20 20:04:18 -0800506 }
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700507 }
508unlock:
509 if (!ret)
510 memcg_shrinker_map_size = size;
511 mutex_unlock(&memcg_shrinker_map_mutex);
512 return ret;
513}
Kirill Tkhaifae91d62018-08-17 15:48:10 -0700514
515void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
516{
517 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
518 struct memcg_shrinker_map *map;
519
520 rcu_read_lock();
521 map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
Kirill Tkhaif90280d2018-08-17 15:48:25 -0700522 /* Pairs with smp mb in shrink_slab() */
523 smp_mb__before_atomic();
Kirill Tkhaifae91d62018-08-17 15:48:10 -0700524 set_bit(shrinker_id, map->map);
525 rcu_read_unlock();
526 }
527}
528
Tejun Heoad7fa852015-05-27 20:00:02 -0400529/**
530 * mem_cgroup_css_from_page - css of the memcg associated with a page
531 * @page: page of interest
532 *
533 * If memcg is bound to the default hierarchy, css of the memcg associated
534 * with @page is returned. The returned css remains associated with @page
535 * until it is released.
536 *
537 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
538 * is returned.
Tejun Heoad7fa852015-05-27 20:00:02 -0400539 */
540struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
541{
542 struct mem_cgroup *memcg;
543
Roman Gushchinbcfe06b2020-12-01 13:58:27 -0800544 memcg = page_memcg(page);
Tejun Heoad7fa852015-05-27 20:00:02 -0400545
Tejun Heo9e10a132015-09-18 11:56:28 -0400546 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Tejun Heoad7fa852015-05-27 20:00:02 -0400547 memcg = root_mem_cgroup;
548
Tejun Heoad7fa852015-05-27 20:00:02 -0400549 return &memcg->css;
550}
551
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700552/**
553 * page_cgroup_ino - return inode number of the memcg a page is charged to
554 * @page: the page
555 *
556 * Look up the closest online ancestor of the memory cgroup @page is charged to
557 * and return its inode number or 0 if @page is not charged to any cgroup. It
558 * is safe to call this function without holding a reference to @page.
559 *
560 * Note, this function is inherently racy, because there is nothing to prevent
561 * the cgroup inode from getting torn down and potentially reallocated a moment
562 * after page_cgroup_ino() returns, so it only should be used by callers that
563 * do not care (such as procfs interfaces).
564 */
565ino_t page_cgroup_ino(struct page *page)
566{
567 struct mem_cgroup *memcg;
568 unsigned long ino = 0;
569
570 rcu_read_lock();
Roman Gushchinbcfe06b2020-12-01 13:58:27 -0800571 memcg = page_memcg_check(page);
Roman Gushchin286e04b2020-08-06 23:20:52 -0700572
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700573 while (memcg && !(memcg->css.flags & CSS_ONLINE))
574 memcg = parent_mem_cgroup(memcg);
575 if (memcg)
576 ino = cgroup_ino(memcg->css.cgroup);
577 rcu_read_unlock();
578 return ino;
579}
580
Mel Gormanef8f2322016-07-28 15:46:05 -0700581static struct mem_cgroup_per_node *
582mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700583{
Johannes Weiner97a6c372011-03-23 16:42:27 -0700584 int nid = page_to_nid(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700585
Mel Gormanef8f2322016-07-28 15:46:05 -0700586 return memcg->nodeinfo[nid];
Balbir Singhf64c3f52009-09-23 15:56:37 -0700587}
588
Mel Gormanef8f2322016-07-28 15:46:05 -0700589static struct mem_cgroup_tree_per_node *
590soft_limit_tree_node(int nid)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700591{
Mel Gormanef8f2322016-07-28 15:46:05 -0700592 return soft_limit_tree.rb_tree_per_node[nid];
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700593}
594
Mel Gormanef8f2322016-07-28 15:46:05 -0700595static struct mem_cgroup_tree_per_node *
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700596soft_limit_tree_from_page(struct page *page)
597{
598 int nid = page_to_nid(page);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700599
Mel Gormanef8f2322016-07-28 15:46:05 -0700600 return soft_limit_tree.rb_tree_per_node[nid];
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700601}
602
Mel Gormanef8f2322016-07-28 15:46:05 -0700603static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
604 struct mem_cgroup_tree_per_node *mctz,
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800605 unsigned long new_usage_in_excess)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700606{
607 struct rb_node **p = &mctz->rb_root.rb_node;
608 struct rb_node *parent = NULL;
Mel Gormanef8f2322016-07-28 15:46:05 -0700609 struct mem_cgroup_per_node *mz_node;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700610 bool rightmost = true;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700611
612 if (mz->on_tree)
613 return;
614
615 mz->usage_in_excess = new_usage_in_excess;
616 if (!mz->usage_in_excess)
617 return;
618 while (*p) {
619 parent = *p;
Mel Gormanef8f2322016-07-28 15:46:05 -0700620 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700621 tree_node);
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700622 if (mz->usage_in_excess < mz_node->usage_in_excess) {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700623 p = &(*p)->rb_left;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700624 rightmost = false;
Miaohe Lin378876b2020-12-14 19:06:28 -0800625 } else {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700626 p = &(*p)->rb_right;
Miaohe Lin378876b2020-12-14 19:06:28 -0800627 }
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700628 }
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700629
630 if (rightmost)
631 mctz->rb_rightmost = &mz->tree_node;
632
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700633 rb_link_node(&mz->tree_node, parent, p);
634 rb_insert_color(&mz->tree_node, &mctz->rb_root);
635 mz->on_tree = true;
636}
637
Mel Gormanef8f2322016-07-28 15:46:05 -0700638static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
639 struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700640{
641 if (!mz->on_tree)
642 return;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700643
644 if (&mz->tree_node == mctz->rb_rightmost)
645 mctz->rb_rightmost = rb_prev(&mz->tree_node);
646
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700647 rb_erase(&mz->tree_node, &mctz->rb_root);
648 mz->on_tree = false;
649}
650
Mel Gormanef8f2322016-07-28 15:46:05 -0700651static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
652 struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700653{
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700654 unsigned long flags;
655
656 spin_lock_irqsave(&mctz->lock, flags);
Johannes Weinercf2c8122014-06-06 14:38:21 -0700657 __mem_cgroup_remove_exceeded(mz, mctz);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700658 spin_unlock_irqrestore(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700659}
660
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800661static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
662{
663 unsigned long nr_pages = page_counter_read(&memcg->memory);
Jason Low4db0c3c2015-04-15 16:14:08 -0700664 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800665 unsigned long excess = 0;
666
667 if (nr_pages > soft_limit)
668 excess = nr_pages - soft_limit;
669
670 return excess;
671}
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700672
673static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
674{
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800675 unsigned long excess;
Mel Gormanef8f2322016-07-28 15:46:05 -0700676 struct mem_cgroup_per_node *mz;
677 struct mem_cgroup_tree_per_node *mctz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700678
Jianyu Zhane2318752014-06-06 14:38:20 -0700679 mctz = soft_limit_tree_from_page(page);
Laurent Dufourbfc72282017-03-09 16:17:06 -0800680 if (!mctz)
681 return;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700682 /*
683 * Necessary to update all ancestors when hierarchy is used.
684 * because their event counter is not touched.
685 */
686 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
Mel Gormanef8f2322016-07-28 15:46:05 -0700687 mz = mem_cgroup_page_nodeinfo(memcg, page);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800688 excess = soft_limit_excess(memcg);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700689 /*
690 * We have to update the tree if mz is on RB-tree or
691 * mem is over its softlimit.
692 */
693 if (excess || mz->on_tree) {
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700694 unsigned long flags;
695
696 spin_lock_irqsave(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700697 /* if on-tree, remove it */
698 if (mz->on_tree)
Johannes Weinercf2c8122014-06-06 14:38:21 -0700699 __mem_cgroup_remove_exceeded(mz, mctz);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700700 /*
701 * Insert again. mz->usage_in_excess will be updated.
702 * If excess is 0, no tree ops.
703 */
Johannes Weinercf2c8122014-06-06 14:38:21 -0700704 __mem_cgroup_insert_exceeded(mz, mctz, excess);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700705 spin_unlock_irqrestore(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700706 }
707 }
708}
709
710static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
711{
Mel Gormanef8f2322016-07-28 15:46:05 -0700712 struct mem_cgroup_tree_per_node *mctz;
713 struct mem_cgroup_per_node *mz;
714 int nid;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700715
Jianyu Zhane2318752014-06-06 14:38:20 -0700716 for_each_node(nid) {
Johannes Weinera3747b52021-04-29 22:56:14 -0700717 mz = memcg->nodeinfo[nid];
Mel Gormanef8f2322016-07-28 15:46:05 -0700718 mctz = soft_limit_tree_node(nid);
Laurent Dufourbfc72282017-03-09 16:17:06 -0800719 if (mctz)
720 mem_cgroup_remove_exceeded(mz, mctz);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700721 }
722}
723
Mel Gormanef8f2322016-07-28 15:46:05 -0700724static struct mem_cgroup_per_node *
725__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700726{
Mel Gormanef8f2322016-07-28 15:46:05 -0700727 struct mem_cgroup_per_node *mz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700728
729retry:
730 mz = NULL;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700731 if (!mctz->rb_rightmost)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700732 goto done; /* Nothing to reclaim from */
733
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700734 mz = rb_entry(mctz->rb_rightmost,
735 struct mem_cgroup_per_node, tree_node);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700736 /*
737 * Remove the node now but someone else can add it back,
738 * we will to add it back at the end of reclaim to its correct
739 * position in the tree.
740 */
Johannes Weinercf2c8122014-06-06 14:38:21 -0700741 __mem_cgroup_remove_exceeded(mz, mctz);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800742 if (!soft_limit_excess(mz->memcg) ||
Shakeel Butt8965aa22020-04-01 21:07:10 -0700743 !css_tryget(&mz->memcg->css))
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700744 goto retry;
745done:
746 return mz;
747}
748
Mel Gormanef8f2322016-07-28 15:46:05 -0700749static struct mem_cgroup_per_node *
750mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700751{
Mel Gormanef8f2322016-07-28 15:46:05 -0700752 struct mem_cgroup_per_node *mz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700753
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700754 spin_lock_irq(&mctz->lock);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700755 mz = __mem_cgroup_largest_soft_limit_node(mctz);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700756 spin_unlock_irq(&mctz->lock);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700757 return mz;
758}
759
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700760/**
761 * __mod_memcg_state - update cgroup memory statistics
762 * @memcg: the memory cgroup
763 * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
764 * @val: delta to add to the counter, can be negative
765 */
766void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
767{
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700768 if (mem_cgroup_disabled())
769 return;
770
Johannes Weiner2d146aa2021-04-29 22:56:26 -0700771 __this_cpu_add(memcg->vmstats_percpu->state[idx], val);
772 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700773}
774
Johannes Weiner2d146aa2021-04-29 22:56:26 -0700775/* idx can be of type enum memcg_stat_item or node_stat_item. */
Johannes Weinera18e6e62021-04-29 22:56:17 -0700776static unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
777{
Johannes Weiner2d146aa2021-04-29 22:56:26 -0700778 long x = READ_ONCE(memcg->vmstats.state[idx]);
Johannes Weinera18e6e62021-04-29 22:56:17 -0700779#ifdef CONFIG_SMP
780 if (x < 0)
781 x = 0;
782#endif
783 return x;
784}
785
Johannes Weiner2d146aa2021-04-29 22:56:26 -0700786/* idx can be of type enum memcg_stat_item or node_stat_item. */
Johannes Weinera18e6e62021-04-29 22:56:17 -0700787static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
788{
789 long x = 0;
790 int cpu;
791
792 for_each_possible_cpu(cpu)
Johannes Weiner2d146aa2021-04-29 22:56:26 -0700793 x += per_cpu(memcg->vmstats_percpu->state[idx], cpu);
Johannes Weinera18e6e62021-04-29 22:56:17 -0700794#ifdef CONFIG_SMP
795 if (x < 0)
796 x = 0;
797#endif
798 return x;
799}
800
Johannes Weiner42a30032019-05-14 15:47:12 -0700801static struct mem_cgroup_per_node *
802parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
803{
804 struct mem_cgroup *parent;
805
806 parent = parent_mem_cgroup(pn->memcg);
807 if (!parent)
808 return NULL;
Johannes Weinera3747b52021-04-29 22:56:14 -0700809 return parent->nodeinfo[nid];
Johannes Weiner42a30032019-05-14 15:47:12 -0700810}
811
Roman Gushchineedc4e52020-08-06 23:20:32 -0700812void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
813 int val)
814{
815 struct mem_cgroup_per_node *pn;
816 struct mem_cgroup *memcg;
Roman Gushchinea426c22020-08-06 23:20:35 -0700817 long x, threshold = MEMCG_CHARGE_BATCH;
Roman Gushchineedc4e52020-08-06 23:20:32 -0700818
819 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
820 memcg = pn->memcg;
821
822 /* Update memcg */
823 __mod_memcg_state(memcg, idx, val);
824
825 /* Update lruvec */
826 __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
827
Roman Gushchinea426c22020-08-06 23:20:35 -0700828 if (vmstat_item_in_bytes(idx))
829 threshold <<= PAGE_SHIFT;
830
Roman Gushchineedc4e52020-08-06 23:20:32 -0700831 x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
Roman Gushchinea426c22020-08-06 23:20:35 -0700832 if (unlikely(abs(x) > threshold)) {
Roman Gushchineedc4e52020-08-06 23:20:32 -0700833 pg_data_t *pgdat = lruvec_pgdat(lruvec);
834 struct mem_cgroup_per_node *pi;
835
836 for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
837 atomic_long_add(x, &pi->lruvec_stat[idx]);
838 x = 0;
839 }
840 __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
841}
842
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700843/**
844 * __mod_lruvec_state - update lruvec memory statistics
845 * @lruvec: the lruvec
846 * @idx: the stat item
847 * @val: delta to add to the counter, can be negative
848 *
849 * The lruvec is the intersection of the NUMA node and a cgroup. This
850 * function updates the all three counters that are affected by a
851 * change of state at this level: per-node, per-cgroup, per-lruvec.
852 */
853void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
854 int val)
855{
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700856 /* Update node */
Roman Gushchineedc4e52020-08-06 23:20:32 -0700857 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700858
Roman Gushchineedc4e52020-08-06 23:20:32 -0700859 /* Update memcg and lruvec */
860 if (!mem_cgroup_disabled())
861 __mod_memcg_lruvec_state(lruvec, idx, val);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700862}
863
Shakeel Buttc47d5032020-12-14 19:07:14 -0800864void __mod_lruvec_page_state(struct page *page, enum node_stat_item idx,
865 int val)
866{
867 struct page *head = compound_head(page); /* rmap on tail pages */
Linus Torvaldsd635a692020-12-15 13:22:29 -0800868 struct mem_cgroup *memcg = page_memcg(head);
Shakeel Buttc47d5032020-12-14 19:07:14 -0800869 pg_data_t *pgdat = page_pgdat(page);
870 struct lruvec *lruvec;
871
872 /* Untracked pages have no memcg, no lruvec. Update only the node */
Linus Torvaldsd635a692020-12-15 13:22:29 -0800873 if (!memcg) {
Shakeel Buttc47d5032020-12-14 19:07:14 -0800874 __mod_node_page_state(pgdat, idx, val);
875 return;
876 }
877
Linus Torvaldsd635a692020-12-15 13:22:29 -0800878 lruvec = mem_cgroup_lruvec(memcg, pgdat);
Shakeel Buttc47d5032020-12-14 19:07:14 -0800879 __mod_lruvec_state(lruvec, idx, val);
880}
Shakeel Buttf0c0c112020-12-14 19:07:17 -0800881EXPORT_SYMBOL(__mod_lruvec_page_state);
Shakeel Buttc47d5032020-12-14 19:07:14 -0800882
Muchun Songda3ceef2020-12-14 19:07:04 -0800883void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val)
Roman Gushchinec9f0232019-08-13 15:37:41 -0700884{
Roman Gushchin4f103c62020-04-01 21:06:36 -0700885 pg_data_t *pgdat = page_pgdat(virt_to_page(p));
Roman Gushchinec9f0232019-08-13 15:37:41 -0700886 struct mem_cgroup *memcg;
887 struct lruvec *lruvec;
888
889 rcu_read_lock();
Roman Gushchin4f103c62020-04-01 21:06:36 -0700890 memcg = mem_cgroup_from_obj(p);
Roman Gushchinec9f0232019-08-13 15:37:41 -0700891
Muchun Song8faeb1f2020-11-21 22:17:12 -0800892 /*
893 * Untracked pages have no memcg, no lruvec. Update only the
894 * node. If we reparent the slab objects to the root memcg,
895 * when we free the slab object, we need to update the per-memcg
896 * vmstats to keep it correct for the root memcg.
897 */
898 if (!memcg) {
Roman Gushchinec9f0232019-08-13 15:37:41 -0700899 __mod_node_page_state(pgdat, idx, val);
900 } else {
Johannes Weiner867e5e12019-11-30 17:55:34 -0800901 lruvec = mem_cgroup_lruvec(memcg, pgdat);
Roman Gushchinec9f0232019-08-13 15:37:41 -0700902 __mod_lruvec_state(lruvec, idx, val);
903 }
904 rcu_read_unlock();
905}
906
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700907/**
908 * __count_memcg_events - account VM events in a cgroup
909 * @memcg: the memory cgroup
910 * @idx: the event item
911 * @count: the number of events that occured
912 */
913void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
914 unsigned long count)
915{
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700916 if (mem_cgroup_disabled())
917 return;
918
Johannes Weiner2d146aa2021-04-29 22:56:26 -0700919 __this_cpu_add(memcg->vmstats_percpu->events[idx], count);
920 cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id());
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700921}
922
Johannes Weiner42a30032019-05-14 15:47:12 -0700923static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
Johannes Weinere9f89742011-03-23 16:42:37 -0700924{
Johannes Weiner2d146aa2021-04-29 22:56:26 -0700925 return READ_ONCE(memcg->vmstats.events[event]);
Johannes Weinere9f89742011-03-23 16:42:37 -0700926}
927
Johannes Weiner42a30032019-05-14 15:47:12 -0700928static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
929{
Johannes Weiner815744d2019-06-13 15:55:46 -0700930 long x = 0;
931 int cpu;
932
933 for_each_possible_cpu(cpu)
Johannes Weiner2d146aa2021-04-29 22:56:26 -0700934 x += per_cpu(memcg->vmstats_percpu->events[event], cpu);
Johannes Weiner815744d2019-06-13 15:55:46 -0700935 return x;
Johannes Weiner42a30032019-05-14 15:47:12 -0700936}
937
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700938static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
David Rientjesb070e652013-05-07 16:18:09 -0700939 struct page *page,
Johannes Weiner3fba69a2020-06-03 16:01:31 -0700940 int nr_pages)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800941{
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800942 /* pagein of a big page is an event. So, ignore page size */
943 if (nr_pages > 0)
Johannes Weinerc9019e92018-01-31 16:16:37 -0800944 __count_memcg_events(memcg, PGPGIN, 1);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800945 else {
Johannes Weinerc9019e92018-01-31 16:16:37 -0800946 __count_memcg_events(memcg, PGPGOUT, 1);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800947 nr_pages = -nr_pages; /* for event */
948 }
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800949
Chris Down871789d2019-05-14 15:46:57 -0700950 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800951}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800952
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800953static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
954 enum mem_cgroup_events_target target)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800955{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700956 unsigned long val, next;
957
Chris Down871789d2019-05-14 15:46:57 -0700958 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
959 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700960 /* from time_after() in jiffies.h */
Michal Hocko6a1a8b82017-07-10 15:48:53 -0700961 if ((long)(next - val) < 0) {
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800962 switch (target) {
963 case MEM_CGROUP_TARGET_THRESH:
964 next = val + THRESHOLDS_EVENTS_TARGET;
965 break;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700966 case MEM_CGROUP_TARGET_SOFTLIMIT:
967 next = val + SOFTLIMIT_EVENTS_TARGET;
968 break;
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800969 default:
970 break;
971 }
Chris Down871789d2019-05-14 15:46:57 -0700972 __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800973 return true;
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700974 }
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800975 return false;
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800976}
977
978/*
979 * Check events in order.
980 *
981 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700982static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800983{
984 /* threshold event is triggered in finer grain than soft limit */
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800985 if (unlikely(mem_cgroup_event_ratelimit(memcg,
986 MEM_CGROUP_TARGET_THRESH))) {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700987 bool do_softlimit;
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800988
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700989 do_softlimit = mem_cgroup_event_ratelimit(memcg,
990 MEM_CGROUP_TARGET_SOFTLIMIT);
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800991 mem_cgroup_threshold(memcg);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700992 if (unlikely(do_softlimit))
993 mem_cgroup_update_tree(memcg, page);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700994 }
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800995}
996
Balbir Singhcf475ad2008-04-29 01:00:16 -0700997struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800998{
Balbir Singh31a78f22008-09-28 23:09:31 +0100999 /*
1000 * mm_update_next_owner() may clear mm->owner to NULL
1001 * if it races with swapoff, page migration, etc.
1002 * So this can be called with p == NULL.
1003 */
1004 if (unlikely(!p))
1005 return NULL;
1006
Tejun Heo073219e2014-02-08 10:36:58 -05001007 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001008}
Michal Hocko33398cf2015-09-08 15:01:02 -07001009EXPORT_SYMBOL(mem_cgroup_from_task);
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001010
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001011/**
1012 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
1013 * @mm: mm from which memcg should be extracted. It can be NULL.
1014 *
1015 * Obtain a reference on mm->memcg and returns it if successful. Otherwise
1016 * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
1017 * returned.
1018 */
1019struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001020{
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001021 struct mem_cgroup *memcg;
1022
1023 if (mem_cgroup_disabled())
1024 return NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001025
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001026 rcu_read_lock();
1027 do {
Michal Hocko6f6acb02014-05-22 11:54:19 -07001028 /*
1029 * Page cache insertions can happen withou an
1030 * actual mm context, e.g. during disk probing
1031 * on boot, loopback IO, acct() writes etc.
1032 */
1033 if (unlikely(!mm))
Johannes Weinerdf381972014-04-07 15:37:43 -07001034 memcg = root_mem_cgroup;
Michal Hocko6f6acb02014-05-22 11:54:19 -07001035 else {
1036 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1037 if (unlikely(!memcg))
1038 memcg = root_mem_cgroup;
1039 }
Roman Gushchin00d484f2019-11-15 17:34:43 -08001040 } while (!css_tryget(&memcg->css));
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001041 rcu_read_unlock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001042 return memcg;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001043}
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001044EXPORT_SYMBOL(get_mem_cgroup_from_mm);
1045
Roman Gushchin37d59852020-10-17 16:13:50 -07001046static __always_inline struct mem_cgroup *active_memcg(void)
1047{
1048 if (in_interrupt())
1049 return this_cpu_read(int_active_memcg);
1050 else
1051 return current->active_memcg;
1052}
1053
1054static __always_inline struct mem_cgroup *get_active_memcg(void)
1055{
1056 struct mem_cgroup *memcg;
1057
1058 rcu_read_lock();
1059 memcg = active_memcg();
Muchun Song1685bde2021-02-24 12:04:22 -08001060 /* remote memcg must hold a ref. */
1061 if (memcg && WARN_ON_ONCE(!css_tryget(&memcg->css)))
1062 memcg = root_mem_cgroup;
Roman Gushchin37d59852020-10-17 16:13:50 -07001063 rcu_read_unlock();
1064
1065 return memcg;
1066}
1067
Roman Gushchin4127c652020-10-17 16:13:53 -07001068static __always_inline bool memcg_kmem_bypass(void)
1069{
1070 /* Allow remote memcg charging from any context. */
1071 if (unlikely(active_memcg()))
1072 return false;
1073
1074 /* Memcg to charge can't be determined. */
1075 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
1076 return true;
1077
1078 return false;
1079}
1080
Shakeel Buttf745c6f2018-08-17 15:46:44 -07001081/**
Roman Gushchin37d59852020-10-17 16:13:50 -07001082 * If active memcg is set, do not fallback to current->mm->memcg.
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001083 */
1084static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
1085{
Roman Gushchin279c3392020-10-17 16:13:44 -07001086 if (memcg_kmem_bypass())
1087 return NULL;
1088
Roman Gushchin37d59852020-10-17 16:13:50 -07001089 if (unlikely(active_memcg()))
1090 return get_active_memcg();
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001091
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001092 return get_mem_cgroup_from_mm(current->mm);
1093}
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001094
Johannes Weiner56600482012-01-12 17:17:59 -08001095/**
1096 * mem_cgroup_iter - iterate over memory cgroup hierarchy
1097 * @root: hierarchy root
1098 * @prev: previously returned memcg, NULL on first invocation
1099 * @reclaim: cookie for shared reclaim walks, NULL for full walks
1100 *
1101 * Returns references to children of the hierarchy below @root, or
1102 * @root itself, or %NULL after a full round-trip.
1103 *
1104 * Caller must pass the return value in @prev on subsequent
1105 * invocations for reference counting, or use mem_cgroup_iter_break()
1106 * to cancel a hierarchy walk before the round-trip is complete.
1107 *
Miaohe Lin05bdc522020-10-13 16:52:45 -07001108 * Reclaimers can specify a node in @reclaim to divide up the memcgs
1109 * in the hierarchy among all concurrent reclaimers operating on the
1110 * same node.
Johannes Weiner56600482012-01-12 17:17:59 -08001111 */
Andrew Morton694fbc02013-09-24 15:27:37 -07001112struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
Johannes Weiner56600482012-01-12 17:17:59 -08001113 struct mem_cgroup *prev,
Andrew Morton694fbc02013-09-24 15:27:37 -07001114 struct mem_cgroup_reclaim_cookie *reclaim)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07001115{
Kees Cook3f649ab2020-06-03 13:09:38 -07001116 struct mem_cgroup_reclaim_iter *iter;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001117 struct cgroup_subsys_state *css = NULL;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001118 struct mem_cgroup *memcg = NULL;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001119 struct mem_cgroup *pos = NULL;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001120
Andrew Morton694fbc02013-09-24 15:27:37 -07001121 if (mem_cgroup_disabled())
1122 return NULL;
Johannes Weiner56600482012-01-12 17:17:59 -08001123
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001124 if (!root)
1125 root = root_mem_cgroup;
1126
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001127 if (prev && !reclaim)
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001128 pos = prev;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001129
Michal Hocko542f85f2013-04-29 15:07:15 -07001130 rcu_read_lock();
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001131
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001132 if (reclaim) {
Mel Gormanef8f2322016-07-28 15:46:05 -07001133 struct mem_cgroup_per_node *mz;
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001134
Johannes Weinera3747b52021-04-29 22:56:14 -07001135 mz = root->nodeinfo[reclaim->pgdat->node_id];
Yafang Shao9da83f32019-11-30 17:50:03 -08001136 iter = &mz->iter;
Michal Hocko5f578162013-04-29 15:07:17 -07001137
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001138 if (prev && reclaim->generation != iter->generation)
Michal Hocko542f85f2013-04-29 15:07:15 -07001139 goto out_unlock;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001140
Vladimir Davydov6df38682015-12-29 14:54:10 -08001141 while (1) {
Jason Low4db0c3c2015-04-15 16:14:08 -07001142 pos = READ_ONCE(iter->position);
Vladimir Davydov6df38682015-12-29 14:54:10 -08001143 if (!pos || css_tryget(&pos->css))
1144 break;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001145 /*
Vladimir Davydov6df38682015-12-29 14:54:10 -08001146 * css reference reached zero, so iter->position will
1147 * be cleared by ->css_released. However, we should not
1148 * rely on this happening soon, because ->css_released
1149 * is called from a work queue, and by busy-waiting we
1150 * might block it. So we clear iter->position right
1151 * away.
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001152 */
Vladimir Davydov6df38682015-12-29 14:54:10 -08001153 (void)cmpxchg(&iter->position, pos, NULL);
1154 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001155 }
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001156
1157 if (pos)
1158 css = &pos->css;
1159
1160 for (;;) {
1161 css = css_next_descendant_pre(css, &root->css);
1162 if (!css) {
1163 /*
1164 * Reclaimers share the hierarchy walk, and a
1165 * new one might jump in right at the end of
1166 * the hierarchy - make sure they see at least
1167 * one group and restart from the beginning.
1168 */
1169 if (!prev)
1170 continue;
1171 break;
1172 }
1173
1174 /*
1175 * Verify the css and acquire a reference. The root
1176 * is provided by the caller, so we know it's alive
1177 * and kicking, and don't take an extra reference.
1178 */
1179 memcg = mem_cgroup_from_css(css);
1180
1181 if (css == &root->css)
1182 break;
1183
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08001184 if (css_tryget(css))
1185 break;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001186
1187 memcg = NULL;
1188 }
1189
1190 if (reclaim) {
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001191 /*
Vladimir Davydov6df38682015-12-29 14:54:10 -08001192 * The position could have already been updated by a competing
1193 * thread, so check that the value hasn't changed since we read
1194 * it to avoid reclaiming from the same cgroup twice.
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001195 */
Vladimir Davydov6df38682015-12-29 14:54:10 -08001196 (void)cmpxchg(&iter->position, pos, memcg);
1197
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001198 if (pos)
1199 css_put(&pos->css);
1200
1201 if (!memcg)
1202 iter->generation++;
1203 else if (!prev)
1204 reclaim->generation = iter->generation;
1205 }
1206
Michal Hocko542f85f2013-04-29 15:07:15 -07001207out_unlock:
1208 rcu_read_unlock();
Michal Hockoc40046f2013-04-29 15:07:14 -07001209 if (prev && prev != root)
1210 css_put(&prev->css);
1211
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001212 return memcg;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001213}
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001214
Johannes Weiner56600482012-01-12 17:17:59 -08001215/**
1216 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1217 * @root: hierarchy root
1218 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1219 */
1220void mem_cgroup_iter_break(struct mem_cgroup *root,
1221 struct mem_cgroup *prev)
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001222{
1223 if (!root)
1224 root = root_mem_cgroup;
1225 if (prev && prev != root)
1226 css_put(&prev->css);
1227}
1228
Miles Chen54a83d62019-08-13 15:37:28 -07001229static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
1230 struct mem_cgroup *dead_memcg)
Vladimir Davydov6df38682015-12-29 14:54:10 -08001231{
Vladimir Davydov6df38682015-12-29 14:54:10 -08001232 struct mem_cgroup_reclaim_iter *iter;
Mel Gormanef8f2322016-07-28 15:46:05 -07001233 struct mem_cgroup_per_node *mz;
1234 int nid;
Vladimir Davydov6df38682015-12-29 14:54:10 -08001235
Miles Chen54a83d62019-08-13 15:37:28 -07001236 for_each_node(nid) {
Johannes Weinera3747b52021-04-29 22:56:14 -07001237 mz = from->nodeinfo[nid];
Yafang Shao9da83f32019-11-30 17:50:03 -08001238 iter = &mz->iter;
1239 cmpxchg(&iter->position, dead_memcg, NULL);
Vladimir Davydov6df38682015-12-29 14:54:10 -08001240 }
1241}
1242
Miles Chen54a83d62019-08-13 15:37:28 -07001243static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1244{
1245 struct mem_cgroup *memcg = dead_memcg;
1246 struct mem_cgroup *last;
1247
1248 do {
1249 __invalidate_reclaim_iterators(memcg, dead_memcg);
1250 last = memcg;
1251 } while ((memcg = parent_mem_cgroup(memcg)));
1252
1253 /*
1254 * When cgruop1 non-hierarchy mode is used,
1255 * parent_mem_cgroup() does not walk all the way up to the
1256 * cgroup root (root_mem_cgroup). So we have to handle
1257 * dead_memcg from cgroup root separately.
1258 */
1259 if (last != root_mem_cgroup)
1260 __invalidate_reclaim_iterators(root_mem_cgroup,
1261 dead_memcg);
1262}
1263
Johannes Weiner925b7672012-01-12 17:18:15 -08001264/**
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001265 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1266 * @memcg: hierarchy root
1267 * @fn: function to call for each task
1268 * @arg: argument passed to @fn
1269 *
1270 * This function iterates over tasks attached to @memcg or to any of its
1271 * descendants and calls @fn for each task. If @fn returns a non-zero
1272 * value, the function breaks the iteration loop and returns the value.
1273 * Otherwise, it will iterate over all tasks and return 0.
1274 *
1275 * This function must not be called for the root memory cgroup.
1276 */
1277int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1278 int (*fn)(struct task_struct *, void *), void *arg)
1279{
1280 struct mem_cgroup *iter;
1281 int ret = 0;
1282
1283 BUG_ON(memcg == root_mem_cgroup);
1284
1285 for_each_mem_cgroup_tree(iter, memcg) {
1286 struct css_task_iter it;
1287 struct task_struct *task;
1288
Tetsuo Handaf168a9a2019-07-11 21:00:20 -07001289 css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001290 while (!ret && (task = css_task_iter_next(&it)))
1291 ret = fn(task, arg);
1292 css_task_iter_end(&it);
1293 if (ret) {
1294 mem_cgroup_iter_break(memcg, iter);
1295 break;
1296 }
1297 }
1298 return ret;
1299}
1300
Alex Shi6168d0d2020-12-15 12:34:29 -08001301#ifdef CONFIG_DEBUG_VM
1302void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
1303{
1304 struct mem_cgroup *memcg;
1305
1306 if (mem_cgroup_disabled())
1307 return;
1308
1309 memcg = page_memcg(page);
1310
1311 if (!memcg)
1312 VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != root_mem_cgroup, page);
1313 else
1314 VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != memcg, page);
1315}
1316#endif
1317
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001318/**
Alex Shi6168d0d2020-12-15 12:34:29 -08001319 * lock_page_lruvec - lock and return lruvec for a given page.
1320 * @page: the page
1321 *
Alex Shid7e3aba2021-02-24 12:03:47 -08001322 * These functions are safe to use under any of the following conditions:
1323 * - page locked
1324 * - PageLRU cleared
1325 * - lock_page_memcg()
1326 * - page->_refcount is zero
Alex Shi6168d0d2020-12-15 12:34:29 -08001327 */
1328struct lruvec *lock_page_lruvec(struct page *page)
1329{
1330 struct lruvec *lruvec;
1331 struct pglist_data *pgdat = page_pgdat(page);
1332
Alex Shi6168d0d2020-12-15 12:34:29 -08001333 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1334 spin_lock(&lruvec->lru_lock);
Alex Shi6168d0d2020-12-15 12:34:29 -08001335
1336 lruvec_memcg_debug(lruvec, page);
1337
1338 return lruvec;
1339}
1340
1341struct lruvec *lock_page_lruvec_irq(struct page *page)
1342{
1343 struct lruvec *lruvec;
1344 struct pglist_data *pgdat = page_pgdat(page);
1345
Alex Shi6168d0d2020-12-15 12:34:29 -08001346 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1347 spin_lock_irq(&lruvec->lru_lock);
Alex Shi6168d0d2020-12-15 12:34:29 -08001348
1349 lruvec_memcg_debug(lruvec, page);
1350
1351 return lruvec;
1352}
1353
1354struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
1355{
1356 struct lruvec *lruvec;
1357 struct pglist_data *pgdat = page_pgdat(page);
1358
Alex Shi6168d0d2020-12-15 12:34:29 -08001359 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1360 spin_lock_irqsave(&lruvec->lru_lock, *flags);
Alex Shi6168d0d2020-12-15 12:34:29 -08001361
1362 lruvec_memcg_debug(lruvec, page);
1363
1364 return lruvec;
1365}
1366
1367/**
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001368 * mem_cgroup_update_lru_size - account for adding or removing an lru page
1369 * @lruvec: mem_cgroup per zone lru vector
1370 * @lru: index of lru list the page is sitting on
Michal Hockob4536f0c82017-01-10 16:58:04 -08001371 * @zid: zone id of the accounted pages
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001372 * @nr_pages: positive when adding or negative when removing
Johannes Weiner925b7672012-01-12 17:18:15 -08001373 *
Hugh Dickinsca707232016-05-19 17:12:35 -07001374 * This function must be called under lru_lock, just before a page is added
1375 * to or just after a page is removed from an lru list (that ordering being
1376 * so as to allow it to check that lru_size 0 is consistent with list_empty).
Johannes Weiner925b7672012-01-12 17:18:15 -08001377 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001378void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
Michal Hockob4536f0c82017-01-10 16:58:04 -08001379 int zid, int nr_pages)
Johannes Weiner925b7672012-01-12 17:18:15 -08001380{
Mel Gormanef8f2322016-07-28 15:46:05 -07001381 struct mem_cgroup_per_node *mz;
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001382 unsigned long *lru_size;
Hugh Dickinsca707232016-05-19 17:12:35 -07001383 long size;
Johannes Weiner925b7672012-01-12 17:18:15 -08001384
1385 if (mem_cgroup_disabled())
1386 return;
1387
Mel Gormanef8f2322016-07-28 15:46:05 -07001388 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Michal Hockob4536f0c82017-01-10 16:58:04 -08001389 lru_size = &mz->lru_zone_size[zid][lru];
Hugh Dickinsca707232016-05-19 17:12:35 -07001390
1391 if (nr_pages < 0)
1392 *lru_size += nr_pages;
1393
1394 size = *lru_size;
Michal Hockob4536f0c82017-01-10 16:58:04 -08001395 if (WARN_ONCE(size < 0,
1396 "%s(%p, %d, %d): lru_size %ld\n",
1397 __func__, lruvec, lru, nr_pages, size)) {
Hugh Dickinsca707232016-05-19 17:12:35 -07001398 VM_BUG_ON(1);
1399 *lru_size = 0;
1400 }
1401
1402 if (nr_pages > 0)
1403 *lru_size += nr_pages;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001404}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001405
Johannes Weiner19942822011-02-01 15:52:43 -08001406/**
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001407 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
Wanpeng Lidad75572012-06-20 12:53:01 -07001408 * @memcg: the memory cgroup
Johannes Weiner19942822011-02-01 15:52:43 -08001409 *
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001410 * Returns the maximum amount of memory @mem can be charged with, in
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001411 * pages.
Johannes Weiner19942822011-02-01 15:52:43 -08001412 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001413static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
Johannes Weiner19942822011-02-01 15:52:43 -08001414{
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001415 unsigned long margin = 0;
1416 unsigned long count;
1417 unsigned long limit;
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001418
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001419 count = page_counter_read(&memcg->memory);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001420 limit = READ_ONCE(memcg->memory.max);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001421 if (count < limit)
1422 margin = limit - count;
1423
Johannes Weiner7941d212016-01-14 15:21:23 -08001424 if (do_memsw_account()) {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001425 count = page_counter_read(&memcg->memsw);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001426 limit = READ_ONCE(memcg->memsw.max);
Kaixu Xia1c4448e2020-06-01 21:49:36 -07001427 if (count < limit)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001428 margin = min(margin, limit - count);
Li RongQingcbedbac2016-05-27 14:27:43 -07001429 else
1430 margin = 0;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001431 }
1432
1433 return margin;
Johannes Weiner19942822011-02-01 15:52:43 -08001434}
1435
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001436/*
Qiang Huangbdcbb652014-06-04 16:08:21 -07001437 * A routine for checking "mem" is under move_account() or not.
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001438 *
Qiang Huangbdcbb652014-06-04 16:08:21 -07001439 * Checking a cgroup is mc.from or mc.to or under hierarchy of
1440 * moving cgroups. This is for waiting at high-memory pressure
1441 * caused by "move".
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001442 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001443static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001444{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001445 struct mem_cgroup *from;
1446 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001447 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001448 /*
1449 * Unlike task_move routines, we access mc.to, mc.from not under
1450 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1451 */
1452 spin_lock(&mc.lock);
1453 from = mc.from;
1454 to = mc.to;
1455 if (!from)
1456 goto unlock;
Michal Hocko3e920412011-07-26 16:08:29 -07001457
Johannes Weiner2314b422014-12-10 15:44:33 -08001458 ret = mem_cgroup_is_descendant(from, memcg) ||
1459 mem_cgroup_is_descendant(to, memcg);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001460unlock:
1461 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001462 return ret;
1463}
1464
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001465static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001466{
1467 if (mc.moving_task && current != mc.moving_task) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001468 if (mem_cgroup_under_move(memcg)) {
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001469 DEFINE_WAIT(wait);
1470 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1471 /* moving charge context might have finished. */
1472 if (mc.moving_task)
1473 schedule();
1474 finish_wait(&mc.waitq, &wait);
1475 return true;
1476 }
1477 }
1478 return false;
1479}
1480
Muchun Song5f9a4f42020-10-13 16:52:59 -07001481struct memory_stat {
1482 const char *name;
Muchun Song5f9a4f42020-10-13 16:52:59 -07001483 unsigned int idx;
1484};
1485
Muchun Song57b28472021-02-24 12:03:31 -08001486static const struct memory_stat memory_stats[] = {
Muchun Songfff66b72021-02-24 12:03:43 -08001487 { "anon", NR_ANON_MAPPED },
1488 { "file", NR_FILE_PAGES },
1489 { "kernel_stack", NR_KERNEL_STACK_KB },
1490 { "pagetables", NR_PAGETABLE },
1491 { "percpu", MEMCG_PERCPU_B },
1492 { "sock", MEMCG_SOCK },
1493 { "shmem", NR_SHMEM },
1494 { "file_mapped", NR_FILE_MAPPED },
1495 { "file_dirty", NR_FILE_DIRTY },
1496 { "file_writeback", NR_WRITEBACK },
Shakeel Buttb6038942021-02-24 12:03:55 -08001497#ifdef CONFIG_SWAP
1498 { "swapcached", NR_SWAPCACHE },
1499#endif
Muchun Song5f9a4f42020-10-13 16:52:59 -07001500#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Muchun Songfff66b72021-02-24 12:03:43 -08001501 { "anon_thp", NR_ANON_THPS },
1502 { "file_thp", NR_FILE_THPS },
1503 { "shmem_thp", NR_SHMEM_THPS },
Muchun Song5f9a4f42020-10-13 16:52:59 -07001504#endif
Muchun Songfff66b72021-02-24 12:03:43 -08001505 { "inactive_anon", NR_INACTIVE_ANON },
1506 { "active_anon", NR_ACTIVE_ANON },
1507 { "inactive_file", NR_INACTIVE_FILE },
1508 { "active_file", NR_ACTIVE_FILE },
1509 { "unevictable", NR_UNEVICTABLE },
1510 { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B },
1511 { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B },
Muchun Song5f9a4f42020-10-13 16:52:59 -07001512
1513 /* The memory events */
Muchun Songfff66b72021-02-24 12:03:43 -08001514 { "workingset_refault_anon", WORKINGSET_REFAULT_ANON },
1515 { "workingset_refault_file", WORKINGSET_REFAULT_FILE },
1516 { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON },
1517 { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE },
1518 { "workingset_restore_anon", WORKINGSET_RESTORE_ANON },
1519 { "workingset_restore_file", WORKINGSET_RESTORE_FILE },
1520 { "workingset_nodereclaim", WORKINGSET_NODERECLAIM },
Muchun Song5f9a4f42020-10-13 16:52:59 -07001521};
1522
Muchun Songfff66b72021-02-24 12:03:43 -08001523/* Translate stat items to the correct unit for memory.stat output */
1524static int memcg_page_state_unit(int item)
1525{
1526 switch (item) {
1527 case MEMCG_PERCPU_B:
1528 case NR_SLAB_RECLAIMABLE_B:
1529 case NR_SLAB_UNRECLAIMABLE_B:
1530 case WORKINGSET_REFAULT_ANON:
1531 case WORKINGSET_REFAULT_FILE:
1532 case WORKINGSET_ACTIVATE_ANON:
1533 case WORKINGSET_ACTIVATE_FILE:
1534 case WORKINGSET_RESTORE_ANON:
1535 case WORKINGSET_RESTORE_FILE:
1536 case WORKINGSET_NODERECLAIM:
1537 return 1;
1538 case NR_KERNEL_STACK_KB:
1539 return SZ_1K;
1540 default:
1541 return PAGE_SIZE;
1542 }
1543}
1544
1545static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg,
1546 int item)
1547{
1548 return memcg_page_state(memcg, item) * memcg_page_state_unit(item);
1549}
1550
Johannes Weinerc8713d02019-07-11 20:55:59 -07001551static char *memory_stat_format(struct mem_cgroup *memcg)
1552{
1553 struct seq_buf s;
1554 int i;
Johannes Weiner71cd3112017-05-03 14:55:13 -07001555
Johannes Weinerc8713d02019-07-11 20:55:59 -07001556 seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
1557 if (!s.buffer)
1558 return NULL;
1559
1560 /*
1561 * Provide statistics on the state of the memory subsystem as
1562 * well as cumulative event counters that show past behavior.
1563 *
1564 * This list is ordered following a combination of these gradients:
1565 * 1) generic big picture -> specifics and details
1566 * 2) reflecting userspace activity -> reflecting kernel heuristics
1567 *
1568 * Current memory state:
1569 */
Johannes Weiner2d146aa2021-04-29 22:56:26 -07001570 cgroup_rstat_flush(memcg->css.cgroup);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001571
Muchun Song5f9a4f42020-10-13 16:52:59 -07001572 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
1573 u64 size;
Johannes Weinerc8713d02019-07-11 20:55:59 -07001574
Muchun Songfff66b72021-02-24 12:03:43 -08001575 size = memcg_page_state_output(memcg, memory_stats[i].idx);
Muchun Song5f9a4f42020-10-13 16:52:59 -07001576 seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001577
Muchun Song5f9a4f42020-10-13 16:52:59 -07001578 if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
Muchun Songfff66b72021-02-24 12:03:43 -08001579 size += memcg_page_state_output(memcg,
1580 NR_SLAB_RECLAIMABLE_B);
Muchun Song5f9a4f42020-10-13 16:52:59 -07001581 seq_buf_printf(&s, "slab %llu\n", size);
1582 }
1583 }
Johannes Weinerc8713d02019-07-11 20:55:59 -07001584
1585 /* Accumulated memory events */
1586
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001587 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGFAULT),
1588 memcg_events(memcg, PGFAULT));
1589 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGMAJFAULT),
1590 memcg_events(memcg, PGMAJFAULT));
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001591 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGREFILL),
1592 memcg_events(memcg, PGREFILL));
Johannes Weinerc8713d02019-07-11 20:55:59 -07001593 seq_buf_printf(&s, "pgscan %lu\n",
1594 memcg_events(memcg, PGSCAN_KSWAPD) +
1595 memcg_events(memcg, PGSCAN_DIRECT));
1596 seq_buf_printf(&s, "pgsteal %lu\n",
1597 memcg_events(memcg, PGSTEAL_KSWAPD) +
1598 memcg_events(memcg, PGSTEAL_DIRECT));
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001599 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGACTIVATE),
1600 memcg_events(memcg, PGACTIVATE));
1601 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGDEACTIVATE),
1602 memcg_events(memcg, PGDEACTIVATE));
1603 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREE),
1604 memcg_events(memcg, PGLAZYFREE));
1605 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREED),
1606 memcg_events(memcg, PGLAZYFREED));
Johannes Weinerc8713d02019-07-11 20:55:59 -07001607
1608#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001609 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_FAULT_ALLOC),
Johannes Weinerc8713d02019-07-11 20:55:59 -07001610 memcg_events(memcg, THP_FAULT_ALLOC));
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001611 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_COLLAPSE_ALLOC),
Johannes Weinerc8713d02019-07-11 20:55:59 -07001612 memcg_events(memcg, THP_COLLAPSE_ALLOC));
1613#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1614
1615 /* The above should easily fit into one page */
1616 WARN_ON_ONCE(seq_buf_has_overflowed(&s));
1617
1618 return s.buffer;
1619}
Johannes Weiner71cd3112017-05-03 14:55:13 -07001620
Sha Zhengju58cf1882013-02-22 16:32:05 -08001621#define K(x) ((x) << (PAGE_SHIFT-10))
Balbir Singhe2224322009-04-02 16:57:39 -07001622/**
yuzhoujianf0c867d2018-12-28 00:36:10 -08001623 * mem_cgroup_print_oom_context: Print OOM information relevant to
1624 * memory controller.
Balbir Singhe2224322009-04-02 16:57:39 -07001625 * @memcg: The memory cgroup that went over limit
1626 * @p: Task that is going to be killed
1627 *
1628 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1629 * enabled
1630 */
yuzhoujianf0c867d2018-12-28 00:36:10 -08001631void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1632{
1633 rcu_read_lock();
1634
1635 if (memcg) {
1636 pr_cont(",oom_memcg=");
1637 pr_cont_cgroup_path(memcg->css.cgroup);
1638 } else
1639 pr_cont(",global_oom");
1640 if (p) {
1641 pr_cont(",task_memcg=");
1642 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1643 }
1644 rcu_read_unlock();
1645}
1646
1647/**
1648 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1649 * memory controller.
1650 * @memcg: The memory cgroup that went over limit
1651 */
1652void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
Balbir Singhe2224322009-04-02 16:57:39 -07001653{
Johannes Weinerc8713d02019-07-11 20:55:59 -07001654 char *buf;
Balbir Singhe2224322009-04-02 16:57:39 -07001655
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001656 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1657 K((u64)page_counter_read(&memcg->memory)),
Chris Down15b42562020-04-01 21:07:20 -07001658 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001659 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1660 pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
1661 K((u64)page_counter_read(&memcg->swap)),
Chris Down32d087c2020-04-01 21:07:30 -07001662 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001663 else {
1664 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1665 K((u64)page_counter_read(&memcg->memsw)),
1666 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1667 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1668 K((u64)page_counter_read(&memcg->kmem)),
1669 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
Sha Zhengju58cf1882013-02-22 16:32:05 -08001670 }
Johannes Weinerc8713d02019-07-11 20:55:59 -07001671
1672 pr_info("Memory cgroup stats for ");
1673 pr_cont_cgroup_path(memcg->css.cgroup);
1674 pr_cont(":");
1675 buf = memory_stat_format(memcg);
1676 if (!buf)
1677 return;
1678 pr_info("%s", buf);
1679 kfree(buf);
Balbir Singhe2224322009-04-02 16:57:39 -07001680}
1681
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001682/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001683 * Return the memory (and swap, if configured) limit for a memcg.
1684 */
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001685unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
David Rientjesa63d83f2010-08-09 17:19:46 -07001686{
Waiman Long8d387a52020-10-13 16:52:52 -07001687 unsigned long max = READ_ONCE(memcg->memory.max);
David Rientjesa63d83f2010-08-09 17:19:46 -07001688
Waiman Long8d387a52020-10-13 16:52:52 -07001689 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
1690 if (mem_cgroup_swappiness(memcg))
1691 max += min(READ_ONCE(memcg->swap.max),
1692 (unsigned long)total_swap_pages);
1693 } else { /* v1 */
1694 if (mem_cgroup_swappiness(memcg)) {
1695 /* Calculate swap excess capacity from memsw limit */
1696 unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001697
Waiman Long8d387a52020-10-13 16:52:52 -07001698 max += min(swap, (unsigned long)total_swap_pages);
1699 }
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001700 }
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001701 return max;
David Rientjesa63d83f2010-08-09 17:19:46 -07001702}
1703
Chris Down9783aa92019-10-06 17:58:32 -07001704unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1705{
1706 return page_counter_read(&memcg->memory);
1707}
1708
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07001709static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
David Rientjes19965462012-12-11 16:00:26 -08001710 int order)
David Rientjes9cbb78b2012-07-31 16:43:44 -07001711{
David Rientjes6e0fc462015-09-08 15:00:36 -07001712 struct oom_control oc = {
1713 .zonelist = NULL,
1714 .nodemask = NULL,
Vladimir Davydov2a966b72016-07-26 15:22:33 -07001715 .memcg = memcg,
David Rientjes6e0fc462015-09-08 15:00:36 -07001716 .gfp_mask = gfp_mask,
1717 .order = order,
David Rientjes6e0fc462015-09-08 15:00:36 -07001718 };
Yafang Shao1378b372020-08-06 23:22:08 -07001719 bool ret = true;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001720
Tetsuo Handa7775fac2019-03-05 15:46:47 -08001721 if (mutex_lock_killable(&oom_lock))
1722 return true;
Yafang Shao1378b372020-08-06 23:22:08 -07001723
1724 if (mem_cgroup_margin(memcg) >= (1 << order))
1725 goto unlock;
1726
Tetsuo Handa7775fac2019-03-05 15:46:47 -08001727 /*
1728 * A few threads which were not waiting at mutex_lock_killable() can
1729 * fail to bail out. Therefore, check again after holding oom_lock.
1730 */
1731 ret = should_force_charge() || out_of_memory(&oc);
Yafang Shao1378b372020-08-06 23:22:08 -07001732
1733unlock:
Johannes Weinerdc564012015-06-24 16:57:19 -07001734 mutex_unlock(&oom_lock);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001735 return ret;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001736}
1737
Andrew Morton0608f432013-09-24 15:27:41 -07001738static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
Mel Gormanef8f2322016-07-28 15:46:05 -07001739 pg_data_t *pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07001740 gfp_t gfp_mask,
1741 unsigned long *total_scanned)
Balbir Singh6d61ef42009-01-07 18:08:06 -08001742{
Andrew Morton0608f432013-09-24 15:27:41 -07001743 struct mem_cgroup *victim = NULL;
1744 int total = 0;
1745 int loop = 0;
1746 unsigned long excess;
1747 unsigned long nr_scanned;
1748 struct mem_cgroup_reclaim_cookie reclaim = {
Mel Gormanef8f2322016-07-28 15:46:05 -07001749 .pgdat = pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07001750 };
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001751
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001752 excess = soft_limit_excess(root_memcg);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001753
Andrew Morton0608f432013-09-24 15:27:41 -07001754 while (1) {
1755 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1756 if (!victim) {
1757 loop++;
1758 if (loop >= 2) {
1759 /*
1760 * If we have not been able to reclaim
1761 * anything, it might because there are
1762 * no reclaimable pages under this hierarchy
1763 */
1764 if (!total)
1765 break;
1766 /*
1767 * We want to do more targeted reclaim.
1768 * excess >> 2 is not to excessive so as to
1769 * reclaim too much, nor too less that we keep
1770 * coming back to reclaim from this cgroup
1771 */
1772 if (total >= (excess >> 2) ||
1773 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1774 break;
1775 }
1776 continue;
1777 }
Mel Gormana9dd0a82016-07-28 15:46:02 -07001778 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
Mel Gormanef8f2322016-07-28 15:46:05 -07001779 pgdat, &nr_scanned);
Andrew Morton0608f432013-09-24 15:27:41 -07001780 *total_scanned += nr_scanned;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001781 if (!soft_limit_excess(root_memcg))
Andrew Morton0608f432013-09-24 15:27:41 -07001782 break;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001783 }
Andrew Morton0608f432013-09-24 15:27:41 -07001784 mem_cgroup_iter_break(root_memcg, victim);
1785 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001786}
1787
Johannes Weiner0056f4e2013-10-31 16:34:14 -07001788#ifdef CONFIG_LOCKDEP
1789static struct lockdep_map memcg_oom_lock_dep_map = {
1790 .name = "memcg_oom_lock",
1791};
1792#endif
1793
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001794static DEFINE_SPINLOCK(memcg_oom_lock);
1795
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001796/*
1797 * Check OOM-Killer is already running under our hierarchy.
1798 * If someone is running, return false.
1799 */
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001800static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001801{
Michal Hocko79dfdac2011-07-26 16:08:23 -07001802 struct mem_cgroup *iter, *failed = NULL;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001803
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001804 spin_lock(&memcg_oom_lock);
1805
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001806 for_each_mem_cgroup_tree(iter, memcg) {
Johannes Weiner23751be2011-08-25 15:59:16 -07001807 if (iter->oom_lock) {
Michal Hocko79dfdac2011-07-26 16:08:23 -07001808 /*
1809 * this subtree of our hierarchy is already locked
1810 * so we cannot give a lock.
1811 */
Michal Hocko79dfdac2011-07-26 16:08:23 -07001812 failed = iter;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001813 mem_cgroup_iter_break(memcg, iter);
1814 break;
Johannes Weiner23751be2011-08-25 15:59:16 -07001815 } else
1816 iter->oom_lock = true;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001817 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001818
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001819 if (failed) {
1820 /*
1821 * OK, we failed to lock the whole subtree so we have
1822 * to clean up what we set up to the failing subtree
1823 */
1824 for_each_mem_cgroup_tree(iter, memcg) {
1825 if (iter == failed) {
1826 mem_cgroup_iter_break(memcg, iter);
1827 break;
1828 }
1829 iter->oom_lock = false;
Michal Hocko79dfdac2011-07-26 16:08:23 -07001830 }
Johannes Weiner0056f4e2013-10-31 16:34:14 -07001831 } else
1832 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001833
1834 spin_unlock(&memcg_oom_lock);
1835
1836 return !failed;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001837}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001838
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001839static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001840{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001841 struct mem_cgroup *iter;
1842
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001843 spin_lock(&memcg_oom_lock);
Qian Cai5facae42019-09-19 12:09:40 -04001844 mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001845 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001846 iter->oom_lock = false;
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001847 spin_unlock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07001848}
1849
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001850static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001851{
1852 struct mem_cgroup *iter;
1853
Tejun Heoc2b42d32015-06-24 16:58:23 -07001854 spin_lock(&memcg_oom_lock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001855 for_each_mem_cgroup_tree(iter, memcg)
Tejun Heoc2b42d32015-06-24 16:58:23 -07001856 iter->under_oom++;
1857 spin_unlock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07001858}
1859
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001860static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001861{
1862 struct mem_cgroup *iter;
1863
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001864 /*
Miaohe Lin7a52d4d82020-10-13 16:53:05 -07001865 * Be careful about under_oom underflows becase a child memcg
1866 * could have been added after mem_cgroup_mark_under_oom.
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001867 */
Tejun Heoc2b42d32015-06-24 16:58:23 -07001868 spin_lock(&memcg_oom_lock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001869 for_each_mem_cgroup_tree(iter, memcg)
Tejun Heoc2b42d32015-06-24 16:58:23 -07001870 if (iter->under_oom > 0)
1871 iter->under_oom--;
1872 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001873}
1874
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001875static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1876
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001877struct oom_wait_info {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001878 struct mem_cgroup *memcg;
Ingo Molnarac6424b2017-06-20 12:06:13 +02001879 wait_queue_entry_t wait;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001880};
1881
Ingo Molnarac6424b2017-06-20 12:06:13 +02001882static int memcg_oom_wake_function(wait_queue_entry_t *wait,
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001883 unsigned mode, int sync, void *arg)
1884{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001885 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1886 struct mem_cgroup *oom_wait_memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001887 struct oom_wait_info *oom_wait_info;
1888
1889 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001890 oom_wait_memcg = oom_wait_info->memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001891
Johannes Weiner2314b422014-12-10 15:44:33 -08001892 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1893 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001894 return 0;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001895 return autoremove_wake_function(wait, mode, sync, arg);
1896}
1897
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001898static void memcg_oom_recover(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001899{
Tejun Heoc2b42d32015-06-24 16:58:23 -07001900 /*
1901 * For the following lockless ->under_oom test, the only required
1902 * guarantee is that it must see the state asserted by an OOM when
1903 * this function is called as a result of userland actions
1904 * triggered by the notification of the OOM. This is trivially
1905 * achieved by invoking mem_cgroup_mark_under_oom() before
1906 * triggering notification.
1907 */
1908 if (memcg && memcg->under_oom)
Tejun Heof4b90b702015-06-24 16:58:21 -07001909 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001910}
1911
Michal Hocko29ef6802018-08-17 15:47:11 -07001912enum oom_status {
1913 OOM_SUCCESS,
1914 OOM_FAILED,
1915 OOM_ASYNC,
1916 OOM_SKIPPED
1917};
1918
1919static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001920{
Michal Hocko7056d3a2018-12-28 00:39:57 -08001921 enum oom_status ret;
1922 bool locked;
1923
Michal Hocko29ef6802018-08-17 15:47:11 -07001924 if (order > PAGE_ALLOC_COSTLY_ORDER)
1925 return OOM_SKIPPED;
1926
Roman Gushchin7a1adfd2018-10-26 15:09:48 -07001927 memcg_memory_event(memcg, MEMCG_OOM);
1928
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001929 /*
Johannes Weiner49426422013-10-16 13:46:59 -07001930 * We are in the middle of the charge context here, so we
1931 * don't want to block when potentially sitting on a callstack
1932 * that holds all kinds of filesystem and mm locks.
1933 *
Michal Hocko29ef6802018-08-17 15:47:11 -07001934 * cgroup1 allows disabling the OOM killer and waiting for outside
1935 * handling until the charge can succeed; remember the context and put
1936 * the task to sleep at the end of the page fault when all locks are
1937 * released.
Johannes Weiner49426422013-10-16 13:46:59 -07001938 *
Michal Hocko29ef6802018-08-17 15:47:11 -07001939 * On the other hand, in-kernel OOM killer allows for an async victim
1940 * memory reclaim (oom_reaper) and that means that we are not solely
1941 * relying on the oom victim to make a forward progress and we can
1942 * invoke the oom killer here.
1943 *
1944 * Please note that mem_cgroup_out_of_memory might fail to find a
1945 * victim and then we have to bail out from the charge path.
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001946 */
Michal Hocko29ef6802018-08-17 15:47:11 -07001947 if (memcg->oom_kill_disable) {
1948 if (!current->in_user_fault)
1949 return OOM_SKIPPED;
1950 css_get(&memcg->css);
1951 current->memcg_in_oom = memcg;
1952 current->memcg_oom_gfp_mask = mask;
1953 current->memcg_oom_order = order;
1954
1955 return OOM_ASYNC;
1956 }
1957
Michal Hocko7056d3a2018-12-28 00:39:57 -08001958 mem_cgroup_mark_under_oom(memcg);
Michal Hocko29ef6802018-08-17 15:47:11 -07001959
Michal Hocko7056d3a2018-12-28 00:39:57 -08001960 locked = mem_cgroup_oom_trylock(memcg);
1961
1962 if (locked)
1963 mem_cgroup_oom_notify(memcg);
1964
1965 mem_cgroup_unmark_under_oom(memcg);
1966 if (mem_cgroup_out_of_memory(memcg, mask, order))
1967 ret = OOM_SUCCESS;
1968 else
1969 ret = OOM_FAILED;
1970
1971 if (locked)
1972 mem_cgroup_oom_unlock(memcg);
1973
1974 return ret;
Johannes Weiner49426422013-10-16 13:46:59 -07001975}
1976
1977/**
1978 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1979 * @handle: actually kill/wait or just clean up the OOM state
1980 *
1981 * This has to be called at the end of a page fault if the memcg OOM
1982 * handler was enabled.
1983 *
1984 * Memcg supports userspace OOM handling where failed allocations must
1985 * sleep on a waitqueue until the userspace task resolves the
1986 * situation. Sleeping directly in the charge context with all kinds
1987 * of locks held is not a good idea, instead we remember an OOM state
1988 * in the task and mem_cgroup_oom_synchronize() has to be called at
1989 * the end of the page fault to complete the OOM handling.
1990 *
1991 * Returns %true if an ongoing memcg OOM situation was detected and
1992 * completed, %false otherwise.
1993 */
1994bool mem_cgroup_oom_synchronize(bool handle)
1995{
Tejun Heo626ebc42015-11-05 18:46:09 -08001996 struct mem_cgroup *memcg = current->memcg_in_oom;
Johannes Weiner49426422013-10-16 13:46:59 -07001997 struct oom_wait_info owait;
1998 bool locked;
1999
2000 /* OOM is global, do not handle */
2001 if (!memcg)
2002 return false;
2003
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07002004 if (!handle)
Johannes Weiner49426422013-10-16 13:46:59 -07002005 goto cleanup;
2006
2007 owait.memcg = memcg;
2008 owait.wait.flags = 0;
2009 owait.wait.func = memcg_oom_wake_function;
2010 owait.wait.private = current;
Ingo Molnar2055da92017-06-20 12:06:46 +02002011 INIT_LIST_HEAD(&owait.wait.entry);
Johannes Weiner49426422013-10-16 13:46:59 -07002012
2013 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002014 mem_cgroup_mark_under_oom(memcg);
2015
2016 locked = mem_cgroup_oom_trylock(memcg);
2017
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002018 if (locked)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002019 mem_cgroup_oom_notify(memcg);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002020
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002021 if (locked && !memcg->oom_kill_disable) {
2022 mem_cgroup_unmark_under_oom(memcg);
Johannes Weiner49426422013-10-16 13:46:59 -07002023 finish_wait(&memcg_oom_waitq, &owait.wait);
Tejun Heo626ebc42015-11-05 18:46:09 -08002024 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
2025 current->memcg_oom_order);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002026 } else {
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002027 schedule();
Johannes Weiner49426422013-10-16 13:46:59 -07002028 mem_cgroup_unmark_under_oom(memcg);
2029 finish_wait(&memcg_oom_waitq, &owait.wait);
2030 }
2031
2032 if (locked) {
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002033 mem_cgroup_oom_unlock(memcg);
2034 /*
2035 * There is no guarantee that an OOM-lock contender
2036 * sees the wakeups triggered by the OOM kill
2037 * uncharges. Wake any sleepers explicitely.
2038 */
2039 memcg_oom_recover(memcg);
2040 }
Johannes Weiner49426422013-10-16 13:46:59 -07002041cleanup:
Tejun Heo626ebc42015-11-05 18:46:09 -08002042 current->memcg_in_oom = NULL;
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002043 css_put(&memcg->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002044 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002045}
2046
Johannes Weinerd7365e72014-10-29 14:50:48 -07002047/**
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07002048 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
2049 * @victim: task to be killed by the OOM killer
2050 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
2051 *
2052 * Returns a pointer to a memory cgroup, which has to be cleaned up
2053 * by killing all belonging OOM-killable tasks.
2054 *
2055 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
2056 */
2057struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
2058 struct mem_cgroup *oom_domain)
2059{
2060 struct mem_cgroup *oom_group = NULL;
2061 struct mem_cgroup *memcg;
2062
2063 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2064 return NULL;
2065
2066 if (!oom_domain)
2067 oom_domain = root_mem_cgroup;
2068
2069 rcu_read_lock();
2070
2071 memcg = mem_cgroup_from_task(victim);
2072 if (memcg == root_mem_cgroup)
2073 goto out;
2074
2075 /*
Roman Gushchin48fe2672020-04-01 21:07:39 -07002076 * If the victim task has been asynchronously moved to a different
2077 * memory cgroup, we might end up killing tasks outside oom_domain.
2078 * In this case it's better to ignore memory.group.oom.
2079 */
2080 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
2081 goto out;
2082
2083 /*
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07002084 * Traverse the memory cgroup hierarchy from the victim task's
2085 * cgroup up to the OOMing cgroup (or root) to find the
2086 * highest-level memory cgroup with oom.group set.
2087 */
2088 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2089 if (memcg->oom_group)
2090 oom_group = memcg;
2091
2092 if (memcg == oom_domain)
2093 break;
2094 }
2095
2096 if (oom_group)
2097 css_get(&oom_group->css);
2098out:
2099 rcu_read_unlock();
2100
2101 return oom_group;
2102}
2103
2104void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2105{
2106 pr_info("Tasks in ");
2107 pr_cont_cgroup_path(memcg->css.cgroup);
2108 pr_cont(" are going to be killed due to memory.oom.group set\n");
2109}
2110
2111/**
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002112 * lock_page_memcg - lock a page and memcg binding
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002113 * @page: the page
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002114 *
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002115 * This function protects unlocked LRU pages from being moved to
Johannes Weiner739f79f2017-08-18 15:15:48 -07002116 * another cgroup.
2117 *
Johannes Weiner1c824a62021-04-29 22:55:32 -07002118 * It ensures lifetime of the locked memcg. Caller is responsible
2119 * for the lifetime of the page.
Balbir Singhd69b0422009-06-17 16:26:34 -07002120 */
Johannes Weiner1c824a62021-04-29 22:55:32 -07002121void lock_page_memcg(struct page *page)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002122{
Johannes Weiner9da7b522020-06-03 16:01:51 -07002123 struct page *head = compound_head(page); /* rmap on tail pages */
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002124 struct mem_cgroup *memcg;
Johannes Weiner6de22612015-02-11 15:25:01 -08002125 unsigned long flags;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002126
Johannes Weiner6de22612015-02-11 15:25:01 -08002127 /*
2128 * The RCU lock is held throughout the transaction. The fast
2129 * path can get away without acquiring the memcg->move_lock
2130 * because page moving starts with an RCU grace period.
Johannes Weiner739f79f2017-08-18 15:15:48 -07002131 */
Johannes Weinerd7365e72014-10-29 14:50:48 -07002132 rcu_read_lock();
2133
2134 if (mem_cgroup_disabled())
Johannes Weiner1c824a62021-04-29 22:55:32 -07002135 return;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002136again:
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002137 memcg = page_memcg(head);
Johannes Weiner29833312014-12-10 15:44:02 -08002138 if (unlikely(!memcg))
Johannes Weiner1c824a62021-04-29 22:55:32 -07002139 return;
Johannes Weinerd7365e72014-10-29 14:50:48 -07002140
Alex Shi20ad50d2020-12-15 12:33:51 -08002141#ifdef CONFIG_PROVE_LOCKING
2142 local_irq_save(flags);
2143 might_lock(&memcg->move_lock);
2144 local_irq_restore(flags);
2145#endif
2146
Qiang Huangbdcbb652014-06-04 16:08:21 -07002147 if (atomic_read(&memcg->moving_account) <= 0)
Johannes Weiner1c824a62021-04-29 22:55:32 -07002148 return;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002149
Johannes Weiner6de22612015-02-11 15:25:01 -08002150 spin_lock_irqsave(&memcg->move_lock, flags);
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002151 if (memcg != page_memcg(head)) {
Johannes Weiner6de22612015-02-11 15:25:01 -08002152 spin_unlock_irqrestore(&memcg->move_lock, flags);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002153 goto again;
2154 }
Johannes Weiner6de22612015-02-11 15:25:01 -08002155
2156 /*
Johannes Weiner1c824a62021-04-29 22:55:32 -07002157 * When charge migration first begins, we can have multiple
2158 * critical sections holding the fast-path RCU lock and one
2159 * holding the slowpath move_lock. Track the task who has the
2160 * move_lock for unlock_page_memcg().
Johannes Weiner6de22612015-02-11 15:25:01 -08002161 */
2162 memcg->move_lock_task = current;
2163 memcg->move_lock_flags = flags;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002164}
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002165EXPORT_SYMBOL(lock_page_memcg);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002166
Johannes Weiner1c824a62021-04-29 22:55:32 -07002167static void __unlock_page_memcg(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002168{
Johannes Weiner6de22612015-02-11 15:25:01 -08002169 if (memcg && memcg->move_lock_task == current) {
2170 unsigned long flags = memcg->move_lock_flags;
2171
2172 memcg->move_lock_task = NULL;
2173 memcg->move_lock_flags = 0;
2174
2175 spin_unlock_irqrestore(&memcg->move_lock, flags);
2176 }
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002177
Johannes Weinerd7365e72014-10-29 14:50:48 -07002178 rcu_read_unlock();
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002179}
Johannes Weiner739f79f2017-08-18 15:15:48 -07002180
2181/**
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002182 * unlock_page_memcg - unlock a page and memcg binding
Johannes Weiner739f79f2017-08-18 15:15:48 -07002183 * @page: the page
2184 */
2185void unlock_page_memcg(struct page *page)
2186{
Johannes Weiner9da7b522020-06-03 16:01:51 -07002187 struct page *head = compound_head(page);
2188
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002189 __unlock_page_memcg(page_memcg(head));
Johannes Weiner739f79f2017-08-18 15:15:48 -07002190}
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002191EXPORT_SYMBOL(unlock_page_memcg);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002192
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002193struct memcg_stock_pcp {
2194 struct mem_cgroup *cached; /* this never be root cgroup */
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002195 unsigned int nr_pages;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002196
2197#ifdef CONFIG_MEMCG_KMEM
2198 struct obj_cgroup *cached_objcg;
2199 unsigned int nr_bytes;
2200#endif
2201
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002202 struct work_struct work;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002203 unsigned long flags;
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -07002204#define FLUSHING_CACHED_CHARGE 0
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002205};
2206static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002207static DEFINE_MUTEX(percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002208
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002209#ifdef CONFIG_MEMCG_KMEM
2210static void drain_obj_stock(struct memcg_stock_pcp *stock);
2211static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2212 struct mem_cgroup *root_memcg);
2213
2214#else
2215static inline void drain_obj_stock(struct memcg_stock_pcp *stock)
2216{
2217}
2218static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2219 struct mem_cgroup *root_memcg)
2220{
2221 return false;
2222}
2223#endif
2224
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002225/**
2226 * consume_stock: Try to consume stocked charge on this cpu.
2227 * @memcg: memcg to consume from.
2228 * @nr_pages: how many pages to charge.
2229 *
2230 * The charges will only happen if @memcg matches the current cpu's memcg
2231 * stock, and at least @nr_pages are available in that stock. Failure to
2232 * service an allocation will refill the stock.
2233 *
2234 * returns true if successful, false otherwise.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002235 */
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002236static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002237{
2238 struct memcg_stock_pcp *stock;
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002239 unsigned long flags;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002240 bool ret = false;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002241
Johannes Weinera983b5e2018-01-31 16:16:45 -08002242 if (nr_pages > MEMCG_CHARGE_BATCH)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002243 return ret;
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002244
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002245 local_irq_save(flags);
2246
2247 stock = this_cpu_ptr(&memcg_stock);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002248 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002249 stock->nr_pages -= nr_pages;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002250 ret = true;
2251 }
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002252
2253 local_irq_restore(flags);
2254
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002255 return ret;
2256}
2257
2258/*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002259 * Returns stocks cached in percpu and reset cached information.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002260 */
2261static void drain_stock(struct memcg_stock_pcp *stock)
2262{
2263 struct mem_cgroup *old = stock->cached;
2264
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07002265 if (!old)
2266 return;
2267
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002268 if (stock->nr_pages) {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002269 page_counter_uncharge(&old->memory, stock->nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002270 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002271 page_counter_uncharge(&old->memsw, stock->nr_pages);
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002272 stock->nr_pages = 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002273 }
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07002274
2275 css_put(&old->css);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002276 stock->cached = NULL;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002277}
2278
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002279static void drain_local_stock(struct work_struct *dummy)
2280{
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002281 struct memcg_stock_pcp *stock;
2282 unsigned long flags;
2283
Michal Hocko72f01842017-10-03 16:14:53 -07002284 /*
2285 * The only protection from memory hotplug vs. drain_stock races is
2286 * that we always operate on local CPU stock here with IRQ disabled
2287 */
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002288 local_irq_save(flags);
2289
2290 stock = this_cpu_ptr(&memcg_stock);
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002291 drain_obj_stock(stock);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002292 drain_stock(stock);
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002293 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002294
2295 local_irq_restore(flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002296}
2297
2298/*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002299 * Cache charges(val) to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01002300 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002301 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002302static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002303{
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002304 struct memcg_stock_pcp *stock;
2305 unsigned long flags;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002306
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002307 local_irq_save(flags);
2308
2309 stock = this_cpu_ptr(&memcg_stock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002310 if (stock->cached != memcg) { /* reset if necessary */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002311 drain_stock(stock);
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07002312 css_get(&memcg->css);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002313 stock->cached = memcg;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002314 }
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002315 stock->nr_pages += nr_pages;
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002316
Johannes Weinera983b5e2018-01-31 16:16:45 -08002317 if (stock->nr_pages > MEMCG_CHARGE_BATCH)
Roman Gushchin475d0482017-09-08 16:13:09 -07002318 drain_stock(stock);
2319
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002320 local_irq_restore(flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002321}
2322
2323/*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002324 * Drains all per-CPU charge caches for given root_memcg resp. subtree
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002325 * of the hierarchy under it.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002326 */
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002327static void drain_all_stock(struct mem_cgroup *root_memcg)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002328{
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002329 int cpu, curcpu;
Michal Hockod38144b2011-07-26 16:08:28 -07002330
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002331 /* If someone's already draining, avoid adding running more workers. */
2332 if (!mutex_trylock(&percpu_charge_mutex))
2333 return;
Michal Hocko72f01842017-10-03 16:14:53 -07002334 /*
2335 * Notify other cpus that system-wide "drain" is running
2336 * We do not care about races with the cpu hotplug because cpu down
2337 * as well as workers from this path always operate on the local
2338 * per-cpu data. CPU up doesn't touch memcg_stock at all.
2339 */
Johannes Weiner5af12d02011-08-25 15:59:07 -07002340 curcpu = get_cpu();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002341 for_each_online_cpu(cpu) {
2342 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002343 struct mem_cgroup *memcg;
Roman Gushchine1a366b2019-09-23 15:34:58 -07002344 bool flush = false;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002345
Roman Gushchine1a366b2019-09-23 15:34:58 -07002346 rcu_read_lock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002347 memcg = stock->cached;
Roman Gushchine1a366b2019-09-23 15:34:58 -07002348 if (memcg && stock->nr_pages &&
2349 mem_cgroup_is_descendant(memcg, root_memcg))
2350 flush = true;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002351 if (obj_stock_flush_required(stock, root_memcg))
2352 flush = true;
Roman Gushchine1a366b2019-09-23 15:34:58 -07002353 rcu_read_unlock();
2354
2355 if (flush &&
2356 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
Michal Hockod1a05b62011-07-26 16:08:27 -07002357 if (cpu == curcpu)
2358 drain_local_stock(&stock->work);
2359 else
2360 schedule_work_on(cpu, &stock->work);
2361 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002362 }
Johannes Weiner5af12d02011-08-25 15:59:07 -07002363 put_cpu();
Michal Hocko9f50fad2011-08-09 11:56:26 +02002364 mutex_unlock(&percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002365}
2366
Johannes Weiner2cd21c82021-04-29 22:56:29 -07002367static void memcg_flush_lruvec_page_state(struct mem_cgroup *memcg, int cpu)
2368{
2369 int nid;
2370
2371 for_each_node(nid) {
2372 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
2373 unsigned long stat[NR_VM_NODE_STAT_ITEMS];
2374 struct batched_lruvec_stat *lstatc;
2375 int i;
2376
2377 lstatc = per_cpu_ptr(pn->lruvec_stat_cpu, cpu);
2378 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
2379 stat[i] = lstatc->count[i];
2380 lstatc->count[i] = 0;
2381 }
2382
2383 do {
2384 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
2385 atomic_long_add(stat[i], &pn->lruvec_stat[i]);
2386 } while ((pn = parent_nodeinfo(pn, nid)));
2387 }
2388}
2389
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01002390static int memcg_hotplug_cpu_dead(unsigned int cpu)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002391{
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002392 struct memcg_stock_pcp *stock;
Johannes Weinera3d4c052021-04-29 22:56:11 -07002393 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002394
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002395 stock = &per_cpu(memcg_stock, cpu);
2396 drain_stock(stock);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002397
Johannes Weiner2cd21c82021-04-29 22:56:29 -07002398 for_each_mem_cgroup(memcg)
2399 memcg_flush_lruvec_page_state(memcg, cpu);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002400
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01002401 return 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002402}
2403
Chris Downb3ff9292020-08-06 23:21:54 -07002404static unsigned long reclaim_high(struct mem_cgroup *memcg,
2405 unsigned int nr_pages,
2406 gfp_t gfp_mask)
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002407{
Chris Downb3ff9292020-08-06 23:21:54 -07002408 unsigned long nr_reclaimed = 0;
2409
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002410 do {
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002411 unsigned long pflags;
2412
Jakub Kicinskid1663a92020-06-01 21:49:49 -07002413 if (page_counter_read(&memcg->memory) <=
2414 READ_ONCE(memcg->memory.high))
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002415 continue;
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002416
Johannes Weinere27be242018-04-10 16:29:45 -07002417 memcg_memory_event(memcg, MEMCG_HIGH);
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002418
2419 psi_memstall_enter(&pflags);
Chris Downb3ff9292020-08-06 23:21:54 -07002420 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2421 gfp_mask, true);
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002422 psi_memstall_leave(&pflags);
Chris Down4bf17302020-04-06 20:03:30 -07002423 } while ((memcg = parent_mem_cgroup(memcg)) &&
2424 !mem_cgroup_is_root(memcg));
Chris Downb3ff9292020-08-06 23:21:54 -07002425
2426 return nr_reclaimed;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002427}
2428
2429static void high_work_func(struct work_struct *work)
2430{
2431 struct mem_cgroup *memcg;
2432
2433 memcg = container_of(work, struct mem_cgroup, high_work);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002434 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002435}
2436
Tejun Heob23afb92015-11-05 18:46:11 -08002437/*
Chris Down0e4b01d2019-09-23 15:34:55 -07002438 * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
2439 * enough to still cause a significant slowdown in most cases, while still
2440 * allowing diagnostics and tracing to proceed without becoming stuck.
2441 */
2442#define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
2443
2444/*
2445 * When calculating the delay, we use these either side of the exponentiation to
2446 * maintain precision and scale to a reasonable number of jiffies (see the table
2447 * below.
2448 *
2449 * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
2450 * overage ratio to a delay.
Randy Dunlapac5ddd02020-08-11 18:33:02 -07002451 * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
Chris Down0e4b01d2019-09-23 15:34:55 -07002452 * proposed penalty in order to reduce to a reasonable number of jiffies, and
2453 * to produce a reasonable delay curve.
2454 *
2455 * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
2456 * reasonable delay curve compared to precision-adjusted overage, not
2457 * penalising heavily at first, but still making sure that growth beyond the
2458 * limit penalises misbehaviour cgroups by slowing them down exponentially. For
2459 * example, with a high of 100 megabytes:
2460 *
2461 * +-------+------------------------+
2462 * | usage | time to allocate in ms |
2463 * +-------+------------------------+
2464 * | 100M | 0 |
2465 * | 101M | 6 |
2466 * | 102M | 25 |
2467 * | 103M | 57 |
2468 * | 104M | 102 |
2469 * | 105M | 159 |
2470 * | 106M | 230 |
2471 * | 107M | 313 |
2472 * | 108M | 409 |
2473 * | 109M | 518 |
2474 * | 110M | 639 |
2475 * | 111M | 774 |
2476 * | 112M | 921 |
2477 * | 113M | 1081 |
2478 * | 114M | 1254 |
2479 * | 115M | 1439 |
2480 * | 116M | 1638 |
2481 * | 117M | 1849 |
2482 * | 118M | 2000 |
2483 * | 119M | 2000 |
2484 * | 120M | 2000 |
2485 * +-------+------------------------+
2486 */
2487 #define MEMCG_DELAY_PRECISION_SHIFT 20
2488 #define MEMCG_DELAY_SCALING_SHIFT 14
2489
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002490static u64 calculate_overage(unsigned long usage, unsigned long high)
2491{
2492 u64 overage;
2493
2494 if (usage <= high)
2495 return 0;
2496
2497 /*
2498 * Prevent division by 0 in overage calculation by acting as if
2499 * it was a threshold of 1 page
2500 */
2501 high = max(high, 1UL);
2502
2503 overage = usage - high;
2504 overage <<= MEMCG_DELAY_PRECISION_SHIFT;
2505 return div64_u64(overage, high);
2506}
2507
2508static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2509{
2510 u64 overage, max_overage = 0;
2511
2512 do {
2513 overage = calculate_overage(page_counter_read(&memcg->memory),
Jakub Kicinskid1663a92020-06-01 21:49:49 -07002514 READ_ONCE(memcg->memory.high));
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002515 max_overage = max(overage, max_overage);
2516 } while ((memcg = parent_mem_cgroup(memcg)) &&
2517 !mem_cgroup_is_root(memcg));
2518
2519 return max_overage;
2520}
2521
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002522static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2523{
2524 u64 overage, max_overage = 0;
2525
2526 do {
2527 overage = calculate_overage(page_counter_read(&memcg->swap),
2528 READ_ONCE(memcg->swap.high));
2529 if (overage)
2530 memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2531 max_overage = max(overage, max_overage);
2532 } while ((memcg = parent_mem_cgroup(memcg)) &&
2533 !mem_cgroup_is_root(memcg));
2534
2535 return max_overage;
2536}
2537
Chris Down0e4b01d2019-09-23 15:34:55 -07002538/*
Chris Downe26733e2020-03-21 18:22:23 -07002539 * Get the number of jiffies that we should penalise a mischievous cgroup which
2540 * is exceeding its memory.high by checking both it and its ancestors.
Tejun Heob23afb92015-11-05 18:46:11 -08002541 */
Chris Downe26733e2020-03-21 18:22:23 -07002542static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002543 unsigned int nr_pages,
2544 u64 max_overage)
Tejun Heob23afb92015-11-05 18:46:11 -08002545{
Chris Downe26733e2020-03-21 18:22:23 -07002546 unsigned long penalty_jiffies;
Chris Downe26733e2020-03-21 18:22:23 -07002547
2548 if (!max_overage)
2549 return 0;
Chris Down0e4b01d2019-09-23 15:34:55 -07002550
2551 /*
Chris Down0e4b01d2019-09-23 15:34:55 -07002552 * We use overage compared to memory.high to calculate the number of
2553 * jiffies to sleep (penalty_jiffies). Ideally this value should be
2554 * fairly lenient on small overages, and increasingly harsh when the
2555 * memcg in question makes it clear that it has no intention of stopping
2556 * its crazy behaviour, so we exponentially increase the delay based on
2557 * overage amount.
2558 */
Chris Downe26733e2020-03-21 18:22:23 -07002559 penalty_jiffies = max_overage * max_overage * HZ;
2560 penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
2561 penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
Chris Down0e4b01d2019-09-23 15:34:55 -07002562
2563 /*
2564 * Factor in the task's own contribution to the overage, such that four
2565 * N-sized allocations are throttled approximately the same as one
2566 * 4N-sized allocation.
2567 *
2568 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
2569 * larger the current charge patch is than that.
2570 */
Jakub Kicinskiff144e62020-06-01 21:49:45 -07002571 return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
Chris Downe26733e2020-03-21 18:22:23 -07002572}
2573
2574/*
2575 * Scheduled by try_charge() to be executed from the userland return path
2576 * and reclaims memory over the high limit.
2577 */
2578void mem_cgroup_handle_over_high(void)
2579{
2580 unsigned long penalty_jiffies;
2581 unsigned long pflags;
Chris Downb3ff9292020-08-06 23:21:54 -07002582 unsigned long nr_reclaimed;
Chris Downe26733e2020-03-21 18:22:23 -07002583 unsigned int nr_pages = current->memcg_nr_pages_over_high;
Chris Downd977aa92020-08-06 23:21:58 -07002584 int nr_retries = MAX_RECLAIM_RETRIES;
Chris Downe26733e2020-03-21 18:22:23 -07002585 struct mem_cgroup *memcg;
Chris Downb3ff9292020-08-06 23:21:54 -07002586 bool in_retry = false;
Chris Downe26733e2020-03-21 18:22:23 -07002587
2588 if (likely(!nr_pages))
2589 return;
2590
2591 memcg = get_mem_cgroup_from_mm(current->mm);
Chris Downe26733e2020-03-21 18:22:23 -07002592 current->memcg_nr_pages_over_high = 0;
2593
Chris Downb3ff9292020-08-06 23:21:54 -07002594retry_reclaim:
2595 /*
2596 * The allocating task should reclaim at least the batch size, but for
2597 * subsequent retries we only want to do what's necessary to prevent oom
2598 * or breaching resource isolation.
2599 *
2600 * This is distinct from memory.max or page allocator behaviour because
2601 * memory.high is currently batched, whereas memory.max and the page
2602 * allocator run every time an allocation is made.
2603 */
2604 nr_reclaimed = reclaim_high(memcg,
2605 in_retry ? SWAP_CLUSTER_MAX : nr_pages,
2606 GFP_KERNEL);
2607
Chris Downe26733e2020-03-21 18:22:23 -07002608 /*
2609 * memory.high is breached and reclaim is unable to keep up. Throttle
2610 * allocators proactively to slow down excessive growth.
2611 */
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002612 penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2613 mem_find_max_overage(memcg));
Chris Down0e4b01d2019-09-23 15:34:55 -07002614
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002615 penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2616 swap_find_max_overage(memcg));
2617
Chris Down0e4b01d2019-09-23 15:34:55 -07002618 /*
Jakub Kicinskiff144e62020-06-01 21:49:45 -07002619 * Clamp the max delay per usermode return so as to still keep the
2620 * application moving forwards and also permit diagnostics, albeit
2621 * extremely slowly.
2622 */
2623 penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
2624
2625 /*
Chris Down0e4b01d2019-09-23 15:34:55 -07002626 * Don't sleep if the amount of jiffies this memcg owes us is so low
2627 * that it's not even worth doing, in an attempt to be nice to those who
2628 * go only a small amount over their memory.high value and maybe haven't
2629 * been aggressively reclaimed enough yet.
2630 */
2631 if (penalty_jiffies <= HZ / 100)
2632 goto out;
2633
2634 /*
Chris Downb3ff9292020-08-06 23:21:54 -07002635 * If reclaim is making forward progress but we're still over
2636 * memory.high, we want to encourage that rather than doing allocator
2637 * throttling.
2638 */
2639 if (nr_reclaimed || nr_retries--) {
2640 in_retry = true;
2641 goto retry_reclaim;
2642 }
2643
2644 /*
Chris Down0e4b01d2019-09-23 15:34:55 -07002645 * If we exit early, we're guaranteed to die (since
2646 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
2647 * need to account for any ill-begotten jiffies to pay them off later.
2648 */
2649 psi_memstall_enter(&pflags);
2650 schedule_timeout_killable(penalty_jiffies);
2651 psi_memstall_leave(&pflags);
2652
2653out:
2654 css_put(&memcg->css);
Tejun Heob23afb92015-11-05 18:46:11 -08002655}
2656
Johannes Weiner00501b52014-08-08 14:19:20 -07002657static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2658 unsigned int nr_pages)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002659{
Johannes Weinera983b5e2018-01-31 16:16:45 -08002660 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
Chris Downd977aa92020-08-06 23:21:58 -07002661 int nr_retries = MAX_RECLAIM_RETRIES;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002662 struct mem_cgroup *mem_over_limit;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002663 struct page_counter *counter;
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002664 enum oom_status oom_status;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002665 unsigned long nr_reclaimed;
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002666 bool may_swap = true;
2667 bool drained = false;
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002668 unsigned long pflags;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002669
Johannes Weinerce00a962014-09-05 08:43:57 -04002670 if (mem_cgroup_is_root(memcg))
Tejun Heo10d53c72015-11-05 18:46:17 -08002671 return 0;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002672retry:
Michal Hockob6b6cc72014-04-07 15:37:44 -07002673 if (consume_stock(memcg, nr_pages))
Tejun Heo10d53c72015-11-05 18:46:17 -08002674 return 0;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002675
Johannes Weiner7941d212016-01-14 15:21:23 -08002676 if (!do_memsw_account() ||
Johannes Weiner6071ca52015-11-05 18:50:26 -08002677 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2678 if (page_counter_try_charge(&memcg->memory, batch, &counter))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002679 goto done_restock;
Johannes Weiner7941d212016-01-14 15:21:23 -08002680 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002681 page_counter_uncharge(&memcg->memsw, batch);
2682 mem_over_limit = mem_cgroup_from_counter(counter, memory);
Johannes Weiner3fbe7242014-10-09 15:28:54 -07002683 } else {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002684 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002685 may_swap = false;
Johannes Weiner3fbe7242014-10-09 15:28:54 -07002686 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002687
Johannes Weiner6539cc02014-08-06 16:05:42 -07002688 if (batch > nr_pages) {
2689 batch = nr_pages;
2690 goto retry;
2691 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002692
Johannes Weiner06b078f2014-08-06 16:05:44 -07002693 /*
Johannes Weiner869712f2019-11-05 21:17:13 -08002694 * Memcg doesn't have a dedicated reserve for atomic
2695 * allocations. But like the global atomic pool, we need to
2696 * put the burden of reclaim on regular allocation requests
2697 * and let these go through as privileged allocations.
2698 */
2699 if (gfp_mask & __GFP_ATOMIC)
2700 goto force;
2701
2702 /*
Johannes Weiner06b078f2014-08-06 16:05:44 -07002703 * Unlike in global OOM situations, memcg is not in a physical
2704 * memory shortage. Allow dying and OOM-killed tasks to
2705 * bypass the last charges so that they can exit quickly and
2706 * free their memory.
2707 */
Tetsuo Handa7775fac2019-03-05 15:46:47 -08002708 if (unlikely(should_force_charge()))
Tejun Heo10d53c72015-11-05 18:46:17 -08002709 goto force;
Johannes Weiner06b078f2014-08-06 16:05:44 -07002710
Johannes Weiner89a28482016-10-27 17:46:56 -07002711 /*
2712 * Prevent unbounded recursion when reclaim operations need to
2713 * allocate memory. This might exceed the limits temporarily,
2714 * but we prefer facilitating memory reclaim and getting back
2715 * under the limit over triggering OOM kills in these cases.
2716 */
2717 if (unlikely(current->flags & PF_MEMALLOC))
2718 goto force;
2719
Johannes Weiner06b078f2014-08-06 16:05:44 -07002720 if (unlikely(task_in_memcg_oom(current)))
2721 goto nomem;
2722
Mel Gormand0164ad2015-11-06 16:28:21 -08002723 if (!gfpflags_allow_blocking(gfp_mask))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002724 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002725
Johannes Weinere27be242018-04-10 16:29:45 -07002726 memcg_memory_event(mem_over_limit, MEMCG_MAX);
Johannes Weiner241994ed2015-02-11 15:26:06 -08002727
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002728 psi_memstall_enter(&pflags);
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002729 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2730 gfp_mask, may_swap);
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002731 psi_memstall_leave(&pflags);
Johannes Weiner6539cc02014-08-06 16:05:42 -07002732
Johannes Weiner61e02c72014-08-06 16:08:16 -07002733 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
Johannes Weiner6539cc02014-08-06 16:05:42 -07002734 goto retry;
Johannes Weiner28c34c22014-08-06 16:05:47 -07002735
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002736 if (!drained) {
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002737 drain_all_stock(mem_over_limit);
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002738 drained = true;
2739 goto retry;
2740 }
2741
Johannes Weiner28c34c22014-08-06 16:05:47 -07002742 if (gfp_mask & __GFP_NORETRY)
2743 goto nomem;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002744 /*
2745 * Even though the limit is exceeded at this point, reclaim
2746 * may have been able to free some pages. Retry the charge
2747 * before killing the task.
2748 *
2749 * Only for regular pages, though: huge pages are rather
2750 * unlikely to succeed so close to the limit, and we fall back
2751 * to regular pages anyway in case of failure.
2752 */
Johannes Weiner61e02c72014-08-06 16:08:16 -07002753 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002754 goto retry;
2755 /*
2756 * At task move, charge accounts can be doubly counted. So, it's
2757 * better to wait until the end of task_move if something is going on.
2758 */
2759 if (mem_cgroup_wait_acct_move(mem_over_limit))
2760 goto retry;
2761
Johannes Weiner9b130612014-08-06 16:05:51 -07002762 if (nr_retries--)
2763 goto retry;
2764
Shakeel Butt38d38492019-07-11 20:55:48 -07002765 if (gfp_mask & __GFP_RETRY_MAYFAIL)
Michal Hocko29ef6802018-08-17 15:47:11 -07002766 goto nomem;
2767
Johannes Weiner6539cc02014-08-06 16:05:42 -07002768 if (fatal_signal_pending(current))
Tejun Heo10d53c72015-11-05 18:46:17 -08002769 goto force;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002770
Michal Hocko29ef6802018-08-17 15:47:11 -07002771 /*
2772 * keep retrying as long as the memcg oom killer is able to make
2773 * a forward progress or bypass the charge if the oom killer
2774 * couldn't make any progress.
2775 */
2776 oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
Jerome Marchand3608de02015-11-05 18:47:29 -08002777 get_order(nr_pages * PAGE_SIZE));
Michal Hocko29ef6802018-08-17 15:47:11 -07002778 switch (oom_status) {
2779 case OOM_SUCCESS:
Chris Downd977aa92020-08-06 23:21:58 -07002780 nr_retries = MAX_RECLAIM_RETRIES;
Michal Hocko29ef6802018-08-17 15:47:11 -07002781 goto retry;
2782 case OOM_FAILED:
2783 goto force;
2784 default:
2785 goto nomem;
2786 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002787nomem:
Johannes Weiner6d1fdc42014-04-07 15:37:45 -07002788 if (!(gfp_mask & __GFP_NOFAIL))
Johannes Weiner3168ecb2013-10-31 16:34:13 -07002789 return -ENOMEM;
Tejun Heo10d53c72015-11-05 18:46:17 -08002790force:
2791 /*
2792 * The allocation either can't fail or will lead to more memory
2793 * being freed very soon. Allow memory usage go over the limit
2794 * temporarily by force charging it.
2795 */
2796 page_counter_charge(&memcg->memory, nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002797 if (do_memsw_account())
Tejun Heo10d53c72015-11-05 18:46:17 -08002798 page_counter_charge(&memcg->memsw, nr_pages);
Tejun Heo10d53c72015-11-05 18:46:17 -08002799
2800 return 0;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002801
2802done_restock:
2803 if (batch > nr_pages)
2804 refill_stock(memcg, batch - nr_pages);
Tejun Heob23afb92015-11-05 18:46:11 -08002805
Johannes Weiner241994ed2015-02-11 15:26:06 -08002806 /*
Tejun Heob23afb92015-11-05 18:46:11 -08002807 * If the hierarchy is above the normal consumption range, schedule
2808 * reclaim on returning to userland. We can perform reclaim here
Mel Gorman71baba42015-11-06 16:28:28 -08002809 * if __GFP_RECLAIM but let's always punt for simplicity and so that
Tejun Heob23afb92015-11-05 18:46:11 -08002810 * GFP_KERNEL can consistently be used during reclaim. @memcg is
2811 * not recorded as it most likely matches current's and won't
2812 * change in the meantime. As high limit is checked again before
2813 * reclaim, the cost of mismatch is negligible.
Johannes Weiner241994ed2015-02-11 15:26:06 -08002814 */
2815 do {
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002816 bool mem_high, swap_high;
2817
2818 mem_high = page_counter_read(&memcg->memory) >
2819 READ_ONCE(memcg->memory.high);
2820 swap_high = page_counter_read(&memcg->swap) >
2821 READ_ONCE(memcg->swap.high);
2822
2823 /* Don't bother a random interrupted task */
2824 if (in_interrupt()) {
2825 if (mem_high) {
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002826 schedule_work(&memcg->high_work);
2827 break;
2828 }
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002829 continue;
2830 }
2831
2832 if (mem_high || swap_high) {
2833 /*
2834 * The allocating tasks in this cgroup will need to do
2835 * reclaim or be throttled to prevent further growth
2836 * of the memory or swap footprints.
2837 *
2838 * Target some best-effort fairness between the tasks,
2839 * and distribute reclaim work and delay penalties
2840 * based on how much each task is actually allocating.
2841 */
Vladimir Davydov9516a182015-12-11 13:40:24 -08002842 current->memcg_nr_pages_over_high += batch;
Tejun Heob23afb92015-11-05 18:46:11 -08002843 set_notify_resume(current);
2844 break;
2845 }
Johannes Weiner241994ed2015-02-11 15:26:06 -08002846 } while ((memcg = parent_mem_cgroup(memcg)));
Tejun Heo10d53c72015-11-05 18:46:17 -08002847
2848 return 0;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002849}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002850
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07002851#if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
Johannes Weiner00501b52014-08-08 14:19:20 -07002852static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002853{
Johannes Weinerce00a962014-09-05 08:43:57 -04002854 if (mem_cgroup_is_root(memcg))
2855 return;
2856
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002857 page_counter_uncharge(&memcg->memory, nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002858 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002859 page_counter_uncharge(&memcg->memsw, nr_pages);
KAMEZAWA Hiroyukid01dd172012-05-29 15:07:03 -07002860}
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07002861#endif
KAMEZAWA Hiroyukid01dd172012-05-29 15:07:03 -07002862
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07002863static void commit_charge(struct page *page, struct mem_cgroup *memcg)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002864{
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002865 VM_BUG_ON_PAGE(page_memcg(page), page);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002866 /*
Alex Shia5eb0112020-12-14 19:06:42 -08002867 * Any of the following ensures page's memcg stability:
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002868 *
Johannes Weinera0b5b412020-06-03 16:02:27 -07002869 * - the page lock
2870 * - LRU isolation
2871 * - lock_page_memcg()
2872 * - exclusive reference
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002873 */
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002874 page->memcg_data = (unsigned long)memcg;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002875}
2876
Kirill Tkhai84c07d12018-08-17 15:47:25 -07002877#ifdef CONFIG_MEMCG_KMEM
Roman Gushchin10befea2020-08-06 23:21:27 -07002878int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002879 gfp_t gfp, bool new_page)
Roman Gushchin10befea2020-08-06 23:21:27 -07002880{
2881 unsigned int objects = objs_per_slab_page(s, page);
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002882 unsigned long memcg_data;
Roman Gushchin10befea2020-08-06 23:21:27 -07002883 void *vec;
2884
2885 vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
2886 page_to_nid(page));
2887 if (!vec)
2888 return -ENOMEM;
2889
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002890 memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
2891 if (new_page) {
2892 /*
2893 * If the slab page is brand new and nobody can yet access
2894 * it's memcg_data, no synchronization is required and
2895 * memcg_data can be simply assigned.
2896 */
2897 page->memcg_data = memcg_data;
2898 } else if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
2899 /*
2900 * If the slab page is already in use, somebody can allocate
2901 * and assign obj_cgroups in parallel. In this case the existing
2902 * objcg vector should be reused.
2903 */
Roman Gushchin10befea2020-08-06 23:21:27 -07002904 kfree(vec);
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002905 return 0;
2906 }
Roman Gushchin10befea2020-08-06 23:21:27 -07002907
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002908 kmemleak_not_leak(vec);
Roman Gushchin10befea2020-08-06 23:21:27 -07002909 return 0;
2910}
2911
Roman Gushchin8380ce42020-03-28 19:17:25 -07002912/*
2913 * Returns a pointer to the memory cgroup to which the kernel object is charged.
2914 *
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002915 * A passed kernel object can be a slab object or a generic kernel page, so
2916 * different mechanisms for getting the memory cgroup pointer should be used.
2917 * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller
2918 * can not know for sure how the kernel object is implemented.
2919 * mem_cgroup_from_obj() can be safely used in such cases.
2920 *
Roman Gushchin8380ce42020-03-28 19:17:25 -07002921 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2922 * cgroup_mutex, etc.
2923 */
2924struct mem_cgroup *mem_cgroup_from_obj(void *p)
2925{
2926 struct page *page;
2927
2928 if (mem_cgroup_disabled())
2929 return NULL;
2930
2931 page = virt_to_head_page(p);
2932
2933 /*
Roman Gushchin98556092020-08-06 23:21:10 -07002934 * Slab objects are accounted individually, not per-page.
2935 * Memcg membership data for each individual object is saved in
2936 * the page->obj_cgroups.
Roman Gushchin8380ce42020-03-28 19:17:25 -07002937 */
Roman Gushchin270c6a72020-12-01 13:58:28 -08002938 if (page_objcgs_check(page)) {
Roman Gushchin98556092020-08-06 23:21:10 -07002939 struct obj_cgroup *objcg;
2940 unsigned int off;
2941
2942 off = obj_to_index(page->slab_cache, page, p);
Roman Gushchin270c6a72020-12-01 13:58:28 -08002943 objcg = page_objcgs(page)[off];
Roman Gushchin10befea2020-08-06 23:21:27 -07002944 if (objcg)
2945 return obj_cgroup_memcg(objcg);
2946
2947 return NULL;
Roman Gushchin98556092020-08-06 23:21:10 -07002948 }
Roman Gushchin8380ce42020-03-28 19:17:25 -07002949
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002950 /*
2951 * page_memcg_check() is used here, because page_has_obj_cgroups()
2952 * check above could fail because the object cgroups vector wasn't set
2953 * at that moment, but it can be set concurrently.
2954 * page_memcg_check(page) will guarantee that a proper memory
2955 * cgroup pointer or NULL will be returned.
2956 */
2957 return page_memcg_check(page);
Roman Gushchin8380ce42020-03-28 19:17:25 -07002958}
2959
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002960__always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
2961{
2962 struct obj_cgroup *objcg = NULL;
2963 struct mem_cgroup *memcg;
2964
Roman Gushchin279c3392020-10-17 16:13:44 -07002965 if (memcg_kmem_bypass())
2966 return NULL;
2967
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002968 rcu_read_lock();
Roman Gushchin37d59852020-10-17 16:13:50 -07002969 if (unlikely(active_memcg()))
2970 memcg = active_memcg();
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002971 else
2972 memcg = mem_cgroup_from_task(current);
2973
2974 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
2975 objcg = rcu_dereference(memcg->objcg);
2976 if (objcg && obj_cgroup_tryget(objcg))
2977 break;
Muchun Song2f7659a2020-12-14 19:06:31 -08002978 objcg = NULL;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002979 }
2980 rcu_read_unlock();
2981
2982 return objcg;
2983}
2984
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002985static int memcg_alloc_cache_id(void)
Glauber Costa55007d82012-12-18 14:22:38 -08002986{
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002987 int id, size;
2988 int err;
Glauber Costa55007d82012-12-18 14:22:38 -08002989
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08002990 id = ida_simple_get(&memcg_cache_ida,
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002991 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2992 if (id < 0)
2993 return id;
2994
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08002995 if (id < memcg_nr_cache_ids)
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07002996 return id;
2997
2998 /*
2999 * There's no space for the new id in memcg_caches arrays,
3000 * so we have to grow them.
3001 */
Vladimir Davydov05257a12015-02-12 14:59:01 -08003002 down_write(&memcg_cache_ids_sem);
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003003
3004 size = 2 * (id + 1);
Glauber Costa55007d82012-12-18 14:22:38 -08003005 if (size < MEMCG_CACHES_MIN_SIZE)
3006 size = MEMCG_CACHES_MIN_SIZE;
3007 else if (size > MEMCG_CACHES_MAX_SIZE)
3008 size = MEMCG_CACHES_MAX_SIZE;
3009
Roman Gushchin98556092020-08-06 23:21:10 -07003010 err = memcg_update_all_list_lrus(size);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -08003011 if (!err)
Vladimir Davydov05257a12015-02-12 14:59:01 -08003012 memcg_nr_cache_ids = size;
3013
3014 up_write(&memcg_cache_ids_sem);
3015
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003016 if (err) {
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08003017 ida_simple_remove(&memcg_cache_ida, id);
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003018 return err;
3019 }
3020 return id;
3021}
3022
3023static void memcg_free_cache_id(int id)
3024{
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08003025 ida_simple_remove(&memcg_cache_ida, id);
Glauber Costa55007d82012-12-18 14:22:38 -08003026}
3027
Vladimir Davydov45264772016-07-26 15:24:21 -07003028/**
Roman Gushchin4b13f642020-04-01 21:06:56 -07003029 * __memcg_kmem_charge: charge a number of kernel pages to a memcg
Roman Gushchin10eaec22020-04-01 21:06:39 -07003030 * @memcg: memory cgroup to charge
Vladimir Davydov45264772016-07-26 15:24:21 -07003031 * @gfp: reclaim mode
Roman Gushchin92d0510c2020-04-01 21:06:49 -07003032 * @nr_pages: number of pages to charge
Vladimir Davydov45264772016-07-26 15:24:21 -07003033 *
3034 * Returns 0 on success, an error code on failure.
3035 */
Roman Gushchinc1a660d2021-02-24 12:03:58 -08003036static int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
3037 unsigned int nr_pages)
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003038{
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003039 struct page_counter *counter;
Johannes Weiner6071ca52015-11-05 18:50:26 -08003040 int ret;
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003041
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003042 ret = try_charge(memcg, gfp, nr_pages);
Johannes Weiner52c29b02016-01-20 15:02:35 -08003043 if (ret)
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003044 return ret;
Johannes Weiner52c29b02016-01-20 15:02:35 -08003045
3046 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
3047 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
Michal Hockoe55d9d92019-09-25 16:45:53 -07003048
3049 /*
3050 * Enforce __GFP_NOFAIL allocation because callers are not
3051 * prepared to see failures and likely do not have any failure
3052 * handling code.
3053 */
3054 if (gfp & __GFP_NOFAIL) {
3055 page_counter_charge(&memcg->kmem, nr_pages);
3056 return 0;
3057 }
Johannes Weiner52c29b02016-01-20 15:02:35 -08003058 cancel_charge(memcg, nr_pages);
3059 return -ENOMEM;
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003060 }
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003061 return 0;
3062}
3063
Vladimir Davydov45264772016-07-26 15:24:21 -07003064/**
Roman Gushchin4b13f642020-04-01 21:06:56 -07003065 * __memcg_kmem_uncharge: uncharge a number of kernel pages from a memcg
3066 * @memcg: memcg to uncharge
3067 * @nr_pages: number of pages to uncharge
3068 */
Roman Gushchinc1a660d2021-02-24 12:03:58 -08003069static void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
Roman Gushchin4b13f642020-04-01 21:06:56 -07003070{
3071 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
3072 page_counter_uncharge(&memcg->kmem, nr_pages);
3073
Roman Gushchin3de7d4f2021-01-23 21:01:07 -08003074 refill_stock(memcg, nr_pages);
Roman Gushchin4b13f642020-04-01 21:06:56 -07003075}
3076
3077/**
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07003078 * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
Vladimir Davydov45264772016-07-26 15:24:21 -07003079 * @page: page to charge
3080 * @gfp: reclaim mode
3081 * @order: allocation order
3082 *
3083 * Returns 0 on success, an error code on failure.
3084 */
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07003085int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003086{
3087 struct mem_cgroup *memcg;
Vladimir Davydovfcff7d72016-03-17 14:17:29 -07003088 int ret = 0;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003089
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07003090 memcg = get_mem_cgroup_from_current();
Roman Gushchin279c3392020-10-17 16:13:44 -07003091 if (memcg && !mem_cgroup_is_root(memcg)) {
Roman Gushchin4b13f642020-04-01 21:06:56 -07003092 ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
Roman Gushchin4d96ba32019-07-11 20:56:31 -07003093 if (!ret) {
Roman Gushchin18b2db32020-12-01 13:58:30 -08003094 page->memcg_data = (unsigned long)memcg |
3095 MEMCG_DATA_KMEM;
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07003096 return 0;
Roman Gushchin4d96ba32019-07-11 20:56:31 -07003097 }
Roman Gushchin279c3392020-10-17 16:13:44 -07003098 css_put(&memcg->css);
Vladimir Davydovc4159a72016-08-08 23:03:12 +03003099 }
Vladimir Davydovd05e83a2015-11-05 18:48:59 -08003100 return ret;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003101}
Roman Gushchin49a18ea2019-07-11 20:56:13 -07003102
3103/**
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07003104 * __memcg_kmem_uncharge_page: uncharge a kmem page
Vladimir Davydov45264772016-07-26 15:24:21 -07003105 * @page: page to uncharge
3106 * @order: allocation order
3107 */
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07003108void __memcg_kmem_uncharge_page(struct page *page, int order)
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003109{
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08003110 struct mem_cgroup *memcg = page_memcg(page);
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003111 unsigned int nr_pages = 1 << order;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003112
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003113 if (!memcg)
3114 return;
3115
Sasha Levin309381fea2014-01-23 15:52:54 -08003116 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
Roman Gushchin4b13f642020-04-01 21:06:56 -07003117 __memcg_kmem_uncharge(memcg, nr_pages);
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08003118 page->memcg_data = 0;
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07003119 css_put(&memcg->css);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -08003120}
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003121
3122static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3123{
3124 struct memcg_stock_pcp *stock;
3125 unsigned long flags;
3126 bool ret = false;
3127
3128 local_irq_save(flags);
3129
3130 stock = this_cpu_ptr(&memcg_stock);
3131 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
3132 stock->nr_bytes -= nr_bytes;
3133 ret = true;
3134 }
3135
3136 local_irq_restore(flags);
3137
3138 return ret;
3139}
3140
3141static void drain_obj_stock(struct memcg_stock_pcp *stock)
3142{
3143 struct obj_cgroup *old = stock->cached_objcg;
3144
3145 if (!old)
3146 return;
3147
3148 if (stock->nr_bytes) {
3149 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3150 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);
3151
3152 if (nr_pages) {
3153 rcu_read_lock();
3154 __memcg_kmem_uncharge(obj_cgroup_memcg(old), nr_pages);
3155 rcu_read_unlock();
3156 }
3157
3158 /*
3159 * The leftover is flushed to the centralized per-memcg value.
3160 * On the next attempt to refill obj stock it will be moved
3161 * to a per-cpu stock (probably, on an other CPU), see
3162 * refill_obj_stock().
3163 *
3164 * How often it's flushed is a trade-off between the memory
3165 * limit enforcement accuracy and potential CPU contention,
3166 * so it might be changed in the future.
3167 */
3168 atomic_add(nr_bytes, &old->nr_charged_bytes);
3169 stock->nr_bytes = 0;
3170 }
3171
3172 obj_cgroup_put(old);
3173 stock->cached_objcg = NULL;
3174}
3175
3176static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
3177 struct mem_cgroup *root_memcg)
3178{
3179 struct mem_cgroup *memcg;
3180
3181 if (stock->cached_objcg) {
3182 memcg = obj_cgroup_memcg(stock->cached_objcg);
3183 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3184 return true;
3185 }
3186
3187 return false;
3188}
3189
3190static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3191{
3192 struct memcg_stock_pcp *stock;
3193 unsigned long flags;
3194
3195 local_irq_save(flags);
3196
3197 stock = this_cpu_ptr(&memcg_stock);
3198 if (stock->cached_objcg != objcg) { /* reset if necessary */
3199 drain_obj_stock(stock);
3200 obj_cgroup_get(objcg);
3201 stock->cached_objcg = objcg;
3202 stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);
3203 }
3204 stock->nr_bytes += nr_bytes;
3205
3206 if (stock->nr_bytes > PAGE_SIZE)
3207 drain_obj_stock(stock);
3208
3209 local_irq_restore(flags);
3210}
3211
3212int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3213{
3214 struct mem_cgroup *memcg;
3215 unsigned int nr_pages, nr_bytes;
3216 int ret;
3217
3218 if (consume_obj_stock(objcg, size))
3219 return 0;
3220
3221 /*
3222 * In theory, memcg->nr_charged_bytes can have enough
3223 * pre-charged bytes to satisfy the allocation. However,
3224 * flushing memcg->nr_charged_bytes requires two atomic
3225 * operations, and memcg->nr_charged_bytes can't be big,
3226 * so it's better to ignore it and try grab some new pages.
3227 * memcg->nr_charged_bytes will be flushed in
3228 * refill_obj_stock(), called from this function or
3229 * independently later.
3230 */
3231 rcu_read_lock();
Muchun Songeefbfa72020-12-14 19:06:35 -08003232retry:
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003233 memcg = obj_cgroup_memcg(objcg);
Muchun Songeefbfa72020-12-14 19:06:35 -08003234 if (unlikely(!css_tryget(&memcg->css)))
3235 goto retry;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003236 rcu_read_unlock();
3237
3238 nr_pages = size >> PAGE_SHIFT;
3239 nr_bytes = size & (PAGE_SIZE - 1);
3240
3241 if (nr_bytes)
3242 nr_pages += 1;
3243
3244 ret = __memcg_kmem_charge(memcg, gfp, nr_pages);
3245 if (!ret && nr_bytes)
3246 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes);
3247
3248 css_put(&memcg->css);
3249 return ret;
3250}
3251
3252void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3253{
3254 refill_obj_stock(objcg, size);
3255}
3256
Kirill Tkhai84c07d12018-08-17 15:47:25 -07003257#endif /* CONFIG_MEMCG_KMEM */
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003258
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003259/*
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08003260 * Because page_memcg(head) is not set on tails, set it now.
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003261 */
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08003262void split_page_memcg(struct page *head, unsigned int nr)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003263{
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08003264 struct mem_cgroup *memcg = page_memcg(head);
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003265 int i;
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003266
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08003267 if (mem_cgroup_disabled() || !memcg)
KAMEZAWA Hiroyuki3d37c4a2011-01-25 15:07:28 -08003268 return;
David Rientjesb070e652013-05-07 16:18:09 -07003269
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08003270 for (i = 1; i < nr; i++)
3271 head[i].memcg_data = head->memcg_data;
3272 css_get_many(&memcg->css, nr - 1);
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003273}
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003274
Andrew Mortonc255a452012-07-31 16:43:02 -07003275#ifdef CONFIG_MEMCG_SWAP
Daisuke Nishimura02491442010-03-10 15:22:17 -08003276/**
3277 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
3278 * @entry: swap entry to be moved
3279 * @from: mem_cgroup which the entry is moved from
3280 * @to: mem_cgroup which the entry is moved to
3281 *
3282 * It succeeds only when the swap_cgroup's record for this entry is the same
3283 * as the mem_cgroup's id of @from.
3284 *
3285 * Returns 0 on success, -EINVAL on failure.
3286 *
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003287 * The caller must have charged to @to, IOW, called page_counter_charge() about
Daisuke Nishimura02491442010-03-10 15:22:17 -08003288 * both res and memsw, and called css_get().
3289 */
3290static int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07003291 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08003292{
3293 unsigned short old_id, new_id;
3294
Li Zefan34c00c32013-09-23 16:56:01 +08003295 old_id = mem_cgroup_id(from);
3296 new_id = mem_cgroup_id(to);
Daisuke Nishimura02491442010-03-10 15:22:17 -08003297
3298 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Johannes Weinerc9019e92018-01-31 16:16:37 -08003299 mod_memcg_state(from, MEMCG_SWAP, -1);
3300 mod_memcg_state(to, MEMCG_SWAP, 1);
Daisuke Nishimura02491442010-03-10 15:22:17 -08003301 return 0;
3302 }
3303 return -EINVAL;
3304}
3305#else
3306static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07003307 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08003308{
3309 return -EINVAL;
3310}
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003311#endif
3312
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003313static DEFINE_MUTEX(memcg_max_mutex);
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07003314
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003315static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3316 unsigned long max, bool memsw)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003317{
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003318 bool enlarge = false;
Shakeel Buttbb4a7ea2018-06-07 17:07:27 -07003319 bool drained = false;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003320 int ret;
Yu Zhaoc054a782018-01-31 16:20:02 -08003321 bool limits_invariant;
3322 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003323
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003324 do {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003325 if (signal_pending(current)) {
3326 ret = -EINTR;
3327 break;
3328 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003329
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003330 mutex_lock(&memcg_max_mutex);
Yu Zhaoc054a782018-01-31 16:20:02 -08003331 /*
3332 * Make sure that the new limit (memsw or memory limit) doesn't
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003333 * break our basic invariant rule memory.max <= memsw.max.
Yu Zhaoc054a782018-01-31 16:20:02 -08003334 */
Chris Down15b42562020-04-01 21:07:20 -07003335 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003336 max <= memcg->memsw.max;
Yu Zhaoc054a782018-01-31 16:20:02 -08003337 if (!limits_invariant) {
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003338 mutex_unlock(&memcg_max_mutex);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003339 ret = -EINVAL;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003340 break;
3341 }
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003342 if (max > counter->max)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003343 enlarge = true;
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003344 ret = page_counter_set_max(counter, max);
3345 mutex_unlock(&memcg_max_mutex);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003346
3347 if (!ret)
3348 break;
3349
Shakeel Buttbb4a7ea2018-06-07 17:07:27 -07003350 if (!drained) {
3351 drain_all_stock(memcg);
3352 drained = true;
3353 continue;
3354 }
3355
Andrey Ryabinin1ab5c052018-01-31 16:20:37 -08003356 if (!try_to_free_mem_cgroup_pages(memcg, 1,
3357 GFP_KERNEL, !memsw)) {
3358 ret = -EBUSY;
3359 break;
3360 }
3361 } while (true);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003362
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003363 if (!ret && enlarge)
3364 memcg_oom_recover(memcg);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003365
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003366 return ret;
3367}
3368
Mel Gormanef8f2322016-07-28 15:46:05 -07003369unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -07003370 gfp_t gfp_mask,
3371 unsigned long *total_scanned)
3372{
3373 unsigned long nr_reclaimed = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003374 struct mem_cgroup_per_node *mz, *next_mz = NULL;
Andrew Morton0608f432013-09-24 15:27:41 -07003375 unsigned long reclaimed;
3376 int loop = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003377 struct mem_cgroup_tree_per_node *mctz;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003378 unsigned long excess;
Andrew Morton0608f432013-09-24 15:27:41 -07003379 unsigned long nr_scanned;
3380
3381 if (order > 0)
3382 return 0;
3383
Mel Gormanef8f2322016-07-28 15:46:05 -07003384 mctz = soft_limit_tree_node(pgdat->node_id);
Michal Hockod6507ff2016-08-02 14:02:37 -07003385
3386 /*
3387 * Do not even bother to check the largest node if the root
3388 * is empty. Do it lockless to prevent lock bouncing. Races
3389 * are acceptable as soft limit is best effort anyway.
3390 */
Laurent Dufourbfc72282017-03-09 16:17:06 -08003391 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
Michal Hockod6507ff2016-08-02 14:02:37 -07003392 return 0;
3393
Andrew Morton0608f432013-09-24 15:27:41 -07003394 /*
3395 * This loop can run a while, specially if mem_cgroup's continuously
3396 * keep exceeding their soft limit and putting the system under
3397 * pressure
3398 */
3399 do {
3400 if (next_mz)
3401 mz = next_mz;
3402 else
3403 mz = mem_cgroup_largest_soft_limit_node(mctz);
3404 if (!mz)
3405 break;
3406
3407 nr_scanned = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003408 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07003409 gfp_mask, &nr_scanned);
3410 nr_reclaimed += reclaimed;
3411 *total_scanned += nr_scanned;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07003412 spin_lock_irq(&mctz->lock);
Vladimir Davydovbc2f2e72014-12-10 15:43:40 -08003413 __mem_cgroup_remove_exceeded(mz, mctz);
Andrew Morton0608f432013-09-24 15:27:41 -07003414
3415 /*
3416 * If we failed to reclaim anything from this memory cgroup
3417 * it is time to move on to the next cgroup
3418 */
3419 next_mz = NULL;
Vladimir Davydovbc2f2e72014-12-10 15:43:40 -08003420 if (!reclaimed)
3421 next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
3422
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003423 excess = soft_limit_excess(mz->memcg);
Andrew Morton0608f432013-09-24 15:27:41 -07003424 /*
3425 * One school of thought says that we should not add
3426 * back the node to the tree if reclaim returns 0.
3427 * But our reclaim could return 0, simply because due
3428 * to priority we are exposing a smaller subset of
3429 * memory to reclaim from. Consider this as a longer
3430 * term TODO.
3431 */
3432 /* If excess == 0, no tree ops */
Johannes Weinercf2c8122014-06-06 14:38:21 -07003433 __mem_cgroup_insert_exceeded(mz, mctz, excess);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07003434 spin_unlock_irq(&mctz->lock);
Andrew Morton0608f432013-09-24 15:27:41 -07003435 css_put(&mz->memcg->css);
3436 loop++;
3437 /*
3438 * Could not reclaim anything and there are no more
3439 * mem cgroups to try or we seem to be looping without
3440 * reclaiming anything.
3441 */
3442 if (!nr_reclaimed &&
3443 (next_mz == NULL ||
3444 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3445 break;
3446 } while (!nr_reclaimed);
3447 if (next_mz)
3448 css_put(&next_mz->memcg->css);
3449 return nr_reclaimed;
3450}
3451
Tejun Heoea280e72014-05-16 13:22:48 -04003452/*
Greg Thelen51038172016-05-20 16:58:18 -07003453 * Reclaims as many pages from the given memcg as possible.
Michal Hockoc26251f2012-10-26 13:37:28 +02003454 *
3455 * Caller is responsible for holding css reference for memcg.
3456 */
3457static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3458{
Chris Downd977aa92020-08-06 23:21:58 -07003459 int nr_retries = MAX_RECLAIM_RETRIES;
Michal Hockoc26251f2012-10-26 13:37:28 +02003460
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003461 /* we call try-to-free pages for make this cgroup empty */
3462 lru_add_drain_all();
Junaid Shahidd12c60f2018-06-07 17:07:31 -07003463
3464 drain_all_stock(memcg);
3465
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003466 /* try to free all pages in this cgroup */
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003467 while (nr_retries && page_counter_read(&memcg->memory)) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003468 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003469
Michal Hockoc26251f2012-10-26 13:37:28 +02003470 if (signal_pending(current))
3471 return -EINTR;
3472
Johannes Weinerb70a2a22014-10-09 15:28:56 -07003473 progress = try_to_free_mem_cgroup_pages(memcg, 1,
3474 GFP_KERNEL, true);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003475 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003476 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003477 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02003478 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003479 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003480
3481 }
Michal Hockoab5196c2012-10-26 13:37:32 +02003482
3483 return 0;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003484}
3485
Tejun Heo6770c642014-05-13 12:16:21 -04003486static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
3487 char *buf, size_t nbytes,
3488 loff_t off)
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003489{
Tejun Heo6770c642014-05-13 12:16:21 -04003490 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Michal Hockoc26251f2012-10-26 13:37:28 +02003491
Michal Hockod8423012012-10-26 13:37:29 +02003492 if (mem_cgroup_is_root(memcg))
3493 return -EINVAL;
Tejun Heo6770c642014-05-13 12:16:21 -04003494 return mem_cgroup_force_empty(memcg) ?: nbytes;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003495}
3496
Tejun Heo182446d2013-08-08 20:11:24 -04003497static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
3498 struct cftype *cft)
Balbir Singh18f59ea2009-01-07 18:08:07 -08003499{
Roman Gushchinbef86202020-12-14 19:06:49 -08003500 return 1;
Balbir Singh18f59ea2009-01-07 18:08:07 -08003501}
3502
Tejun Heo182446d2013-08-08 20:11:24 -04003503static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
3504 struct cftype *cft, u64 val)
Balbir Singh18f59ea2009-01-07 18:08:07 -08003505{
Roman Gushchinbef86202020-12-14 19:06:49 -08003506 if (val == 1)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003507 return 0;
Glauber Costa567fb432012-07-31 16:43:07 -07003508
Roman Gushchinbef86202020-12-14 19:06:49 -08003509 pr_warn_once("Non-hierarchical mode is deprecated. "
3510 "Please report your usecase to linux-mm@kvack.org if you "
3511 "depend on this functionality.\n");
Glauber Costa567fb432012-07-31 16:43:07 -07003512
Roman Gushchinbef86202020-12-14 19:06:49 -08003513 return -EINVAL;
Balbir Singh18f59ea2009-01-07 18:08:07 -08003514}
3515
Andrew Morton6f646152015-11-06 16:28:58 -08003516static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
Johannes Weinerce00a962014-09-05 08:43:57 -04003517{
Johannes Weiner42a30032019-05-14 15:47:12 -07003518 unsigned long val;
Johannes Weinerce00a962014-09-05 08:43:57 -04003519
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003520 if (mem_cgroup_is_root(memcg)) {
Johannes Weiner2d146aa2021-04-29 22:56:26 -07003521 cgroup_rstat_flush(memcg->css.cgroup);
Johannes Weiner0d1c2072020-06-03 16:01:54 -07003522 val = memcg_page_state(memcg, NR_FILE_PAGES) +
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07003523 memcg_page_state(memcg, NR_ANON_MAPPED);
Johannes Weiner42a30032019-05-14 15:47:12 -07003524 if (swap)
3525 val += memcg_page_state(memcg, MEMCG_SWAP);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003526 } else {
Johannes Weinerce00a962014-09-05 08:43:57 -04003527 if (!swap)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003528 val = page_counter_read(&memcg->memory);
Johannes Weinerce00a962014-09-05 08:43:57 -04003529 else
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003530 val = page_counter_read(&memcg->memsw);
Johannes Weinerce00a962014-09-05 08:43:57 -04003531 }
Michal Hockoc12176d2015-11-05 18:50:29 -08003532 return val;
Johannes Weinerce00a962014-09-05 08:43:57 -04003533}
3534
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003535enum {
3536 RES_USAGE,
3537 RES_LIMIT,
3538 RES_MAX_USAGE,
3539 RES_FAILCNT,
3540 RES_SOFT_LIMIT,
3541};
Johannes Weinerce00a962014-09-05 08:43:57 -04003542
Tejun Heo791badb2013-12-05 12:28:02 -05003543static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
Johannes Weiner05b84302014-08-06 16:05:59 -07003544 struct cftype *cft)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003545{
Tejun Heo182446d2013-08-08 20:11:24 -04003546 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003547 struct page_counter *counter;
Tejun Heoaf36f902012-04-01 12:09:55 -07003548
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003549 switch (MEMFILE_TYPE(cft->private)) {
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003550 case _MEM:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003551 counter = &memcg->memory;
Glauber Costa510fc4e2012-12-18 14:21:47 -08003552 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003553 case _MEMSWAP:
3554 counter = &memcg->memsw;
3555 break;
3556 case _KMEM:
3557 counter = &memcg->kmem;
3558 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003559 case _TCP:
Johannes Weiner0db15292016-01-20 15:02:50 -08003560 counter = &memcg->tcpmem;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003561 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003562 default:
3563 BUG();
3564 }
3565
3566 switch (MEMFILE_ATTR(cft->private)) {
3567 case RES_USAGE:
3568 if (counter == &memcg->memory)
Michal Hockoc12176d2015-11-05 18:50:29 -08003569 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003570 if (counter == &memcg->memsw)
Michal Hockoc12176d2015-11-05 18:50:29 -08003571 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003572 return (u64)page_counter_read(counter) * PAGE_SIZE;
3573 case RES_LIMIT:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003574 return (u64)counter->max * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003575 case RES_MAX_USAGE:
3576 return (u64)counter->watermark * PAGE_SIZE;
3577 case RES_FAILCNT:
3578 return counter->failcnt;
3579 case RES_SOFT_LIMIT:
3580 return (u64)memcg->soft_limit * PAGE_SIZE;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003581 default:
3582 BUG();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003583 }
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003584}
Glauber Costa510fc4e2012-12-18 14:21:47 -08003585
Kirill Tkhai84c07d12018-08-17 15:47:25 -07003586#ifdef CONFIG_MEMCG_KMEM
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003587static int memcg_online_kmem(struct mem_cgroup *memcg)
Vladimir Davydovd6441632014-01-23 15:53:09 -08003588{
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003589 struct obj_cgroup *objcg;
Vladimir Davydovd6441632014-01-23 15:53:09 -08003590 int memcg_id;
3591
Vladimir Davydovb313aee2016-03-17 14:18:27 -07003592 if (cgroup_memory_nokmem)
3593 return 0;
3594
Vladimir Davydov2a4db7e2015-02-12 14:59:32 -08003595 BUG_ON(memcg->kmemcg_id >= 0);
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003596 BUG_ON(memcg->kmem_state);
Vladimir Davydovd6441632014-01-23 15:53:09 -08003597
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003598 memcg_id = memcg_alloc_cache_id();
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003599 if (memcg_id < 0)
3600 return memcg_id;
Vladimir Davydovd6441632014-01-23 15:53:09 -08003601
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003602 objcg = obj_cgroup_alloc();
3603 if (!objcg) {
3604 memcg_free_cache_id(memcg_id);
3605 return -ENOMEM;
3606 }
3607 objcg->memcg = memcg;
3608 rcu_assign_pointer(memcg->objcg, objcg);
3609
Roman Gushchind648bcc2020-08-06 23:20:28 -07003610 static_branch_enable(&memcg_kmem_enabled_key);
3611
Vladimir Davydov900a38f2014-12-12 16:55:10 -08003612 memcg->kmemcg_id = memcg_id;
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003613 memcg->kmem_state = KMEM_ONLINE;
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003614
3615 return 0;
Vladimir Davydovd6441632014-01-23 15:53:09 -08003616}
3617
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003618static void memcg_offline_kmem(struct mem_cgroup *memcg)
3619{
3620 struct cgroup_subsys_state *css;
3621 struct mem_cgroup *parent, *child;
3622 int kmemcg_id;
3623
3624 if (memcg->kmem_state != KMEM_ONLINE)
3625 return;
Roman Gushchin98556092020-08-06 23:21:10 -07003626
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003627 memcg->kmem_state = KMEM_ALLOCATED;
3628
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003629 parent = parent_mem_cgroup(memcg);
3630 if (!parent)
3631 parent = root_mem_cgroup;
3632
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003633 memcg_reparent_objcgs(memcg, parent);
Roman Gushchinfb2f2b02019-07-11 20:56:34 -07003634
3635 kmemcg_id = memcg->kmemcg_id;
3636 BUG_ON(kmemcg_id < 0);
3637
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003638 /*
3639 * Change kmemcg_id of this cgroup and all its descendants to the
3640 * parent's id, and then move all entries from this cgroup's list_lrus
3641 * to ones of the parent. After we have finished, all list_lrus
3642 * corresponding to this cgroup are guaranteed to remain empty. The
3643 * ordering is imposed by list_lru_node->lock taken by
3644 * memcg_drain_all_list_lrus().
3645 */
Tejun Heo3a06bb72016-06-03 14:55:44 -07003646 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003647 css_for_each_descendant_pre(css, &memcg->css) {
3648 child = mem_cgroup_from_css(css);
3649 BUG_ON(child->kmemcg_id != kmemcg_id);
3650 child->kmemcg_id = parent->kmemcg_id;
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003651 }
Tejun Heo3a06bb72016-06-03 14:55:44 -07003652 rcu_read_unlock();
3653
Kirill Tkhai9bec5c32018-08-17 15:47:58 -07003654 memcg_drain_all_list_lrus(kmemcg_id, parent);
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003655
3656 memcg_free_cache_id(kmemcg_id);
3657}
3658
3659static void memcg_free_kmem(struct mem_cgroup *memcg)
3660{
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003661 /* css_alloc() failed, offlining didn't happen */
3662 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3663 memcg_offline_kmem(memcg);
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003664}
Vladimir Davydovd6441632014-01-23 15:53:09 -08003665#else
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003666static int memcg_online_kmem(struct mem_cgroup *memcg)
Johannes Weiner127424c2016-01-20 15:02:32 -08003667{
3668 return 0;
3669}
3670static void memcg_offline_kmem(struct mem_cgroup *memcg)
3671{
3672}
3673static void memcg_free_kmem(struct mem_cgroup *memcg)
3674{
3675}
Kirill Tkhai84c07d12018-08-17 15:47:25 -07003676#endif /* CONFIG_MEMCG_KMEM */
Johannes Weiner127424c2016-01-20 15:02:32 -08003677
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003678static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3679 unsigned long max)
Johannes Weiner127424c2016-01-20 15:02:32 -08003680{
Vladimir Davydovb313aee2016-03-17 14:18:27 -07003681 int ret;
Johannes Weiner127424c2016-01-20 15:02:32 -08003682
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003683 mutex_lock(&memcg_max_mutex);
3684 ret = page_counter_set_max(&memcg->kmem, max);
3685 mutex_unlock(&memcg_max_mutex);
Johannes Weiner127424c2016-01-20 15:02:32 -08003686 return ret;
3687}
Glauber Costa510fc4e2012-12-18 14:21:47 -08003688
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003689static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003690{
3691 int ret;
3692
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003693 mutex_lock(&memcg_max_mutex);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003694
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003695 ret = page_counter_set_max(&memcg->tcpmem, max);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003696 if (ret)
3697 goto out;
3698
Johannes Weiner0db15292016-01-20 15:02:50 -08003699 if (!memcg->tcpmem_active) {
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003700 /*
3701 * The active flag needs to be written after the static_key
3702 * update. This is what guarantees that the socket activation
Johannes Weiner2d758072016-10-07 17:00:58 -07003703 * function is the last one to run. See mem_cgroup_sk_alloc()
3704 * for details, and note that we don't mark any socket as
3705 * belonging to this memcg until that flag is up.
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003706 *
3707 * We need to do this, because static_keys will span multiple
3708 * sites, but we can't control their order. If we mark a socket
3709 * as accounted, but the accounting functions are not patched in
3710 * yet, we'll lose accounting.
3711 *
Johannes Weiner2d758072016-10-07 17:00:58 -07003712 * We never race with the readers in mem_cgroup_sk_alloc(),
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003713 * because when this value change, the code to process it is not
3714 * patched in yet.
3715 */
3716 static_branch_inc(&memcg_sockets_enabled_key);
Johannes Weiner0db15292016-01-20 15:02:50 -08003717 memcg->tcpmem_active = true;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003718 }
3719out:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003720 mutex_unlock(&memcg_max_mutex);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003721 return ret;
3722}
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003723
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003724/*
3725 * The user of this function is...
3726 * RES_LIMIT.
3727 */
Tejun Heo451af502014-05-13 12:16:21 -04003728static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3729 char *buf, size_t nbytes, loff_t off)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003730{
Tejun Heo451af502014-05-13 12:16:21 -04003731 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003732 unsigned long nr_pages;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003733 int ret;
3734
Tejun Heo451af502014-05-13 12:16:21 -04003735 buf = strstrip(buf);
Johannes Weiner650c5e52015-02-11 15:26:03 -08003736 ret = page_counter_memparse(buf, "-1", &nr_pages);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003737 if (ret)
3738 return ret;
Tejun Heoaf36f902012-04-01 12:09:55 -07003739
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003740 switch (MEMFILE_ATTR(of_cft(of)->private)) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003741 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07003742 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3743 ret = -EINVAL;
3744 break;
3745 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003746 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3747 case _MEM:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003748 ret = mem_cgroup_resize_max(memcg, nr_pages, false);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003749 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003750 case _MEMSWAP:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003751 ret = mem_cgroup_resize_max(memcg, nr_pages, true);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003752 break;
3753 case _KMEM:
Michal Hocko01581152019-09-23 15:37:22 -07003754 pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
3755 "Please report your usecase to linux-mm@kvack.org if you "
3756 "depend on this functionality.\n");
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003757 ret = memcg_update_kmem_max(memcg, nr_pages);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003758 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003759 case _TCP:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003760 ret = memcg_update_tcp_max(memcg, nr_pages);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003761 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003762 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003763 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07003764 case RES_SOFT_LIMIT:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003765 memcg->soft_limit = nr_pages;
3766 ret = 0;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003767 break;
3768 }
Tejun Heo451af502014-05-13 12:16:21 -04003769 return ret ?: nbytes;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003770}
3771
Tejun Heo6770c642014-05-13 12:16:21 -04003772static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3773 size_t nbytes, loff_t off)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003774{
Tejun Heo6770c642014-05-13 12:16:21 -04003775 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003776 struct page_counter *counter;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003777
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003778 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3779 case _MEM:
3780 counter = &memcg->memory;
3781 break;
3782 case _MEMSWAP:
3783 counter = &memcg->memsw;
3784 break;
3785 case _KMEM:
3786 counter = &memcg->kmem;
3787 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003788 case _TCP:
Johannes Weiner0db15292016-01-20 15:02:50 -08003789 counter = &memcg->tcpmem;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003790 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003791 default:
3792 BUG();
3793 }
Tejun Heoaf36f902012-04-01 12:09:55 -07003794
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003795 switch (MEMFILE_ATTR(of_cft(of)->private)) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003796 case RES_MAX_USAGE:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003797 page_counter_reset_watermark(counter);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003798 break;
3799 case RES_FAILCNT:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003800 counter->failcnt = 0;
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003801 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003802 default:
3803 BUG();
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003804 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07003805
Tejun Heo6770c642014-05-13 12:16:21 -04003806 return nbytes;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003807}
3808
Tejun Heo182446d2013-08-08 20:11:24 -04003809static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003810 struct cftype *cft)
3811{
Tejun Heo182446d2013-08-08 20:11:24 -04003812 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003813}
3814
Daisuke Nishimura02491442010-03-10 15:22:17 -08003815#ifdef CONFIG_MMU
Tejun Heo182446d2013-08-08 20:11:24 -04003816static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003817 struct cftype *cft, u64 val)
3818{
Tejun Heo182446d2013-08-08 20:11:24 -04003819 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003820
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08003821 if (val & ~MOVE_MASK)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003822 return -EINVAL;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003823
Glauber Costaee5e8472013-02-22 16:34:50 -08003824 /*
3825 * No kind of locking is needed in here, because ->can_attach() will
3826 * check this value once in the beginning of the process, and then carry
3827 * on with stale data. This means that changes to this value will only
3828 * affect task migrations starting after the change.
3829 */
3830 memcg->move_charge_at_immigrate = val;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003831 return 0;
3832}
Daisuke Nishimura02491442010-03-10 15:22:17 -08003833#else
Tejun Heo182446d2013-08-08 20:11:24 -04003834static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
Daisuke Nishimura02491442010-03-10 15:22:17 -08003835 struct cftype *cft, u64 val)
3836{
3837 return -ENOSYS;
3838}
3839#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003840
Ying Han406eb0c2011-05-26 16:25:37 -07003841#ifdef CONFIG_NUMA
Johannes Weiner113b7df2019-05-13 17:18:11 -07003842
3843#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
3844#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
3845#define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
3846
3847static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003848 int nid, unsigned int lru_mask, bool tree)
Johannes Weiner113b7df2019-05-13 17:18:11 -07003849{
Johannes Weiner867e5e12019-11-30 17:55:34 -08003850 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
Johannes Weiner113b7df2019-05-13 17:18:11 -07003851 unsigned long nr = 0;
3852 enum lru_list lru;
3853
3854 VM_BUG_ON((unsigned)nid >= nr_node_ids);
3855
3856 for_each_lru(lru) {
3857 if (!(BIT(lru) & lru_mask))
3858 continue;
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003859 if (tree)
3860 nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
3861 else
3862 nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
Johannes Weiner113b7df2019-05-13 17:18:11 -07003863 }
3864 return nr;
3865}
3866
3867static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003868 unsigned int lru_mask,
3869 bool tree)
Johannes Weiner113b7df2019-05-13 17:18:11 -07003870{
3871 unsigned long nr = 0;
3872 enum lru_list lru;
3873
3874 for_each_lru(lru) {
3875 if (!(BIT(lru) & lru_mask))
3876 continue;
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003877 if (tree)
3878 nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
3879 else
3880 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
Johannes Weiner113b7df2019-05-13 17:18:11 -07003881 }
3882 return nr;
3883}
3884
Tejun Heo2da8ca82013-12-05 12:28:04 -05003885static int memcg_numa_stat_show(struct seq_file *m, void *v)
Ying Han406eb0c2011-05-26 16:25:37 -07003886{
Greg Thelen25485de2013-11-12 15:07:40 -08003887 struct numa_stat {
3888 const char *name;
3889 unsigned int lru_mask;
3890 };
3891
3892 static const struct numa_stat stats[] = {
3893 { "total", LRU_ALL },
3894 { "file", LRU_ALL_FILE },
3895 { "anon", LRU_ALL_ANON },
3896 { "unevictable", BIT(LRU_UNEVICTABLE) },
3897 };
3898 const struct numa_stat *stat;
Ying Han406eb0c2011-05-26 16:25:37 -07003899 int nid;
Chris Downaa9694b2019-03-05 15:45:52 -08003900 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Ying Han406eb0c2011-05-26 16:25:37 -07003901
Johannes Weiner2d146aa2021-04-29 22:56:26 -07003902 cgroup_rstat_flush(memcg->css.cgroup);
3903
Greg Thelen25485de2013-11-12 15:07:40 -08003904 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003905 seq_printf(m, "%s=%lu", stat->name,
3906 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3907 false));
3908 for_each_node_state(nid, N_MEMORY)
3909 seq_printf(m, " N%d=%lu", nid,
3910 mem_cgroup_node_nr_lru_pages(memcg, nid,
3911 stat->lru_mask, false));
Greg Thelen25485de2013-11-12 15:07:40 -08003912 seq_putc(m, '\n');
Ying Han406eb0c2011-05-26 16:25:37 -07003913 }
Ying Han406eb0c2011-05-26 16:25:37 -07003914
Ying Han071aee12013-11-12 15:07:41 -08003915 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
Ying Han406eb0c2011-05-26 16:25:37 -07003916
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003917 seq_printf(m, "hierarchical_%s=%lu", stat->name,
3918 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3919 true));
3920 for_each_node_state(nid, N_MEMORY)
3921 seq_printf(m, " N%d=%lu", nid,
3922 mem_cgroup_node_nr_lru_pages(memcg, nid,
3923 stat->lru_mask, true));
Ying Han071aee12013-11-12 15:07:41 -08003924 seq_putc(m, '\n');
Ying Han406eb0c2011-05-26 16:25:37 -07003925 }
Ying Han406eb0c2011-05-26 16:25:37 -07003926
Ying Han406eb0c2011-05-26 16:25:37 -07003927 return 0;
3928}
3929#endif /* CONFIG_NUMA */
3930
Johannes Weinerc8713d02019-07-11 20:55:59 -07003931static const unsigned int memcg1_stats[] = {
Johannes Weiner0d1c2072020-06-03 16:01:54 -07003932 NR_FILE_PAGES,
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07003933 NR_ANON_MAPPED,
Johannes Weiner468c3982020-06-03 16:02:01 -07003934#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3935 NR_ANON_THPS,
3936#endif
Johannes Weinerc8713d02019-07-11 20:55:59 -07003937 NR_SHMEM,
3938 NR_FILE_MAPPED,
3939 NR_FILE_DIRTY,
3940 NR_WRITEBACK,
3941 MEMCG_SWAP,
3942};
3943
3944static const char *const memcg1_stat_names[] = {
3945 "cache",
3946 "rss",
Johannes Weiner468c3982020-06-03 16:02:01 -07003947#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Johannes Weinerc8713d02019-07-11 20:55:59 -07003948 "rss_huge",
Johannes Weiner468c3982020-06-03 16:02:01 -07003949#endif
Johannes Weinerc8713d02019-07-11 20:55:59 -07003950 "shmem",
3951 "mapped_file",
3952 "dirty",
3953 "writeback",
3954 "swap",
3955};
3956
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07003957/* Universal VM events cgroup1 shows, original sort order */
Greg Thelen8dd53fd2018-06-07 17:07:23 -07003958static const unsigned int memcg1_events[] = {
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07003959 PGPGIN,
3960 PGPGOUT,
3961 PGFAULT,
3962 PGMAJFAULT,
3963};
3964
Tejun Heo2da8ca82013-12-05 12:28:04 -05003965static int memcg_stat_show(struct seq_file *m, void *v)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003966{
Chris Downaa9694b2019-03-05 15:45:52 -08003967 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003968 unsigned long memory, memsw;
Johannes Weineraf7c4b02012-05-29 15:07:08 -07003969 struct mem_cgroup *mi;
3970 unsigned int i;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003971
Johannes Weiner71cd3112017-05-03 14:55:13 -07003972 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
Rickard Strandqvist70bc0682014-12-12 16:56:41 -08003973
Johannes Weiner2d146aa2021-04-29 22:56:26 -07003974 cgroup_rstat_flush(memcg->css.cgroup);
3975
Johannes Weiner71cd3112017-05-03 14:55:13 -07003976 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
Johannes Weiner468c3982020-06-03 16:02:01 -07003977 unsigned long nr;
3978
Johannes Weiner71cd3112017-05-03 14:55:13 -07003979 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003980 continue;
Johannes Weiner468c3982020-06-03 16:02:01 -07003981 nr = memcg_page_state_local(memcg, memcg1_stats[i]);
Johannes Weiner468c3982020-06-03 16:02:01 -07003982 seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003983 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003984
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07003985 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08003986 seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]),
Johannes Weiner205b20c2019-05-14 15:47:06 -07003987 memcg_events_local(memcg, memcg1_events[i]));
Johannes Weineraf7c4b02012-05-29 15:07:08 -07003988
3989 for (i = 0; i < NR_LRU_LISTS; i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08003990 seq_printf(m, "%s %lu\n", lru_list_name(i),
Johannes Weiner205b20c2019-05-14 15:47:06 -07003991 memcg_page_state_local(memcg, NR_LRU_BASE + i) *
Johannes Weiner21d89d12019-05-13 17:18:08 -07003992 PAGE_SIZE);
Johannes Weineraf7c4b02012-05-29 15:07:08 -07003993
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003994 /* Hierarchical information */
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003995 memory = memsw = PAGE_COUNTER_MAX;
3996 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
Chris Down15b42562020-04-01 21:07:20 -07003997 memory = min(memory, READ_ONCE(mi->memory.max));
3998 memsw = min(memsw, READ_ONCE(mi->memsw.max));
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08003999 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004000 seq_printf(m, "hierarchical_memory_limit %llu\n",
4001 (u64)memory * PAGE_SIZE);
Johannes Weiner7941d212016-01-14 15:21:23 -08004002 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004003 seq_printf(m, "hierarchical_memsw_limit %llu\n",
4004 (u64)memsw * PAGE_SIZE);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004005
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07004006 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
zhongjiang-ali7de2e9f2020-11-01 17:07:30 -08004007 unsigned long nr;
4008
Johannes Weiner71cd3112017-05-03 14:55:13 -07004009 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004010 continue;
zhongjiang-ali7de2e9f2020-11-01 17:07:30 -08004011 nr = memcg_page_state(memcg, memcg1_stats[i]);
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07004012 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
zhongjiang-ali7de2e9f2020-11-01 17:07:30 -08004013 (u64)nr * PAGE_SIZE);
Johannes Weineraf7c4b02012-05-29 15:07:08 -07004014 }
4015
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07004016 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08004017 seq_printf(m, "total_%s %llu\n",
4018 vm_event_name(memcg1_events[i]),
Yafang Shaodd923992019-07-11 20:52:11 -07004019 (u64)memcg_events(memcg, memcg1_events[i]));
Johannes Weineraf7c4b02012-05-29 15:07:08 -07004020
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07004021 for (i = 0; i < NR_LRU_LISTS; i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08004022 seq_printf(m, "total_%s %llu\n", lru_list_name(i),
Johannes Weiner42a30032019-05-14 15:47:12 -07004023 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
4024 PAGE_SIZE);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004025
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004026#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004027 {
Mel Gormanef8f2322016-07-28 15:46:05 -07004028 pg_data_t *pgdat;
4029 struct mem_cgroup_per_node *mz;
Johannes Weiner1431d4d2020-06-03 16:02:53 -07004030 unsigned long anon_cost = 0;
4031 unsigned long file_cost = 0;
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004032
Mel Gormanef8f2322016-07-28 15:46:05 -07004033 for_each_online_pgdat(pgdat) {
Johannes Weinera3747b52021-04-29 22:56:14 -07004034 mz = memcg->nodeinfo[pgdat->node_id];
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004035
Johannes Weiner1431d4d2020-06-03 16:02:53 -07004036 anon_cost += mz->lruvec.anon_cost;
4037 file_cost += mz->lruvec.file_cost;
Mel Gormanef8f2322016-07-28 15:46:05 -07004038 }
Johannes Weiner1431d4d2020-06-03 16:02:53 -07004039 seq_printf(m, "anon_cost %lu\n", anon_cost);
4040 seq_printf(m, "file_cost %lu\n", file_cost);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004041 }
4042#endif
4043
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004044 return 0;
4045}
4046
Tejun Heo182446d2013-08-08 20:11:24 -04004047static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
4048 struct cftype *cft)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004049{
Tejun Heo182446d2013-08-08 20:11:24 -04004050 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004051
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07004052 return mem_cgroup_swappiness(memcg);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004053}
4054
Tejun Heo182446d2013-08-08 20:11:24 -04004055static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
4056 struct cftype *cft, u64 val)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004057{
Tejun Heo182446d2013-08-08 20:11:24 -04004058 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Li Zefan068b38c2009-01-15 13:51:26 -08004059
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07004060 if (val > 100)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004061 return -EINVAL;
4062
Shakeel Butta4792032021-04-29 22:56:05 -07004063 if (!mem_cgroup_is_root(memcg))
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07004064 memcg->swappiness = val;
4065 else
4066 vm_swappiness = val;
Li Zefan068b38c2009-01-15 13:51:26 -08004067
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004068 return 0;
4069}
4070
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004071static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4072{
4073 struct mem_cgroup_threshold_ary *t;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004074 unsigned long usage;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004075 int i;
4076
4077 rcu_read_lock();
4078 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004079 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004080 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004081 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004082
4083 if (!t)
4084 goto unlock;
4085
Johannes Weinerce00a962014-09-05 08:43:57 -04004086 usage = mem_cgroup_usage(memcg, swap);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004087
4088 /*
Sha Zhengju748dad32012-05-29 15:06:57 -07004089 * current_threshold points to threshold just below or equal to usage.
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004090 * If it's not true, a threshold was crossed after last
4091 * call of __mem_cgroup_threshold().
4092 */
Phil Carmody5407a562010-05-26 14:42:42 -07004093 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004094
4095 /*
4096 * Iterate backward over array of thresholds starting from
4097 * current_threshold and check if a threshold is crossed.
4098 * If none of thresholds below usage is crossed, we read
4099 * only one element of the array here.
4100 */
4101 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
4102 eventfd_signal(t->entries[i].eventfd, 1);
4103
4104 /* i = current_threshold + 1 */
4105 i++;
4106
4107 /*
4108 * Iterate forward over array of thresholds starting from
4109 * current_threshold+1 and check if a threshold is crossed.
4110 * If none of thresholds above usage is crossed, we read
4111 * only one element of the array here.
4112 */
4113 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
4114 eventfd_signal(t->entries[i].eventfd, 1);
4115
4116 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07004117 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004118unlock:
4119 rcu_read_unlock();
4120}
4121
4122static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4123{
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07004124 while (memcg) {
4125 __mem_cgroup_threshold(memcg, false);
Johannes Weiner7941d212016-01-14 15:21:23 -08004126 if (do_memsw_account())
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07004127 __mem_cgroup_threshold(memcg, true);
4128
4129 memcg = parent_mem_cgroup(memcg);
4130 }
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004131}
4132
4133static int compare_thresholds(const void *a, const void *b)
4134{
4135 const struct mem_cgroup_threshold *_a = a;
4136 const struct mem_cgroup_threshold *_b = b;
4137
Greg Thelen2bff24a2013-09-11 14:23:08 -07004138 if (_a->threshold > _b->threshold)
4139 return 1;
4140
4141 if (_a->threshold < _b->threshold)
4142 return -1;
4143
4144 return 0;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004145}
4146
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004147static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004148{
4149 struct mem_cgroup_eventfd_list *ev;
4150
Michal Hocko2bcf2e92014-07-30 16:08:33 -07004151 spin_lock(&memcg_oom_lock);
4152
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004153 list_for_each_entry(ev, &memcg->oom_notify, list)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004154 eventfd_signal(ev->eventfd, 1);
Michal Hocko2bcf2e92014-07-30 16:08:33 -07004155
4156 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004157 return 0;
4158}
4159
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004160static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004161{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004162 struct mem_cgroup *iter;
4163
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004164 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004165 mem_cgroup_oom_notify_cb(iter);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004166}
4167
Tejun Heo59b6f872013-11-22 18:20:43 -05004168static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004169 struct eventfd_ctx *eventfd, const char *args, enum res_type type)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004170{
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004171 struct mem_cgroup_thresholds *thresholds;
4172 struct mem_cgroup_threshold_ary *new;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004173 unsigned long threshold;
4174 unsigned long usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004175 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004176
Johannes Weiner650c5e52015-02-11 15:26:03 -08004177 ret = page_counter_memparse(args, "-1", &threshold);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004178 if (ret)
4179 return ret;
4180
4181 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004182
Johannes Weiner05b84302014-08-06 16:05:59 -07004183 if (type == _MEM) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004184 thresholds = &memcg->thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004185 usage = mem_cgroup_usage(memcg, false);
Johannes Weiner05b84302014-08-06 16:05:59 -07004186 } else if (type == _MEMSWAP) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004187 thresholds = &memcg->memsw_thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004188 usage = mem_cgroup_usage(memcg, true);
Johannes Weiner05b84302014-08-06 16:05:59 -07004189 } else
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004190 BUG();
4191
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004192 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004193 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004194 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4195
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004196 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004197
4198 /* Allocate memory for new array of thresholds */
Gustavo A. R. Silva67b80462019-03-05 15:44:05 -08004199 new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004200 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004201 ret = -ENOMEM;
4202 goto unlock;
4203 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004204 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004205
4206 /* Copy thresholds (if any) to new array */
Gustavo A. R. Silvae90342e2020-10-13 16:52:36 -07004207 if (thresholds->primary)
4208 memcpy(new->entries, thresholds->primary->entries,
4209 flex_array_size(new, entries, size - 1));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004210
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004211 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004212 new->entries[size - 1].eventfd = eventfd;
4213 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004214
4215 /* Sort thresholds. Registering of new threshold isn't time-critical */
Gustavo A. R. Silva61e604e62020-10-13 16:52:39 -07004216 sort(new->entries, size, sizeof(*new->entries),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004217 compare_thresholds, NULL);
4218
4219 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004220 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004221 for (i = 0; i < size; i++) {
Sha Zhengju748dad32012-05-29 15:06:57 -07004222 if (new->entries[i].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004223 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004224 * new->current_threshold will not be used until
4225 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004226 * it here.
4227 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004228 ++new->current_threshold;
Sha Zhengju748dad32012-05-29 15:06:57 -07004229 } else
4230 break;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004231 }
4232
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004233 /* Free old spare buffer and save old primary buffer as spare */
4234 kfree(thresholds->spare);
4235 thresholds->spare = thresholds->primary;
4236
4237 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004238
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004239 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004240 synchronize_rcu();
4241
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004242unlock:
4243 mutex_unlock(&memcg->thresholds_lock);
4244
4245 return ret;
4246}
4247
Tejun Heo59b6f872013-11-22 18:20:43 -05004248static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004249 struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004250{
Tejun Heo59b6f872013-11-22 18:20:43 -05004251 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
Tejun Heo347c4a82013-11-22 18:20:43 -05004252}
4253
Tejun Heo59b6f872013-11-22 18:20:43 -05004254static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004255 struct eventfd_ctx *eventfd, const char *args)
4256{
Tejun Heo59b6f872013-11-22 18:20:43 -05004257 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
Tejun Heo347c4a82013-11-22 18:20:43 -05004258}
4259
Tejun Heo59b6f872013-11-22 18:20:43 -05004260static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004261 struct eventfd_ctx *eventfd, enum res_type type)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004262{
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004263 struct mem_cgroup_thresholds *thresholds;
4264 struct mem_cgroup_threshold_ary *new;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004265 unsigned long usage;
Chunguang Xu7d366652020-03-21 18:22:10 -07004266 int i, j, size, entries;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004267
4268 mutex_lock(&memcg->thresholds_lock);
Johannes Weiner05b84302014-08-06 16:05:59 -07004269
4270 if (type == _MEM) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004271 thresholds = &memcg->thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004272 usage = mem_cgroup_usage(memcg, false);
Johannes Weiner05b84302014-08-06 16:05:59 -07004273 } else if (type == _MEMSWAP) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004274 thresholds = &memcg->memsw_thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004275 usage = mem_cgroup_usage(memcg, true);
Johannes Weiner05b84302014-08-06 16:05:59 -07004276 } else
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004277 BUG();
4278
Anton Vorontsov371528c2012-02-24 05:14:46 +04004279 if (!thresholds->primary)
4280 goto unlock;
4281
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004282 /* Check if a threshold crossed before removing */
4283 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4284
4285 /* Calculate new number of threshold */
Chunguang Xu7d366652020-03-21 18:22:10 -07004286 size = entries = 0;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004287 for (i = 0; i < thresholds->primary->size; i++) {
4288 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004289 size++;
Chunguang Xu7d366652020-03-21 18:22:10 -07004290 else
4291 entries++;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004292 }
4293
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004294 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004295
Chunguang Xu7d366652020-03-21 18:22:10 -07004296 /* If no items related to eventfd have been cleared, nothing to do */
4297 if (!entries)
4298 goto unlock;
4299
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004300 /* Set thresholds array to NULL if we don't have thresholds */
4301 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004302 kfree(new);
4303 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004304 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004305 }
4306
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004307 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004308
4309 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004310 new->current_threshold = -1;
4311 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
4312 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004313 continue;
4314
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004315 new->entries[j] = thresholds->primary->entries[i];
Sha Zhengju748dad32012-05-29 15:06:57 -07004316 if (new->entries[j].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004317 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004318 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004319 * until rcu_assign_pointer(), so it's safe to increment
4320 * it here.
4321 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004322 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004323 }
4324 j++;
4325 }
4326
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004327swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004328 /* Swap primary and spare array */
4329 thresholds->spare = thresholds->primary;
Sha Zhengju8c757762012-05-10 13:01:45 -07004330
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004331 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004332
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004333 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004334 synchronize_rcu();
Martijn Coenen6611d8d2016-01-15 16:57:49 -08004335
4336 /* If all events are unregistered, free the spare array */
4337 if (!new) {
4338 kfree(thresholds->spare);
4339 thresholds->spare = NULL;
4340 }
Anton Vorontsov371528c2012-02-24 05:14:46 +04004341unlock:
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004342 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004343}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004344
Tejun Heo59b6f872013-11-22 18:20:43 -05004345static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004346 struct eventfd_ctx *eventfd)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004347{
Tejun Heo59b6f872013-11-22 18:20:43 -05004348 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
Tejun Heo347c4a82013-11-22 18:20:43 -05004349}
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004350
Tejun Heo59b6f872013-11-22 18:20:43 -05004351static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004352 struct eventfd_ctx *eventfd)
4353{
Tejun Heo59b6f872013-11-22 18:20:43 -05004354 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
Tejun Heo347c4a82013-11-22 18:20:43 -05004355}
4356
Tejun Heo59b6f872013-11-22 18:20:43 -05004357static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004358 struct eventfd_ctx *eventfd, const char *args)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004359{
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004360 struct mem_cgroup_eventfd_list *event;
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004361
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004362 event = kmalloc(sizeof(*event), GFP_KERNEL);
4363 if (!event)
4364 return -ENOMEM;
4365
Michal Hocko1af8efe2011-07-26 16:08:24 -07004366 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004367
4368 event->eventfd = eventfd;
4369 list_add(&event->list, &memcg->oom_notify);
4370
4371 /* already in OOM ? */
Tejun Heoc2b42d32015-06-24 16:58:23 -07004372 if (memcg->under_oom)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004373 eventfd_signal(eventfd, 1);
Michal Hocko1af8efe2011-07-26 16:08:24 -07004374 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004375
4376 return 0;
4377}
4378
Tejun Heo59b6f872013-11-22 18:20:43 -05004379static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004380 struct eventfd_ctx *eventfd)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004381{
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004382 struct mem_cgroup_eventfd_list *ev, *tmp;
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004383
Michal Hocko1af8efe2011-07-26 16:08:24 -07004384 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004385
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004386 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004387 if (ev->eventfd == eventfd) {
4388 list_del(&ev->list);
4389 kfree(ev);
4390 }
4391 }
4392
Michal Hocko1af8efe2011-07-26 16:08:24 -07004393 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004394}
4395
Tejun Heo2da8ca82013-12-05 12:28:04 -05004396static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004397{
Chris Downaa9694b2019-03-05 15:45:52 -08004398 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004399
Tejun Heo791badb2013-12-05 12:28:02 -05004400 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
Tejun Heoc2b42d32015-06-24 16:58:23 -07004401 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
Roman Gushchinfe6bdfc2018-06-14 15:28:05 -07004402 seq_printf(sf, "oom_kill %lu\n",
4403 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004404 return 0;
4405}
4406
Tejun Heo182446d2013-08-08 20:11:24 -04004407static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004408 struct cftype *cft, u64 val)
4409{
Tejun Heo182446d2013-08-08 20:11:24 -04004410 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004411
4412 /* cannot set to root cgroup and only 0 and 1 are allowed */
Shakeel Butta4792032021-04-29 22:56:05 -07004413 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1)))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004414 return -EINVAL;
4415
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004416 memcg->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07004417 if (!val)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004418 memcg_oom_recover(memcg);
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07004419
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004420 return 0;
4421}
4422
Tejun Heo52ebea72015-05-22 17:13:37 -04004423#ifdef CONFIG_CGROUP_WRITEBACK
4424
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004425#include <trace/events/writeback.h>
4426
Tejun Heo841710a2015-05-22 18:23:33 -04004427static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4428{
4429 return wb_domain_init(&memcg->cgwb_domain, gfp);
4430}
4431
4432static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4433{
4434 wb_domain_exit(&memcg->cgwb_domain);
4435}
4436
Tejun Heo2529bb32015-05-22 18:23:34 -04004437static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4438{
4439 wb_domain_size_changed(&memcg->cgwb_domain);
4440}
4441
Tejun Heo841710a2015-05-22 18:23:33 -04004442struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
4443{
4444 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4445
4446 if (!memcg->css.parent)
4447 return NULL;
4448
4449 return &memcg->cgwb_domain;
4450}
4451
Tejun Heoc2aa7232015-05-22 18:23:35 -04004452/**
4453 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4454 * @wb: bdi_writeback in question
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004455 * @pfilepages: out parameter for number of file pages
4456 * @pheadroom: out parameter for number of allocatable pages according to memcg
Tejun Heoc2aa7232015-05-22 18:23:35 -04004457 * @pdirty: out parameter for number of dirty pages
4458 * @pwriteback: out parameter for number of pages under writeback
4459 *
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004460 * Determine the numbers of file, headroom, dirty, and writeback pages in
4461 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
4462 * is a bit more involved.
Tejun Heoc2aa7232015-05-22 18:23:35 -04004463 *
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004464 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
4465 * headroom is calculated as the lowest headroom of itself and the
4466 * ancestors. Note that this doesn't consider the actual amount of
4467 * available memory in the system. The caller should further cap
4468 * *@pheadroom accordingly.
Tejun Heoc2aa7232015-05-22 18:23:35 -04004469 */
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004470void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
4471 unsigned long *pheadroom, unsigned long *pdirty,
4472 unsigned long *pwriteback)
Tejun Heoc2aa7232015-05-22 18:23:35 -04004473{
4474 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4475 struct mem_cgroup *parent;
Tejun Heoc2aa7232015-05-22 18:23:35 -04004476
Johannes Weiner2d146aa2021-04-29 22:56:26 -07004477 cgroup_rstat_flush_irqsafe(memcg->css.cgroup);
Tejun Heoc2aa7232015-05-22 18:23:35 -04004478
Johannes Weiner2d146aa2021-04-29 22:56:26 -07004479 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
4480 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
4481 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) +
4482 memcg_page_state(memcg, NR_ACTIVE_FILE);
4483
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004484 *pheadroom = PAGE_COUNTER_MAX;
Tejun Heoc2aa7232015-05-22 18:23:35 -04004485 while ((parent = parent_mem_cgroup(memcg))) {
Chris Down15b42562020-04-01 21:07:20 -07004486 unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
Jakub Kicinskid1663a92020-06-01 21:49:49 -07004487 READ_ONCE(memcg->memory.high));
Tejun Heoc2aa7232015-05-22 18:23:35 -04004488 unsigned long used = page_counter_read(&memcg->memory);
4489
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004490 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
Tejun Heoc2aa7232015-05-22 18:23:35 -04004491 memcg = parent;
4492 }
Tejun Heoc2aa7232015-05-22 18:23:35 -04004493}
4494
Tejun Heo97b27822019-08-26 09:06:56 -07004495/*
4496 * Foreign dirty flushing
4497 *
4498 * There's an inherent mismatch between memcg and writeback. The former
4499 * trackes ownership per-page while the latter per-inode. This was a
4500 * deliberate design decision because honoring per-page ownership in the
4501 * writeback path is complicated, may lead to higher CPU and IO overheads
4502 * and deemed unnecessary given that write-sharing an inode across
4503 * different cgroups isn't a common use-case.
4504 *
4505 * Combined with inode majority-writer ownership switching, this works well
4506 * enough in most cases but there are some pathological cases. For
4507 * example, let's say there are two cgroups A and B which keep writing to
4508 * different but confined parts of the same inode. B owns the inode and
4509 * A's memory is limited far below B's. A's dirty ratio can rise enough to
4510 * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
4511 * triggering background writeback. A will be slowed down without a way to
4512 * make writeback of the dirty pages happen.
4513 *
4514 * Conditions like the above can lead to a cgroup getting repatedly and
4515 * severely throttled after making some progress after each
4516 * dirty_expire_interval while the underyling IO device is almost
4517 * completely idle.
4518 *
4519 * Solving this problem completely requires matching the ownership tracking
4520 * granularities between memcg and writeback in either direction. However,
4521 * the more egregious behaviors can be avoided by simply remembering the
4522 * most recent foreign dirtying events and initiating remote flushes on
4523 * them when local writeback isn't enough to keep the memory clean enough.
4524 *
4525 * The following two functions implement such mechanism. When a foreign
4526 * page - a page whose memcg and writeback ownerships don't match - is
4527 * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
4528 * bdi_writeback on the page owning memcg. When balance_dirty_pages()
4529 * decides that the memcg needs to sleep due to high dirty ratio, it calls
4530 * mem_cgroup_flush_foreign() which queues writeback on the recorded
4531 * foreign bdi_writebacks which haven't expired. Both the numbers of
4532 * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
4533 * limited to MEMCG_CGWB_FRN_CNT.
4534 *
4535 * The mechanism only remembers IDs and doesn't hold any object references.
4536 * As being wrong occasionally doesn't matter, updates and accesses to the
4537 * records are lockless and racy.
4538 */
4539void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
4540 struct bdi_writeback *wb)
4541{
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08004542 struct mem_cgroup *memcg = page_memcg(page);
Tejun Heo97b27822019-08-26 09:06:56 -07004543 struct memcg_cgwb_frn *frn;
4544 u64 now = get_jiffies_64();
4545 u64 oldest_at = now;
4546 int oldest = -1;
4547 int i;
4548
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004549 trace_track_foreign_dirty(page, wb);
4550
Tejun Heo97b27822019-08-26 09:06:56 -07004551 /*
4552 * Pick the slot to use. If there is already a slot for @wb, keep
4553 * using it. If not replace the oldest one which isn't being
4554 * written out.
4555 */
4556 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4557 frn = &memcg->cgwb_frn[i];
4558 if (frn->bdi_id == wb->bdi->id &&
4559 frn->memcg_id == wb->memcg_css->id)
4560 break;
4561 if (time_before64(frn->at, oldest_at) &&
4562 atomic_read(&frn->done.cnt) == 1) {
4563 oldest = i;
4564 oldest_at = frn->at;
4565 }
4566 }
4567
4568 if (i < MEMCG_CGWB_FRN_CNT) {
4569 /*
4570 * Re-using an existing one. Update timestamp lazily to
4571 * avoid making the cacheline hot. We want them to be
4572 * reasonably up-to-date and significantly shorter than
4573 * dirty_expire_interval as that's what expires the record.
4574 * Use the shorter of 1s and dirty_expire_interval / 8.
4575 */
4576 unsigned long update_intv =
4577 min_t(unsigned long, HZ,
4578 msecs_to_jiffies(dirty_expire_interval * 10) / 8);
4579
4580 if (time_before64(frn->at, now - update_intv))
4581 frn->at = now;
4582 } else if (oldest >= 0) {
4583 /* replace the oldest free one */
4584 frn = &memcg->cgwb_frn[oldest];
4585 frn->bdi_id = wb->bdi->id;
4586 frn->memcg_id = wb->memcg_css->id;
4587 frn->at = now;
4588 }
4589}
4590
4591/* issue foreign writeback flushes for recorded foreign dirtying events */
4592void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
4593{
4594 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4595 unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
4596 u64 now = jiffies_64;
4597 int i;
4598
4599 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4600 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
4601
4602 /*
4603 * If the record is older than dirty_expire_interval,
4604 * writeback on it has already started. No need to kick it
4605 * off again. Also, don't start a new one if there's
4606 * already one in flight.
4607 */
4608 if (time_after64(frn->at, now - intv) &&
4609 atomic_read(&frn->done.cnt) == 1) {
4610 frn->at = 0;
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004611 trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
Tejun Heo97b27822019-08-26 09:06:56 -07004612 cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,
4613 WB_REASON_FOREIGN_FLUSH,
4614 &frn->done);
4615 }
4616 }
4617}
4618
Tejun Heo841710a2015-05-22 18:23:33 -04004619#else /* CONFIG_CGROUP_WRITEBACK */
4620
4621static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4622{
4623 return 0;
4624}
4625
4626static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4627{
4628}
4629
Tejun Heo2529bb32015-05-22 18:23:34 -04004630static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4631{
4632}
4633
Tejun Heo52ebea72015-05-22 17:13:37 -04004634#endif /* CONFIG_CGROUP_WRITEBACK */
4635
Tejun Heo79bd9812013-11-22 18:20:42 -05004636/*
Tejun Heo3bc942f2013-11-22 18:20:44 -05004637 * DO NOT USE IN NEW FILES.
4638 *
4639 * "cgroup.event_control" implementation.
4640 *
4641 * This is way over-engineered. It tries to support fully configurable
4642 * events for each user. Such level of flexibility is completely
4643 * unnecessary especially in the light of the planned unified hierarchy.
4644 *
4645 * Please deprecate this and replace with something simpler if at all
4646 * possible.
4647 */
4648
4649/*
Tejun Heo79bd9812013-11-22 18:20:42 -05004650 * Unregister event and free resources.
4651 *
4652 * Gets called from workqueue.
4653 */
Tejun Heo3bc942f2013-11-22 18:20:44 -05004654static void memcg_event_remove(struct work_struct *work)
Tejun Heo79bd9812013-11-22 18:20:42 -05004655{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004656 struct mem_cgroup_event *event =
4657 container_of(work, struct mem_cgroup_event, remove);
Tejun Heo59b6f872013-11-22 18:20:43 -05004658 struct mem_cgroup *memcg = event->memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -05004659
4660 remove_wait_queue(event->wqh, &event->wait);
4661
Tejun Heo59b6f872013-11-22 18:20:43 -05004662 event->unregister_event(memcg, event->eventfd);
Tejun Heo79bd9812013-11-22 18:20:42 -05004663
4664 /* Notify userspace the event is going away. */
4665 eventfd_signal(event->eventfd, 1);
4666
4667 eventfd_ctx_put(event->eventfd);
4668 kfree(event);
Tejun Heo59b6f872013-11-22 18:20:43 -05004669 css_put(&memcg->css);
Tejun Heo79bd9812013-11-22 18:20:42 -05004670}
4671
4672/*
Linus Torvaldsa9a08842018-02-11 14:34:03 -08004673 * Gets called on EPOLLHUP on eventfd when user closes it.
Tejun Heo79bd9812013-11-22 18:20:42 -05004674 *
4675 * Called with wqh->lock held and interrupts disabled.
4676 */
Ingo Molnarac6424b2017-06-20 12:06:13 +02004677static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
Tejun Heo3bc942f2013-11-22 18:20:44 -05004678 int sync, void *key)
Tejun Heo79bd9812013-11-22 18:20:42 -05004679{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004680 struct mem_cgroup_event *event =
4681 container_of(wait, struct mem_cgroup_event, wait);
Tejun Heo59b6f872013-11-22 18:20:43 -05004682 struct mem_cgroup *memcg = event->memcg;
Al Viro3ad6f932017-07-03 20:14:56 -04004683 __poll_t flags = key_to_poll(key);
Tejun Heo79bd9812013-11-22 18:20:42 -05004684
Linus Torvaldsa9a08842018-02-11 14:34:03 -08004685 if (flags & EPOLLHUP) {
Tejun Heo79bd9812013-11-22 18:20:42 -05004686 /*
4687 * If the event has been detached at cgroup removal, we
4688 * can simply return knowing the other side will cleanup
4689 * for us.
4690 *
4691 * We can't race against event freeing since the other
4692 * side will require wqh->lock via remove_wait_queue(),
4693 * which we hold.
4694 */
Tejun Heofba94802013-11-22 18:20:43 -05004695 spin_lock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004696 if (!list_empty(&event->list)) {
4697 list_del_init(&event->list);
4698 /*
4699 * We are in atomic context, but cgroup_event_remove()
4700 * may sleep, so we have to call it in workqueue.
4701 */
4702 schedule_work(&event->remove);
4703 }
Tejun Heofba94802013-11-22 18:20:43 -05004704 spin_unlock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004705 }
4706
4707 return 0;
4708}
4709
Tejun Heo3bc942f2013-11-22 18:20:44 -05004710static void memcg_event_ptable_queue_proc(struct file *file,
Tejun Heo79bd9812013-11-22 18:20:42 -05004711 wait_queue_head_t *wqh, poll_table *pt)
4712{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004713 struct mem_cgroup_event *event =
4714 container_of(pt, struct mem_cgroup_event, pt);
Tejun Heo79bd9812013-11-22 18:20:42 -05004715
4716 event->wqh = wqh;
4717 add_wait_queue(wqh, &event->wait);
4718}
4719
4720/*
Tejun Heo3bc942f2013-11-22 18:20:44 -05004721 * DO NOT USE IN NEW FILES.
4722 *
Tejun Heo79bd9812013-11-22 18:20:42 -05004723 * Parse input and register new cgroup event handler.
4724 *
4725 * Input must be in format '<event_fd> <control_fd> <args>'.
4726 * Interpretation of args is defined by control file implementation.
4727 */
Tejun Heo451af502014-05-13 12:16:21 -04004728static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4729 char *buf, size_t nbytes, loff_t off)
Tejun Heo79bd9812013-11-22 18:20:42 -05004730{
Tejun Heo451af502014-05-13 12:16:21 -04004731 struct cgroup_subsys_state *css = of_css(of);
Tejun Heofba94802013-11-22 18:20:43 -05004732 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo3bc942f2013-11-22 18:20:44 -05004733 struct mem_cgroup_event *event;
Tejun Heo79bd9812013-11-22 18:20:42 -05004734 struct cgroup_subsys_state *cfile_css;
4735 unsigned int efd, cfd;
4736 struct fd efile;
4737 struct fd cfile;
Tejun Heofba94802013-11-22 18:20:43 -05004738 const char *name;
Tejun Heo79bd9812013-11-22 18:20:42 -05004739 char *endp;
4740 int ret;
4741
Tejun Heo451af502014-05-13 12:16:21 -04004742 buf = strstrip(buf);
4743
4744 efd = simple_strtoul(buf, &endp, 10);
Tejun Heo79bd9812013-11-22 18:20:42 -05004745 if (*endp != ' ')
4746 return -EINVAL;
Tejun Heo451af502014-05-13 12:16:21 -04004747 buf = endp + 1;
Tejun Heo79bd9812013-11-22 18:20:42 -05004748
Tejun Heo451af502014-05-13 12:16:21 -04004749 cfd = simple_strtoul(buf, &endp, 10);
Tejun Heo79bd9812013-11-22 18:20:42 -05004750 if ((*endp != ' ') && (*endp != '\0'))
4751 return -EINVAL;
Tejun Heo451af502014-05-13 12:16:21 -04004752 buf = endp + 1;
Tejun Heo79bd9812013-11-22 18:20:42 -05004753
4754 event = kzalloc(sizeof(*event), GFP_KERNEL);
4755 if (!event)
4756 return -ENOMEM;
4757
Tejun Heo59b6f872013-11-22 18:20:43 -05004758 event->memcg = memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -05004759 INIT_LIST_HEAD(&event->list);
Tejun Heo3bc942f2013-11-22 18:20:44 -05004760 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4761 init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4762 INIT_WORK(&event->remove, memcg_event_remove);
Tejun Heo79bd9812013-11-22 18:20:42 -05004763
4764 efile = fdget(efd);
4765 if (!efile.file) {
4766 ret = -EBADF;
4767 goto out_kfree;
4768 }
4769
4770 event->eventfd = eventfd_ctx_fileget(efile.file);
4771 if (IS_ERR(event->eventfd)) {
4772 ret = PTR_ERR(event->eventfd);
4773 goto out_put_efile;
4774 }
4775
4776 cfile = fdget(cfd);
4777 if (!cfile.file) {
4778 ret = -EBADF;
4779 goto out_put_eventfd;
4780 }
4781
4782 /* the process need read permission on control file */
4783 /* AV: shouldn't we check that it's been opened for read instead? */
Christian Brauner02f92b32021-01-21 14:19:22 +01004784 ret = file_permission(cfile.file, MAY_READ);
Tejun Heo79bd9812013-11-22 18:20:42 -05004785 if (ret < 0)
4786 goto out_put_cfile;
4787
Tejun Heo79bd9812013-11-22 18:20:42 -05004788 /*
Tejun Heofba94802013-11-22 18:20:43 -05004789 * Determine the event callbacks and set them in @event. This used
4790 * to be done via struct cftype but cgroup core no longer knows
4791 * about these events. The following is crude but the whole thing
4792 * is for compatibility anyway.
Tejun Heo3bc942f2013-11-22 18:20:44 -05004793 *
4794 * DO NOT ADD NEW FILES.
Tejun Heofba94802013-11-22 18:20:43 -05004795 */
Al Virob5830432014-10-31 01:22:04 -04004796 name = cfile.file->f_path.dentry->d_name.name;
Tejun Heofba94802013-11-22 18:20:43 -05004797
4798 if (!strcmp(name, "memory.usage_in_bytes")) {
4799 event->register_event = mem_cgroup_usage_register_event;
4800 event->unregister_event = mem_cgroup_usage_unregister_event;
4801 } else if (!strcmp(name, "memory.oom_control")) {
4802 event->register_event = mem_cgroup_oom_register_event;
4803 event->unregister_event = mem_cgroup_oom_unregister_event;
4804 } else if (!strcmp(name, "memory.pressure_level")) {
4805 event->register_event = vmpressure_register_event;
4806 event->unregister_event = vmpressure_unregister_event;
4807 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
Tejun Heo347c4a82013-11-22 18:20:43 -05004808 event->register_event = memsw_cgroup_usage_register_event;
4809 event->unregister_event = memsw_cgroup_usage_unregister_event;
Tejun Heofba94802013-11-22 18:20:43 -05004810 } else {
4811 ret = -EINVAL;
4812 goto out_put_cfile;
4813 }
4814
4815 /*
Tejun Heob5557c42013-11-22 18:20:42 -05004816 * Verify @cfile should belong to @css. Also, remaining events are
4817 * automatically removed on cgroup destruction but the removal is
4818 * asynchronous, so take an extra ref on @css.
Tejun Heo79bd9812013-11-22 18:20:42 -05004819 */
Al Virob5830432014-10-31 01:22:04 -04004820 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
Tejun Heoec903c02014-05-13 12:11:01 -04004821 &memory_cgrp_subsys);
Tejun Heo79bd9812013-11-22 18:20:42 -05004822 ret = -EINVAL;
Tejun Heo5a17f542014-02-11 11:52:47 -05004823 if (IS_ERR(cfile_css))
Tejun Heo79bd9812013-11-22 18:20:42 -05004824 goto out_put_cfile;
Tejun Heo5a17f542014-02-11 11:52:47 -05004825 if (cfile_css != css) {
4826 css_put(cfile_css);
4827 goto out_put_cfile;
4828 }
Tejun Heo79bd9812013-11-22 18:20:42 -05004829
Tejun Heo451af502014-05-13 12:16:21 -04004830 ret = event->register_event(memcg, event->eventfd, buf);
Tejun Heo79bd9812013-11-22 18:20:42 -05004831 if (ret)
4832 goto out_put_css;
4833
Christoph Hellwig9965ed172018-03-05 07:26:05 -08004834 vfs_poll(efile.file, &event->pt);
Tejun Heo79bd9812013-11-22 18:20:42 -05004835
Tejun Heofba94802013-11-22 18:20:43 -05004836 spin_lock(&memcg->event_list_lock);
4837 list_add(&event->list, &memcg->event_list);
4838 spin_unlock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004839
4840 fdput(cfile);
4841 fdput(efile);
4842
Tejun Heo451af502014-05-13 12:16:21 -04004843 return nbytes;
Tejun Heo79bd9812013-11-22 18:20:42 -05004844
4845out_put_css:
Tejun Heob5557c42013-11-22 18:20:42 -05004846 css_put(css);
Tejun Heo79bd9812013-11-22 18:20:42 -05004847out_put_cfile:
4848 fdput(cfile);
4849out_put_eventfd:
4850 eventfd_ctx_put(event->eventfd);
4851out_put_efile:
4852 fdput(efile);
4853out_kfree:
4854 kfree(event);
4855
4856 return ret;
4857}
4858
Johannes Weiner241994ed2015-02-11 15:26:06 -08004859static struct cftype mem_cgroup_legacy_files[] = {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004860 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004861 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004862 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Tejun Heo791badb2013-12-05 12:28:02 -05004863 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004864 },
4865 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004866 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004867 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Tejun Heo6770c642014-05-13 12:16:21 -04004868 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004869 .read_u64 = mem_cgroup_read_u64,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004870 },
4871 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004872 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004873 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Tejun Heo451af502014-05-13 12:16:21 -04004874 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05004875 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004876 },
4877 {
Balbir Singh296c81d2009-09-23 15:56:36 -07004878 .name = "soft_limit_in_bytes",
4879 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
Tejun Heo451af502014-05-13 12:16:21 -04004880 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05004881 .read_u64 = mem_cgroup_read_u64,
Balbir Singh296c81d2009-09-23 15:56:36 -07004882 },
4883 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004884 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004885 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Tejun Heo6770c642014-05-13 12:16:21 -04004886 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004887 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004888 },
Balbir Singh8697d332008-02-07 00:13:59 -08004889 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004890 .name = "stat",
Tejun Heo2da8ca82013-12-05 12:28:04 -05004891 .seq_show = memcg_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004892 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004893 {
4894 .name = "force_empty",
Tejun Heo6770c642014-05-13 12:16:21 -04004895 .write = mem_cgroup_force_empty_write,
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004896 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08004897 {
4898 .name = "use_hierarchy",
4899 .write_u64 = mem_cgroup_hierarchy_write,
4900 .read_u64 = mem_cgroup_hierarchy_read,
4901 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004902 {
Tejun Heo3bc942f2013-11-22 18:20:44 -05004903 .name = "cgroup.event_control", /* XXX: for compat */
Tejun Heo451af502014-05-13 12:16:21 -04004904 .write = memcg_write_event_control,
Tejun Heo7dbdb192015-09-18 17:54:23 -04004905 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
Tejun Heo79bd9812013-11-22 18:20:42 -05004906 },
4907 {
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004908 .name = "swappiness",
4909 .read_u64 = mem_cgroup_swappiness_read,
4910 .write_u64 = mem_cgroup_swappiness_write,
4911 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004912 {
4913 .name = "move_charge_at_immigrate",
4914 .read_u64 = mem_cgroup_move_charge_read,
4915 .write_u64 = mem_cgroup_move_charge_write,
4916 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004917 {
4918 .name = "oom_control",
Tejun Heo2da8ca82013-12-05 12:28:04 -05004919 .seq_show = mem_cgroup_oom_control_read,
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004920 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004921 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4922 },
Anton Vorontsov70ddf632013-04-29 15:08:31 -07004923 {
4924 .name = "pressure_level",
Anton Vorontsov70ddf632013-04-29 15:08:31 -07004925 },
Ying Han406eb0c2011-05-26 16:25:37 -07004926#ifdef CONFIG_NUMA
4927 {
4928 .name = "numa_stat",
Tejun Heo2da8ca82013-12-05 12:28:04 -05004929 .seq_show = memcg_numa_stat_show,
Ying Han406eb0c2011-05-26 16:25:37 -07004930 },
4931#endif
Glauber Costa510fc4e2012-12-18 14:21:47 -08004932 {
4933 .name = "kmem.limit_in_bytes",
4934 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
Tejun Heo451af502014-05-13 12:16:21 -04004935 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05004936 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08004937 },
4938 {
4939 .name = "kmem.usage_in_bytes",
4940 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
Tejun Heo791badb2013-12-05 12:28:02 -05004941 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08004942 },
4943 {
4944 .name = "kmem.failcnt",
4945 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
Tejun Heo6770c642014-05-13 12:16:21 -04004946 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004947 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08004948 },
4949 {
4950 .name = "kmem.max_usage_in_bytes",
4951 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
Tejun Heo6770c642014-05-13 12:16:21 -04004952 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004953 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08004954 },
Yafang Shaoa87425a2020-04-01 21:06:30 -07004955#if defined(CONFIG_MEMCG_KMEM) && \
4956 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
Glauber Costa749c5412012-12-18 14:23:01 -08004957 {
4958 .name = "kmem.slabinfo",
Vladimir Davydovb0475012014-12-10 15:44:19 -08004959 .seq_show = memcg_slab_show,
Glauber Costa749c5412012-12-18 14:23:01 -08004960 },
4961#endif
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08004962 {
4963 .name = "kmem.tcp.limit_in_bytes",
4964 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4965 .write = mem_cgroup_write,
4966 .read_u64 = mem_cgroup_read_u64,
4967 },
4968 {
4969 .name = "kmem.tcp.usage_in_bytes",
4970 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4971 .read_u64 = mem_cgroup_read_u64,
4972 },
4973 {
4974 .name = "kmem.tcp.failcnt",
4975 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4976 .write = mem_cgroup_reset,
4977 .read_u64 = mem_cgroup_read_u64,
4978 },
4979 {
4980 .name = "kmem.tcp.max_usage_in_bytes",
4981 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4982 .write = mem_cgroup_reset,
4983 .read_u64 = mem_cgroup_read_u64,
4984 },
Tejun Heo6bc10342012-04-01 12:09:55 -07004985 { }, /* terminate */
Tejun Heoaf36f902012-04-01 12:09:55 -07004986};
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004987
Johannes Weiner73f576c2016-07-20 15:44:57 -07004988/*
4989 * Private memory cgroup IDR
4990 *
4991 * Swap-out records and page cache shadow entries need to store memcg
4992 * references in constrained space, so we maintain an ID space that is
4993 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4994 * memory-controlled cgroups to 64k.
4995 *
Ethon Paulb8f29352020-06-04 16:49:28 -07004996 * However, there usually are many references to the offline CSS after
Johannes Weiner73f576c2016-07-20 15:44:57 -07004997 * the cgroup has been destroyed, such as page cache or reclaimable
4998 * slab objects, that don't need to hang on to the ID. We want to keep
4999 * those dead CSS from occupying IDs, or we might quickly exhaust the
5000 * relatively small ID space and prevent the creation of new cgroups
5001 * even when there are much fewer than 64k cgroups - possibly none.
5002 *
5003 * Maintain a private 16-bit ID space for memcg, and allow the ID to
5004 * be freed and recycled when it's no longer needed, which is usually
5005 * when the CSS is offlined.
5006 *
5007 * The only exception to that are records of swapped out tmpfs/shmem
5008 * pages that need to be attributed to live ancestors on swapin. But
5009 * those references are manageable from userspace.
5010 */
5011
5012static DEFINE_IDR(mem_cgroup_idr);
5013
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005014static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5015{
5016 if (memcg->id.id > 0) {
5017 idr_remove(&mem_cgroup_idr, memcg->id.id);
5018 memcg->id.id = 0;
5019 }
5020}
5021
Vincenzo Frascinoc1514c02020-04-01 21:07:13 -07005022static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5023 unsigned int n)
Johannes Weiner73f576c2016-07-20 15:44:57 -07005024{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07005025 refcount_add(n, &memcg->id.ref);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005026}
5027
Vladimir Davydov615d66c2016-08-11 15:33:03 -07005028static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
Johannes Weiner73f576c2016-07-20 15:44:57 -07005029{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07005030 if (refcount_sub_and_test(n, &memcg->id.ref)) {
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005031 mem_cgroup_id_remove(memcg);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005032
5033 /* Memcg ID pins CSS */
5034 css_put(&memcg->css);
5035 }
5036}
5037
Vladimir Davydov615d66c2016-08-11 15:33:03 -07005038static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5039{
5040 mem_cgroup_id_put_many(memcg, 1);
5041}
5042
Johannes Weiner73f576c2016-07-20 15:44:57 -07005043/**
5044 * mem_cgroup_from_id - look up a memcg from a memcg id
5045 * @id: the memcg id to look up
5046 *
5047 * Caller must hold rcu_read_lock().
5048 */
5049struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
5050{
5051 WARN_ON_ONCE(!rcu_read_lock_held());
5052 return idr_find(&mem_cgroup_idr, id);
5053}
5054
Mel Gormanef8f2322016-07-28 15:46:05 -07005055static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005056{
5057 struct mem_cgroup_per_node *pn;
Mel Gormanef8f2322016-07-28 15:46:05 -07005058 int tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005059 /*
5060 * This routine is called against possible nodes.
5061 * But it's BUG to call kmalloc() against offline node.
5062 *
5063 * TODO: this routine can waste much memory for nodes which will
5064 * never be onlined. It's better to use memory hotplug callback
5065 * function.
5066 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07005067 if (!node_state(node, N_NORMAL_MEMORY))
5068 tmp = -1;
Jesper Juhl17295c82011-01-13 15:47:42 -08005069 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005070 if (!pn)
5071 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005072
Roman Gushchin3e38e0a2020-08-11 18:30:25 -07005073 pn->lruvec_stat_local = alloc_percpu_gfp(struct lruvec_stat,
5074 GFP_KERNEL_ACCOUNT);
Johannes Weiner815744d2019-06-13 15:55:46 -07005075 if (!pn->lruvec_stat_local) {
5076 kfree(pn);
5077 return 1;
5078 }
5079
Muchun Songf3344ad2021-02-24 12:03:15 -08005080 pn->lruvec_stat_cpu = alloc_percpu_gfp(struct batched_lruvec_stat,
Roman Gushchin3e38e0a2020-08-11 18:30:25 -07005081 GFP_KERNEL_ACCOUNT);
Johannes Weinera983b5e2018-01-31 16:16:45 -08005082 if (!pn->lruvec_stat_cpu) {
Johannes Weiner815744d2019-06-13 15:55:46 -07005083 free_percpu(pn->lruvec_stat_local);
Johannes Weiner00f3ca22017-07-06 15:40:52 -07005084 kfree(pn);
5085 return 1;
5086 }
5087
Mel Gormanef8f2322016-07-28 15:46:05 -07005088 lruvec_init(&pn->lruvec);
5089 pn->usage_in_excess = 0;
5090 pn->on_tree = false;
5091 pn->memcg = memcg;
5092
Johannes Weiner54f72fe2013-07-08 15:59:49 -07005093 memcg->nodeinfo[node] = pn;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005094 return 0;
5095}
5096
Mel Gormanef8f2322016-07-28 15:46:05 -07005097static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005098{
Johannes Weiner00f3ca22017-07-06 15:40:52 -07005099 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
5100
Michal Hocko4eaf4312018-04-10 16:29:52 -07005101 if (!pn)
5102 return;
5103
Johannes Weinera983b5e2018-01-31 16:16:45 -08005104 free_percpu(pn->lruvec_stat_cpu);
Johannes Weiner815744d2019-06-13 15:55:46 -07005105 free_percpu(pn->lruvec_stat_local);
Johannes Weiner00f3ca22017-07-06 15:40:52 -07005106 kfree(pn);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005107}
5108
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005109static void __mem_cgroup_free(struct mem_cgroup *memcg)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005110{
5111 int node;
5112
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005113 for_each_node(node)
Mel Gormanef8f2322016-07-28 15:46:05 -07005114 free_mem_cgroup_per_node_info(memcg, node);
Chris Down871789d2019-05-14 15:46:57 -07005115 free_percpu(memcg->vmstats_percpu);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005116 kfree(memcg);
5117}
5118
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005119static void mem_cgroup_free(struct mem_cgroup *memcg)
5120{
Johannes Weiner2cd21c82021-04-29 22:56:29 -07005121 int cpu;
5122
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005123 memcg_wb_domain_exit(memcg);
Shakeel Butt7961eee2019-11-05 21:16:21 -08005124 /*
Johannes Weiner2d146aa2021-04-29 22:56:26 -07005125 * Flush percpu lruvec stats to guarantee the value
5126 * correctness on parent's and all ancestor levels.
Shakeel Butt7961eee2019-11-05 21:16:21 -08005127 */
Johannes Weiner2cd21c82021-04-29 22:56:29 -07005128 for_each_online_cpu(cpu)
5129 memcg_flush_lruvec_page_state(memcg, cpu);
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005130 __mem_cgroup_free(memcg);
5131}
5132
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005133static struct mem_cgroup *mem_cgroup_alloc(void)
5134{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005135 struct mem_cgroup *memcg;
Alexey Dobriyanb9726c22019-03-05 15:48:26 -08005136 unsigned int size;
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005137 int node;
Tejun Heo97b27822019-08-26 09:06:56 -07005138 int __maybe_unused i;
Yafang Shao11d67612020-05-07 18:35:43 -07005139 long error = -ENOMEM;
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005140
Vladimir Davydov8ff69e22014-01-23 15:52:52 -08005141 size = sizeof(struct mem_cgroup);
5142 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005143
Vladimir Davydov8ff69e22014-01-23 15:52:52 -08005144 memcg = kzalloc(size, GFP_KERNEL);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005145 if (!memcg)
Yafang Shao11d67612020-05-07 18:35:43 -07005146 return ERR_PTR(error);
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07005147
Johannes Weiner73f576c2016-07-20 15:44:57 -07005148 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
5149 1, MEM_CGROUP_ID_MAX,
5150 GFP_KERNEL);
Yafang Shao11d67612020-05-07 18:35:43 -07005151 if (memcg->id.id < 0) {
5152 error = memcg->id.id;
Johannes Weiner73f576c2016-07-20 15:44:57 -07005153 goto fail;
Yafang Shao11d67612020-05-07 18:35:43 -07005154 }
Johannes Weiner73f576c2016-07-20 15:44:57 -07005155
Roman Gushchin3e38e0a2020-08-11 18:30:25 -07005156 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5157 GFP_KERNEL_ACCOUNT);
Chris Down871789d2019-05-14 15:46:57 -07005158 if (!memcg->vmstats_percpu)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005159 goto fail;
Pavel Emelianov78fb7462008-02-07 00:13:51 -08005160
Bob Liu3ed28fa2012-01-12 17:19:04 -08005161 for_each_node(node)
Mel Gormanef8f2322016-07-28 15:46:05 -07005162 if (alloc_mem_cgroup_per_node_info(memcg, node))
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005163 goto fail;
Balbir Singhf64c3f52009-09-23 15:56:37 -07005164
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005165 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5166 goto fail;
Balbir Singh28dbc4b2009-01-07 18:08:05 -08005167
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005168 INIT_WORK(&memcg->high_work, high_work_func);
Glauber Costad142e3e2013-02-22 16:34:52 -08005169 INIT_LIST_HEAD(&memcg->oom_notify);
Glauber Costad142e3e2013-02-22 16:34:52 -08005170 mutex_init(&memcg->thresholds_lock);
5171 spin_lock_init(&memcg->move_lock);
Anton Vorontsov70ddf632013-04-29 15:08:31 -07005172 vmpressure_init(&memcg->vmpressure);
Tejun Heofba94802013-11-22 18:20:43 -05005173 INIT_LIST_HEAD(&memcg->event_list);
5174 spin_lock_init(&memcg->event_list_lock);
Johannes Weinerd886f4e2016-01-20 15:02:47 -08005175 memcg->socket_pressure = jiffies;
Kirill Tkhai84c07d12018-08-17 15:47:25 -07005176#ifdef CONFIG_MEMCG_KMEM
Vladimir Davydov900a38f2014-12-12 16:55:10 -08005177 memcg->kmemcg_id = -1;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07005178 INIT_LIST_HEAD(&memcg->objcg_list);
Vladimir Davydov900a38f2014-12-12 16:55:10 -08005179#endif
Tejun Heo52ebea72015-05-22 17:13:37 -04005180#ifdef CONFIG_CGROUP_WRITEBACK
5181 INIT_LIST_HEAD(&memcg->cgwb_list);
Tejun Heo97b27822019-08-26 09:06:56 -07005182 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5183 memcg->cgwb_frn[i].done =
5184 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
Tejun Heo52ebea72015-05-22 17:13:37 -04005185#endif
Yang Shi87eaceb2019-09-23 15:38:15 -07005186#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5187 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5188 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5189 memcg->deferred_split_queue.split_queue_len = 0;
5190#endif
Johannes Weiner73f576c2016-07-20 15:44:57 -07005191 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005192 return memcg;
5193fail:
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005194 mem_cgroup_id_remove(memcg);
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005195 __mem_cgroup_free(memcg);
Yafang Shao11d67612020-05-07 18:35:43 -07005196 return ERR_PTR(error);
Glauber Costad142e3e2013-02-22 16:34:52 -08005197}
5198
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005199static struct cgroup_subsys_state * __ref
5200mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
Glauber Costad142e3e2013-02-22 16:34:52 -08005201{
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005202 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
Roman Gushchinb87d8ce2020-10-17 16:13:40 -07005203 struct mem_cgroup *memcg, *old_memcg;
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005204 long error = -ENOMEM;
Glauber Costad142e3e2013-02-22 16:34:52 -08005205
Roman Gushchinb87d8ce2020-10-17 16:13:40 -07005206 old_memcg = set_active_memcg(parent);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005207 memcg = mem_cgroup_alloc();
Roman Gushchinb87d8ce2020-10-17 16:13:40 -07005208 set_active_memcg(old_memcg);
Yafang Shao11d67612020-05-07 18:35:43 -07005209 if (IS_ERR(memcg))
5210 return ERR_CAST(memcg);
Li Zefan4219b2d2013-09-23 16:56:29 +08005211
Jakub Kicinskid1663a92020-06-01 21:49:49 -07005212 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005213 memcg->soft_limit = PAGE_COUNTER_MAX;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07005214 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005215 if (parent) {
5216 memcg->swappiness = mem_cgroup_swappiness(parent);
5217 memcg->oom_kill_disable = parent->oom_kill_disable;
Roman Gushchinbef86202020-12-14 19:06:49 -08005218
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005219 page_counter_init(&memcg->memory, &parent->memory);
Vladimir Davydov37e84352016-01-20 15:02:56 -08005220 page_counter_init(&memcg->swap, &parent->swap);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005221 page_counter_init(&memcg->kmem, &parent->kmem);
Johannes Weiner0db15292016-01-20 15:02:50 -08005222 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
Balbir Singh18f59ea2009-01-07 18:08:07 -08005223 } else {
Roman Gushchinbef86202020-12-14 19:06:49 -08005224 page_counter_init(&memcg->memory, NULL);
5225 page_counter_init(&memcg->swap, NULL);
5226 page_counter_init(&memcg->kmem, NULL);
5227 page_counter_init(&memcg->tcpmem, NULL);
Vladimir Davydovd6441632014-01-23 15:53:09 -08005228
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005229 root_mem_cgroup = memcg;
5230 return &memcg->css;
5231 }
5232
Roman Gushchinbef86202020-12-14 19:06:49 -08005233 /* The following stuff does not apply to the root */
Vladimir Davydovb313aee2016-03-17 14:18:27 -07005234 error = memcg_online_kmem(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005235 if (error)
5236 goto fail;
Johannes Weiner127424c2016-01-20 15:02:32 -08005237
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005238 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
Johannes Weineref129472016-01-14 15:21:34 -08005239 static_branch_inc(&memcg_sockets_enabled_key);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005240
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005241 return &memcg->css;
5242fail:
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005243 mem_cgroup_id_remove(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005244 mem_cgroup_free(memcg);
Yafang Shao11d67612020-05-07 18:35:43 -07005245 return ERR_PTR(error);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005246}
5247
Johannes Weiner73f576c2016-07-20 15:44:57 -07005248static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005249{
Vladimir Davydov58fa2a52016-10-07 16:57:29 -07005250 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5251
Kirill Tkhai0a4465d2018-08-17 15:47:37 -07005252 /*
5253 * A memcg must be visible for memcg_expand_shrinker_maps()
5254 * by the time the maps are allocated. So, we allocate maps
5255 * here, when for_each_mem_cgroup() can't skip it.
5256 */
5257 if (memcg_alloc_shrinker_maps(memcg)) {
5258 mem_cgroup_id_remove(memcg);
5259 return -ENOMEM;
5260 }
5261
Johannes Weiner73f576c2016-07-20 15:44:57 -07005262 /* Online state pins memcg ID, memcg ID pins CSS */
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07005263 refcount_set(&memcg->id.ref, 1);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005264 css_get(css);
Johannes Weiner2f7dd7a2014-10-02 16:16:57 -07005265 return 0;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005266}
5267
Tejun Heoeb954192013-08-08 20:11:23 -04005268static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005269{
Tejun Heoeb954192013-08-08 20:11:23 -04005270 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo3bc942f2013-11-22 18:20:44 -05005271 struct mem_cgroup_event *event, *tmp;
Tejun Heo79bd9812013-11-22 18:20:42 -05005272
5273 /*
5274 * Unregister events and notify userspace.
5275 * Notify userspace about cgroup removing only after rmdir of cgroup
5276 * directory to avoid race between userspace and kernelspace.
5277 */
Tejun Heofba94802013-11-22 18:20:43 -05005278 spin_lock(&memcg->event_list_lock);
5279 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
Tejun Heo79bd9812013-11-22 18:20:42 -05005280 list_del_init(&event->list);
5281 schedule_work(&event->remove);
5282 }
Tejun Heofba94802013-11-22 18:20:43 -05005283 spin_unlock(&memcg->event_list_lock);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07005284
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07005285 page_counter_set_min(&memcg->memory, 0);
Roman Gushchin23067152018-06-07 17:06:22 -07005286 page_counter_set_low(&memcg->memory, 0);
Roman Gushchin63677c742017-09-06 16:21:47 -07005287
Johannes Weiner567e9ab2016-01-20 15:02:24 -08005288 memcg_offline_kmem(memcg);
Tejun Heo52ebea72015-05-22 17:13:37 -04005289 wb_memcg_offline(memcg);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005290
Roman Gushchin591edfb2018-10-26 15:03:23 -07005291 drain_all_stock(memcg);
5292
Johannes Weiner73f576c2016-07-20 15:44:57 -07005293 mem_cgroup_id_put(memcg);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005294}
5295
Vladimir Davydov6df38682015-12-29 14:54:10 -08005296static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
5297{
5298 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5299
5300 invalidate_reclaim_iterators(memcg);
5301}
5302
Tejun Heoeb954192013-08-08 20:11:23 -04005303static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005304{
Tejun Heoeb954192013-08-08 20:11:23 -04005305 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo97b27822019-08-26 09:06:56 -07005306 int __maybe_unused i;
Daisuke Nishimurac268e992009-01-15 13:51:13 -08005307
Tejun Heo97b27822019-08-26 09:06:56 -07005308#ifdef CONFIG_CGROUP_WRITEBACK
5309 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5310 wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5311#endif
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005312 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
Johannes Weineref129472016-01-14 15:21:34 -08005313 static_branch_dec(&memcg_sockets_enabled_key);
Johannes Weiner3893e302016-01-20 15:02:29 -08005314
Johannes Weiner0db15292016-01-20 15:02:50 -08005315 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08005316 static_branch_dec(&memcg_sockets_enabled_key);
Johannes Weiner3893e302016-01-20 15:02:29 -08005317
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005318 vmpressure_cleanup(&memcg->vmpressure);
5319 cancel_work_sync(&memcg->high_work);
5320 mem_cgroup_remove_from_trees(memcg);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -07005321 memcg_free_shrinker_maps(memcg);
Johannes Weinerd886f4e2016-01-20 15:02:47 -08005322 memcg_free_kmem(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005323 mem_cgroup_free(memcg);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005324}
5325
Tejun Heo1ced9532014-07-08 18:02:57 -04005326/**
5327 * mem_cgroup_css_reset - reset the states of a mem_cgroup
5328 * @css: the target css
5329 *
5330 * Reset the states of the mem_cgroup associated with @css. This is
5331 * invoked when the userland requests disabling on the default hierarchy
5332 * but the memcg is pinned through dependency. The memcg should stop
5333 * applying policies and should revert to the vanilla state as it may be
5334 * made visible again.
5335 *
5336 * The current implementation only resets the essential configurations.
5337 * This needs to be expanded to cover all the visible parts.
5338 */
5339static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
5340{
5341 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5342
Roman Gushchinbbec2e12018-06-07 17:06:18 -07005343 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5344 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07005345 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5346 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07005347 page_counter_set_min(&memcg->memory, 0);
Roman Gushchin23067152018-06-07 17:06:22 -07005348 page_counter_set_low(&memcg->memory, 0);
Jakub Kicinskid1663a92020-06-01 21:49:49 -07005349 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
Johannes Weiner24d404d2015-01-08 14:32:35 -08005350 memcg->soft_limit = PAGE_COUNTER_MAX;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07005351 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
Tejun Heo2529bb32015-05-22 18:23:34 -04005352 memcg_wb_domain_size_changed(memcg);
Tejun Heo1ced9532014-07-08 18:02:57 -04005353}
5354
Johannes Weiner2d146aa2021-04-29 22:56:26 -07005355static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
5356{
5357 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5358 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5359 struct memcg_vmstats_percpu *statc;
5360 long delta, v;
5361 int i;
5362
5363 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
5364
5365 for (i = 0; i < MEMCG_NR_STAT; i++) {
5366 /*
5367 * Collect the aggregated propagation counts of groups
5368 * below us. We're in a per-cpu loop here and this is
5369 * a global counter, so the first cycle will get them.
5370 */
5371 delta = memcg->vmstats.state_pending[i];
5372 if (delta)
5373 memcg->vmstats.state_pending[i] = 0;
5374
5375 /* Add CPU changes on this level since the last flush */
5376 v = READ_ONCE(statc->state[i]);
5377 if (v != statc->state_prev[i]) {
5378 delta += v - statc->state_prev[i];
5379 statc->state_prev[i] = v;
5380 }
5381
5382 if (!delta)
5383 continue;
5384
5385 /* Aggregate counts on this level and propagate upwards */
5386 memcg->vmstats.state[i] += delta;
5387 if (parent)
5388 parent->vmstats.state_pending[i] += delta;
5389 }
5390
5391 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
5392 delta = memcg->vmstats.events_pending[i];
5393 if (delta)
5394 memcg->vmstats.events_pending[i] = 0;
5395
5396 v = READ_ONCE(statc->events[i]);
5397 if (v != statc->events_prev[i]) {
5398 delta += v - statc->events_prev[i];
5399 statc->events_prev[i] = v;
5400 }
5401
5402 if (!delta)
5403 continue;
5404
5405 memcg->vmstats.events[i] += delta;
5406 if (parent)
5407 parent->vmstats.events_pending[i] += delta;
5408 }
5409}
5410
Daisuke Nishimura02491442010-03-10 15:22:17 -08005411#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005412/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005413static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005414{
Johannes Weiner05b84302014-08-06 16:05:59 -07005415 int ret;
Johannes Weiner9476db92014-08-06 16:05:55 -07005416
Mel Gormand0164ad2015-11-06 16:28:21 -08005417 /* Try a single bulk charge without reclaim first, kswapd may wake */
5418 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
Johannes Weiner9476db92014-08-06 16:05:55 -07005419 if (!ret) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005420 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005421 return ret;
5422 }
Johannes Weiner9476db92014-08-06 16:05:55 -07005423
David Rientjes36745342017-01-24 15:18:10 -08005424 /* Try charges one by one with reclaim, but do not retry */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005425 while (count--) {
David Rientjes36745342017-01-24 15:18:10 -08005426 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08005427 if (ret)
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08005428 return ret;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005429 mc.precharge++;
Johannes Weiner9476db92014-08-06 16:05:55 -07005430 cond_resched();
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005431 }
Johannes Weiner9476db92014-08-06 16:05:55 -07005432 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005433}
5434
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005435union mc_target {
5436 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005437 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005438};
5439
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005440enum mc_target_type {
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005441 MC_TARGET_NONE = 0,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005442 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08005443 MC_TARGET_SWAP,
Jérôme Glissec733a822017-09-08 16:11:54 -07005444 MC_TARGET_DEVICE,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005445};
5446
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005447static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
5448 unsigned long addr, pte_t ptent)
5449{
Christoph Hellwig25b29952019-06-13 22:50:49 +02005450 struct page *page = vm_normal_page(vma, addr, ptent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005451
5452 if (!page || !page_mapped(page))
5453 return NULL;
5454 if (PageAnon(page)) {
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005455 if (!(mc.flags & MOVE_ANON))
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005456 return NULL;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005457 } else {
5458 if (!(mc.flags & MOVE_FILE))
5459 return NULL;
5460 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005461 if (!get_page_unless_zero(page))
5462 return NULL;
5463
5464 return page;
5465}
5466
Jérôme Glissec733a822017-09-08 16:11:54 -07005467#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005468static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
Li RongQing48406ef2016-07-26 15:22:14 -07005469 pte_t ptent, swp_entry_t *entry)
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005470{
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005471 struct page *page = NULL;
5472 swp_entry_t ent = pte_to_swp_entry(ptent);
5473
Ralph Campbell9a137152020-10-13 16:53:13 -07005474 if (!(mc.flags & MOVE_ANON))
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005475 return NULL;
Jérôme Glissec733a822017-09-08 16:11:54 -07005476
5477 /*
5478 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
5479 * a device and because they are not accessible by CPU they are store
5480 * as special swap entry in the CPU page table.
5481 */
5482 if (is_device_private_entry(ent)) {
5483 page = device_private_entry_to_page(ent);
5484 /*
5485 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
5486 * a refcount of 1 when free (unlike normal page)
5487 */
5488 if (!page_ref_add_unless(page, 1, 1))
5489 return NULL;
5490 return page;
5491 }
5492
Ralph Campbell9a137152020-10-13 16:53:13 -07005493 if (non_swap_entry(ent))
5494 return NULL;
5495
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005496 /*
5497 * Because lookup_swap_cache() updates some statistics counter,
5498 * we call find_get_page() with swapper_space directly.
5499 */
Huang Yingf6ab1f72016-10-07 17:00:21 -07005500 page = find_get_page(swap_address_space(ent), swp_offset(ent));
Johannes Weiner2d1c4982020-06-03 16:02:14 -07005501 entry->val = ent.val;
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005502
5503 return page;
5504}
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005505#else
5506static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
Li RongQing48406ef2016-07-26 15:22:14 -07005507 pte_t ptent, swp_entry_t *entry)
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005508{
5509 return NULL;
5510}
5511#endif
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005512
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005513static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
5514 unsigned long addr, pte_t ptent, swp_entry_t *entry)
5515{
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005516 if (!vma->vm_file) /* anonymous vma */
5517 return NULL;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005518 if (!(mc.flags & MOVE_FILE))
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005519 return NULL;
5520
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005521 /* page is moved even if it's not RSS of this task(page-faulted). */
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07005522 /* shmem/tmpfs may report page out on swap: account for that too. */
Matthew Wilcox (Oracle)f5df8632020-10-13 16:51:21 -07005523 return find_get_incore_page(vma->vm_file->f_mapping,
5524 linear_page_index(vma, addr));
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005525}
5526
Chen Gangb1b0dea2015-04-14 15:47:35 -07005527/**
5528 * mem_cgroup_move_account - move account of the page
5529 * @page: the page
Li RongQing25843c22016-07-26 15:26:56 -07005530 * @compound: charge the page as compound or small page
Chen Gangb1b0dea2015-04-14 15:47:35 -07005531 * @from: mem_cgroup which the page is moved from.
5532 * @to: mem_cgroup which the page is moved to. @from != @to.
5533 *
Kirill A. Shutemov3ac808f2016-01-15 16:53:07 -08005534 * The caller must make sure the page is not on LRU (isolate_page() is useful.)
Chen Gangb1b0dea2015-04-14 15:47:35 -07005535 *
5536 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
5537 * from old cgroup.
5538 */
5539static int mem_cgroup_move_account(struct page *page,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005540 bool compound,
Chen Gangb1b0dea2015-04-14 15:47:35 -07005541 struct mem_cgroup *from,
5542 struct mem_cgroup *to)
5543{
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005544 struct lruvec *from_vec, *to_vec;
5545 struct pglist_data *pgdat;
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07005546 unsigned int nr_pages = compound ? thp_nr_pages(page) : 1;
Chen Gangb1b0dea2015-04-14 15:47:35 -07005547 int ret;
5548
5549 VM_BUG_ON(from == to);
5550 VM_BUG_ON_PAGE(PageLRU(page), page);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005551 VM_BUG_ON(compound && !PageTransHuge(page));
Chen Gangb1b0dea2015-04-14 15:47:35 -07005552
5553 /*
Johannes Weiner6a93ca82016-03-15 14:57:19 -07005554 * Prevent mem_cgroup_migrate() from looking at
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005555 * page's memory cgroup of its source page while we change it.
Chen Gangb1b0dea2015-04-14 15:47:35 -07005556 */
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005557 ret = -EBUSY;
Chen Gangb1b0dea2015-04-14 15:47:35 -07005558 if (!trylock_page(page))
5559 goto out;
5560
5561 ret = -EINVAL;
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005562 if (page_memcg(page) != from)
Chen Gangb1b0dea2015-04-14 15:47:35 -07005563 goto out_unlock;
5564
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005565 pgdat = page_pgdat(page);
Johannes Weiner867e5e12019-11-30 17:55:34 -08005566 from_vec = mem_cgroup_lruvec(from, pgdat);
5567 to_vec = mem_cgroup_lruvec(to, pgdat);
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005568
Johannes Weinerabb242f2020-06-03 16:01:28 -07005569 lock_page_memcg(page);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005570
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07005571 if (PageAnon(page)) {
5572 if (page_mapped(page)) {
5573 __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
5574 __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
Johannes Weiner468c3982020-06-03 16:02:01 -07005575 if (PageTransHuge(page)) {
Muchun Song69473e52021-02-24 12:03:23 -08005576 __mod_lruvec_state(from_vec, NR_ANON_THPS,
5577 -nr_pages);
5578 __mod_lruvec_state(to_vec, NR_ANON_THPS,
5579 nr_pages);
Johannes Weiner468c3982020-06-03 16:02:01 -07005580 }
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07005581 }
5582 } else {
Johannes Weiner0d1c2072020-06-03 16:01:54 -07005583 __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
5584 __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
5585
5586 if (PageSwapBacked(page)) {
5587 __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
5588 __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
5589 }
5590
Johannes Weiner49e50d22020-06-03 16:01:47 -07005591 if (page_mapped(page)) {
5592 __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
5593 __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
5594 }
Chen Gangb1b0dea2015-04-14 15:47:35 -07005595
Johannes Weiner49e50d22020-06-03 16:01:47 -07005596 if (PageDirty(page)) {
5597 struct address_space *mapping = page_mapping(page);
Greg Thelenc4843a72015-05-22 17:13:16 -04005598
Christoph Hellwigf56753a2020-09-24 08:51:40 +02005599 if (mapping_can_writeback(mapping)) {
Johannes Weiner49e50d22020-06-03 16:01:47 -07005600 __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
5601 -nr_pages);
5602 __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
5603 nr_pages);
5604 }
Greg Thelenc4843a72015-05-22 17:13:16 -04005605 }
5606 }
5607
Chen Gangb1b0dea2015-04-14 15:47:35 -07005608 if (PageWriteback(page)) {
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005609 __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
5610 __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005611 }
5612
5613 /*
Johannes Weinerabb242f2020-06-03 16:01:28 -07005614 * All state has been migrated, let's switch to the new memcg.
5615 *
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005616 * It is safe to change page's memcg here because the page
Johannes Weinerabb242f2020-06-03 16:01:28 -07005617 * is referenced, charged, isolated, and locked: we can't race
5618 * with (un)charging, migration, LRU putback, or anything else
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005619 * that would rely on a stable page's memory cgroup.
Johannes Weinerabb242f2020-06-03 16:01:28 -07005620 *
5621 * Note that lock_page_memcg is a memcg lock, not a page lock,
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005622 * to save space. As soon as we switch page's memory cgroup to a
Johannes Weinerabb242f2020-06-03 16:01:28 -07005623 * new memcg that isn't locked, the above state can change
5624 * concurrently again. Make sure we're truly done with it.
Chen Gangb1b0dea2015-04-14 15:47:35 -07005625 */
Johannes Weinerabb242f2020-06-03 16:01:28 -07005626 smp_mb();
Chen Gangb1b0dea2015-04-14 15:47:35 -07005627
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07005628 css_get(&to->css);
5629 css_put(&from->css);
5630
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005631 page->memcg_data = (unsigned long)to;
Yang Shi87eaceb2019-09-23 15:38:15 -07005632
Johannes Weinerabb242f2020-06-03 16:01:28 -07005633 __unlock_page_memcg(from);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005634
5635 ret = 0;
5636
5637 local_irq_disable();
Johannes Weiner3fba69a2020-06-03 16:01:31 -07005638 mem_cgroup_charge_statistics(to, page, nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005639 memcg_check_events(to, page);
Johannes Weiner3fba69a2020-06-03 16:01:31 -07005640 mem_cgroup_charge_statistics(from, page, -nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005641 memcg_check_events(from, page);
5642 local_irq_enable();
5643out_unlock:
5644 unlock_page(page);
5645out:
5646 return ret;
5647}
5648
Li RongQing7cf78062016-05-27 14:27:46 -07005649/**
5650 * get_mctgt_type - get target type of moving charge
5651 * @vma: the vma the pte to be checked belongs
5652 * @addr: the address corresponding to the pte to be checked
5653 * @ptent: the pte to be checked
5654 * @target: the pointer the target page or swap ent will be stored(can be NULL)
5655 *
5656 * Returns
5657 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
5658 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
5659 * move charge. if @target is not NULL, the page is stored in target->page
5660 * with extra refcnt got(Callers should handle it).
5661 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
5662 * target for charge migration. if @target is not NULL, the entry is stored
5663 * in target->ent.
Christoph Hellwig25b29952019-06-13 22:50:49 +02005664 * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE
5665 * (so ZONE_DEVICE page and thus not on the lru).
Jérôme Glissedf6ad692017-09-08 16:12:24 -07005666 * For now we such page is charge like a regular page would be as for all
5667 * intent and purposes it is just special memory taking the place of a
5668 * regular page.
Jérôme Glissec733a822017-09-08 16:11:54 -07005669 *
5670 * See Documentations/vm/hmm.txt and include/linux/hmm.h
Li RongQing7cf78062016-05-27 14:27:46 -07005671 *
5672 * Called with pte lock held.
5673 */
5674
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005675static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005676 unsigned long addr, pte_t ptent, union mc_target *target)
5677{
Daisuke Nishimura02491442010-03-10 15:22:17 -08005678 struct page *page = NULL;
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005679 enum mc_target_type ret = MC_TARGET_NONE;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005680 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005681
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005682 if (pte_present(ptent))
5683 page = mc_handle_present_pte(vma, addr, ptent);
5684 else if (is_swap_pte(ptent))
Li RongQing48406ef2016-07-26 15:22:14 -07005685 page = mc_handle_swap_pte(vma, ptent, &ent);
Kirill A. Shutemov0661a332015-02-10 14:10:04 -08005686 else if (pte_none(ptent))
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005687 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005688
5689 if (!page && !ent.val)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005690 return ret;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005691 if (page) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08005692 /*
Johannes Weiner0a31bc92014-08-08 14:19:22 -07005693 * Do only loose check w/o serialization.
Johannes Weiner1306a852014-12-10 15:44:52 -08005694 * mem_cgroup_move_account() checks the page is valid or
Johannes Weiner0a31bc92014-08-08 14:19:22 -07005695 * not under LRU exclusion.
Daisuke Nishimura02491442010-03-10 15:22:17 -08005696 */
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005697 if (page_memcg(page) == mc.from) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08005698 ret = MC_TARGET_PAGE;
Christoph Hellwig25b29952019-06-13 22:50:49 +02005699 if (is_device_private_page(page))
Jérôme Glissec733a822017-09-08 16:11:54 -07005700 ret = MC_TARGET_DEVICE;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005701 if (target)
5702 target->page = page;
5703 }
5704 if (!ret || !target)
5705 put_page(page);
5706 }
Huang Ying3e14a572017-09-06 16:22:37 -07005707 /*
5708 * There is a swap entry and a page doesn't exist or isn't charged.
5709 * But we cannot move a tail-page in a THP.
5710 */
5711 if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
Li Zefan34c00c32013-09-23 16:56:01 +08005712 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07005713 ret = MC_TARGET_SWAP;
5714 if (target)
5715 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005716 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005717 return ret;
5718}
5719
Naoya Horiguchi12724852012-03-21 16:34:28 -07005720#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5721/*
Huang Yingd6810d72017-09-06 16:22:45 -07005722 * We don't consider PMD mapped swapping or file mapped pages because THP does
5723 * not support them for now.
Naoya Horiguchi12724852012-03-21 16:34:28 -07005724 * Caller should make sure that pmd_trans_huge(pmd) is true.
5725 */
5726static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5727 unsigned long addr, pmd_t pmd, union mc_target *target)
5728{
5729 struct page *page = NULL;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005730 enum mc_target_type ret = MC_TARGET_NONE;
5731
Zi Yan84c3fc42017-09-08 16:11:01 -07005732 if (unlikely(is_swap_pmd(pmd))) {
5733 VM_BUG_ON(thp_migration_supported() &&
5734 !is_pmd_migration_entry(pmd));
5735 return ret;
5736 }
Naoya Horiguchi12724852012-03-21 16:34:28 -07005737 page = pmd_page(pmd);
Sasha Levin309381fea2014-01-23 15:52:54 -08005738 VM_BUG_ON_PAGE(!page || !PageHead(page), page);
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005739 if (!(mc.flags & MOVE_ANON))
Naoya Horiguchi12724852012-03-21 16:34:28 -07005740 return ret;
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005741 if (page_memcg(page) == mc.from) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07005742 ret = MC_TARGET_PAGE;
5743 if (target) {
5744 get_page(page);
5745 target->page = page;
5746 }
5747 }
5748 return ret;
5749}
5750#else
5751static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5752 unsigned long addr, pmd_t pmd, union mc_target *target)
5753{
5754 return MC_TARGET_NONE;
5755}
5756#endif
5757
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005758static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
5759 unsigned long addr, unsigned long end,
5760 struct mm_walk *walk)
5761{
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08005762 struct vm_area_struct *vma = walk->vma;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005763 pte_t *pte;
5764 spinlock_t *ptl;
5765
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08005766 ptl = pmd_trans_huge_lock(pmd, vma);
5767 if (ptl) {
Jérôme Glissec733a822017-09-08 16:11:54 -07005768 /*
5769 * Note their can not be MC_TARGET_DEVICE for now as we do not
Christoph Hellwig25b29952019-06-13 22:50:49 +02005770 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
5771 * this might change.
Jérôme Glissec733a822017-09-08 16:11:54 -07005772 */
Naoya Horiguchi12724852012-03-21 16:34:28 -07005773 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
5774 mc.precharge += HPAGE_PMD_NR;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08005775 spin_unlock(ptl);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07005776 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005777 }
Dave Hansen03319322011-03-22 16:32:56 -07005778
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07005779 if (pmd_trans_unstable(pmd))
5780 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005781 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5782 for (; addr != end; pte++, addr += PAGE_SIZE)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005783 if (get_mctgt_type(vma, addr, *pte, NULL))
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005784 mc.precharge++; /* increment precharge temporarily */
5785 pte_unmap_unlock(pte - 1, ptl);
5786 cond_resched();
5787
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005788 return 0;
5789}
5790
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02005791static const struct mm_walk_ops precharge_walk_ops = {
5792 .pmd_entry = mem_cgroup_count_precharge_pte_range,
5793};
5794
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005795static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5796{
5797 unsigned long precharge;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005798
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07005799 mmap_read_lock(mm);
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02005800 walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07005801 mmap_read_unlock(mm);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005802
5803 precharge = mc.precharge;
5804 mc.precharge = 0;
5805
5806 return precharge;
5807}
5808
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005809static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5810{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005811 unsigned long precharge = mem_cgroup_count_precharge(mm);
5812
5813 VM_BUG_ON(mc.moving_task);
5814 mc.moving_task = current;
5815 return mem_cgroup_do_precharge(precharge);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005816}
5817
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005818/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
5819static void __mem_cgroup_clear_mc(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005820{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005821 struct mem_cgroup *from = mc.from;
5822 struct mem_cgroup *to = mc.to;
5823
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005824 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005825 if (mc.precharge) {
Johannes Weiner00501b52014-08-08 14:19:20 -07005826 cancel_charge(mc.to, mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005827 mc.precharge = 0;
5828 }
5829 /*
5830 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
5831 * we must uncharge here.
5832 */
5833 if (mc.moved_charge) {
Johannes Weiner00501b52014-08-08 14:19:20 -07005834 cancel_charge(mc.from, mc.moved_charge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005835 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005836 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005837 /* we must fixup refcnts and charges */
5838 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005839 /* uncharge swap account from the old cgroup */
Johannes Weinerce00a962014-09-05 08:43:57 -04005840 if (!mem_cgroup_is_root(mc.from))
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005841 page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005842
Vladimir Davydov615d66c2016-08-11 15:33:03 -07005843 mem_cgroup_id_put_many(mc.from, mc.moved_swap);
5844
Johannes Weiner05b84302014-08-06 16:05:59 -07005845 /*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005846 * we charged both to->memory and to->memsw, so we
5847 * should uncharge to->memory.
Johannes Weiner05b84302014-08-06 16:05:59 -07005848 */
Johannes Weinerce00a962014-09-05 08:43:57 -04005849 if (!mem_cgroup_is_root(mc.to))
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005850 page_counter_uncharge(&mc.to->memory, mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005851
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005852 mc.moved_swap = 0;
5853 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005854 memcg_oom_recover(from);
5855 memcg_oom_recover(to);
5856 wake_up_all(&mc.waitq);
5857}
5858
5859static void mem_cgroup_clear_mc(void)
5860{
Tejun Heo264a0ae2016-04-21 19:09:02 -04005861 struct mm_struct *mm = mc.mm;
5862
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005863 /*
5864 * we must clear moving_task before waking up waiters at the end of
5865 * task migration.
5866 */
5867 mc.moving_task = NULL;
5868 __mem_cgroup_clear_mc();
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005869 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005870 mc.from = NULL;
5871 mc.to = NULL;
Tejun Heo264a0ae2016-04-21 19:09:02 -04005872 mc.mm = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005873 spin_unlock(&mc.lock);
Tejun Heo264a0ae2016-04-21 19:09:02 -04005874
5875 mmput(mm);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005876}
5877
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005878static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005879{
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005880 struct cgroup_subsys_state *css;
Ross Zwislereed67d72015-12-23 14:53:27 -07005881 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
Tejun Heo9f2115f2015-09-08 15:01:10 -07005882 struct mem_cgroup *from;
Tejun Heo4530edd2015-09-11 15:00:19 -04005883 struct task_struct *leader, *p;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005884 struct mm_struct *mm;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005885 unsigned long move_flags;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005886 int ret = 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005887
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005888 /* charge immigration isn't supported on the default hierarchy */
5889 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Tejun Heo9f2115f2015-09-08 15:01:10 -07005890 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005891
Tejun Heo4530edd2015-09-11 15:00:19 -04005892 /*
5893 * Multi-process migrations only happen on the default hierarchy
5894 * where charge immigration is not used. Perform charge
5895 * immigration if @tset contains a leader and whine if there are
5896 * multiple.
5897 */
5898 p = NULL;
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005899 cgroup_taskset_for_each_leader(leader, css, tset) {
Tejun Heo4530edd2015-09-11 15:00:19 -04005900 WARN_ON_ONCE(p);
5901 p = leader;
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005902 memcg = mem_cgroup_from_css(css);
Tejun Heo4530edd2015-09-11 15:00:19 -04005903 }
5904 if (!p)
5905 return 0;
5906
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005907 /*
5908 * We are now commited to this value whatever it is. Changes in this
5909 * tunable will only affect upcoming migrations, not the current one.
5910 * So we need to save it, and keep it going.
5911 */
5912 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
5913 if (!move_flags)
5914 return 0;
5915
Tejun Heo9f2115f2015-09-08 15:01:10 -07005916 from = mem_cgroup_from_task(p);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005917
Tejun Heo9f2115f2015-09-08 15:01:10 -07005918 VM_BUG_ON(from == memcg);
Johannes Weiner247b1442014-12-10 15:44:11 -08005919
Tejun Heo9f2115f2015-09-08 15:01:10 -07005920 mm = get_task_mm(p);
5921 if (!mm)
5922 return 0;
5923 /* We move charges only when we move a owner of the mm */
5924 if (mm->owner == p) {
5925 VM_BUG_ON(mc.from);
5926 VM_BUG_ON(mc.to);
5927 VM_BUG_ON(mc.precharge);
5928 VM_BUG_ON(mc.moved_charge);
5929 VM_BUG_ON(mc.moved_swap);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005930
Tejun Heo9f2115f2015-09-08 15:01:10 -07005931 spin_lock(&mc.lock);
Tejun Heo264a0ae2016-04-21 19:09:02 -04005932 mc.mm = mm;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005933 mc.from = from;
5934 mc.to = memcg;
5935 mc.flags = move_flags;
5936 spin_unlock(&mc.lock);
5937 /* We set mc.moving_task later */
5938
5939 ret = mem_cgroup_precharge_mc(mm);
5940 if (ret)
5941 mem_cgroup_clear_mc();
Tejun Heo264a0ae2016-04-21 19:09:02 -04005942 } else {
5943 mmput(mm);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005944 }
5945 return ret;
5946}
5947
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005948static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005949{
Johannes Weiner4e2f2452014-12-10 15:44:08 -08005950 if (mc.to)
5951 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005952}
5953
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005954static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
5955 unsigned long addr, unsigned long end,
5956 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005957{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005958 int ret = 0;
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08005959 struct vm_area_struct *vma = walk->vma;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005960 pte_t *pte;
5961 spinlock_t *ptl;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005962 enum mc_target_type target_type;
5963 union mc_target target;
5964 struct page *page;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005965
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08005966 ptl = pmd_trans_huge_lock(pmd, vma);
5967 if (ptl) {
Hugh Dickins62ade862012-05-18 11:28:34 -07005968 if (mc.precharge < HPAGE_PMD_NR) {
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08005969 spin_unlock(ptl);
Naoya Horiguchi12724852012-03-21 16:34:28 -07005970 return 0;
5971 }
5972 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
5973 if (target_type == MC_TARGET_PAGE) {
5974 page = target.page;
5975 if (!isolate_lru_page(page)) {
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005976 if (!mem_cgroup_move_account(page, true,
Johannes Weiner1306a852014-12-10 15:44:52 -08005977 mc.from, mc.to)) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07005978 mc.precharge -= HPAGE_PMD_NR;
5979 mc.moved_charge += HPAGE_PMD_NR;
5980 }
5981 putback_lru_page(page);
5982 }
5983 put_page(page);
Jérôme Glissec733a822017-09-08 16:11:54 -07005984 } else if (target_type == MC_TARGET_DEVICE) {
5985 page = target.page;
5986 if (!mem_cgroup_move_account(page, true,
5987 mc.from, mc.to)) {
5988 mc.precharge -= HPAGE_PMD_NR;
5989 mc.moved_charge += HPAGE_PMD_NR;
5990 }
5991 put_page(page);
Naoya Horiguchi12724852012-03-21 16:34:28 -07005992 }
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08005993 spin_unlock(ptl);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07005994 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005995 }
5996
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07005997 if (pmd_trans_unstable(pmd))
5998 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005999retry:
6000 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6001 for (; addr != end; addr += PAGE_SIZE) {
6002 pte_t ptent = *(pte++);
Jérôme Glissec733a822017-09-08 16:11:54 -07006003 bool device = false;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006004 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006005
6006 if (!mc.precharge)
6007 break;
6008
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006009 switch (get_mctgt_type(vma, addr, ptent, &target)) {
Jérôme Glissec733a822017-09-08 16:11:54 -07006010 case MC_TARGET_DEVICE:
6011 device = true;
Joe Perchese4a9bc52020-04-06 20:08:39 -07006012 fallthrough;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006013 case MC_TARGET_PAGE:
6014 page = target.page;
Kirill A. Shutemov53f92632016-01-15 16:53:42 -08006015 /*
6016 * We can have a part of the split pmd here. Moving it
6017 * can be done but it would be too convoluted so simply
6018 * ignore such a partial THP and keep it in original
6019 * memcg. There should be somebody mapping the head.
6020 */
6021 if (PageTransCompound(page))
6022 goto put;
Jérôme Glissec733a822017-09-08 16:11:54 -07006023 if (!device && isolate_lru_page(page))
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006024 goto put;
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08006025 if (!mem_cgroup_move_account(page, false,
6026 mc.from, mc.to)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006027 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006028 /* we uncharge from mc.from later. */
6029 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006030 }
Jérôme Glissec733a822017-09-08 16:11:54 -07006031 if (!device)
6032 putback_lru_page(page);
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006033put: /* get_mctgt_type() gets the page */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006034 put_page(page);
6035 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006036 case MC_TARGET_SWAP:
6037 ent = target.ent;
Hugh Dickinse91cbb42012-05-29 15:06:51 -07006038 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08006039 mc.precharge--;
Hugh Dickins8d22a932020-07-23 21:15:24 -07006040 mem_cgroup_id_get_many(mc.to, 1);
6041 /* we fixup other refcnts and charges later. */
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006042 mc.moved_swap++;
6043 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08006044 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006045 default:
6046 break;
6047 }
6048 }
6049 pte_unmap_unlock(pte - 1, ptl);
6050 cond_resched();
6051
6052 if (addr != end) {
6053 /*
6054 * We have consumed all precharges we got in can_attach().
6055 * We try charge one by one, but don't do any additional
6056 * charges to mc.to if we have failed in charge once in attach()
6057 * phase.
6058 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006059 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006060 if (!ret)
6061 goto retry;
6062 }
6063
6064 return ret;
6065}
6066
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02006067static const struct mm_walk_ops charge_walk_ops = {
6068 .pmd_entry = mem_cgroup_move_charge_pte_range,
6069};
6070
Tejun Heo264a0ae2016-04-21 19:09:02 -04006071static void mem_cgroup_move_charge(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006072{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006073 lru_add_drain_all();
Johannes Weiner312722c2014-12-10 15:44:25 -08006074 /*
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07006075 * Signal lock_page_memcg() to take the memcg's move_lock
6076 * while we're moving its pages to another memcg. Then wait
6077 * for already started RCU-only updates to finish.
Johannes Weiner312722c2014-12-10 15:44:25 -08006078 */
6079 atomic_inc(&mc.from->moving_account);
6080 synchronize_rcu();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006081retry:
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07006082 if (unlikely(!mmap_read_trylock(mc.mm))) {
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006083 /*
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -07006084 * Someone who are holding the mmap_lock might be waiting in
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006085 * waitq. So we cancel all extra charges, wake up all waiters,
6086 * and retry. Because we cancel precharges, we might not be able
6087 * to move enough charges, but moving charge is a best-effort
6088 * feature anyway, so it wouldn't be a big problem.
6089 */
6090 __mem_cgroup_clear_mc();
6091 cond_resched();
6092 goto retry;
6093 }
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08006094 /*
6095 * When we have consumed all precharges and failed in doing
6096 * additional charge, the page walk just aborts.
6097 */
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02006098 walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
6099 NULL);
James Morse0247f3f2016-10-07 17:00:12 -07006100
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07006101 mmap_read_unlock(mc.mm);
Johannes Weiner312722c2014-12-10 15:44:25 -08006102 atomic_dec(&mc.from->moving_account);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006103}
6104
Tejun Heo264a0ae2016-04-21 19:09:02 -04006105static void mem_cgroup_move_task(void)
Balbir Singh67e465a2008-02-07 00:13:54 -08006106{
Tejun Heo264a0ae2016-04-21 19:09:02 -04006107 if (mc.to) {
6108 mem_cgroup_move_charge();
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07006109 mem_cgroup_clear_mc();
Tejun Heo264a0ae2016-04-21 19:09:02 -04006110 }
Balbir Singh67e465a2008-02-07 00:13:54 -08006111}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006112#else /* !CONFIG_MMU */
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05006113static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006114{
6115 return 0;
6116}
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05006117static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006118{
6119}
Tejun Heo264a0ae2016-04-21 19:09:02 -04006120static void mem_cgroup_move_task(void)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006121{
6122}
6123#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08006124
Chris Down677dc972019-03-05 15:45:55 -08006125static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
6126{
6127 if (value == PAGE_COUNTER_MAX)
6128 seq_puts(m, "max\n");
6129 else
6130 seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
6131
6132 return 0;
6133}
6134
Johannes Weiner241994ed2015-02-11 15:26:06 -08006135static u64 memory_current_read(struct cgroup_subsys_state *css,
6136 struct cftype *cft)
6137{
Johannes Weinerf5fc3c5d2015-11-05 18:50:23 -08006138 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6139
6140 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006141}
6142
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006143static int memory_min_show(struct seq_file *m, void *v)
6144{
Chris Down677dc972019-03-05 15:45:55 -08006145 return seq_puts_memcg_tunable(m,
6146 READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006147}
6148
6149static ssize_t memory_min_write(struct kernfs_open_file *of,
6150 char *buf, size_t nbytes, loff_t off)
6151{
6152 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6153 unsigned long min;
6154 int err;
6155
6156 buf = strstrip(buf);
6157 err = page_counter_memparse(buf, "max", &min);
6158 if (err)
6159 return err;
6160
6161 page_counter_set_min(&memcg->memory, min);
6162
6163 return nbytes;
6164}
6165
Johannes Weiner241994ed2015-02-11 15:26:06 -08006166static int memory_low_show(struct seq_file *m, void *v)
6167{
Chris Down677dc972019-03-05 15:45:55 -08006168 return seq_puts_memcg_tunable(m,
6169 READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006170}
6171
6172static ssize_t memory_low_write(struct kernfs_open_file *of,
6173 char *buf, size_t nbytes, loff_t off)
6174{
6175 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6176 unsigned long low;
6177 int err;
6178
6179 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006180 err = page_counter_memparse(buf, "max", &low);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006181 if (err)
6182 return err;
6183
Roman Gushchin23067152018-06-07 17:06:22 -07006184 page_counter_set_low(&memcg->memory, low);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006185
6186 return nbytes;
6187}
6188
6189static int memory_high_show(struct seq_file *m, void *v)
6190{
Jakub Kicinskid1663a92020-06-01 21:49:49 -07006191 return seq_puts_memcg_tunable(m,
6192 READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006193}
6194
6195static ssize_t memory_high_write(struct kernfs_open_file *of,
6196 char *buf, size_t nbytes, loff_t off)
6197{
6198 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Chris Downd977aa92020-08-06 23:21:58 -07006199 unsigned int nr_retries = MAX_RECLAIM_RETRIES;
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006200 bool drained = false;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006201 unsigned long high;
6202 int err;
6203
6204 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006205 err = page_counter_memparse(buf, "max", &high);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006206 if (err)
6207 return err;
6208
Johannes Weinere82553c2021-02-09 13:42:28 -08006209 page_counter_set_high(&memcg->memory, high);
6210
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006211 for (;;) {
6212 unsigned long nr_pages = page_counter_read(&memcg->memory);
6213 unsigned long reclaimed;
Johannes Weiner588083b2016-03-17 14:20:25 -07006214
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006215 if (nr_pages <= high)
6216 break;
6217
6218 if (signal_pending(current))
6219 break;
6220
6221 if (!drained) {
6222 drain_all_stock(memcg);
6223 drained = true;
6224 continue;
6225 }
6226
6227 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6228 GFP_KERNEL, true);
6229
6230 if (!reclaimed && !nr_retries--)
6231 break;
6232 }
6233
Johannes Weiner19ce33a2020-08-06 23:22:12 -07006234 memcg_wb_domain_size_changed(memcg);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006235 return nbytes;
6236}
6237
6238static int memory_max_show(struct seq_file *m, void *v)
6239{
Chris Down677dc972019-03-05 15:45:55 -08006240 return seq_puts_memcg_tunable(m,
6241 READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006242}
6243
6244static ssize_t memory_max_write(struct kernfs_open_file *of,
6245 char *buf, size_t nbytes, loff_t off)
6246{
6247 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Chris Downd977aa92020-08-06 23:21:58 -07006248 unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006249 bool drained = false;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006250 unsigned long max;
6251 int err;
6252
6253 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006254 err = page_counter_memparse(buf, "max", &max);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006255 if (err)
6256 return err;
6257
Roman Gushchinbbec2e12018-06-07 17:06:18 -07006258 xchg(&memcg->memory.max, max);
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006259
6260 for (;;) {
6261 unsigned long nr_pages = page_counter_read(&memcg->memory);
6262
6263 if (nr_pages <= max)
6264 break;
6265
Johannes Weiner7249c9f2019-11-30 17:50:06 -08006266 if (signal_pending(current))
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006267 break;
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006268
6269 if (!drained) {
6270 drain_all_stock(memcg);
6271 drained = true;
6272 continue;
6273 }
6274
6275 if (nr_reclaims) {
6276 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
6277 GFP_KERNEL, true))
6278 nr_reclaims--;
6279 continue;
6280 }
6281
Johannes Weinere27be242018-04-10 16:29:45 -07006282 memcg_memory_event(memcg, MEMCG_OOM);
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006283 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6284 break;
6285 }
Johannes Weiner241994ed2015-02-11 15:26:06 -08006286
Tejun Heo2529bb32015-05-22 18:23:34 -04006287 memcg_wb_domain_size_changed(memcg);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006288 return nbytes;
6289}
6290
Shakeel Butt1e577f92019-07-11 20:55:55 -07006291static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
6292{
6293 seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
6294 seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
6295 seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
6296 seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
6297 seq_printf(m, "oom_kill %lu\n",
6298 atomic_long_read(&events[MEMCG_OOM_KILL]));
6299}
6300
Johannes Weiner241994ed2015-02-11 15:26:06 -08006301static int memory_events_show(struct seq_file *m, void *v)
6302{
Chris Downaa9694b2019-03-05 15:45:52 -08006303 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006304
Shakeel Butt1e577f92019-07-11 20:55:55 -07006305 __memory_events_show(m, memcg->memory_events);
6306 return 0;
6307}
Johannes Weiner241994ed2015-02-11 15:26:06 -08006308
Shakeel Butt1e577f92019-07-11 20:55:55 -07006309static int memory_events_local_show(struct seq_file *m, void *v)
6310{
6311 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6312
6313 __memory_events_show(m, memcg->memory_events_local);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006314 return 0;
6315}
6316
Johannes Weiner587d9f72016-01-20 15:03:19 -08006317static int memory_stat_show(struct seq_file *m, void *v)
6318{
Chris Downaa9694b2019-03-05 15:45:52 -08006319 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weinerc8713d02019-07-11 20:55:59 -07006320 char *buf;
Johannes Weiner587d9f72016-01-20 15:03:19 -08006321
Johannes Weinerc8713d02019-07-11 20:55:59 -07006322 buf = memory_stat_format(memcg);
6323 if (!buf)
6324 return -ENOMEM;
6325 seq_puts(m, buf);
6326 kfree(buf);
Johannes Weiner587d9f72016-01-20 15:03:19 -08006327 return 0;
6328}
6329
Muchun Song5f9a4f42020-10-13 16:52:59 -07006330#ifdef CONFIG_NUMA
Muchun Songfff66b72021-02-24 12:03:43 -08006331static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec,
6332 int item)
6333{
6334 return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item);
6335}
6336
Muchun Song5f9a4f42020-10-13 16:52:59 -07006337static int memory_numa_stat_show(struct seq_file *m, void *v)
6338{
6339 int i;
6340 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6341
6342 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
6343 int nid;
6344
6345 if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
6346 continue;
6347
6348 seq_printf(m, "%s", memory_stats[i].name);
6349 for_each_node_state(nid, N_MEMORY) {
6350 u64 size;
6351 struct lruvec *lruvec;
6352
6353 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
Muchun Songfff66b72021-02-24 12:03:43 -08006354 size = lruvec_page_state_output(lruvec,
6355 memory_stats[i].idx);
Muchun Song5f9a4f42020-10-13 16:52:59 -07006356 seq_printf(m, " N%d=%llu", nid, size);
6357 }
6358 seq_putc(m, '\n');
6359 }
6360
6361 return 0;
6362}
6363#endif
6364
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006365static int memory_oom_group_show(struct seq_file *m, void *v)
6366{
Chris Downaa9694b2019-03-05 15:45:52 -08006367 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006368
6369 seq_printf(m, "%d\n", memcg->oom_group);
6370
6371 return 0;
6372}
6373
6374static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
6375 char *buf, size_t nbytes, loff_t off)
6376{
6377 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6378 int ret, oom_group;
6379
6380 buf = strstrip(buf);
6381 if (!buf)
6382 return -EINVAL;
6383
6384 ret = kstrtoint(buf, 0, &oom_group);
6385 if (ret)
6386 return ret;
6387
6388 if (oom_group != 0 && oom_group != 1)
6389 return -EINVAL;
6390
6391 memcg->oom_group = oom_group;
6392
6393 return nbytes;
6394}
6395
Johannes Weiner241994ed2015-02-11 15:26:06 -08006396static struct cftype memory_files[] = {
6397 {
6398 .name = "current",
Johannes Weinerf5fc3c5d2015-11-05 18:50:23 -08006399 .flags = CFTYPE_NOT_ON_ROOT,
Johannes Weiner241994ed2015-02-11 15:26:06 -08006400 .read_u64 = memory_current_read,
6401 },
6402 {
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006403 .name = "min",
6404 .flags = CFTYPE_NOT_ON_ROOT,
6405 .seq_show = memory_min_show,
6406 .write = memory_min_write,
6407 },
6408 {
Johannes Weiner241994ed2015-02-11 15:26:06 -08006409 .name = "low",
6410 .flags = CFTYPE_NOT_ON_ROOT,
6411 .seq_show = memory_low_show,
6412 .write = memory_low_write,
6413 },
6414 {
6415 .name = "high",
6416 .flags = CFTYPE_NOT_ON_ROOT,
6417 .seq_show = memory_high_show,
6418 .write = memory_high_write,
6419 },
6420 {
6421 .name = "max",
6422 .flags = CFTYPE_NOT_ON_ROOT,
6423 .seq_show = memory_max_show,
6424 .write = memory_max_write,
6425 },
6426 {
6427 .name = "events",
6428 .flags = CFTYPE_NOT_ON_ROOT,
Tejun Heo472912a2015-09-18 18:01:59 -04006429 .file_offset = offsetof(struct mem_cgroup, events_file),
Johannes Weiner241994ed2015-02-11 15:26:06 -08006430 .seq_show = memory_events_show,
6431 },
Johannes Weiner587d9f72016-01-20 15:03:19 -08006432 {
Shakeel Butt1e577f92019-07-11 20:55:55 -07006433 .name = "events.local",
6434 .flags = CFTYPE_NOT_ON_ROOT,
6435 .file_offset = offsetof(struct mem_cgroup, events_local_file),
6436 .seq_show = memory_events_local_show,
6437 },
6438 {
Johannes Weiner587d9f72016-01-20 15:03:19 -08006439 .name = "stat",
Johannes Weiner587d9f72016-01-20 15:03:19 -08006440 .seq_show = memory_stat_show,
6441 },
Muchun Song5f9a4f42020-10-13 16:52:59 -07006442#ifdef CONFIG_NUMA
6443 {
6444 .name = "numa_stat",
6445 .seq_show = memory_numa_stat_show,
6446 },
6447#endif
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006448 {
6449 .name = "oom.group",
6450 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
6451 .seq_show = memory_oom_group_show,
6452 .write = memory_oom_group_write,
6453 },
Johannes Weiner241994ed2015-02-11 15:26:06 -08006454 { } /* terminate */
6455};
6456
Tejun Heo073219e2014-02-08 10:36:58 -05006457struct cgroup_subsys memory_cgrp_subsys = {
Tejun Heo92fb9742012-11-19 08:13:38 -08006458 .css_alloc = mem_cgroup_css_alloc,
Glauber Costad142e3e2013-02-22 16:34:52 -08006459 .css_online = mem_cgroup_css_online,
Tejun Heo92fb9742012-11-19 08:13:38 -08006460 .css_offline = mem_cgroup_css_offline,
Vladimir Davydov6df38682015-12-29 14:54:10 -08006461 .css_released = mem_cgroup_css_released,
Tejun Heo92fb9742012-11-19 08:13:38 -08006462 .css_free = mem_cgroup_css_free,
Tejun Heo1ced9532014-07-08 18:02:57 -04006463 .css_reset = mem_cgroup_css_reset,
Johannes Weiner2d146aa2021-04-29 22:56:26 -07006464 .css_rstat_flush = mem_cgroup_css_rstat_flush,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006465 .can_attach = mem_cgroup_can_attach,
6466 .cancel_attach = mem_cgroup_cancel_attach,
Tejun Heo264a0ae2016-04-21 19:09:02 -04006467 .post_attach = mem_cgroup_move_task,
Johannes Weiner241994ed2015-02-11 15:26:06 -08006468 .dfl_cftypes = memory_files,
6469 .legacy_cftypes = mem_cgroup_legacy_files,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006470 .early_init = 0,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006471};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006472
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006473/*
6474 * This function calculates an individual cgroup's effective
6475 * protection which is derived from its own memory.min/low, its
6476 * parent's and siblings' settings, as well as the actual memory
6477 * distribution in the tree.
6478 *
6479 * The following rules apply to the effective protection values:
6480 *
6481 * 1. At the first level of reclaim, effective protection is equal to
6482 * the declared protection in memory.min and memory.low.
6483 *
6484 * 2. To enable safe delegation of the protection configuration, at
6485 * subsequent levels the effective protection is capped to the
6486 * parent's effective protection.
6487 *
6488 * 3. To make complex and dynamic subtrees easier to configure, the
6489 * user is allowed to overcommit the declared protection at a given
6490 * level. If that is the case, the parent's effective protection is
6491 * distributed to the children in proportion to how much protection
6492 * they have declared and how much of it they are utilizing.
6493 *
6494 * This makes distribution proportional, but also work-conserving:
6495 * if one cgroup claims much more protection than it uses memory,
6496 * the unused remainder is available to its siblings.
6497 *
6498 * 4. Conversely, when the declared protection is undercommitted at a
6499 * given level, the distribution of the larger parental protection
6500 * budget is NOT proportional. A cgroup's protection from a sibling
6501 * is capped to its own memory.min/low setting.
6502 *
Johannes Weiner8a931f82020-04-01 21:07:07 -07006503 * 5. However, to allow protecting recursive subtrees from each other
6504 * without having to declare each individual cgroup's fixed share
6505 * of the ancestor's claim to protection, any unutilized -
6506 * "floating" - protection from up the tree is distributed in
6507 * proportion to each cgroup's *usage*. This makes the protection
6508 * neutral wrt sibling cgroups and lets them compete freely over
6509 * the shared parental protection budget, but it protects the
6510 * subtree as a whole from neighboring subtrees.
6511 *
6512 * Note that 4. and 5. are not in conflict: 4. is about protecting
6513 * against immediate siblings whereas 5. is about protecting against
6514 * neighboring subtrees.
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006515 */
6516static unsigned long effective_protection(unsigned long usage,
Johannes Weiner8a931f82020-04-01 21:07:07 -07006517 unsigned long parent_usage,
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006518 unsigned long setting,
6519 unsigned long parent_effective,
6520 unsigned long siblings_protected)
6521{
6522 unsigned long protected;
Johannes Weiner8a931f82020-04-01 21:07:07 -07006523 unsigned long ep;
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006524
6525 protected = min(usage, setting);
6526 /*
6527 * If all cgroups at this level combined claim and use more
6528 * protection then what the parent affords them, distribute
6529 * shares in proportion to utilization.
6530 *
6531 * We are using actual utilization rather than the statically
6532 * claimed protection in order to be work-conserving: claimed
6533 * but unused protection is available to siblings that would
6534 * otherwise get a smaller chunk than what they claimed.
6535 */
6536 if (siblings_protected > parent_effective)
6537 return protected * parent_effective / siblings_protected;
6538
6539 /*
6540 * Ok, utilized protection of all children is within what the
6541 * parent affords them, so we know whatever this child claims
6542 * and utilizes is effectively protected.
6543 *
6544 * If there is unprotected usage beyond this value, reclaim
6545 * will apply pressure in proportion to that amount.
6546 *
6547 * If there is unutilized protection, the cgroup will be fully
6548 * shielded from reclaim, but we do return a smaller value for
6549 * protection than what the group could enjoy in theory. This
6550 * is okay. With the overcommit distribution above, effective
6551 * protection is always dependent on how memory is actually
6552 * consumed among the siblings anyway.
6553 */
Johannes Weiner8a931f82020-04-01 21:07:07 -07006554 ep = protected;
6555
6556 /*
6557 * If the children aren't claiming (all of) the protection
6558 * afforded to them by the parent, distribute the remainder in
6559 * proportion to the (unprotected) memory of each cgroup. That
6560 * way, cgroups that aren't explicitly prioritized wrt each
6561 * other compete freely over the allowance, but they are
6562 * collectively protected from neighboring trees.
6563 *
6564 * We're using unprotected memory for the weight so that if
6565 * some cgroups DO claim explicit protection, we don't protect
6566 * the same bytes twice.
Johannes Weinercd324ed2020-06-25 20:30:16 -07006567 *
6568 * Check both usage and parent_usage against the respective
6569 * protected values. One should imply the other, but they
6570 * aren't read atomically - make sure the division is sane.
Johannes Weiner8a931f82020-04-01 21:07:07 -07006571 */
6572 if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
6573 return ep;
Johannes Weinercd324ed2020-06-25 20:30:16 -07006574 if (parent_effective > siblings_protected &&
6575 parent_usage > siblings_protected &&
6576 usage > protected) {
Johannes Weiner8a931f82020-04-01 21:07:07 -07006577 unsigned long unclaimed;
6578
6579 unclaimed = parent_effective - siblings_protected;
6580 unclaimed *= usage - protected;
6581 unclaimed /= parent_usage - siblings_protected;
6582
6583 ep += unclaimed;
6584 }
6585
6586 return ep;
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006587}
6588
Johannes Weiner241994ed2015-02-11 15:26:06 -08006589/**
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006590 * mem_cgroup_protected - check if memory consumption is in the normal range
Sean Christopherson34c81052017-07-10 15:48:05 -07006591 * @root: the top ancestor of the sub-tree being checked
Johannes Weiner241994ed2015-02-11 15:26:06 -08006592 * @memcg: the memory cgroup to check
6593 *
Roman Gushchin23067152018-06-07 17:06:22 -07006594 * WARNING: This function is not stateless! It can only be used as part
6595 * of a top-down tree iteration, not for isolated queries.
Johannes Weiner241994ed2015-02-11 15:26:06 -08006596 */
Chris Down45c7f7e2020-08-06 23:22:05 -07006597void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6598 struct mem_cgroup *memcg)
Johannes Weiner241994ed2015-02-11 15:26:06 -08006599{
Johannes Weiner8a931f82020-04-01 21:07:07 -07006600 unsigned long usage, parent_usage;
Roman Gushchin23067152018-06-07 17:06:22 -07006601 struct mem_cgroup *parent;
6602
Johannes Weiner241994ed2015-02-11 15:26:06 -08006603 if (mem_cgroup_disabled())
Chris Down45c7f7e2020-08-06 23:22:05 -07006604 return;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006605
Sean Christopherson34c81052017-07-10 15:48:05 -07006606 if (!root)
6607 root = root_mem_cgroup;
Yafang Shao22f74962020-08-06 23:22:01 -07006608
6609 /*
6610 * Effective values of the reclaim targets are ignored so they
6611 * can be stale. Have a look at mem_cgroup_protection for more
6612 * details.
6613 * TODO: calculation should be more robust so that we do not need
6614 * that special casing.
6615 */
Sean Christopherson34c81052017-07-10 15:48:05 -07006616 if (memcg == root)
Chris Down45c7f7e2020-08-06 23:22:05 -07006617 return;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006618
Roman Gushchin23067152018-06-07 17:06:22 -07006619 usage = page_counter_read(&memcg->memory);
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006620 if (!usage)
Chris Down45c7f7e2020-08-06 23:22:05 -07006621 return;
Sean Christopherson34c81052017-07-10 15:48:05 -07006622
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006623 parent = parent_mem_cgroup(memcg);
Roman Gushchindf2a4192018-06-14 15:26:17 -07006624 /* No parent means a non-hierarchical mode on v1 memcg */
6625 if (!parent)
Chris Down45c7f7e2020-08-06 23:22:05 -07006626 return;
Roman Gushchindf2a4192018-06-14 15:26:17 -07006627
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006628 if (parent == root) {
Chris Downc3d53202020-04-01 21:07:27 -07006629 memcg->memory.emin = READ_ONCE(memcg->memory.min);
Chris Down03960e32020-06-25 20:30:22 -07006630 memcg->memory.elow = READ_ONCE(memcg->memory.low);
Chris Down45c7f7e2020-08-06 23:22:05 -07006631 return;
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006632 }
6633
Johannes Weiner8a931f82020-04-01 21:07:07 -07006634 parent_usage = page_counter_read(&parent->memory);
6635
Chris Downb3a78222020-04-01 21:07:33 -07006636 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
Chris Downc3d53202020-04-01 21:07:27 -07006637 READ_ONCE(memcg->memory.min),
6638 READ_ONCE(parent->memory.emin),
Chris Downb3a78222020-04-01 21:07:33 -07006639 atomic_long_read(&parent->memory.children_min_usage)));
Roman Gushchin23067152018-06-07 17:06:22 -07006640
Chris Downb3a78222020-04-01 21:07:33 -07006641 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
Chris Down03960e32020-06-25 20:30:22 -07006642 READ_ONCE(memcg->memory.low),
6643 READ_ONCE(parent->memory.elow),
Chris Downb3a78222020-04-01 21:07:33 -07006644 atomic_long_read(&parent->memory.children_low_usage)));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006645}
6646
Shakeel Butt0add0c72021-04-29 22:56:36 -07006647static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
6648 gfp_t gfp)
6649{
6650 unsigned int nr_pages = thp_nr_pages(page);
6651 int ret;
6652
6653 ret = try_charge(memcg, gfp, nr_pages);
6654 if (ret)
6655 goto out;
6656
6657 css_get(&memcg->css);
6658 commit_charge(page, memcg);
6659
6660 local_irq_disable();
6661 mem_cgroup_charge_statistics(memcg, page, nr_pages);
6662 memcg_check_events(memcg, page);
6663 local_irq_enable();
6664out:
6665 return ret;
6666}
6667
Johannes Weiner00501b52014-08-08 14:19:20 -07006668/**
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006669 * mem_cgroup_charge - charge a newly allocated page to a cgroup
Johannes Weiner00501b52014-08-08 14:19:20 -07006670 * @page: page to charge
6671 * @mm: mm context of the victim
6672 * @gfp_mask: reclaim mode
Johannes Weiner00501b52014-08-08 14:19:20 -07006673 *
6674 * Try to charge @page to the memcg that @mm belongs to, reclaiming
6675 * pages according to @gfp_mask if necessary.
6676 *
Shakeel Butt0add0c72021-04-29 22:56:36 -07006677 * Do not use this for pages allocated for swapin.
6678 *
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006679 * Returns 0 on success. Otherwise, an error code is returned.
Johannes Weiner00501b52014-08-08 14:19:20 -07006680 */
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07006681int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
Johannes Weiner00501b52014-08-08 14:19:20 -07006682{
Shakeel Butt0add0c72021-04-29 22:56:36 -07006683 struct mem_cgroup *memcg;
6684 int ret;
Johannes Weiner00501b52014-08-08 14:19:20 -07006685
6686 if (mem_cgroup_disabled())
Shakeel Butt0add0c72021-04-29 22:56:36 -07006687 return 0;
Johannes Weiner00501b52014-08-08 14:19:20 -07006688
Shakeel Butt0add0c72021-04-29 22:56:36 -07006689 memcg = get_mem_cgroup_from_mm(mm);
6690 ret = __mem_cgroup_charge(page, memcg, gfp_mask);
6691 css_put(&memcg->css);
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006692
Shakeel Butt0add0c72021-04-29 22:56:36 -07006693 return ret;
6694}
Vladimir Davydove993d902015-09-09 15:35:35 -07006695
Shakeel Butt0add0c72021-04-29 22:56:36 -07006696/**
6697 * mem_cgroup_swapin_charge_page - charge a newly allocated page for swapin
6698 * @page: page to charge
6699 * @mm: mm context of the victim
6700 * @gfp: reclaim mode
6701 * @entry: swap entry for which the page is allocated
6702 *
6703 * This function charges a page allocated for swapin. Please call this before
6704 * adding the page to the swapcache.
6705 *
6706 * Returns 0 on success. Otherwise, an error code is returned.
6707 */
6708int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
6709 gfp_t gfp, swp_entry_t entry)
6710{
6711 struct mem_cgroup *memcg;
6712 unsigned short id;
6713 int ret;
Johannes Weiner00501b52014-08-08 14:19:20 -07006714
Shakeel Butt0add0c72021-04-29 22:56:36 -07006715 if (mem_cgroup_disabled())
6716 return 0;
6717
6718 id = lookup_swap_cgroup_id(entry);
6719 rcu_read_lock();
6720 memcg = mem_cgroup_from_id(id);
6721 if (!memcg || !css_tryget_online(&memcg->css))
Johannes Weiner00501b52014-08-08 14:19:20 -07006722 memcg = get_mem_cgroup_from_mm(mm);
Shakeel Butt0add0c72021-04-29 22:56:36 -07006723 rcu_read_unlock();
Johannes Weiner00501b52014-08-08 14:19:20 -07006724
Shakeel Butt0add0c72021-04-29 22:56:36 -07006725 ret = __mem_cgroup_charge(page, memcg, gfp);
Johannes Weiner00501b52014-08-08 14:19:20 -07006726
Shakeel Butt0add0c72021-04-29 22:56:36 -07006727 css_put(&memcg->css);
6728 return ret;
6729}
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006730
Shakeel Butt0add0c72021-04-29 22:56:36 -07006731/*
6732 * mem_cgroup_swapin_uncharge_swap - uncharge swap slot
6733 * @entry: swap entry for which the page is charged
6734 *
6735 * Call this function after successfully adding the charged page to swapcache.
6736 *
6737 * Note: This function assumes the page for which swap slot is being uncharged
6738 * is order 0 page.
6739 */
6740void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry)
6741{
Muchun Songcae3af62021-02-24 12:04:19 -08006742 /*
6743 * Cgroup1's unified memory+swap counter has been charged with the
6744 * new swapcache page, finish the transfer by uncharging the swap
6745 * slot. The swap slot would also get uncharged when it dies, but
6746 * it can stick around indefinitely and we'd count the page twice
6747 * the entire time.
6748 *
6749 * Cgroup2 has separate resource counters for memory and swap,
6750 * so this is a non-issue here. Memory and swap charge lifetimes
6751 * correspond 1:1 to page and swap slot lifetimes: we charge the
6752 * page to memory here, and uncharge swap when the slot is freed.
6753 */
Shakeel Butt0add0c72021-04-29 22:56:36 -07006754 if (!mem_cgroup_disabled() && do_memsw_account()) {
Johannes Weiner00501b52014-08-08 14:19:20 -07006755 /*
6756 * The swap entry might not get freed for a long time,
6757 * let's not wait for it. The page already received a
6758 * memory+swap charge, drop the swap entry duplicate.
6759 */
Shakeel Butt0add0c72021-04-29 22:56:36 -07006760 mem_cgroup_uncharge_swap(entry, 1);
Johannes Weiner00501b52014-08-08 14:19:20 -07006761 }
Johannes Weiner3fea5a42020-06-03 16:01:41 -07006762}
6763
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006764struct uncharge_gather {
6765 struct mem_cgroup *memcg;
Johannes Weiner9f762db2020-06-03 16:01:44 -07006766 unsigned long nr_pages;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006767 unsigned long pgpgout;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006768 unsigned long nr_kmem;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006769 struct page *dummy_page;
6770};
6771
6772static inline void uncharge_gather_clear(struct uncharge_gather *ug)
Johannes Weiner747db952014-08-08 14:19:24 -07006773{
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006774 memset(ug, 0, sizeof(*ug));
6775}
6776
6777static void uncharge_batch(const struct uncharge_gather *ug)
6778{
Johannes Weiner747db952014-08-08 14:19:24 -07006779 unsigned long flags;
6780
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006781 if (!mem_cgroup_is_root(ug->memcg)) {
Johannes Weiner9f762db2020-06-03 16:01:44 -07006782 page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08006783 if (do_memsw_account())
Johannes Weiner9f762db2020-06-03 16:01:44 -07006784 page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006785 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
6786 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
6787 memcg_oom_recover(ug->memcg);
Johannes Weinerce00a962014-09-05 08:43:57 -04006788 }
Johannes Weiner747db952014-08-08 14:19:24 -07006789
6790 local_irq_save(flags);
Johannes Weinerc9019e92018-01-31 16:16:37 -08006791 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
Johannes Weiner9f762db2020-06-03 16:01:44 -07006792 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006793 memcg_check_events(ug->memcg, ug->dummy_page);
Johannes Weiner747db952014-08-08 14:19:24 -07006794 local_irq_restore(flags);
Michal Hockof1796542020-09-04 16:35:24 -07006795
6796 /* drop reference from uncharge_page */
6797 css_put(&ug->memcg->css);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006798}
6799
6800static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6801{
Johannes Weiner9f762db2020-06-03 16:01:44 -07006802 unsigned long nr_pages;
6803
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006804 VM_BUG_ON_PAGE(PageLRU(page), page);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006805
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006806 if (!page_memcg(page))
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006807 return;
6808
6809 /*
6810 * Nobody should be changing or seriously looking at
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006811 * page_memcg(page) at this point, we have fully
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006812 * exclusive access to the page.
6813 */
6814
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006815 if (ug->memcg != page_memcg(page)) {
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006816 if (ug->memcg) {
6817 uncharge_batch(ug);
6818 uncharge_gather_clear(ug);
6819 }
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006820 ug->memcg = page_memcg(page);
Michal Hockof1796542020-09-04 16:35:24 -07006821
6822 /* pairs with css_put in uncharge_batch */
6823 css_get(&ug->memcg->css);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006824 }
6825
Johannes Weiner9f762db2020-06-03 16:01:44 -07006826 nr_pages = compound_nr(page);
6827 ug->nr_pages += nr_pages;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006828
Roman Gushchin18b2db32020-12-01 13:58:30 -08006829 if (PageMemcgKmem(page))
Johannes Weiner9f762db2020-06-03 16:01:44 -07006830 ug->nr_kmem += nr_pages;
Roman Gushchin18b2db32020-12-01 13:58:30 -08006831 else
6832 ug->pgpgout++;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006833
6834 ug->dummy_page = page;
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006835 page->memcg_data = 0;
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07006836 css_put(&ug->memcg->css);
Johannes Weiner747db952014-08-08 14:19:24 -07006837}
6838
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006839/**
6840 * mem_cgroup_uncharge - uncharge a page
6841 * @page: page to uncharge
6842 *
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006843 * Uncharge a page previously charged with mem_cgroup_charge().
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006844 */
6845void mem_cgroup_uncharge(struct page *page)
6846{
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006847 struct uncharge_gather ug;
6848
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006849 if (mem_cgroup_disabled())
6850 return;
6851
Johannes Weiner747db952014-08-08 14:19:24 -07006852 /* Don't touch page->lru of any random page, pre-check: */
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006853 if (!page_memcg(page))
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006854 return;
6855
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006856 uncharge_gather_clear(&ug);
6857 uncharge_page(page, &ug);
6858 uncharge_batch(&ug);
Johannes Weiner747db952014-08-08 14:19:24 -07006859}
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006860
Johannes Weiner747db952014-08-08 14:19:24 -07006861/**
6862 * mem_cgroup_uncharge_list - uncharge a list of page
6863 * @page_list: list of pages to uncharge
6864 *
6865 * Uncharge a list of pages previously charged with
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006866 * mem_cgroup_charge().
Johannes Weiner747db952014-08-08 14:19:24 -07006867 */
6868void mem_cgroup_uncharge_list(struct list_head *page_list)
6869{
Muchun Songc41a40b2021-02-24 12:04:08 -08006870 struct uncharge_gather ug;
6871 struct page *page;
6872
Johannes Weiner747db952014-08-08 14:19:24 -07006873 if (mem_cgroup_disabled())
6874 return;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006875
Muchun Songc41a40b2021-02-24 12:04:08 -08006876 uncharge_gather_clear(&ug);
6877 list_for_each_entry(page, page_list, lru)
6878 uncharge_page(page, &ug);
6879 if (ug.memcg)
6880 uncharge_batch(&ug);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006881}
6882
6883/**
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006884 * mem_cgroup_migrate - charge a page's replacement
6885 * @oldpage: currently circulating page
6886 * @newpage: replacement page
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006887 *
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006888 * Charge @newpage as a replacement page for @oldpage. @oldpage will
6889 * be uncharged upon free.
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006890 *
6891 * Both pages must be locked, @newpage->mapping must be set up.
6892 */
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006893void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006894{
Johannes Weiner29833312014-12-10 15:44:02 -08006895 struct mem_cgroup *memcg;
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006896 unsigned int nr_pages;
Tejun Heod93c4132016-06-24 14:49:54 -07006897 unsigned long flags;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006898
6899 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
6900 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006901 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006902 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
6903 newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006904
6905 if (mem_cgroup_disabled())
6906 return;
6907
6908 /* Page cache replacement: new page already charged? */
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006909 if (page_memcg(newpage))
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006910 return;
6911
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006912 memcg = page_memcg(oldpage);
Alex Shia4055882020-12-18 14:01:31 -08006913 VM_WARN_ON_ONCE_PAGE(!memcg, oldpage);
Johannes Weiner29833312014-12-10 15:44:02 -08006914 if (!memcg)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006915 return;
6916
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006917 /* Force-charge the new page. The old one will be freed soon */
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07006918 nr_pages = thp_nr_pages(newpage);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006919
6920 page_counter_charge(&memcg->memory, nr_pages);
6921 if (do_memsw_account())
6922 page_counter_charge(&memcg->memsw, nr_pages);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006923
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07006924 css_get(&memcg->css);
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07006925 commit_charge(newpage, memcg);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006926
Tejun Heod93c4132016-06-24 14:49:54 -07006927 local_irq_save(flags);
Johannes Weiner3fba69a2020-06-03 16:01:31 -07006928 mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006929 memcg_check_events(memcg, newpage);
Tejun Heod93c4132016-06-24 14:49:54 -07006930 local_irq_restore(flags);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006931}
6932
Johannes Weineref129472016-01-14 15:21:34 -08006933DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
Johannes Weiner11092082016-01-14 15:21:26 -08006934EXPORT_SYMBOL(memcg_sockets_enabled_key);
6935
Johannes Weiner2d758072016-10-07 17:00:58 -07006936void mem_cgroup_sk_alloc(struct sock *sk)
Johannes Weiner11092082016-01-14 15:21:26 -08006937{
6938 struct mem_cgroup *memcg;
6939
Johannes Weiner2d758072016-10-07 17:00:58 -07006940 if (!mem_cgroup_sockets_enabled)
6941 return;
6942
Shakeel Butte876ecc2020-03-09 22:16:05 -07006943 /* Do not associate the sock with unrelated interrupted task's memcg. */
6944 if (in_interrupt())
6945 return;
6946
Johannes Weiner11092082016-01-14 15:21:26 -08006947 rcu_read_lock();
6948 memcg = mem_cgroup_from_task(current);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006949 if (memcg == root_mem_cgroup)
6950 goto out;
Johannes Weiner0db15292016-01-20 15:02:50 -08006951 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006952 goto out;
Shakeel Butt8965aa22020-04-01 21:07:10 -07006953 if (css_tryget(&memcg->css))
Johannes Weiner11092082016-01-14 15:21:26 -08006954 sk->sk_memcg = memcg;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006955out:
Johannes Weiner11092082016-01-14 15:21:26 -08006956 rcu_read_unlock();
6957}
Johannes Weiner11092082016-01-14 15:21:26 -08006958
Johannes Weiner2d758072016-10-07 17:00:58 -07006959void mem_cgroup_sk_free(struct sock *sk)
Johannes Weiner11092082016-01-14 15:21:26 -08006960{
Johannes Weiner2d758072016-10-07 17:00:58 -07006961 if (sk->sk_memcg)
6962 css_put(&sk->sk_memcg->css);
Johannes Weiner11092082016-01-14 15:21:26 -08006963}
6964
6965/**
6966 * mem_cgroup_charge_skmem - charge socket memory
6967 * @memcg: memcg to charge
6968 * @nr_pages: number of pages to charge
6969 *
6970 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
6971 * @memcg's configured limit, %false if the charge had to be forced.
6972 */
6973bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6974{
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006975 gfp_t gfp_mask = GFP_KERNEL;
Johannes Weiner11092082016-01-14 15:21:26 -08006976
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006977 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
Johannes Weiner0db15292016-01-20 15:02:50 -08006978 struct page_counter *fail;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006979
Johannes Weiner0db15292016-01-20 15:02:50 -08006980 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
6981 memcg->tcpmem_pressure = 0;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006982 return true;
6983 }
Johannes Weiner0db15292016-01-20 15:02:50 -08006984 page_counter_charge(&memcg->tcpmem, nr_pages);
6985 memcg->tcpmem_pressure = 1;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006986 return false;
Johannes Weiner11092082016-01-14 15:21:26 -08006987 }
Johannes Weinerd886f4e2016-01-20 15:02:47 -08006988
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006989 /* Don't block in the packet receive path */
6990 if (in_softirq())
6991 gfp_mask = GFP_NOWAIT;
6992
Johannes Weinerc9019e92018-01-31 16:16:37 -08006993 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
Johannes Weinerb2807f02016-01-20 15:03:22 -08006994
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006995 if (try_charge(memcg, gfp_mask, nr_pages) == 0)
6996 return true;
6997
6998 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
Johannes Weiner11092082016-01-14 15:21:26 -08006999 return false;
7000}
7001
7002/**
7003 * mem_cgroup_uncharge_skmem - uncharge socket memory
Mike Rapoportb7701a52018-02-06 15:42:13 -08007004 * @memcg: memcg to uncharge
7005 * @nr_pages: number of pages to uncharge
Johannes Weiner11092082016-01-14 15:21:26 -08007006 */
7007void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7008{
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08007009 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
Johannes Weiner0db15292016-01-20 15:02:50 -08007010 page_counter_uncharge(&memcg->tcpmem, nr_pages);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08007011 return;
7012 }
Johannes Weinerd886f4e2016-01-20 15:02:47 -08007013
Johannes Weinerc9019e92018-01-31 16:16:37 -08007014 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
Johannes Weinerb2807f02016-01-20 15:03:22 -08007015
Roman Gushchin475d0482017-09-08 16:13:09 -07007016 refill_stock(memcg, nr_pages);
Johannes Weiner11092082016-01-14 15:21:26 -08007017}
7018
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08007019static int __init cgroup_memory(char *s)
7020{
7021 char *token;
7022
7023 while ((token = strsep(&s, ",")) != NULL) {
7024 if (!*token)
7025 continue;
7026 if (!strcmp(token, "nosocket"))
7027 cgroup_memory_nosocket = true;
Vladimir Davydov04823c82016-01-20 15:02:38 -08007028 if (!strcmp(token, "nokmem"))
7029 cgroup_memory_nokmem = true;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08007030 }
7031 return 0;
7032}
7033__setup("cgroup.memory=", cgroup_memory);
Johannes Weiner11092082016-01-14 15:21:26 -08007034
Michal Hocko2d110852013-02-22 16:34:43 -08007035/*
Michal Hocko10813122013-02-22 16:35:41 -08007036 * subsys_initcall() for memory controller.
7037 *
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01007038 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
7039 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
7040 * basically everything that doesn't depend on a specific mem_cgroup structure
7041 * should be initialized from here.
Michal Hocko2d110852013-02-22 16:34:43 -08007042 */
7043static int __init mem_cgroup_init(void)
7044{
Johannes Weiner95a045f2015-02-11 15:26:33 -08007045 int cpu, node;
7046
Muchun Songf3344ad2021-02-24 12:03:15 -08007047 /*
7048 * Currently s32 type (can refer to struct batched_lruvec_stat) is
7049 * used for per-memcg-per-cpu caching of per-node statistics. In order
7050 * to work fine, we should make sure that the overfill threshold can't
7051 * exceed S32_MAX / PAGE_SIZE.
7052 */
7053 BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE);
7054
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01007055 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
7056 memcg_hotplug_cpu_dead);
Johannes Weiner95a045f2015-02-11 15:26:33 -08007057
7058 for_each_possible_cpu(cpu)
7059 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
7060 drain_local_stock);
7061
7062 for_each_node(node) {
7063 struct mem_cgroup_tree_per_node *rtpn;
Johannes Weiner95a045f2015-02-11 15:26:33 -08007064
7065 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
7066 node_online(node) ? node : NUMA_NO_NODE);
7067
Mel Gormanef8f2322016-07-28 15:46:05 -07007068 rtpn->rb_root = RB_ROOT;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -07007069 rtpn->rb_rightmost = NULL;
Mel Gormanef8f2322016-07-28 15:46:05 -07007070 spin_lock_init(&rtpn->lock);
Johannes Weiner95a045f2015-02-11 15:26:33 -08007071 soft_limit_tree.rb_tree_per_node[node] = rtpn;
7072 }
7073
Michal Hocko2d110852013-02-22 16:34:43 -08007074 return 0;
7075}
7076subsys_initcall(mem_cgroup_init);
Johannes Weiner21afa382015-02-11 15:26:36 -08007077
7078#ifdef CONFIG_MEMCG_SWAP
Arnd Bergmann358c07f2016-08-25 15:17:08 -07007079static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7080{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07007081 while (!refcount_inc_not_zero(&memcg->id.ref)) {
Arnd Bergmann358c07f2016-08-25 15:17:08 -07007082 /*
7083 * The root cgroup cannot be destroyed, so it's refcount must
7084 * always be >= 1.
7085 */
7086 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
7087 VM_BUG_ON(1);
7088 break;
7089 }
7090 memcg = parent_mem_cgroup(memcg);
7091 if (!memcg)
7092 memcg = root_mem_cgroup;
7093 }
7094 return memcg;
7095}
7096
Johannes Weiner21afa382015-02-11 15:26:36 -08007097/**
7098 * mem_cgroup_swapout - transfer a memsw charge to swap
7099 * @page: page whose memsw charge to transfer
7100 * @entry: swap entry to move the charge to
7101 *
7102 * Transfer the memsw charge of @page to @entry.
7103 */
7104void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
7105{
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007106 struct mem_cgroup *memcg, *swap_memcg;
Huang Yingd6810d72017-09-06 16:22:45 -07007107 unsigned int nr_entries;
Johannes Weiner21afa382015-02-11 15:26:36 -08007108 unsigned short oldid;
7109
7110 VM_BUG_ON_PAGE(PageLRU(page), page);
7111 VM_BUG_ON_PAGE(page_count(page), page);
7112
Alex Shi76358ab2020-12-18 14:01:28 -08007113 if (mem_cgroup_disabled())
7114 return;
7115
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007116 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Johannes Weiner21afa382015-02-11 15:26:36 -08007117 return;
7118
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08007119 memcg = page_memcg(page);
Johannes Weiner21afa382015-02-11 15:26:36 -08007120
Alex Shia4055882020-12-18 14:01:31 -08007121 VM_WARN_ON_ONCE_PAGE(!memcg, page);
Johannes Weiner21afa382015-02-11 15:26:36 -08007122 if (!memcg)
7123 return;
7124
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007125 /*
7126 * In case the memcg owning these pages has been offlined and doesn't
7127 * have an ID allocated to it anymore, charge the closest online
7128 * ancestor for the swap instead and transfer the memory+swap charge.
7129 */
7130 swap_memcg = mem_cgroup_id_get_online(memcg);
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07007131 nr_entries = thp_nr_pages(page);
Huang Yingd6810d72017-09-06 16:22:45 -07007132 /* Get references for the tail pages, too */
7133 if (nr_entries > 1)
7134 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
7135 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
7136 nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08007137 VM_BUG_ON_PAGE(oldid, page);
Johannes Weinerc9019e92018-01-31 16:16:37 -08007138 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08007139
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08007140 page->memcg_data = 0;
Johannes Weiner21afa382015-02-11 15:26:36 -08007141
7142 if (!mem_cgroup_is_root(memcg))
Huang Yingd6810d72017-09-06 16:22:45 -07007143 page_counter_uncharge(&memcg->memory, nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08007144
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007145 if (!cgroup_memory_noswap && memcg != swap_memcg) {
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007146 if (!mem_cgroup_is_root(swap_memcg))
Huang Yingd6810d72017-09-06 16:22:45 -07007147 page_counter_charge(&swap_memcg->memsw, nr_entries);
7148 page_counter_uncharge(&memcg->memsw, nr_entries);
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007149 }
7150
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07007151 /*
7152 * Interrupts should be disabled here because the caller holds the
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07007153 * i_pages lock which is taken with interrupts-off. It is
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07007154 * important here to have the interrupts disabled because it is the
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07007155 * only synchronisation we have for updating the per-CPU variables.
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07007156 */
7157 VM_BUG_ON(!irqs_disabled());
Johannes Weiner3fba69a2020-06-03 16:01:31 -07007158 mem_cgroup_charge_statistics(memcg, page, -nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08007159 memcg_check_events(memcg, page);
Johannes Weiner73f576c2016-07-20 15:44:57 -07007160
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07007161 css_put(&memcg->css);
Johannes Weiner21afa382015-02-11 15:26:36 -08007162}
7163
Huang Ying38d8b4e2017-07-06 15:37:18 -07007164/**
7165 * mem_cgroup_try_charge_swap - try charging swap space for a page
Vladimir Davydov37e84352016-01-20 15:02:56 -08007166 * @page: page being added to swap
7167 * @entry: swap entry to charge
7168 *
Huang Ying38d8b4e2017-07-06 15:37:18 -07007169 * Try to charge @page's memcg for the swap space at @entry.
Vladimir Davydov37e84352016-01-20 15:02:56 -08007170 *
7171 * Returns 0 on success, -ENOMEM on failure.
7172 */
7173int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
7174{
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07007175 unsigned int nr_pages = thp_nr_pages(page);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007176 struct page_counter *counter;
Huang Ying38d8b4e2017-07-06 15:37:18 -07007177 struct mem_cgroup *memcg;
Vladimir Davydov37e84352016-01-20 15:02:56 -08007178 unsigned short oldid;
7179
Alex Shi76358ab2020-12-18 14:01:28 -08007180 if (mem_cgroup_disabled())
7181 return 0;
7182
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007183 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydov37e84352016-01-20 15:02:56 -08007184 return 0;
7185
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08007186 memcg = page_memcg(page);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007187
Alex Shia4055882020-12-18 14:01:31 -08007188 VM_WARN_ON_ONCE_PAGE(!memcg, page);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007189 if (!memcg)
7190 return 0;
7191
Tejun Heof3a53a32018-06-07 17:05:35 -07007192 if (!entry.val) {
7193 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
Tejun Heobb98f2c2018-06-07 17:05:31 -07007194 return 0;
Tejun Heof3a53a32018-06-07 17:05:35 -07007195 }
Tejun Heobb98f2c2018-06-07 17:05:31 -07007196
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007197 memcg = mem_cgroup_id_get_online(memcg);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007198
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007199 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
Huang Ying38d8b4e2017-07-06 15:37:18 -07007200 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
Tejun Heof3a53a32018-06-07 17:05:35 -07007201 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
7202 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007203 mem_cgroup_id_put(memcg);
7204 return -ENOMEM;
7205 }
7206
Huang Ying38d8b4e2017-07-06 15:37:18 -07007207 /* Get references for the tail pages, too */
7208 if (nr_pages > 1)
7209 mem_cgroup_id_get_many(memcg, nr_pages - 1);
7210 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007211 VM_BUG_ON_PAGE(oldid, page);
Johannes Weinerc9019e92018-01-31 16:16:37 -08007212 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007213
Vladimir Davydov37e84352016-01-20 15:02:56 -08007214 return 0;
7215}
7216
Johannes Weiner21afa382015-02-11 15:26:36 -08007217/**
Huang Ying38d8b4e2017-07-06 15:37:18 -07007218 * mem_cgroup_uncharge_swap - uncharge swap space
Johannes Weiner21afa382015-02-11 15:26:36 -08007219 * @entry: swap entry to uncharge
Huang Ying38d8b4e2017-07-06 15:37:18 -07007220 * @nr_pages: the amount of swap space to uncharge
Johannes Weiner21afa382015-02-11 15:26:36 -08007221 */
Huang Ying38d8b4e2017-07-06 15:37:18 -07007222void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
Johannes Weiner21afa382015-02-11 15:26:36 -08007223{
7224 struct mem_cgroup *memcg;
7225 unsigned short id;
7226
Huang Ying38d8b4e2017-07-06 15:37:18 -07007227 id = swap_cgroup_record(entry, 0, nr_pages);
Johannes Weiner21afa382015-02-11 15:26:36 -08007228 rcu_read_lock();
Vladimir Davydovadbe4272015-04-15 16:13:00 -07007229 memcg = mem_cgroup_from_id(id);
Johannes Weiner21afa382015-02-11 15:26:36 -08007230 if (memcg) {
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007231 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
Vladimir Davydov37e84352016-01-20 15:02:56 -08007232 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Huang Ying38d8b4e2017-07-06 15:37:18 -07007233 page_counter_uncharge(&memcg->swap, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007234 else
Huang Ying38d8b4e2017-07-06 15:37:18 -07007235 page_counter_uncharge(&memcg->memsw, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007236 }
Johannes Weinerc9019e92018-01-31 16:16:37 -08007237 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
Huang Ying38d8b4e2017-07-06 15:37:18 -07007238 mem_cgroup_id_put_many(memcg, nr_pages);
Johannes Weiner21afa382015-02-11 15:26:36 -08007239 }
7240 rcu_read_unlock();
7241}
7242
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007243long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7244{
7245 long nr_swap_pages = get_nr_swap_pages();
7246
Johannes Weinereccb52e2020-06-03 16:02:11 -07007247 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007248 return nr_swap_pages;
7249 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
7250 nr_swap_pages = min_t(long, nr_swap_pages,
Roman Gushchinbbec2e12018-06-07 17:06:18 -07007251 READ_ONCE(memcg->swap.max) -
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007252 page_counter_read(&memcg->swap));
7253 return nr_swap_pages;
7254}
7255
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007256bool mem_cgroup_swap_full(struct page *page)
7257{
7258 struct mem_cgroup *memcg;
7259
7260 VM_BUG_ON_PAGE(!PageLocked(page), page);
7261
7262 if (vm_swap_full())
7263 return true;
Johannes Weinereccb52e2020-06-03 16:02:11 -07007264 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007265 return false;
7266
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08007267 memcg = page_memcg(page);
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007268 if (!memcg)
7269 return false;
7270
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007271 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
7272 unsigned long usage = page_counter_read(&memcg->swap);
7273
7274 if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
7275 usage * 2 >= READ_ONCE(memcg->swap.max))
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007276 return true;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007277 }
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007278
7279 return false;
7280}
7281
Johannes Weinereccb52e2020-06-03 16:02:11 -07007282static int __init setup_swap_account(char *s)
Johannes Weiner21afa382015-02-11 15:26:36 -08007283{
7284 if (!strcmp(s, "1"))
Kaixu Xia5ab92902020-12-14 19:07:07 -08007285 cgroup_memory_noswap = false;
Johannes Weiner21afa382015-02-11 15:26:36 -08007286 else if (!strcmp(s, "0"))
Kaixu Xia5ab92902020-12-14 19:07:07 -08007287 cgroup_memory_noswap = true;
Johannes Weiner21afa382015-02-11 15:26:36 -08007288 return 1;
7289}
Johannes Weinereccb52e2020-06-03 16:02:11 -07007290__setup("swapaccount=", setup_swap_account);
Johannes Weiner21afa382015-02-11 15:26:36 -08007291
Vladimir Davydov37e84352016-01-20 15:02:56 -08007292static u64 swap_current_read(struct cgroup_subsys_state *css,
7293 struct cftype *cft)
7294{
7295 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7296
7297 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
7298}
7299
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007300static int swap_high_show(struct seq_file *m, void *v)
7301{
7302 return seq_puts_memcg_tunable(m,
7303 READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
7304}
7305
7306static ssize_t swap_high_write(struct kernfs_open_file *of,
7307 char *buf, size_t nbytes, loff_t off)
7308{
7309 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7310 unsigned long high;
7311 int err;
7312
7313 buf = strstrip(buf);
7314 err = page_counter_memparse(buf, "max", &high);
7315 if (err)
7316 return err;
7317
7318 page_counter_set_high(&memcg->swap, high);
7319
7320 return nbytes;
7321}
7322
Vladimir Davydov37e84352016-01-20 15:02:56 -08007323static int swap_max_show(struct seq_file *m, void *v)
7324{
Chris Down677dc972019-03-05 15:45:55 -08007325 return seq_puts_memcg_tunable(m,
7326 READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
Vladimir Davydov37e84352016-01-20 15:02:56 -08007327}
7328
7329static ssize_t swap_max_write(struct kernfs_open_file *of,
7330 char *buf, size_t nbytes, loff_t off)
7331{
7332 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7333 unsigned long max;
7334 int err;
7335
7336 buf = strstrip(buf);
7337 err = page_counter_memparse(buf, "max", &max);
7338 if (err)
7339 return err;
7340
Tejun Heobe091022018-06-07 17:09:21 -07007341 xchg(&memcg->swap.max, max);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007342
7343 return nbytes;
7344}
7345
Tejun Heof3a53a32018-06-07 17:05:35 -07007346static int swap_events_show(struct seq_file *m, void *v)
7347{
Chris Downaa9694b2019-03-05 15:45:52 -08007348 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Tejun Heof3a53a32018-06-07 17:05:35 -07007349
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007350 seq_printf(m, "high %lu\n",
7351 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
Tejun Heof3a53a32018-06-07 17:05:35 -07007352 seq_printf(m, "max %lu\n",
7353 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7354 seq_printf(m, "fail %lu\n",
7355 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7356
7357 return 0;
7358}
7359
Vladimir Davydov37e84352016-01-20 15:02:56 -08007360static struct cftype swap_files[] = {
7361 {
7362 .name = "swap.current",
7363 .flags = CFTYPE_NOT_ON_ROOT,
7364 .read_u64 = swap_current_read,
7365 },
7366 {
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007367 .name = "swap.high",
7368 .flags = CFTYPE_NOT_ON_ROOT,
7369 .seq_show = swap_high_show,
7370 .write = swap_high_write,
7371 },
7372 {
Vladimir Davydov37e84352016-01-20 15:02:56 -08007373 .name = "swap.max",
7374 .flags = CFTYPE_NOT_ON_ROOT,
7375 .seq_show = swap_max_show,
7376 .write = swap_max_write,
7377 },
Tejun Heof3a53a32018-06-07 17:05:35 -07007378 {
7379 .name = "swap.events",
7380 .flags = CFTYPE_NOT_ON_ROOT,
7381 .file_offset = offsetof(struct mem_cgroup, swap_events_file),
7382 .seq_show = swap_events_show,
7383 },
Vladimir Davydov37e84352016-01-20 15:02:56 -08007384 { } /* terminate */
7385};
7386
Johannes Weinereccb52e2020-06-03 16:02:11 -07007387static struct cftype memsw_files[] = {
Johannes Weiner21afa382015-02-11 15:26:36 -08007388 {
7389 .name = "memsw.usage_in_bytes",
7390 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
7391 .read_u64 = mem_cgroup_read_u64,
7392 },
7393 {
7394 .name = "memsw.max_usage_in_bytes",
7395 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
7396 .write = mem_cgroup_reset,
7397 .read_u64 = mem_cgroup_read_u64,
7398 },
7399 {
7400 .name = "memsw.limit_in_bytes",
7401 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
7402 .write = mem_cgroup_write,
7403 .read_u64 = mem_cgroup_read_u64,
7404 },
7405 {
7406 .name = "memsw.failcnt",
7407 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
7408 .write = mem_cgroup_reset,
7409 .read_u64 = mem_cgroup_read_u64,
7410 },
7411 { }, /* terminate */
7412};
7413
Bhupesh Sharma82ff1652020-07-23 21:15:21 -07007414/*
7415 * If mem_cgroup_swap_init() is implemented as a subsys_initcall()
7416 * instead of a core_initcall(), this could mean cgroup_memory_noswap still
7417 * remains set to false even when memcg is disabled via "cgroup_disable=memory"
7418 * boot parameter. This may result in premature OOPS inside
7419 * mem_cgroup_get_nr_swap_pages() function in corner cases.
7420 */
Johannes Weiner21afa382015-02-11 15:26:36 -08007421static int __init mem_cgroup_swap_init(void)
7422{
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007423 /* No memory control -> no swap control */
7424 if (mem_cgroup_disabled())
7425 cgroup_memory_noswap = true;
7426
7427 if (cgroup_memory_noswap)
Johannes Weinereccb52e2020-06-03 16:02:11 -07007428 return 0;
7429
7430 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
7431 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
7432
Johannes Weiner21afa382015-02-11 15:26:36 -08007433 return 0;
7434}
Bhupesh Sharma82ff1652020-07-23 21:15:21 -07007435core_initcall(mem_cgroup_swap_init);
Johannes Weiner21afa382015-02-11 15:26:36 -08007436
7437#endif /* CONFIG_MEMCG_SWAP */