blob: b238403f95b2ac6055ec505c2ef72ea5f52fc860 [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.h - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -08009 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 */
19
20#ifndef _LINUX_MEMCONTROL_H
21#define _LINUX_MEMCONTROL_H
Hirokazu Takahashif8d665422009-01-07 18:08:02 -080022#include <linux/cgroup.h>
Ying Han456f9982011-05-26 16:25:38 -070023#include <linux/vm_event_item.h>
Glauber Costa7ae1e1d2012-12-18 14:21:56 -080024#include <linux/hardirq.h>
Glauber Costaa8964b92012-12-18 14:22:09 -080025#include <linux/jump_label.h>
Michal Hocko33398cf2015-09-08 15:01:02 -070026#include <linux/page_counter.h>
27#include <linux/vmpressure.h>
28#include <linux/eventfd.h>
Johannes Weiner00f3ca22017-07-06 15:40:52 -070029#include <linux/mm.h>
30#include <linux/vmstat.h>
Michal Hocko33398cf2015-09-08 15:01:02 -070031#include <linux/writeback.h>
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -070032#include <linux/page-flags.h>
Ying Han456f9982011-05-26 16:25:38 -070033
Pavel Emelianov78fb7462008-02-07 00:13:51 -080034struct mem_cgroup;
Balbir Singh8697d332008-02-07 00:13:59 -080035struct page;
36struct mm_struct;
Glauber Costa2633d7a2012-12-18 14:22:34 -080037struct kmem_cache;
Pavel Emelianov78fb7462008-02-07 00:13:51 -080038
Johannes Weiner71cd3112017-05-03 14:55:13 -070039/* Cgroup-specific page state, on top of universal node page state */
40enum memcg_stat_item {
41 MEMCG_CACHE = NR_VM_NODE_STAT_ITEMS,
42 MEMCG_RSS,
43 MEMCG_RSS_HUGE,
44 MEMCG_SWAP,
45 MEMCG_SOCK,
46 /* XXX: why are these zone and not node counters? */
47 MEMCG_KERNEL_STACK_KB,
Johannes Weinerb2807f02016-01-20 15:03:22 -080048 MEMCG_NR_STAT,
Greg Thelen2a7106f2011-01-13 15:47:37 -080049};
50
Johannes Weinere27be242018-04-10 16:29:45 -070051enum memcg_memory_event {
52 MEMCG_LOW,
Johannes Weiner71cd3112017-05-03 14:55:13 -070053 MEMCG_HIGH,
54 MEMCG_MAX,
55 MEMCG_OOM,
Roman Gushchinfe6bdfc2018-06-14 15:28:05 -070056 MEMCG_OOM_KILL,
Tejun Heof3a53a32018-06-07 17:05:35 -070057 MEMCG_SWAP_MAX,
58 MEMCG_SWAP_FAIL,
Johannes Weinere27be242018-04-10 16:29:45 -070059 MEMCG_NR_MEMORY_EVENTS,
Johannes Weiner71cd3112017-05-03 14:55:13 -070060};
61
Roman Gushchinbf8d5d52018-06-07 17:07:46 -070062enum mem_cgroup_protection {
63 MEMCG_PROT_NONE,
64 MEMCG_PROT_LOW,
65 MEMCG_PROT_MIN,
66};
67
Johannes Weiner56600482012-01-12 17:17:59 -080068struct mem_cgroup_reclaim_cookie {
Mel Gormanef8f2322016-07-28 15:46:05 -070069 pg_data_t *pgdat;
Johannes Weiner56600482012-01-12 17:17:59 -080070 int priority;
71 unsigned int generation;
72};
73
Johannes Weiner71cd3112017-05-03 14:55:13 -070074#ifdef CONFIG_MEMCG
75
76#define MEM_CGROUP_ID_SHIFT 16
77#define MEM_CGROUP_ID_MAX USHRT_MAX
78
79struct mem_cgroup_id {
80 int id;
Kirill Tkhai1c2d4792018-10-26 15:09:28 -070081 refcount_t ref;
Johannes Weiner71cd3112017-05-03 14:55:13 -070082};
83
Michal Hocko33398cf2015-09-08 15:01:02 -070084/*
85 * Per memcg event counter is incremented at every pagein/pageout. With THP,
86 * it will be incremated by the number of pages. This counter is used for
87 * for trigger some periodic events. This is straightforward and better
88 * than using jiffies etc. to handle periodic memcg event.
89 */
90enum mem_cgroup_events_target {
91 MEM_CGROUP_TARGET_THRESH,
92 MEM_CGROUP_TARGET_SOFTLIMIT,
93 MEM_CGROUP_TARGET_NUMAINFO,
94 MEM_CGROUP_NTARGETS,
95};
96
Michal Hocko33398cf2015-09-08 15:01:02 -070097struct mem_cgroup_stat_cpu {
Johannes Weinerb2807f02016-01-20 15:03:22 -080098 long count[MEMCG_NR_STAT];
Johannes Weinere27be242018-04-10 16:29:45 -070099 unsigned long events[NR_VM_EVENT_ITEMS];
Michal Hocko33398cf2015-09-08 15:01:02 -0700100 unsigned long nr_page_events;
101 unsigned long targets[MEM_CGROUP_NTARGETS];
102};
103
104struct mem_cgroup_reclaim_iter {
105 struct mem_cgroup *position;
106 /* scan generation, increased every round-trip */
107 unsigned int generation;
108};
109
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700110struct lruvec_stat {
111 long count[NR_VM_NODE_STAT_ITEMS];
112};
113
Michal Hocko33398cf2015-09-08 15:01:02 -0700114/*
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700115 * Bitmap of shrinker::id corresponding to memcg-aware shrinkers,
116 * which have elements charged to this memcg.
117 */
118struct memcg_shrinker_map {
119 struct rcu_head rcu;
120 unsigned long map[0];
121};
122
123/*
Michal Hocko33398cf2015-09-08 15:01:02 -0700124 * per-zone information in memory controller.
125 */
Mel Gormanef8f2322016-07-28 15:46:05 -0700126struct mem_cgroup_per_node {
Michal Hocko33398cf2015-09-08 15:01:02 -0700127 struct lruvec lruvec;
Johannes Weinera983b5e2018-01-31 16:16:45 -0800128
129 struct lruvec_stat __percpu *lruvec_stat_cpu;
130 atomic_long_t lruvec_stat[NR_VM_NODE_STAT_ITEMS];
131
Michal Hockob4536f0c82017-01-10 16:58:04 -0800132 unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
Michal Hocko33398cf2015-09-08 15:01:02 -0700133
134 struct mem_cgroup_reclaim_iter iter[DEF_PRIORITY + 1];
135
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700136#ifdef CONFIG_MEMCG_KMEM
137 struct memcg_shrinker_map __rcu *shrinker_map;
138#endif
Michal Hocko33398cf2015-09-08 15:01:02 -0700139 struct rb_node tree_node; /* RB tree node */
140 unsigned long usage_in_excess;/* Set to the value by which */
141 /* the soft limit is exceeded*/
142 bool on_tree;
Andrey Ryabinine3c1ac52018-04-10 16:28:03 -0700143 bool congested; /* memcg has many dirty pages */
144 /* backed by a congested BDI */
145
Michal Hocko33398cf2015-09-08 15:01:02 -0700146 struct mem_cgroup *memcg; /* Back pointer, we cannot */
147 /* use container_of */
148};
149
Michal Hocko33398cf2015-09-08 15:01:02 -0700150struct mem_cgroup_threshold {
151 struct eventfd_ctx *eventfd;
152 unsigned long threshold;
153};
154
155/* For threshold */
156struct mem_cgroup_threshold_ary {
157 /* An array index points to threshold just below or equal to usage. */
158 int current_threshold;
159 /* Size of entries[] */
160 unsigned int size;
161 /* Array of thresholds */
162 struct mem_cgroup_threshold entries[0];
163};
164
165struct mem_cgroup_thresholds {
166 /* Primary thresholds array */
167 struct mem_cgroup_threshold_ary *primary;
168 /*
169 * Spare threshold array.
170 * This is needed to make mem_cgroup_unregister_event() "never fail".
171 * It must be able to store at least primary->size - 1 entries.
172 */
173 struct mem_cgroup_threshold_ary *spare;
174};
175
Johannes Weiner567e9ab2016-01-20 15:02:24 -0800176enum memcg_kmem_state {
177 KMEM_NONE,
178 KMEM_ALLOCATED,
179 KMEM_ONLINE,
180};
181
Aaron Lue81bf972018-06-07 17:09:44 -0700182#if defined(CONFIG_SMP)
183struct memcg_padding {
184 char x[0];
185} ____cacheline_internodealigned_in_smp;
186#define MEMCG_PADDING(name) struct memcg_padding name;
187#else
188#define MEMCG_PADDING(name)
189#endif
190
Michal Hocko33398cf2015-09-08 15:01:02 -0700191/*
192 * The memory controller data structure. The memory controller controls both
193 * page cache and RSS per cgroup. We would eventually like to provide
194 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
195 * to help the administrator determine what knobs to tune.
196 */
197struct mem_cgroup {
198 struct cgroup_subsys_state css;
199
Johannes Weiner73f576c2016-07-20 15:44:57 -0700200 /* Private memcg ID. Used to ID objects that outlive the cgroup */
201 struct mem_cgroup_id id;
202
Michal Hocko33398cf2015-09-08 15:01:02 -0700203 /* Accounted resources */
204 struct page_counter memory;
Vladimir Davydov37e84352016-01-20 15:02:56 -0800205 struct page_counter swap;
Johannes Weiner0db15292016-01-20 15:02:50 -0800206
207 /* Legacy consumer-oriented counters */
Michal Hocko33398cf2015-09-08 15:01:02 -0700208 struct page_counter memsw;
209 struct page_counter kmem;
Johannes Weiner0db15292016-01-20 15:02:50 -0800210 struct page_counter tcpmem;
Michal Hocko33398cf2015-09-08 15:01:02 -0700211
Roman Gushchin23067152018-06-07 17:06:22 -0700212 /* Upper bound of normal memory consumption range */
Michal Hocko33398cf2015-09-08 15:01:02 -0700213 unsigned long high;
214
Johannes Weinerf7e1cb62016-01-14 15:21:29 -0800215 /* Range enforcement for interrupt charges */
216 struct work_struct high_work;
217
Michal Hocko33398cf2015-09-08 15:01:02 -0700218 unsigned long soft_limit;
219
220 /* vmpressure notifications */
221 struct vmpressure vmpressure;
222
Michal Hocko33398cf2015-09-08 15:01:02 -0700223 /*
224 * Should the accounting and control be hierarchical, per subtree?
225 */
226 bool use_hierarchy;
227
Roman Gushchin3d8b38e2018-08-21 21:53:54 -0700228 /*
229 * Should the OOM killer kill all belonging tasks, had it kill one?
230 */
231 bool oom_group;
232
Michal Hocko33398cf2015-09-08 15:01:02 -0700233 /* protected by memcg_oom_lock */
234 bool oom_lock;
235 int under_oom;
236
237 int swappiness;
238 /* OOM-Killer disable */
239 int oom_kill_disable;
240
Johannes Weinere27be242018-04-10 16:29:45 -0700241 /* memory.events */
Tejun Heo472912a2015-09-18 18:01:59 -0400242 struct cgroup_file events_file;
243
Tejun Heof3a53a32018-06-07 17:05:35 -0700244 /* handle for "memory.swap.events" */
245 struct cgroup_file swap_events_file;
246
Michal Hocko33398cf2015-09-08 15:01:02 -0700247 /* protect arrays of thresholds */
248 struct mutex thresholds_lock;
249
250 /* thresholds for memory usage. RCU-protected */
251 struct mem_cgroup_thresholds thresholds;
252
253 /* thresholds for mem+swap usage. RCU-protected */
254 struct mem_cgroup_thresholds memsw_thresholds;
255
256 /* For oom notifier event fd */
257 struct list_head oom_notify;
258
259 /*
260 * Should we move charges of a task when a task is moved into this
261 * mem_cgroup ? And what type of charges should we move ?
262 */
263 unsigned long move_charge_at_immigrate;
Aaron Lue81bf972018-06-07 17:09:44 -0700264 /* taken only while moving_account > 0 */
265 spinlock_t move_lock;
266 unsigned long move_lock_flags;
267
268 MEMCG_PADDING(_pad1_);
269
Michal Hocko33398cf2015-09-08 15:01:02 -0700270 /*
271 * set > 0 if pages under this cgroup are moving to other cgroup.
272 */
273 atomic_t moving_account;
Michal Hocko33398cf2015-09-08 15:01:02 -0700274 struct task_struct *move_lock_task;
Johannes Weinera983b5e2018-01-31 16:16:45 -0800275
Johannes Weinere27be242018-04-10 16:29:45 -0700276 /* memory.stat */
Johannes Weinera983b5e2018-01-31 16:16:45 -0800277 struct mem_cgroup_stat_cpu __percpu *stat_cpu;
Aaron Lue81bf972018-06-07 17:09:44 -0700278
279 MEMCG_PADDING(_pad2_);
280
Johannes Weinera983b5e2018-01-31 16:16:45 -0800281 atomic_long_t stat[MEMCG_NR_STAT];
Johannes Weinere27be242018-04-10 16:29:45 -0700282 atomic_long_t events[NR_VM_EVENT_ITEMS];
Aaron Lue81bf972018-06-07 17:09:44 -0700283 atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS];
Michal Hocko33398cf2015-09-08 15:01:02 -0700284
Johannes Weinerd886f4e2016-01-20 15:02:47 -0800285 unsigned long socket_pressure;
286
287 /* Legacy tcp memory accounting */
Johannes Weiner0db15292016-01-20 15:02:50 -0800288 bool tcpmem_active;
289 int tcpmem_pressure;
Johannes Weinerd886f4e2016-01-20 15:02:47 -0800290
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700291#ifdef CONFIG_MEMCG_KMEM
Michal Hocko33398cf2015-09-08 15:01:02 -0700292 /* Index in the kmem_cache->memcg_params.memcg_caches array */
293 int kmemcg_id;
Johannes Weiner567e9ab2016-01-20 15:02:24 -0800294 enum memcg_kmem_state kmem_state;
Tejun Heobc2791f2017-02-22 15:41:21 -0800295 struct list_head kmem_caches;
Michal Hocko33398cf2015-09-08 15:01:02 -0700296#endif
297
298 int last_scanned_node;
299#if MAX_NUMNODES > 1
300 nodemask_t scan_nodes;
301 atomic_t numainfo_events;
302 atomic_t numainfo_updating;
303#endif
304
305#ifdef CONFIG_CGROUP_WRITEBACK
306 struct list_head cgwb_list;
307 struct wb_domain cgwb_domain;
308#endif
309
310 /* List of events which userspace want to receive */
311 struct list_head event_list;
312 spinlock_t event_list_lock;
313
314 struct mem_cgroup_per_node *nodeinfo[0];
315 /* WARNING: nodeinfo must be the last member here */
316};
Johannes Weiner7d828602016-01-14 15:20:56 -0800317
Johannes Weinera983b5e2018-01-31 16:16:45 -0800318/*
319 * size of first charge trial. "32" comes from vmscan.c's magic value.
320 * TODO: maybe necessary to use big numbers in big irons.
321 */
322#define MEMCG_CHARGE_BATCH 32U
323
Johannes Weiner7d828602016-01-14 15:20:56 -0800324extern struct mem_cgroup *root_mem_cgroup;
Tejun Heo56161632015-05-22 17:13:20 -0400325
Kirill Tkhaidfd2f102018-08-17 15:48:06 -0700326static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
327{
328 return (memcg == root_mem_cgroup);
329}
330
Johannes Weiner23047a92016-03-15 14:57:16 -0700331static inline bool mem_cgroup_disabled(void)
332{
333 return !cgroup_subsys_enabled(memory_cgrp_subsys);
334}
335
Roman Gushchinbf8d5d52018-06-07 17:07:46 -0700336enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
337 struct mem_cgroup *memcg);
Johannes Weiner241994ed2015-02-11 15:26:06 -0800338
Johannes Weiner00501b52014-08-08 14:19:20 -0700339int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800340 gfp_t gfp_mask, struct mem_cgroup **memcgp,
341 bool compound);
Tejun Heo2cf85582018-07-03 11:14:56 -0400342int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
343 gfp_t gfp_mask, struct mem_cgroup **memcgp,
344 bool compound);
Johannes Weiner00501b52014-08-08 14:19:20 -0700345void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800346 bool lrucare, bool compound);
347void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
348 bool compound);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700349void mem_cgroup_uncharge(struct page *page);
Johannes Weiner747db952014-08-08 14:19:24 -0700350void mem_cgroup_uncharge_list(struct list_head *page_list);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700351
Johannes Weiner6a93ca82016-03-15 14:57:19 -0700352void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700353
Mel Gormanef8f2322016-07-28 15:46:05 -0700354static struct mem_cgroup_per_node *
355mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
Johannes Weiner55779ec2016-07-28 15:45:10 -0700356{
Mel Gormanef8f2322016-07-28 15:46:05 -0700357 return memcg->nodeinfo[nid];
Johannes Weiner55779ec2016-07-28 15:45:10 -0700358}
359
360/**
Mel Gormana9dd0a82016-07-28 15:46:02 -0700361 * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
362 * @node: node of the wanted lruvec
Johannes Weiner55779ec2016-07-28 15:45:10 -0700363 * @memcg: memcg of the wanted lruvec
364 *
Mel Gormana9dd0a82016-07-28 15:46:02 -0700365 * Returns the lru list vector holding pages for a given @node or a given
366 * @memcg and @zone. This can be the node lruvec, if the memory controller
Johannes Weiner55779ec2016-07-28 15:45:10 -0700367 * is disabled.
368 */
Mel Gormana9dd0a82016-07-28 15:46:02 -0700369static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
Mel Gormanef8f2322016-07-28 15:46:05 -0700370 struct mem_cgroup *memcg)
Johannes Weiner55779ec2016-07-28 15:45:10 -0700371{
Mel Gormanef8f2322016-07-28 15:46:05 -0700372 struct mem_cgroup_per_node *mz;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700373 struct lruvec *lruvec;
374
375 if (mem_cgroup_disabled()) {
Mel Gormana9dd0a82016-07-28 15:46:02 -0700376 lruvec = node_lruvec(pgdat);
Johannes Weiner55779ec2016-07-28 15:45:10 -0700377 goto out;
378 }
379
Mel Gormanef8f2322016-07-28 15:46:05 -0700380 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
Johannes Weiner55779ec2016-07-28 15:45:10 -0700381 lruvec = &mz->lruvec;
382out:
383 /*
384 * Since a node can be onlined after the mem_cgroup was created,
Mel Gorman599d0c92016-07-28 15:45:31 -0700385 * we have to be prepared to initialize lruvec->pgdat here;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700386 * and if offlined then reonlined, we need to reinitialize it.
387 */
Mel Gormanef8f2322016-07-28 15:46:05 -0700388 if (unlikely(lruvec->pgdat != pgdat))
389 lruvec->pgdat = pgdat;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700390 return lruvec;
391}
392
Mel Gorman599d0c92016-07-28 15:45:31 -0700393struct lruvec *mem_cgroup_page_lruvec(struct page *, struct pglist_data *);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -0800394
Johannes Weiner2314b422014-12-10 15:44:33 -0800395bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
Michal Hocko64219992015-09-08 15:01:07 -0700396struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
Vladimir Davydove993d902015-09-09 15:35:35 -0700397
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700398struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
399
Shakeel Buttf745c6f2018-08-17 15:46:44 -0700400struct mem_cgroup *get_mem_cgroup_from_page(struct page *page);
401
Michal Hocko33398cf2015-09-08 15:01:02 -0700402static inline
403struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
404 return css ? container_of(css, struct mem_cgroup, css) : NULL;
405}
406
Roman Gushchindc0b5862018-08-17 15:46:36 -0700407static inline void mem_cgroup_put(struct mem_cgroup *memcg)
408{
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700409 if (memcg)
410 css_put(&memcg->css);
Roman Gushchindc0b5862018-08-17 15:46:36 -0700411}
412
Johannes Weiner8e8ae642016-01-14 15:21:32 -0800413#define mem_cgroup_from_counter(counter, member) \
414 container_of(counter, struct mem_cgroup, member)
415
Michal Hocko33398cf2015-09-08 15:01:02 -0700416struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
417 struct mem_cgroup *,
418 struct mem_cgroup_reclaim_cookie *);
419void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700420int mem_cgroup_scan_tasks(struct mem_cgroup *,
421 int (*)(struct task_struct *, void *), void *);
Michal Hocko33398cf2015-09-08 15:01:02 -0700422
Johannes Weiner23047a92016-03-15 14:57:16 -0700423static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
424{
425 if (mem_cgroup_disabled())
426 return 0;
427
Johannes Weiner73f576c2016-07-20 15:44:57 -0700428 return memcg->id.id;
Johannes Weiner23047a92016-03-15 14:57:16 -0700429}
Johannes Weiner73f576c2016-07-20 15:44:57 -0700430struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
Johannes Weiner23047a92016-03-15 14:57:16 -0700431
Chris Downaa9694b2019-03-05 15:45:52 -0800432static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
433{
434 return mem_cgroup_from_css(seq_css(m));
435}
436
Roman Gushchin22621852017-07-06 15:40:25 -0700437static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
438{
439 struct mem_cgroup_per_node *mz;
440
441 if (mem_cgroup_disabled())
442 return NULL;
443
444 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
445 return mz->memcg;
446}
447
Johannes Weiner8e8ae642016-01-14 15:21:32 -0800448/**
449 * parent_mem_cgroup - find the accounting parent of a memcg
450 * @memcg: memcg whose parent to find
451 *
452 * Returns the parent memcg, or NULL if this is the root or the memory
453 * controller is in legacy no-hierarchy mode.
454 */
455static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
456{
457 if (!memcg->memory.parent)
458 return NULL;
459 return mem_cgroup_from_counter(memcg->memory.parent, memory);
460}
461
Michal Hocko33398cf2015-09-08 15:01:02 -0700462static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
463 struct mem_cgroup *root)
464{
465 if (root == memcg)
466 return true;
467 if (!root->use_hierarchy)
468 return false;
469 return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
470}
Glauber Costae1aab162011-12-11 21:47:03 +0000471
Johannes Weiner2314b422014-12-10 15:44:33 -0800472static inline bool mm_match_cgroup(struct mm_struct *mm,
473 struct mem_cgroup *memcg)
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800474{
Johannes Weiner587af302012-10-08 16:34:12 -0700475 struct mem_cgroup *task_memcg;
Johannes Weiner413918b2014-12-10 15:44:30 -0800476 bool match = false;
Johannes Weinerc3ac9a82012-05-29 15:06:25 -0700477
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800478 rcu_read_lock();
Johannes Weiner587af302012-10-08 16:34:12 -0700479 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
Johannes Weiner413918b2014-12-10 15:44:30 -0800480 if (task_memcg)
Johannes Weiner2314b422014-12-10 15:44:33 -0800481 match = mem_cgroup_is_descendant(task_memcg, memcg);
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800482 rcu_read_unlock();
Johannes Weinerc3ac9a82012-05-29 15:06:25 -0700483 return match;
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800484}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800485
Michal Hocko64219992015-09-08 15:01:07 -0700486struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700487ino_t page_cgroup_ino(struct page *page);
Wu Fengguangd3242362009-12-16 12:19:59 +0100488
Vladimir Davydoveb01aaa2016-01-20 15:03:02 -0800489static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
490{
491 if (mem_cgroup_disabled())
492 return true;
493 return !!(memcg->css.flags & CSS_ONLINE);
494}
495
KAMEZAWA Hiroyuki58ae83d2008-02-07 00:14:32 -0800496/*
497 * For memory reclaim.
498 */
Ying Han889976d2011-05-26 16:25:33 -0700499int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
Michal Hocko33398cf2015-09-08 15:01:02 -0700500
501void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
Michal Hockob4536f0c82017-01-10 16:58:04 -0800502 int zid, int nr_pages);
Michal Hocko33398cf2015-09-08 15:01:02 -0700503
Vladimir Davydov0a6b76d2016-03-17 14:18:42 -0700504unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
505 int nid, unsigned int lru_mask);
506
Michal Hocko33398cf2015-09-08 15:01:02 -0700507static inline
508unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
509{
Mel Gormanef8f2322016-07-28 15:46:05 -0700510 struct mem_cgroup_per_node *mz;
Michal Hockob4536f0c82017-01-10 16:58:04 -0800511 unsigned long nr_pages = 0;
512 int zid;
Michal Hocko33398cf2015-09-08 15:01:02 -0700513
Mel Gormanef8f2322016-07-28 15:46:05 -0700514 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Michal Hockob4536f0c82017-01-10 16:58:04 -0800515 for (zid = 0; zid < MAX_NR_ZONES; zid++)
516 nr_pages += mz->lru_zone_size[zid][lru];
517 return nr_pages;
518}
519
520static inline
521unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
522 enum lru_list lru, int zone_idx)
523{
524 struct mem_cgroup_per_node *mz;
525
526 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
527 return mz->lru_zone_size[zone_idx][lru];
Michal Hocko33398cf2015-09-08 15:01:02 -0700528}
529
Tejun Heob23afb92015-11-05 18:46:11 -0800530void mem_cgroup_handle_over_high(void);
531
Roman Gushchinbbec2e12018-06-07 17:06:18 -0700532unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700533
yuzhoujianf0c867d2018-12-28 00:36:10 -0800534void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
Michal Hocko64219992015-09-08 15:01:07 -0700535 struct task_struct *p);
KAMEZAWA Hiroyuki58ae83d2008-02-07 00:14:32 -0800536
yuzhoujianf0c867d2018-12-28 00:36:10 -0800537void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);
538
Michal Hocko29ef6802018-08-17 15:47:11 -0700539static inline void mem_cgroup_enter_user_fault(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700540{
Michal Hocko29ef6802018-08-17 15:47:11 -0700541 WARN_ON(current->in_user_fault);
542 current->in_user_fault = 1;
Johannes Weiner519e5242013-09-12 15:13:42 -0700543}
544
Michal Hocko29ef6802018-08-17 15:47:11 -0700545static inline void mem_cgroup_exit_user_fault(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700546{
Michal Hocko29ef6802018-08-17 15:47:11 -0700547 WARN_ON(!current->in_user_fault);
548 current->in_user_fault = 0;
Johannes Weiner519e5242013-09-12 15:13:42 -0700549}
550
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700551static inline bool task_in_memcg_oom(struct task_struct *p)
552{
Tejun Heo626ebc42015-11-05 18:46:09 -0800553 return p->memcg_in_oom;
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700554}
555
Johannes Weiner49426422013-10-16 13:46:59 -0700556bool mem_cgroup_oom_synchronize(bool wait);
Roman Gushchin3d8b38e2018-08-21 21:53:54 -0700557struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
558 struct mem_cgroup *oom_domain);
559void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700560
Andrew Mortonc255a452012-07-31 16:43:02 -0700561#ifdef CONFIG_MEMCG_SWAP
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -0800562extern int do_swap_account;
563#endif
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800564
Johannes Weiner739f79f2017-08-18 15:15:48 -0700565struct mem_cgroup *lock_page_memcg(struct page *page);
566void __unlock_page_memcg(struct mem_cgroup *memcg);
Johannes Weiner62cccb82016-03-15 14:57:22 -0700567void unlock_page_memcg(struct page *page);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -0700568
Greg Thelen0b3d6e62019-04-05 18:39:18 -0700569/*
570 * idx can be of type enum memcg_stat_item or node_stat_item.
571 * Keep in sync with memcg_exact_page_state().
572 */
Johannes Weinerccda7f42017-05-03 14:55:16 -0700573static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700574 int idx)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700575{
Johannes Weinera983b5e2018-01-31 16:16:45 -0800576 long x = atomic_long_read(&memcg->stat[idx]);
577#ifdef CONFIG_SMP
578 if (x < 0)
579 x = 0;
580#endif
581 return x;
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700582}
583
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700584/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700585static inline void __mod_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700586 int idx, int val)
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700587{
Johannes Weinera983b5e2018-01-31 16:16:45 -0800588 long x;
589
590 if (mem_cgroup_disabled())
591 return;
592
593 x = val + __this_cpu_read(memcg->stat_cpu->count[idx]);
594 if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
595 atomic_long_add(x, &memcg->stat[idx]);
596 x = 0;
597 }
598 __this_cpu_write(memcg->stat_cpu->count[idx], x);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700599}
600
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700601/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weinerccda7f42017-05-03 14:55:16 -0700602static inline void mod_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700603 int idx, int val)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700604{
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800605 unsigned long flags;
606
607 local_irq_save(flags);
Johannes Weinera983b5e2018-01-31 16:16:45 -0800608 __mod_memcg_state(memcg, idx, val);
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800609 local_irq_restore(flags);
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700610}
611
Michal Hocko33398cf2015-09-08 15:01:02 -0700612/**
Johannes Weinerccda7f42017-05-03 14:55:16 -0700613 * mod_memcg_page_state - update page state statistics
Johannes Weiner62cccb82016-03-15 14:57:22 -0700614 * @page: the page
Michal Hocko33398cf2015-09-08 15:01:02 -0700615 * @idx: page state item to account
616 * @val: number of pages (positive or negative)
617 *
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700618 * The @page must be locked or the caller must use lock_page_memcg()
619 * to prevent double accounting when the page is concurrently being
620 * moved to another memcg:
Johannes Weiner81f8c3a2016-03-15 14:57:04 -0700621 *
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700622 * lock_page(page) or lock_page_memcg(page)
Johannes Weiner81f8c3a2016-03-15 14:57:04 -0700623 * if (TestClearPageState(page))
Johannes Weinerccda7f42017-05-03 14:55:16 -0700624 * mod_memcg_page_state(page, state, -1);
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700625 * unlock_page(page) or unlock_page_memcg(page)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700626 *
627 * Kernel pages are an exception to this, since they'll never move.
Michal Hocko33398cf2015-09-08 15:01:02 -0700628 */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700629static inline void __mod_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700630 int idx, int val)
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700631{
632 if (page->mem_cgroup)
633 __mod_memcg_state(page->mem_cgroup, idx, val);
634}
635
Johannes Weinerccda7f42017-05-03 14:55:16 -0700636static inline void mod_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700637 int idx, int val)
Michal Hocko33398cf2015-09-08 15:01:02 -0700638{
Johannes Weiner62cccb82016-03-15 14:57:22 -0700639 if (page->mem_cgroup)
Johannes Weinerccda7f42017-05-03 14:55:16 -0700640 mod_memcg_state(page->mem_cgroup, idx, val);
Michal Hocko33398cf2015-09-08 15:01:02 -0700641}
642
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700643static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
644 enum node_stat_item idx)
Greg Thelen2a7106f2011-01-13 15:47:37 -0800645{
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700646 struct mem_cgroup_per_node *pn;
Johannes Weinera983b5e2018-01-31 16:16:45 -0800647 long x;
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700648
649 if (mem_cgroup_disabled())
650 return node_page_state(lruvec_pgdat(lruvec), idx);
651
652 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Johannes Weinera983b5e2018-01-31 16:16:45 -0800653 x = atomic_long_read(&pn->lruvec_stat[idx]);
654#ifdef CONFIG_SMP
655 if (x < 0)
656 x = 0;
657#endif
658 return x;
Greg Thelen2a7106f2011-01-13 15:47:37 -0800659}
660
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700661static inline void __mod_lruvec_state(struct lruvec *lruvec,
662 enum node_stat_item idx, int val)
Greg Thelen2a7106f2011-01-13 15:47:37 -0800663{
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700664 struct mem_cgroup_per_node *pn;
Johannes Weinera983b5e2018-01-31 16:16:45 -0800665 long x;
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700666
Johannes Weiner28454262018-01-31 16:16:41 -0800667 /* Update node */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700668 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
Johannes Weiner28454262018-01-31 16:16:41 -0800669
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700670 if (mem_cgroup_disabled())
671 return;
Johannes Weiner28454262018-01-31 16:16:41 -0800672
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700673 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Johannes Weiner28454262018-01-31 16:16:41 -0800674
675 /* Update memcg */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700676 __mod_memcg_state(pn->memcg, idx, val);
Johannes Weiner28454262018-01-31 16:16:41 -0800677
678 /* Update lruvec */
Johannes Weinera983b5e2018-01-31 16:16:45 -0800679 x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
680 if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
681 atomic_long_add(x, &pn->lruvec_stat[idx]);
682 x = 0;
683 }
684 __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700685}
686
687static inline void mod_lruvec_state(struct lruvec *lruvec,
688 enum node_stat_item idx, int val)
689{
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800690 unsigned long flags;
691
692 local_irq_save(flags);
Johannes Weiner28454262018-01-31 16:16:41 -0800693 __mod_lruvec_state(lruvec, idx, val);
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800694 local_irq_restore(flags);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700695}
696
697static inline void __mod_lruvec_page_state(struct page *page,
698 enum node_stat_item idx, int val)
699{
Johannes Weiner28454262018-01-31 16:16:41 -0800700 pg_data_t *pgdat = page_pgdat(page);
701 struct lruvec *lruvec;
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700702
Johannes Weiner28454262018-01-31 16:16:41 -0800703 /* Untracked pages have no memcg, no lruvec. Update only the node */
704 if (!page->mem_cgroup) {
705 __mod_node_page_state(pgdat, idx, val);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700706 return;
Johannes Weiner28454262018-01-31 16:16:41 -0800707 }
708
709 lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
710 __mod_lruvec_state(lruvec, idx, val);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700711}
712
713static inline void mod_lruvec_page_state(struct page *page,
714 enum node_stat_item idx, int val)
715{
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800716 unsigned long flags;
717
718 local_irq_save(flags);
Johannes Weiner28454262018-01-31 16:16:41 -0800719 __mod_lruvec_page_state(page, idx, val);
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800720 local_irq_restore(flags);
Greg Thelen2a7106f2011-01-13 15:47:37 -0800721}
722
Mel Gormanef8f2322016-07-28 15:46:05 -0700723unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -0700724 gfp_t gfp_mask,
725 unsigned long *total_scanned);
David Rientjesa63d83f2010-08-09 17:19:46 -0700726
Johannes Weinerc9019e92018-01-31 16:16:37 -0800727static inline void __count_memcg_events(struct mem_cgroup *memcg,
Johannes Weinere27be242018-04-10 16:29:45 -0700728 enum vm_event_item idx,
729 unsigned long count)
Johannes Weinerc9019e92018-01-31 16:16:37 -0800730{
Johannes Weinera983b5e2018-01-31 16:16:45 -0800731 unsigned long x;
732
733 if (mem_cgroup_disabled())
734 return;
735
736 x = count + __this_cpu_read(memcg->stat_cpu->events[idx]);
737 if (unlikely(x > MEMCG_CHARGE_BATCH)) {
738 atomic_long_add(x, &memcg->events[idx]);
739 x = 0;
740 }
741 __this_cpu_write(memcg->stat_cpu->events[idx], x);
Johannes Weinerc9019e92018-01-31 16:16:37 -0800742}
743
Roman Gushchin22621852017-07-06 15:40:25 -0700744static inline void count_memcg_events(struct mem_cgroup *memcg,
Johannes Weinere27be242018-04-10 16:29:45 -0700745 enum vm_event_item idx,
746 unsigned long count)
Roman Gushchin22621852017-07-06 15:40:25 -0700747{
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800748 unsigned long flags;
749
750 local_irq_save(flags);
Johannes Weinera983b5e2018-01-31 16:16:45 -0800751 __count_memcg_events(memcg, idx, count);
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800752 local_irq_restore(flags);
Roman Gushchin22621852017-07-06 15:40:25 -0700753}
754
755static inline void count_memcg_page_event(struct page *page,
Johannes Weinere27be242018-04-10 16:29:45 -0700756 enum vm_event_item idx)
Roman Gushchin22621852017-07-06 15:40:25 -0700757{
758 if (page->mem_cgroup)
759 count_memcg_events(page->mem_cgroup, idx, 1);
760}
761
762static inline void count_memcg_event_mm(struct mm_struct *mm,
763 enum vm_event_item idx)
David Rientjes68ae5642012-12-12 13:51:57 -0800764{
Michal Hocko33398cf2015-09-08 15:01:02 -0700765 struct mem_cgroup *memcg;
766
David Rientjes68ae5642012-12-12 13:51:57 -0800767 if (mem_cgroup_disabled())
768 return;
Michal Hocko33398cf2015-09-08 15:01:02 -0700769
770 rcu_read_lock();
771 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
Roman Gushchinfe6bdfc2018-06-14 15:28:05 -0700772 if (likely(memcg))
Johannes Weinerc9019e92018-01-31 16:16:37 -0800773 count_memcg_events(memcg, idx, 1);
Michal Hocko33398cf2015-09-08 15:01:02 -0700774 rcu_read_unlock();
David Rientjes68ae5642012-12-12 13:51:57 -0800775}
Johannes Weinerc9019e92018-01-31 16:16:37 -0800776
Johannes Weinere27be242018-04-10 16:29:45 -0700777static inline void memcg_memory_event(struct mem_cgroup *memcg,
778 enum memcg_memory_event event)
Johannes Weinerc9019e92018-01-31 16:16:37 -0800779{
Johannes Weinere27be242018-04-10 16:29:45 -0700780 atomic_long_inc(&memcg->memory_events[event]);
Johannes Weinerc9019e92018-01-31 16:16:37 -0800781 cgroup_file_notify(&memcg->events_file);
782}
783
Roman Gushchinfe6bdfc2018-06-14 15:28:05 -0700784static inline void memcg_memory_event_mm(struct mm_struct *mm,
785 enum memcg_memory_event event)
786{
787 struct mem_cgroup *memcg;
788
789 if (mem_cgroup_disabled())
790 return;
791
792 rcu_read_lock();
793 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
794 if (likely(memcg))
795 memcg_memory_event(memcg, event);
796 rcu_read_unlock();
797}
798
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -0800799#ifdef CONFIG_TRANSPARENT_HUGEPAGE
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -0800800void mem_cgroup_split_huge_fixup(struct page *head);
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -0800801#endif
802
Andrew Mortonc255a452012-07-31 16:43:02 -0700803#else /* CONFIG_MEMCG */
Johannes Weiner23047a92016-03-15 14:57:16 -0700804
805#define MEM_CGROUP_ID_SHIFT 0
806#define MEM_CGROUP_ID_MAX 0
807
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800808struct mem_cgroup;
809
Kirill Tkhaidfd2f102018-08-17 15:48:06 -0700810static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
811{
812 return true;
813}
814
Johannes Weiner23047a92016-03-15 14:57:16 -0700815static inline bool mem_cgroup_disabled(void)
816{
817 return true;
818}
819
Johannes Weinere27be242018-04-10 16:29:45 -0700820static inline void memcg_memory_event(struct mem_cgroup *memcg,
821 enum memcg_memory_event event)
Johannes Weiner241994ed2015-02-11 15:26:06 -0800822{
823}
824
Roman Gushchinfe6bdfc2018-06-14 15:28:05 -0700825static inline void memcg_memory_event_mm(struct mm_struct *mm,
826 enum memcg_memory_event event)
827{
828}
829
Roman Gushchinbf8d5d52018-06-07 17:07:46 -0700830static inline enum mem_cgroup_protection mem_cgroup_protected(
831 struct mem_cgroup *root, struct mem_cgroup *memcg)
Johannes Weiner241994ed2015-02-11 15:26:06 -0800832{
Roman Gushchinbf8d5d52018-06-07 17:07:46 -0700833 return MEMCG_PROT_NONE;
Johannes Weiner241994ed2015-02-11 15:26:06 -0800834}
835
Johannes Weiner00501b52014-08-08 14:19:20 -0700836static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
837 gfp_t gfp_mask,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800838 struct mem_cgroup **memcgp,
839 bool compound)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800840{
Johannes Weiner00501b52014-08-08 14:19:20 -0700841 *memcgp = NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800842 return 0;
843}
844
Tejun Heo2cf85582018-07-03 11:14:56 -0400845static inline int mem_cgroup_try_charge_delay(struct page *page,
846 struct mm_struct *mm,
847 gfp_t gfp_mask,
848 struct mem_cgroup **memcgp,
849 bool compound)
850{
851 *memcgp = NULL;
852 return 0;
853}
854
Johannes Weiner00501b52014-08-08 14:19:20 -0700855static inline void mem_cgroup_commit_charge(struct page *page,
856 struct mem_cgroup *memcg,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800857 bool lrucare, bool compound)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800858{
859}
860
Johannes Weiner00501b52014-08-08 14:19:20 -0700861static inline void mem_cgroup_cancel_charge(struct page *page,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800862 struct mem_cgroup *memcg,
863 bool compound)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800864{
865}
866
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700867static inline void mem_cgroup_uncharge(struct page *page)
868{
869}
870
Johannes Weiner747db952014-08-08 14:19:24 -0700871static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -0800872{
873}
874
Johannes Weiner6a93ca82016-03-15 14:57:19 -0700875static inline void mem_cgroup_migrate(struct page *old, struct page *new)
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700876{
877}
878
Mel Gormana9dd0a82016-07-28 15:46:02 -0700879static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
Mel Gormanef8f2322016-07-28 15:46:05 -0700880 struct mem_cgroup *memcg)
Johannes Weiner925b7672012-01-12 17:18:15 -0800881{
Mel Gormana9dd0a82016-07-28 15:46:02 -0700882 return node_lruvec(pgdat);
Johannes Weiner925b7672012-01-12 17:18:15 -0800883}
884
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700885static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
Mel Gorman599d0c92016-07-28 15:45:31 -0700886 struct pglist_data *pgdat)
Minchan Kim3f58a822011-03-22 16:32:53 -0700887{
Mel Gorman599d0c92016-07-28 15:45:31 -0700888 return &pgdat->lruvec;
Balbir Singh66e17072008-02-07 00:13:56 -0800889}
890
Johannes Weiner587af302012-10-08 16:34:12 -0700891static inline bool mm_match_cgroup(struct mm_struct *mm,
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700892 struct mem_cgroup *memcg)
Balbir Singhbed71612008-02-07 00:14:01 -0800893{
Johannes Weiner587af302012-10-08 16:34:12 -0700894 return true;
Balbir Singhbed71612008-02-07 00:14:01 -0800895}
896
David Rientjesffbdccf2013-07-03 15:01:23 -0700897static inline bool task_in_mem_cgroup(struct task_struct *task,
898 const struct mem_cgroup *memcg)
David Rientjes4c4a2212008-02-07 00:14:06 -0800899{
David Rientjesffbdccf2013-07-03 15:01:23 -0700900 return true;
David Rientjes4c4a2212008-02-07 00:14:06 -0800901}
902
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700903static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
904{
905 return NULL;
906}
907
Shakeel Buttf745c6f2018-08-17 15:46:44 -0700908static inline struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
909{
910 return NULL;
911}
912
Roman Gushchindc0b5862018-08-17 15:46:36 -0700913static inline void mem_cgroup_put(struct mem_cgroup *memcg)
914{
915}
916
Johannes Weiner56600482012-01-12 17:17:59 -0800917static inline struct mem_cgroup *
918mem_cgroup_iter(struct mem_cgroup *root,
919 struct mem_cgroup *prev,
920 struct mem_cgroup_reclaim_cookie *reclaim)
921{
922 return NULL;
923}
924
925static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
926 struct mem_cgroup *prev)
927{
928}
929
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700930static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
931 int (*fn)(struct task_struct *, void *), void *arg)
932{
933 return 0;
934}
935
Johannes Weiner23047a92016-03-15 14:57:16 -0700936static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800937{
Johannes Weiner23047a92016-03-15 14:57:16 -0700938 return 0;
939}
940
941static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
942{
943 WARN_ON_ONCE(id);
944 /* XXX: This should always return root_mem_cgroup */
945 return NULL;
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800946}
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -0800947
Chris Downaa9694b2019-03-05 15:45:52 -0800948static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
949{
950 return NULL;
951}
952
Roman Gushchin22621852017-07-06 15:40:25 -0700953static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
954{
955 return NULL;
956}
957
Vladimir Davydoveb01aaa2016-01-20 15:03:02 -0800958static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800959{
Yaowei Bai13308ca2015-11-05 18:47:40 -0800960 return true;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800961}
962
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800963static inline unsigned long
Hugh Dickins4d7dcca2012-05-29 15:07:08 -0700964mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800965{
966 return 0;
967}
Michal Hockob4536f0c82017-01-10 16:58:04 -0800968static inline
969unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
970 enum lru_list lru, int zone_idx)
971{
972 return 0;
973}
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800974
Vladimir Davydov0a6b76d2016-03-17 14:18:42 -0700975static inline unsigned long
976mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
977 int nid, unsigned int lru_mask)
978{
979 return 0;
980}
981
Roman Gushchinbbec2e12018-06-07 17:06:18 -0700982static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700983{
984 return 0;
985}
986
Balbir Singhe2224322009-04-02 16:57:39 -0700987static inline void
yuzhoujianf0c867d2018-12-28 00:36:10 -0800988mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
989{
990}
991
992static inline void
993mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
Balbir Singhe2224322009-04-02 16:57:39 -0700994{
995}
996
Johannes Weiner739f79f2017-08-18 15:15:48 -0700997static inline struct mem_cgroup *lock_page_memcg(struct page *page)
998{
999 return NULL;
1000}
1001
1002static inline void __unlock_page_memcg(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07001003{
1004}
1005
Johannes Weiner62cccb82016-03-15 14:57:22 -07001006static inline void unlock_page_memcg(struct page *page)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07001007{
1008}
1009
Tejun Heob23afb92015-11-05 18:46:11 -08001010static inline void mem_cgroup_handle_over_high(void)
1011{
1012}
1013
Michal Hocko29ef6802018-08-17 15:47:11 -07001014static inline void mem_cgroup_enter_user_fault(void)
Johannes Weiner519e5242013-09-12 15:13:42 -07001015{
1016}
1017
Michal Hocko29ef6802018-08-17 15:47:11 -07001018static inline void mem_cgroup_exit_user_fault(void)
Johannes Weiner519e5242013-09-12 15:13:42 -07001019{
1020}
1021
Johannes Weiner3812c8c2013-09-12 15:13:44 -07001022static inline bool task_in_memcg_oom(struct task_struct *p)
1023{
1024 return false;
1025}
1026
Johannes Weiner49426422013-10-16 13:46:59 -07001027static inline bool mem_cgroup_oom_synchronize(bool wait)
Johannes Weiner3812c8c2013-09-12 15:13:44 -07001028{
1029 return false;
1030}
1031
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07001032static inline struct mem_cgroup *mem_cgroup_get_oom_group(
1033 struct task_struct *victim, struct mem_cgroup *oom_domain)
1034{
1035 return NULL;
1036}
1037
1038static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1039{
1040}
1041
Johannes Weinerccda7f42017-05-03 14:55:16 -07001042static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001043 int idx)
Johannes Weiner2a2e4882017-05-03 14:55:03 -07001044{
1045 return 0;
1046}
1047
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001048static inline void __mod_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001049 int idx,
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001050 int nr)
1051{
1052}
1053
Johannes Weinerccda7f42017-05-03 14:55:16 -07001054static inline void mod_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001055 int idx,
Johannes Weinerccda7f42017-05-03 14:55:16 -07001056 int nr)
Johannes Weiner2a2e4882017-05-03 14:55:03 -07001057{
1058}
1059
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001060static inline void __mod_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001061 int idx,
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001062 int nr)
Johannes Weiner2a2e4882017-05-03 14:55:03 -07001063{
1064}
1065
Johannes Weinerccda7f42017-05-03 14:55:16 -07001066static inline void mod_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001067 int idx,
Johannes Weinerccda7f42017-05-03 14:55:16 -07001068 int nr)
Johannes Weiner553af432017-03-31 15:11:50 -07001069{
1070}
1071
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001072static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
1073 enum node_stat_item idx)
Greg Thelen2a7106f2011-01-13 15:47:37 -08001074{
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001075 return node_page_state(lruvec_pgdat(lruvec), idx);
Greg Thelen2a7106f2011-01-13 15:47:37 -08001076}
1077
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001078static inline void __mod_lruvec_state(struct lruvec *lruvec,
1079 enum node_stat_item idx, int val)
Balbir Singhd69b0422009-06-17 16:26:34 -07001080{
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001081 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
1082}
1083
1084static inline void mod_lruvec_state(struct lruvec *lruvec,
1085 enum node_stat_item idx, int val)
1086{
1087 mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
1088}
1089
1090static inline void __mod_lruvec_page_state(struct page *page,
1091 enum node_stat_item idx, int val)
1092{
1093 __mod_node_page_state(page_pgdat(page), idx, val);
1094}
1095
1096static inline void mod_lruvec_page_state(struct page *page,
1097 enum node_stat_item idx, int val)
1098{
1099 mod_node_page_state(page_pgdat(page), idx, val);
Balbir Singhd69b0422009-06-17 16:26:34 -07001100}
1101
Balbir Singh4e416952009-09-23 15:56:39 -07001102static inline
Mel Gormanef8f2322016-07-28 15:46:05 -07001103unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -07001104 gfp_t gfp_mask,
1105 unsigned long *total_scanned)
Balbir Singh4e416952009-09-23 15:56:39 -07001106{
Andrew Morton0608f432013-09-24 15:27:41 -07001107 return 0;
Balbir Singh4e416952009-09-23 15:56:39 -07001108}
1109
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08001110static inline void mem_cgroup_split_huge_fixup(struct page *head)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08001111{
1112}
1113
Roman Gushchin22621852017-07-06 15:40:25 -07001114static inline void count_memcg_events(struct mem_cgroup *memcg,
1115 enum vm_event_item idx,
1116 unsigned long count)
1117{
1118}
1119
Kirill Tkhai9851ac12019-05-13 17:16:54 -07001120static inline void __count_memcg_events(struct mem_cgroup *memcg,
1121 enum vm_event_item idx,
1122 unsigned long count)
1123{
1124}
1125
Roman Gushchin22621852017-07-06 15:40:25 -07001126static inline void count_memcg_page_event(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001127 int idx)
Roman Gushchin22621852017-07-06 15:40:25 -07001128{
1129}
1130
Ying Han456f9982011-05-26 16:25:38 -07001131static inline
Roman Gushchin22621852017-07-06 15:40:25 -07001132void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
Ying Han456f9982011-05-26 16:25:38 -07001133{
1134}
Andrew Mortonc255a452012-07-31 16:43:02 -07001135#endif /* CONFIG_MEMCG */
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001136
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001137/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001138static inline void __inc_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001139 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001140{
1141 __mod_memcg_state(memcg, idx, 1);
1142}
1143
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001144/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001145static inline void __dec_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001146 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001147{
1148 __mod_memcg_state(memcg, idx, -1);
1149}
1150
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001151/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001152static inline void __inc_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001153 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001154{
1155 __mod_memcg_page_state(page, idx, 1);
1156}
1157
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001158/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001159static inline void __dec_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001160 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001161{
1162 __mod_memcg_page_state(page, idx, -1);
1163}
1164
1165static inline void __inc_lruvec_state(struct lruvec *lruvec,
1166 enum node_stat_item idx)
1167{
1168 __mod_lruvec_state(lruvec, idx, 1);
1169}
1170
1171static inline void __dec_lruvec_state(struct lruvec *lruvec,
1172 enum node_stat_item idx)
1173{
1174 __mod_lruvec_state(lruvec, idx, -1);
1175}
1176
1177static inline void __inc_lruvec_page_state(struct page *page,
1178 enum node_stat_item idx)
1179{
1180 __mod_lruvec_page_state(page, idx, 1);
1181}
1182
1183static inline void __dec_lruvec_page_state(struct page *page,
1184 enum node_stat_item idx)
1185{
1186 __mod_lruvec_page_state(page, idx, -1);
1187}
1188
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001189/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001190static inline void inc_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001191 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001192{
1193 mod_memcg_state(memcg, idx, 1);
1194}
1195
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001196/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001197static inline void dec_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001198 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001199{
1200 mod_memcg_state(memcg, idx, -1);
1201}
1202
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001203/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001204static inline void inc_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001205 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001206{
1207 mod_memcg_page_state(page, idx, 1);
1208}
1209
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001210/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001211static inline void dec_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001212 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001213{
1214 mod_memcg_page_state(page, idx, -1);
1215}
1216
1217static inline void inc_lruvec_state(struct lruvec *lruvec,
1218 enum node_stat_item idx)
1219{
1220 mod_lruvec_state(lruvec, idx, 1);
1221}
1222
1223static inline void dec_lruvec_state(struct lruvec *lruvec,
1224 enum node_stat_item idx)
1225{
1226 mod_lruvec_state(lruvec, idx, -1);
1227}
1228
1229static inline void inc_lruvec_page_state(struct page *page,
1230 enum node_stat_item idx)
1231{
1232 mod_lruvec_page_state(page, idx, 1);
1233}
1234
1235static inline void dec_lruvec_page_state(struct page *page,
1236 enum node_stat_item idx)
1237{
1238 mod_lruvec_page_state(page, idx, -1);
1239}
1240
Tejun Heo52ebea72015-05-22 17:13:37 -04001241#ifdef CONFIG_CGROUP_WRITEBACK
Tejun Heo841710a2015-05-22 18:23:33 -04001242
Tejun Heo841710a2015-05-22 18:23:33 -04001243struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
Tejun Heoc5edf9c2015-09-29 13:04:26 -04001244void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
1245 unsigned long *pheadroom, unsigned long *pdirty,
1246 unsigned long *pwriteback);
Tejun Heo841710a2015-05-22 18:23:33 -04001247
1248#else /* CONFIG_CGROUP_WRITEBACK */
1249
1250static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
1251{
1252 return NULL;
1253}
1254
Tejun Heoc2aa7232015-05-22 18:23:35 -04001255static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
Tejun Heoc5edf9c2015-09-29 13:04:26 -04001256 unsigned long *pfilepages,
1257 unsigned long *pheadroom,
Tejun Heoc2aa7232015-05-22 18:23:35 -04001258 unsigned long *pdirty,
1259 unsigned long *pwriteback)
1260{
1261}
1262
Tejun Heo841710a2015-05-22 18:23:33 -04001263#endif /* CONFIG_CGROUP_WRITEBACK */
Tejun Heo52ebea72015-05-22 17:13:37 -04001264
Glauber Costae1aab162011-12-11 21:47:03 +00001265struct sock;
Johannes Weinerbaac50b2016-01-14 15:21:17 -08001266bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1267void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
Johannes Weinerd886f4e2016-01-20 15:02:47 -08001268#ifdef CONFIG_MEMCG
Johannes Weineref129472016-01-14 15:21:34 -08001269extern struct static_key_false memcg_sockets_enabled_key;
1270#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
Johannes Weiner2d758072016-10-07 17:00:58 -07001271void mem_cgroup_sk_alloc(struct sock *sk);
1272void mem_cgroup_sk_free(struct sock *sk);
Johannes Weinerbaac50b2016-01-14 15:21:17 -08001273static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
Johannes Weinere8056052016-01-14 15:21:14 -08001274{
Johannes Weiner0db15292016-01-20 15:02:50 -08001275 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
Johannes Weiner8e8ae642016-01-14 15:21:32 -08001276 return true;
Johannes Weiner8e8ae642016-01-14 15:21:32 -08001277 do {
1278 if (time_before(jiffies, memcg->socket_pressure))
1279 return true;
1280 } while ((memcg = parent_mem_cgroup(memcg)));
1281 return false;
Johannes Weinere8056052016-01-14 15:21:14 -08001282}
1283#else
Johannes Weiner80e95fe2016-01-14 15:21:20 -08001284#define mem_cgroup_sockets_enabled 0
Johannes Weiner2d758072016-10-07 17:00:58 -07001285static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
1286static inline void mem_cgroup_sk_free(struct sock *sk) { };
Johannes Weinerbaac50b2016-01-14 15:21:17 -08001287static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
Johannes Weinere8056052016-01-14 15:21:14 -08001288{
1289 return false;
1290}
1291#endif
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001292
Vladimir Davydov45264772016-07-26 15:24:21 -07001293struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep);
1294void memcg_kmem_put_cache(struct kmem_cache *cachep);
Roman Gushchin9b6f7e12018-10-26 15:03:19 -07001295
1296#ifdef CONFIG_MEMCG_KMEM
Shakeel Butt60cd4bc2019-03-05 15:43:13 -08001297int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
1298void __memcg_kmem_uncharge(struct page *page, int order);
1299int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
1300 struct mem_cgroup *memcg);
Vladimir Davydov45264772016-07-26 15:24:21 -07001301
Johannes Weineref129472016-01-14 15:21:34 -08001302extern struct static_key_false memcg_kmem_enabled_key;
Tejun Heo17cc4df2017-02-22 15:41:36 -08001303extern struct workqueue_struct *memcg_kmem_cache_wq;
Glauber Costa749c5412012-12-18 14:23:01 -08001304
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08001305extern int memcg_nr_cache_ids;
Michal Hocko64219992015-09-08 15:01:07 -07001306void memcg_get_cache_ids(void);
1307void memcg_put_cache_ids(void);
Glauber Costaebe945c2012-12-18 14:23:10 -08001308
1309/*
1310 * Helper macro to loop through all memcg-specific caches. Callers must still
1311 * check if the cache is valid (it is either valid or NULL).
1312 * the slab_mutex must be held when looping through those caches
1313 */
Glauber Costa749c5412012-12-18 14:23:01 -08001314#define for_each_memcg_cache_index(_idx) \
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08001315 for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
Glauber Costa749c5412012-12-18 14:23:01 -08001316
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001317static inline bool memcg_kmem_enabled(void)
1318{
Johannes Weineref129472016-01-14 15:21:34 -08001319 return static_branch_unlikely(&memcg_kmem_enabled_key);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001320}
1321
Shakeel Butt60cd4bc2019-03-05 15:43:13 -08001322static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
1323{
1324 if (memcg_kmem_enabled())
1325 return __memcg_kmem_charge(page, gfp, order);
1326 return 0;
1327}
1328
1329static inline void memcg_kmem_uncharge(struct page *page, int order)
1330{
1331 if (memcg_kmem_enabled())
1332 __memcg_kmem_uncharge(page, order);
1333}
1334
1335static inline int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp,
1336 int order, struct mem_cgroup *memcg)
1337{
1338 if (memcg_kmem_enabled())
1339 return __memcg_kmem_charge_memcg(page, gfp, order, memcg);
1340 return 0;
1341}
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001342/*
Jesper Dangaard Brouer9f706d62016-03-15 14:54:03 -07001343 * helper for accessing a memcg's index. It will be used as an index in the
Michal Hocko33398cf2015-09-08 15:01:02 -07001344 * child cache array in kmem_cache, and also to derive its name. This function
1345 * will return -1 when this is not a kmem-limited memcg.
1346 */
1347static inline int memcg_cache_id(struct mem_cgroup *memcg)
1348{
1349 return memcg ? memcg->kmemcg_id : -1;
1350}
Vladimir Davydov5722d092014-04-07 15:39:24 -07001351
Kirill Tkhai0a4465d2018-08-17 15:47:37 -07001352extern int memcg_expand_shrinker_maps(int new_id);
1353
Kirill Tkhaifae91d62018-08-17 15:48:10 -07001354extern void memcg_set_shrinker_bit(struct mem_cgroup *memcg,
1355 int nid, int shrinker_id);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001356#else
Roman Gushchin9b6f7e12018-10-26 15:03:19 -07001357
1358static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
1359{
1360 return 0;
1361}
1362
1363static inline void memcg_kmem_uncharge(struct page *page, int order)
1364{
1365}
1366
Shakeel Butt60cd4bc2019-03-05 15:43:13 -08001367static inline int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
1368{
1369 return 0;
1370}
1371
1372static inline void __memcg_kmem_uncharge(struct page *page, int order)
1373{
1374}
1375
Glauber Costa749c5412012-12-18 14:23:01 -08001376#define for_each_memcg_cache_index(_idx) \
1377 for (; NULL; )
1378
Glauber Costab9ce5ef2012-12-18 14:22:46 -08001379static inline bool memcg_kmem_enabled(void)
1380{
1381 return false;
1382}
1383
Glauber Costa2633d7a2012-12-18 14:22:34 -08001384static inline int memcg_cache_id(struct mem_cgroup *memcg)
1385{
1386 return -1;
1387}
1388
Vladimir Davydov05257a12015-02-12 14:59:01 -08001389static inline void memcg_get_cache_ids(void)
1390{
1391}
1392
1393static inline void memcg_put_cache_ids(void)
1394{
1395}
1396
Kirill Tkhaifae91d62018-08-17 15:48:10 -07001397static inline void memcg_set_shrinker_bit(struct mem_cgroup *memcg,
1398 int nid, int shrinker_id) { }
Kirill Tkhai84c07d12018-08-17 15:47:25 -07001399#endif /* CONFIG_MEMCG_KMEM */
Johannes Weiner127424c2016-01-20 15:02:32 -08001400
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001401#endif /* _LINUX_MEMCONTROL_H */