blob: 7f6537b84ef5ee5b518ca8cf5c0328c224e320f4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/ipc/shm.c
3 * Copyright (C) 1992, 1993 Krishna Balasubramanian
4 * Many improvements/fixes by Bruno Haible.
5 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
6 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
7 *
8 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
9 * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
10 * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
11 * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
12 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
13 * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
14 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
15 *
Steve Grubb073115d2006-04-02 17:07:33 -040016 * support for audit of ipc object properties and permission changes
17 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
Kirill Korotaev4e982312006-10-02 02:18:22 -070018 *
19 * namespaces support
20 * OpenVZ, SWsoft Inc.
21 * Pavel Emelianov <xemul@openvz.org>
Davidlohr Buesoc2c737a2013-09-11 14:26:23 -070022 *
23 * Better ipc lock (kern_ipc_perm.lock) handling
24 * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
Linus Torvalds1da177e2005-04-16 15:20:36 -070025 */
26
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/slab.h>
28#include <linux/mm.h>
29#include <linux/hugetlb.h>
30#include <linux/shm.h>
31#include <linux/init.h>
32#include <linux/file.h>
33#include <linux/mman.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <linux/shmem_fs.h>
35#include <linux/security.h>
36#include <linux/syscalls.h>
37#include <linux/audit.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080038#include <linux/capability.h>
Stephen Rothwell7d87e14c2005-05-01 08:59:12 -070039#include <linux/ptrace.h>
Mike Waychison19b49462005-09-06 15:17:10 -070040#include <linux/seq_file.h>
Nadia Derbey3e148c72007-10-18 23:40:54 -070041#include <linux/rwsem.h>
Kirill Korotaev4e982312006-10-02 02:18:22 -070042#include <linux/nsproxy.h>
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -080043#include <linux/mount.h>
Pavel Emelyanovae5e1b22008-02-08 04:18:22 -080044#include <linux/ipc_namespace.h>
Stephen Rothwell7d87e14c2005-05-01 08:59:12 -070045
Paul McQuade7153e402014-06-06 14:37:37 -070046#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047
48#include "util.h"
49
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -080050struct shm_file_data {
51 int id;
52 struct ipc_namespace *ns;
53 struct file *file;
54 const struct vm_operations_struct *vm_ops;
55};
56
57#define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))
58
Arjan van de Ven9a321442007-02-12 00:55:35 -080059static const struct file_operations shm_file_operations;
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +040060static const struct vm_operations_struct shm_vm_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
Pierre Peiffered2ddbf2008-02-08 04:18:57 -080062#define shm_ids(ns) ((ns)->ids[IPC_SHM_IDS])
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
Kirill Korotaev4e982312006-10-02 02:18:22 -070064#define shm_unlock(shp) \
65 ipc_unlock(&(shp)->shm_perm)
Kirill Korotaev4e982312006-10-02 02:18:22 -070066
Nadia Derbey7748dbf2007-10-18 23:40:49 -070067static int newseg(struct ipc_namespace *, struct ipc_params *);
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -080068static void shm_open(struct vm_area_struct *vma);
69static void shm_close(struct vm_area_struct *vma);
Manfred Spraul239521f2014-01-27 17:07:04 -080070static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
Linus Torvalds1da177e2005-04-16 15:20:36 -070071#ifdef CONFIG_PROC_FS
Mike Waychison19b49462005-09-06 15:17:10 -070072static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
Linus Torvalds1da177e2005-04-16 15:20:36 -070073#endif
74
Pierre Peiffered2ddbf2008-02-08 04:18:57 -080075void shm_init_ns(struct ipc_namespace *ns)
Kirill Korotaev4e982312006-10-02 02:18:22 -070076{
Kirill Korotaev4e982312006-10-02 02:18:22 -070077 ns->shm_ctlmax = SHMMAX;
78 ns->shm_ctlall = SHMALL;
79 ns->shm_ctlmni = SHMMNI;
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -070080 ns->shm_rmid_forced = 0;
Kirill Korotaev4e982312006-10-02 02:18:22 -070081 ns->shm_tot = 0;
WANG Conge8148f72009-01-06 14:42:49 -080082 ipc_init_ids(&shm_ids(ns));
Kirill Korotaev4e982312006-10-02 02:18:22 -070083}
Linus Torvalds1da177e2005-04-16 15:20:36 -070084
Nadia Derbeyf4566f02007-10-18 23:40:53 -070085/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -070086 * Called with shm_ids.rwsem (writer) and the shp structure locked.
87 * Only shm_ids.rwsem remains locked on exit.
Nadia Derbeyf4566f02007-10-18 23:40:53 -070088 */
Pierre Peiffer01b8b072008-02-08 04:18:57 -080089static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
Kirill Korotaev4e982312006-10-02 02:18:22 -070090{
Pierre Peiffer01b8b072008-02-08 04:18:57 -080091 struct shmid_kernel *shp;
Shailesh Pandey63980c82016-12-14 15:06:10 -080092
Pierre Peiffer01b8b072008-02-08 04:18:57 -080093 shp = container_of(ipcp, struct shmid_kernel, shm_perm);
94
Manfred Spraul239521f2014-01-27 17:07:04 -080095 if (shp->shm_nattch) {
Kirill Korotaev4e982312006-10-02 02:18:22 -070096 shp->shm_perm.mode |= SHM_DEST;
97 /* Do not find it any more */
98 shp->shm_perm.key = IPC_PRIVATE;
99 shm_unlock(shp);
100 } else
101 shm_destroy(ns, shp);
102}
103
Pavel Emelyanovae5e1b22008-02-08 04:18:22 -0800104#ifdef CONFIG_IPC_NS
Kirill Korotaev4e982312006-10-02 02:18:22 -0700105void shm_exit_ns(struct ipc_namespace *ns)
106{
Pierre Peiffer01b8b072008-02-08 04:18:57 -0800107 free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
Serge E. Hallyn7d6feeb2009-12-15 16:47:27 -0800108 idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
Kirill Korotaev4e982312006-10-02 02:18:22 -0700109}
Pavel Emelyanovae5e1b22008-02-08 04:18:22 -0800110#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111
Linus Torvalds140d0b22011-08-04 19:35:59 -1000112static int __init ipc_ns_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113{
Pierre Peiffered2ddbf2008-02-08 04:18:57 -0800114 shm_init_ns(&init_ipc_ns);
Linus Torvalds140d0b22011-08-04 19:35:59 -1000115 return 0;
116}
117
118pure_initcall(ipc_ns_init);
119
Manfred Spraul239521f2014-01-27 17:07:04 -0800120void __init shm_init(void)
Linus Torvalds140d0b22011-08-04 19:35:59 -1000121{
Mike Waychison19b49462005-09-06 15:17:10 -0700122 ipc_init_proc_interface("sysvipc/shm",
Helge Dellerb7952182010-10-27 15:34:16 -0700123#if BITS_PER_LONG <= 32
124 " key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime rss swap\n",
125#else
126 " key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime rss swap\n",
127#endif
Kirill Korotaev4e982312006-10-02 02:18:22 -0700128 IPC_SHM_IDS, sysvipc_shm_proc_show);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129}
130
Davidlohr Bueso8b8d52a2013-09-11 14:26:15 -0700131static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
132{
Davidlohr Bueso55b7ae52015-06-30 14:58:42 -0700133 struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
Davidlohr Bueso8b8d52a2013-09-11 14:26:15 -0700134
135 if (IS_ERR(ipcp))
136 return ERR_CAST(ipcp);
137
138 return container_of(ipcp, struct shmid_kernel, shm_perm);
139}
140
141static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
142{
143 struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);
144
145 if (IS_ERR(ipcp))
146 return ERR_CAST(ipcp);
147
148 return container_of(ipcp, struct shmid_kernel, shm_perm);
149}
150
Nadia Derbey3e148c72007-10-18 23:40:54 -0700151/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700152 * shm_lock_(check_) routines are called in the paths where the rwsem
Nadia Derbey00c2bf82008-07-25 01:48:03 -0700153 * is not necessarily held.
Nadia Derbey3e148c72007-10-18 23:40:54 -0700154 */
Nadia Derbey023a5352007-10-18 23:40:51 -0700155static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156{
Nadia Derbey03f02c72007-10-18 23:40:51 -0700157 struct kern_ipc_perm *ipcp = ipc_lock(&shm_ids(ns), id);
158
Davidlohr Buesoc5c89752015-06-30 14:58:36 -0700159 /*
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800160 * Callers of shm_lock() must validate the status of the returned ipc
161 * object pointer (as returned by ipc_lock()), and error out as
162 * appropriate.
Davidlohr Buesoc5c89752015-06-30 14:58:36 -0700163 */
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800164 if (IS_ERR(ipcp))
165 return (void *)ipcp;
Nadia Derbey03f02c72007-10-18 23:40:51 -0700166 return container_of(ipcp, struct shmid_kernel, shm_perm);
Nadia Derbey023a5352007-10-18 23:40:51 -0700167}
168
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400169static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
170{
171 rcu_read_lock();
Davidlohr Buesocf9d5d72013-07-08 16:01:11 -0700172 ipc_lock_object(&ipcp->shm_perm);
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400173}
174
Davidlohr Bueso53dad6d2013-09-23 17:04:45 -0700175static void shm_rcu_free(struct rcu_head *head)
176{
177 struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu);
178 struct shmid_kernel *shp = ipc_rcu_to_struct(p);
179
180 security_shm_free(shp);
181 ipc_rcu_free(head);
182}
183
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700184static inline void shm_rmid(struct ipc_namespace *ns, struct shmid_kernel *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185{
Jack Millerab602f72014-08-08 14:23:19 -0700186 list_del(&s->shm_clist);
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700187 ipc_rmid(&shm_ids(ns), &s->shm_perm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188}
189
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800191static int __shm_open(struct vm_area_struct *vma)
Kirill Korotaev4e982312006-10-02 02:18:22 -0700192{
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800193 struct file *file = vma->vm_file;
194 struct shm_file_data *sfd = shm_file_data(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 struct shmid_kernel *shp;
196
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800197 shp = shm_lock(sfd->ns, sfd->id);
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800198
199 if (IS_ERR(shp))
200 return PTR_ERR(shp);
201
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 shp->shm_atim = get_seconds();
Pavel Emelyanovb4888932007-10-18 23:40:14 -0700203 shp->shm_lprid = task_tgid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 shp->shm_nattch++;
205 shm_unlock(shp);
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800206 return 0;
207}
208
209/* This is called by fork, once for every shm attach. */
210static void shm_open(struct vm_area_struct *vma)
211{
212 int err = __shm_open(vma);
213 /*
214 * We raced in the idr lookup or with shm_destroy().
215 * Either way, the ID is busted.
216 */
217 WARN_ON_ONCE(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218}
219
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220/*
221 * shm_destroy - free the struct shmid_kernel
222 *
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700223 * @ns: namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 * @shp: struct to free
225 *
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700226 * It has to be called with shp and shm_ids.rwsem (writer) locked,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 * but returns with shp unlocked and freed.
228 */
Kirill Korotaev4e982312006-10-02 02:18:22 -0700229static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230{
Greg Thelena399b292013-11-21 14:32:00 -0800231 struct file *shm_file;
232
233 shm_file = shp->shm_file;
234 shp->shm_file = NULL;
Kirill Korotaev4e982312006-10-02 02:18:22 -0700235 ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700236 shm_rmid(ns, shp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 shm_unlock(shp);
Greg Thelena399b292013-11-21 14:32:00 -0800238 if (!is_file_hugepages(shm_file))
239 shmem_lock(shm_file, 0, shp->mlock_user);
Hugh Dickins353d5c32009-08-24 16:30:28 +0100240 else if (shp->mlock_user)
Dave Hansen07a46ed2014-12-12 16:58:22 -0800241 user_shm_unlock(i_size_read(file_inode(shm_file)),
242 shp->mlock_user);
Greg Thelena399b292013-11-21 14:32:00 -0800243 fput(shm_file);
Davidlohr Bueso53dad6d2013-09-23 17:04:45 -0700244 ipc_rcu_putref(shp, shm_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245}
246
247/*
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700248 * shm_may_destroy - identifies whether shm segment should be destroyed now
249 *
250 * Returns true if and only if there are no active users of the segment and
251 * one of the following is true:
252 *
253 * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
254 *
255 * 2) sysctl kernel.shm_rmid_forced is set to 1.
256 */
257static bool shm_may_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
258{
259 return (shp->shm_nattch == 0) &&
260 (ns->shm_rmid_forced ||
261 (shp->shm_perm.mode & SHM_DEST));
262}
263
264/*
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800265 * remove the attach descriptor vma.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 * free memory for segment if it is marked destroyed.
267 * The descriptor has already been removed from the current->mm->mmap list
268 * and will later be kfree()d.
269 */
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800270static void shm_close(struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271{
Manfred Spraul239521f2014-01-27 17:07:04 -0800272 struct file *file = vma->vm_file;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800273 struct shm_file_data *sfd = shm_file_data(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 struct shmid_kernel *shp;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800275 struct ipc_namespace *ns = sfd->ns;
Kirill Korotaev4e982312006-10-02 02:18:22 -0700276
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700277 down_write(&shm_ids(ns).rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 /* remove from the list of attaches of the shm segment */
Nadia Derbey00c2bf82008-07-25 01:48:03 -0700279 shp = shm_lock(ns, sfd->id);
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800280
281 /*
282 * We raced in the idr lookup or with shm_destroy().
283 * Either way, the ID is busted.
284 */
285 if (WARN_ON_ONCE(IS_ERR(shp)))
286 goto done; /* no-op */
287
Pavel Emelyanovb4888932007-10-18 23:40:14 -0700288 shp->shm_lprid = task_tgid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 shp->shm_dtim = get_seconds();
290 shp->shm_nattch--;
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700291 if (shm_may_destroy(ns, shp))
Kirill Korotaev4e982312006-10-02 02:18:22 -0700292 shm_destroy(ns, shp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 else
294 shm_unlock(shp);
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800295done:
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700296 up_write(&shm_ids(ns).rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297}
298
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700299/* Called with ns->shm_ids(ns).rwsem locked */
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700300static int shm_try_destroy_orphaned(int id, void *p, void *data)
301{
302 struct ipc_namespace *ns = data;
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400303 struct kern_ipc_perm *ipcp = p;
304 struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700305
306 /*
307 * We want to destroy segments without users and with already
308 * exit'ed originating process.
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400309 *
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700310 * As shp->* are changed under rwsem, it's safe to skip shp locking.
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700311 */
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400312 if (shp->shm_creator != NULL)
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700313 return 0;
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700314
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400315 if (shm_may_destroy(ns, shp)) {
316 shm_lock_by_ptr(shp);
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700317 shm_destroy(ns, shp);
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400318 }
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700319 return 0;
320}
321
322void shm_destroy_orphaned(struct ipc_namespace *ns)
323{
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700324 down_write(&shm_ids(ns).rwsem);
Vasiliy Kulikov33a30ed2011-08-03 22:26:55 +0400325 if (shm_ids(ns).in_use)
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400326 idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700327 up_write(&shm_ids(ns).rwsem);
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700328}
329
Jack Miller83293c0f52014-08-08 14:23:21 -0700330/* Locking assumes this will only be called with task == current */
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700331void exit_shm(struct task_struct *task)
332{
Vasiliy Kulikov4c677e22011-07-29 03:56:40 +0400333 struct ipc_namespace *ns = task->nsproxy->ipc_ns;
Jack Millerab602f72014-08-08 14:23:19 -0700334 struct shmid_kernel *shp, *n;
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700335
Jack Miller83293c0f52014-08-08 14:23:21 -0700336 if (list_empty(&task->sysvshm.shm_clist))
Vasiliy Kulikov298507d2011-08-03 22:28:26 +0400337 return;
338
Jack Miller83293c0f52014-08-08 14:23:21 -0700339 /*
340 * If kernel.shm_rmid_forced is not set then only keep track of
341 * which shmids are orphaned, so that a later set of the sysctl
342 * can clean them up.
343 */
344 if (!ns->shm_rmid_forced) {
345 down_read(&shm_ids(ns).rwsem);
346 list_for_each_entry(shp, &task->sysvshm.shm_clist, shm_clist)
347 shp->shm_creator = NULL;
348 /*
349 * Only under read lock but we are only called on current
350 * so no entry on the list will be shared.
351 */
352 list_del(&task->sysvshm.shm_clist);
353 up_read(&shm_ids(ns).rwsem);
354 return;
355 }
356
357 /*
358 * Destroy all already created segments, that were not yet mapped,
359 * and mark any mapped as orphan to cover the sysctl toggling.
360 * Destroy is skipped if shm_may_destroy() returns false.
361 */
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700362 down_write(&shm_ids(ns).rwsem);
Jack Miller83293c0f52014-08-08 14:23:21 -0700363 list_for_each_entry_safe(shp, n, &task->sysvshm.shm_clist, shm_clist) {
364 shp->shm_creator = NULL;
365
366 if (shm_may_destroy(ns, shp)) {
367 shm_lock_by_ptr(shp);
368 shm_destroy(ns, shp);
369 }
370 }
371
372 /* Remove the list head from any segments still attached. */
Jack Millerab602f72014-08-08 14:23:19 -0700373 list_del(&task->sysvshm.shm_clist);
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700374 up_write(&shm_ids(ns).rwsem);
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -0700375}
376
Dave Jiang11bac802017-02-24 14:56:41 -0800377static int shm_fault(struct vm_fault *vmf)
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800378{
Dave Jiang11bac802017-02-24 14:56:41 -0800379 struct file *file = vmf->vma->vm_file;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800380 struct shm_file_data *sfd = shm_file_data(file);
381
Dave Jiang11bac802017-02-24 14:56:41 -0800382 return sfd->vm_ops->fault(vmf);
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800383}
384
385#ifdef CONFIG_NUMA
Adrian Bunkd823e3e2007-10-16 23:26:42 -0700386static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800387{
388 struct file *file = vma->vm_file;
389 struct shm_file_data *sfd = shm_file_data(file);
390 int err = 0;
Shailesh Pandey63980c82016-12-14 15:06:10 -0800391
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800392 if (sfd->vm_ops->set_policy)
393 err = sfd->vm_ops->set_policy(vma, new);
394 return err;
395}
396
Adrian Bunkd823e3e2007-10-16 23:26:42 -0700397static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
398 unsigned long addr)
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800399{
400 struct file *file = vma->vm_file;
401 struct shm_file_data *sfd = shm_file_data(file);
402 struct mempolicy *pol = NULL;
403
404 if (sfd->vm_ops->get_policy)
405 pol = sfd->vm_ops->get_policy(vma, addr);
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700406 else if (vma->vm_policy)
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800407 pol = vma->vm_policy;
Lee Schermerhorn52cd3b02008-04-28 02:13:16 -0700408
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800409 return pol;
410}
411#endif
412
Manfred Spraul239521f2014-01-27 17:07:04 -0800413static int shm_mmap(struct file *file, struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414{
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800415 struct shm_file_data *sfd = shm_file_data(file);
David Howellsb0e15192006-01-06 00:11:42 -0800416 int ret;
417
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800418 /*
419 * In case of remap_file_pages() emulation, the file can represent
420 * removed IPC ID: propogate shm_lock() error to caller.
421 */
Shailesh Pandey63980c82016-12-14 15:06:10 -0800422 ret = __shm_open(vma);
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800423 if (ret)
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800424 return ret;
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800425
426 ret = sfd->file->f_op->mmap(sfd->file, vma);
427 if (ret) {
428 shm_close(vma);
429 return ret;
430 }
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800431 sfd->vm_ops = vma->vm_ops;
David Howells2e92a3b2007-07-31 00:37:24 -0700432#ifdef CONFIG_MMU
Davidlohr Buesod0edd852015-09-09 15:39:20 -0700433 WARN_ON(!sfd->vm_ops->fault);
David Howells2e92a3b2007-07-31 00:37:24 -0700434#endif
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800435 vma->vm_ops = &shm_vm_ops;
Kirill A. Shutemov1ac0b6d2016-02-17 13:11:35 -0800436 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437}
438
Kirill Korotaev4e982312006-10-02 02:18:22 -0700439static int shm_release(struct inode *ino, struct file *file)
440{
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800441 struct shm_file_data *sfd = shm_file_data(file);
Kirill Korotaev4e982312006-10-02 02:18:22 -0700442
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800443 put_ipc_ns(sfd->ns);
444 shm_file_data(file) = NULL;
445 kfree(sfd);
Kirill Korotaev4e982312006-10-02 02:18:22 -0700446 return 0;
447}
448
Josef Bacik02c24a82011-07-16 20:44:56 -0400449static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
Adam Litke516dffd2007-03-01 15:46:08 -0800450{
Adam Litke516dffd2007-03-01 15:46:08 -0800451 struct shm_file_data *sfd = shm_file_data(file);
Adam Litke516dffd2007-03-01 15:46:08 -0800452
Christoph Hellwig7ea80852010-05-26 17:53:25 +0200453 if (!sfd->file->f_op->fsync)
454 return -EINVAL;
Josef Bacik02c24a82011-07-16 20:44:56 -0400455 return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
Adam Litke516dffd2007-03-01 15:46:08 -0800456}
457
Will Deacon7d8a4562012-06-07 14:21:13 -0700458static long shm_fallocate(struct file *file, int mode, loff_t offset,
459 loff_t len)
460{
461 struct shm_file_data *sfd = shm_file_data(file);
462
463 if (!sfd->file->f_op->fallocate)
464 return -EOPNOTSUPP;
465 return sfd->file->f_op->fallocate(file, mode, offset, len);
466}
467
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800468static unsigned long shm_get_unmapped_area(struct file *file,
469 unsigned long addr, unsigned long len, unsigned long pgoff,
470 unsigned long flags)
471{
472 struct shm_file_data *sfd = shm_file_data(file);
Shailesh Pandey63980c82016-12-14 15:06:10 -0800473
Al Viroc4caa772009-11-30 08:38:43 -0500474 return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
475 pgoff, flags);
Adam Litke516dffd2007-03-01 15:46:08 -0800476}
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800477
Arjan van de Ven9a321442007-02-12 00:55:35 -0800478static const struct file_operations shm_file_operations = {
Kirill Korotaev4e982312006-10-02 02:18:22 -0700479 .mmap = shm_mmap,
Adam Litke516dffd2007-03-01 15:46:08 -0800480 .fsync = shm_fsync,
Kirill Korotaev4e982312006-10-02 02:18:22 -0700481 .release = shm_release,
David Howellsed5e5892010-01-15 17:01:32 -0800482 .get_unmapped_area = shm_get_unmapped_area,
Arnd Bergmann6038f372010-08-15 18:52:59 +0200483 .llseek = noop_llseek,
Will Deacon7d8a4562012-06-07 14:21:13 -0700484 .fallocate = shm_fallocate,
Al Viroc4caa772009-11-30 08:38:43 -0500485};
486
Hugh Dickinsc01d5b32016-07-26 15:26:15 -0700487/*
488 * shm_file_operations_huge is now identical to shm_file_operations,
489 * but we keep it distinct for the sake of is_file_shm_hugepages().
490 */
Al Viroc4caa772009-11-30 08:38:43 -0500491static const struct file_operations shm_file_operations_huge = {
492 .mmap = shm_mmap,
493 .fsync = shm_fsync,
494 .release = shm_release,
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800495 .get_unmapped_area = shm_get_unmapped_area,
Arnd Bergmann6038f372010-08-15 18:52:59 +0200496 .llseek = noop_llseek,
Will Deacon7d8a4562012-06-07 14:21:13 -0700497 .fallocate = shm_fallocate,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498};
499
Yaowei Bai2954e4402016-01-20 15:01:11 -0800500bool is_file_shm_hugepages(struct file *file)
Al Viroc4caa772009-11-30 08:38:43 -0500501{
502 return file->f_op == &shm_file_operations_huge;
503}
504
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +0400505static const struct vm_operations_struct shm_vm_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 .open = shm_open, /* callback for a new vm-area open */
507 .close = shm_close, /* callback for when the vm-area is released */
Nick Piggin54cb8822007-07-19 01:46:59 -0700508 .fault = shm_fault,
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800509#if defined(CONFIG_NUMA)
510 .set_policy = shm_set_policy,
511 .get_policy = shm_get_policy,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512#endif
513};
514
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700515/**
516 * newseg - Create a new shared memory segment
517 * @ns: namespace
518 * @params: ptr to the structure that contains key, size and shmflg
519 *
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700520 * Called with shm_ids.rwsem held as a writer.
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700521 */
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700522static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523{
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700524 key_t key = params->key;
525 int shmflg = params->flg;
526 size_t size = params->u.size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 int error;
528 struct shmid_kernel *shp;
Robin Holtd69f3ba2013-04-30 19:15:54 -0700529 size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Manfred Spraul239521f2014-01-27 17:07:04 -0800530 struct file *file;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 char name[13];
532 int id;
KOSAKI Motohiroca16d142011-05-26 19:16:19 +0900533 vm_flags_t acctflag = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534
Kirill Korotaev4e982312006-10-02 02:18:22 -0700535 if (size < SHMMIN || size > ns->shm_ctlmax)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 return -EINVAL;
537
Manfred Spraul13763272014-06-06 14:37:41 -0700538 if (numpages << PAGE_SHIFT < size)
539 return -ENOSPC;
540
Manfred Spraul09c6eb12014-06-06 14:37:40 -0700541 if (ns->shm_tot + numpages < ns->shm_tot ||
542 ns->shm_tot + numpages > ns->shm_ctlall)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 return -ENOSPC;
544
545 shp = ipc_rcu_alloc(sizeof(*shp));
546 if (!shp)
547 return -ENOMEM;
548
549 shp->shm_perm.key = key;
Andrew Mortonb33291c2006-01-08 01:02:21 -0800550 shp->shm_perm.mode = (shmflg & S_IRWXUGO);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 shp->mlock_user = NULL;
552
553 shp->shm_perm.security = NULL;
554 error = security_shm_alloc(shp);
555 if (error) {
Davidlohr Bueso53dad6d2013-09-23 17:04:45 -0700556 ipc_rcu_putref(shp, ipc_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 return error;
558 }
559
Manfred Spraul239521f2014-01-27 17:07:04 -0800560 sprintf(name, "SYSV%08x", key);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 if (shmflg & SHM_HUGETLB) {
Andrew Mortonc103a4d2013-07-08 16:01:08 -0700562 struct hstate *hs;
Li Zefan091d0d52013-05-09 15:08:15 +0800563 size_t hugesize;
564
Andrew Mortonc103a4d2013-07-08 16:01:08 -0700565 hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
Li Zefan091d0d52013-05-09 15:08:15 +0800566 if (!hs) {
567 error = -EINVAL;
568 goto no_file;
569 }
570 hugesize = ALIGN(size, huge_page_size(hs));
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700571
Mel Gorman5a6fe122009-02-10 14:02:27 +0000572 /* hugetlb_file_setup applies strict accounting */
573 if (shmflg & SHM_NORESERVE)
574 acctflag = VM_NORESERVE;
Naoya Horiguchiaf73e4d2013-05-07 16:18:13 -0700575 file = hugetlb_file_setup(name, hugesize, acctflag,
Andi Kleen42d73952012-12-11 16:01:34 -0800576 &shp->mlock_user, HUGETLB_SHMFS_INODE,
577 (shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 } else {
Badari Pulavartybf8f9722005-11-07 00:59:27 -0800579 /*
580 * Do not allow no accounting for OVERCOMMIT_NEVER, even
Manfred Spraul239521f2014-01-27 17:07:04 -0800581 * if it's asked for.
Badari Pulavartybf8f9722005-11-07 00:59:27 -0800582 */
583 if ((shmflg & SHM_NORESERVE) &&
584 sysctl_overcommit_memory != OVERCOMMIT_NEVER)
Linus Torvaldsfc8744a2009-01-31 15:08:56 -0800585 acctflag = VM_NORESERVE;
Stephen Smalleye1832f22015-08-06 15:46:55 -0700586 file = shmem_kernel_file_setup(name, size, acctflag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 }
588 error = PTR_ERR(file);
589 if (IS_ERR(file))
590 goto no_file;
591
Pavel Emelyanovb4888932007-10-18 23:40:14 -0700592 shp->shm_cprid = task_tgid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 shp->shm_lprid = 0;
594 shp->shm_atim = shp->shm_dtim = 0;
595 shp->shm_ctim = get_seconds();
596 shp->shm_segsz = size;
597 shp->shm_nattch = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 shp->shm_file = file;
Vasiliy Kulikov5774ed02011-07-29 03:55:31 +0400599 shp->shm_creator = current;
Linus Torvaldsb9a53222015-09-30 12:48:40 -0400600
601 id = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
602 if (id < 0) {
603 error = id;
604 goto no_id;
605 }
606
Jack Millerab602f72014-08-08 14:23:19 -0700607 list_add(&shp->shm_clist, &current->sysvshm.shm_clist);
Davidlohr Buesodbfcd912013-07-08 16:01:09 -0700608
Badari Pulavarty30475cc2007-06-16 10:15:59 -0700609 /*
610 * shmid gets reported as "inode#" in /proc/pid/maps.
611 * proc-ps tools use this. Changing this will break them.
612 */
Al Viro496ad9a2013-01-23 17:07:38 -0500613 file_inode(file)->i_ino = shp->shm_perm.id;
Krishnakumar R551110a2005-10-29 18:16:45 -0700614
Kirill Korotaev4e982312006-10-02 02:18:22 -0700615 ns->shm_tot += numpages;
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700616 error = shp->shm_perm.id;
Davidlohr Buesodbfcd912013-07-08 16:01:09 -0700617
Davidlohr Buesocf9d5d72013-07-08 16:01:11 -0700618 ipc_unlock_object(&shp->shm_perm);
Davidlohr Buesodbfcd912013-07-08 16:01:09 -0700619 rcu_read_unlock();
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700620 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621
622no_id:
Hugh Dickins2195d282009-09-12 12:21:27 +0100623 if (is_file_hugepages(file) && shp->mlock_user)
Hugh Dickins353d5c32009-08-24 16:30:28 +0100624 user_shm_unlock(size, shp->mlock_user);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 fput(file);
626no_file:
Davidlohr Bueso53dad6d2013-09-23 17:04:45 -0700627 ipc_rcu_putref(shp, shm_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 return error;
629}
630
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700631/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700632 * Called with shm_ids.rwsem and ipcp locked.
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700633 */
Nadia Derbey03f02c72007-10-18 23:40:51 -0700634static inline int shm_security(struct kern_ipc_perm *ipcp, int shmflg)
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700635{
Nadia Derbey03f02c72007-10-18 23:40:51 -0700636 struct shmid_kernel *shp;
637
638 shp = container_of(ipcp, struct shmid_kernel, shm_perm);
639 return security_shm_associate(shp, shmflg);
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700640}
641
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700642/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700643 * Called with shm_ids.rwsem and ipcp locked.
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700644 */
Nadia Derbey03f02c72007-10-18 23:40:51 -0700645static inline int shm_more_checks(struct kern_ipc_perm *ipcp,
646 struct ipc_params *params)
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700647{
Nadia Derbey03f02c72007-10-18 23:40:51 -0700648 struct shmid_kernel *shp;
649
650 shp = container_of(ipcp, struct shmid_kernel, shm_perm);
651 if (shp->shm_segsz < params->u.size)
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700652 return -EINVAL;
653
654 return 0;
655}
656
Heiko Carstensd5460c92009-01-14 14:14:27 +0100657SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658{
Kirill Korotaev4e982312006-10-02 02:18:22 -0700659 struct ipc_namespace *ns;
Mathias Krauseeb66ec42014-06-06 14:37:36 -0700660 static const struct ipc_ops shm_ops = {
661 .getnew = newseg,
662 .associate = shm_security,
663 .more_checks = shm_more_checks,
664 };
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700665 struct ipc_params shm_params;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
Kirill Korotaev4e982312006-10-02 02:18:22 -0700667 ns = current->nsproxy->ipc_ns;
668
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700669 shm_params.key = key;
670 shm_params.flg = shmflg;
671 shm_params.u.size = size;
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700672
Nadia Derbey7748dbf2007-10-18 23:40:49 -0700673 return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674}
675
676static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
677{
Manfred Spraul239521f2014-01-27 17:07:04 -0800678 switch (version) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 case IPC_64:
680 return copy_to_user(buf, in, sizeof(*in));
681 case IPC_OLD:
682 {
683 struct shmid_ds out;
684
Vasiliy Kulikov3af54c92010-10-30 18:22:49 +0400685 memset(&out, 0, sizeof(out));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
687 out.shm_segsz = in->shm_segsz;
688 out.shm_atime = in->shm_atime;
689 out.shm_dtime = in->shm_dtime;
690 out.shm_ctime = in->shm_ctime;
691 out.shm_cpid = in->shm_cpid;
692 out.shm_lpid = in->shm_lpid;
693 out.shm_nattch = in->shm_nattch;
694
695 return copy_to_user(buf, &out, sizeof(out));
696 }
697 default:
698 return -EINVAL;
699 }
700}
701
Pierre Peiffer016d7132008-04-29 01:00:50 -0700702static inline unsigned long
703copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704{
Manfred Spraul239521f2014-01-27 17:07:04 -0800705 switch (version) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 case IPC_64:
Pierre Peiffer016d7132008-04-29 01:00:50 -0700707 if (copy_from_user(out, buf, sizeof(*out)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 case IPC_OLD:
711 {
712 struct shmid_ds tbuf_old;
713
714 if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
715 return -EFAULT;
716
Pierre Peiffer016d7132008-04-29 01:00:50 -0700717 out->shm_perm.uid = tbuf_old.shm_perm.uid;
718 out->shm_perm.gid = tbuf_old.shm_perm.gid;
719 out->shm_perm.mode = tbuf_old.shm_perm.mode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
721 return 0;
722 }
723 default:
724 return -EINVAL;
725 }
726}
727
728static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
729{
Manfred Spraul239521f2014-01-27 17:07:04 -0800730 switch (version) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 case IPC_64:
732 return copy_to_user(buf, in, sizeof(*in));
733 case IPC_OLD:
734 {
735 struct shminfo out;
736
Manfred Spraul239521f2014-01-27 17:07:04 -0800737 if (in->shmmax > INT_MAX)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 out.shmmax = INT_MAX;
739 else
740 out.shmmax = (int)in->shmmax;
741
742 out.shmmin = in->shmmin;
743 out.shmmni = in->shmmni;
744 out.shmseg = in->shmseg;
Paul McQuade46c0a8c2014-06-06 14:37:37 -0700745 out.shmall = in->shmall;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
747 return copy_to_user(buf, &out, sizeof(out));
748 }
749 default:
750 return -EINVAL;
751 }
752}
753
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700754/*
Helge Dellerb7952182010-10-27 15:34:16 -0700755 * Calculate and add used RSS and swap pages of a shm.
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700756 * Called with shm_ids.rwsem held as a reader
Helge Dellerb7952182010-10-27 15:34:16 -0700757 */
758static void shm_add_rss_swap(struct shmid_kernel *shp,
759 unsigned long *rss_add, unsigned long *swp_add)
760{
761 struct inode *inode;
762
Al Viro496ad9a2013-01-23 17:07:38 -0500763 inode = file_inode(shp->shm_file);
Helge Dellerb7952182010-10-27 15:34:16 -0700764
765 if (is_file_hugepages(shp->shm_file)) {
766 struct address_space *mapping = inode->i_mapping;
767 struct hstate *h = hstate_file(shp->shm_file);
768 *rss_add += pages_per_huge_page(h) * mapping->nrpages;
769 } else {
770#ifdef CONFIG_SHMEM
771 struct shmem_inode_info *info = SHMEM_I(inode);
Shailesh Pandey63980c82016-12-14 15:06:10 -0800772
Kirill A. Shutemov4595ef82016-07-26 15:26:29 -0700773 spin_lock_irq(&info->lock);
Helge Dellerb7952182010-10-27 15:34:16 -0700774 *rss_add += inode->i_mapping->nrpages;
775 *swp_add += info->swapped;
Kirill A. Shutemov4595ef82016-07-26 15:26:29 -0700776 spin_unlock_irq(&info->lock);
Helge Dellerb7952182010-10-27 15:34:16 -0700777#else
778 *rss_add += inode->i_mapping->nrpages;
779#endif
780 }
781}
782
783/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700784 * Called with shm_ids.rwsem held as a reader
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700785 */
Kirill Korotaev4e982312006-10-02 02:18:22 -0700786static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
787 unsigned long *swp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788{
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700789 int next_id;
790 int total, in_use;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791
792 *rss = 0;
793 *swp = 0;
794
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700795 in_use = shm_ids(ns).in_use;
796
797 for (total = 0, next_id = 0; total < in_use; next_id++) {
Tony Battersbye562aeb2009-04-02 16:58:26 -0700798 struct kern_ipc_perm *ipc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 struct shmid_kernel *shp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800
Tony Battersbye562aeb2009-04-02 16:58:26 -0700801 ipc = idr_find(&shm_ids(ns).ipcs_idr, next_id);
802 if (ipc == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 continue;
Tony Battersbye562aeb2009-04-02 16:58:26 -0700804 shp = container_of(ipc, struct shmid_kernel, shm_perm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805
Helge Dellerb7952182010-10-27 15:34:16 -0700806 shm_add_rss_swap(shp, rss, swp);
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700807
808 total++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 }
810}
811
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700812/*
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700813 * This function handles some shmctl commands which require the rwsem
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700814 * to be held in write mode.
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700815 * NOTE: no locks must be held, the rwsem is taken inside this function.
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700816 */
817static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
818 struct shmid_ds __user *buf, int version)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819{
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700820 struct kern_ipc_perm *ipcp;
Pierre Peiffer016d7132008-04-29 01:00:50 -0700821 struct shmid64_ds shmid64;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 struct shmid_kernel *shp;
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700823 int err;
824
825 if (cmd == IPC_SET) {
Pierre Peiffer016d7132008-04-29 01:00:50 -0700826 if (copy_shmid_from_user(&shmid64, buf, version))
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700827 return -EFAULT;
828 }
829
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700830 down_write(&shm_ids(ns).rwsem);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -0700831 rcu_read_lock();
832
Davidlohr Bueso79ccf0f2013-09-11 14:26:16 -0700833 ipcp = ipcctl_pre_down_nolock(ns, &shm_ids(ns), shmid, cmd,
834 &shmid64.shm_perm, 0);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -0700835 if (IS_ERR(ipcp)) {
836 err = PTR_ERR(ipcp);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -0700837 goto out_unlock1;
838 }
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700839
Pierre Peiffera5f75e72008-04-29 01:00:54 -0700840 shp = container_of(ipcp, struct shmid_kernel, shm_perm);
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700841
842 err = security_shm_shmctl(shp, cmd);
843 if (err)
Davidlohr Bueso79ccf0f2013-09-11 14:26:16 -0700844 goto out_unlock1;
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -0700845
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700846 switch (cmd) {
847 case IPC_RMID:
Davidlohr Bueso79ccf0f2013-09-11 14:26:16 -0700848 ipc_lock_object(&shp->shm_perm);
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -0700849 /* do_shm_rmid unlocks the ipc object and rcu */
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700850 do_shm_rmid(ns, ipcp);
851 goto out_up;
852 case IPC_SET:
Davidlohr Bueso79ccf0f2013-09-11 14:26:16 -0700853 ipc_lock_object(&shp->shm_perm);
Eric W. Biederman1efdb692012-02-07 16:54:11 -0800854 err = ipc_update_perm(&shmid64.shm_perm, ipcp);
855 if (err)
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -0700856 goto out_unlock0;
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700857 shp->shm_ctim = get_seconds();
858 break;
859 default:
860 err = -EINVAL;
Davidlohr Bueso79ccf0f2013-09-11 14:26:16 -0700861 goto out_unlock1;
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700862 }
Davidlohr Bueso7b4cc5d2013-07-08 16:01:12 -0700863
864out_unlock0:
865 ipc_unlock_object(&shp->shm_perm);
866out_unlock1:
867 rcu_read_unlock();
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700868out_up:
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700869 up_write(&shm_ids(ns).rwsem);
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700870 return err;
871}
872
Davidlohr Bueso68eccc12013-09-11 14:26:18 -0700873static int shmctl_nolock(struct ipc_namespace *ns, int shmid,
874 int cmd, int version, void __user *buf)
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700875{
Davidlohr Bueso68eccc12013-09-11 14:26:18 -0700876 int err;
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -0700877 struct shmid_kernel *shp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878
Davidlohr Bueso68eccc12013-09-11 14:26:18 -0700879 /* preliminary security checks for *_INFO */
880 if (cmd == IPC_INFO || cmd == SHM_INFO) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881 err = security_shm_shmctl(NULL, cmd);
882 if (err)
883 return err;
Davidlohr Bueso68eccc12013-09-11 14:26:18 -0700884 }
885
886 switch (cmd) {
887 case IPC_INFO:
888 {
889 struct shminfo64 shminfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890
WANG Conge8148f72009-01-06 14:42:49 -0800891 memset(&shminfo, 0, sizeof(shminfo));
Kirill Korotaev4e982312006-10-02 02:18:22 -0700892 shminfo.shmmni = shminfo.shmseg = ns->shm_ctlmni;
893 shminfo.shmmax = ns->shm_ctlmax;
894 shminfo.shmall = ns->shm_ctlall;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895
896 shminfo.shmmin = SHMMIN;
Manfred Spraul239521f2014-01-27 17:07:04 -0800897 if (copy_shminfo_to_user(buf, &shminfo, version))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898 return -EFAULT;
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700899
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700900 down_read(&shm_ids(ns).rwsem);
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700901 err = ipc_get_maxid(&shm_ids(ns));
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700902 up_read(&shm_ids(ns).rwsem);
Nadia Derbeyf4566f02007-10-18 23:40:53 -0700903
Manfred Spraul239521f2014-01-27 17:07:04 -0800904 if (err < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905 err = 0;
906 goto out;
907 }
908 case SHM_INFO:
909 {
910 struct shm_info shm_info;
911
WANG Conge8148f72009-01-06 14:42:49 -0800912 memset(&shm_info, 0, sizeof(shm_info));
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700913 down_read(&shm_ids(ns).rwsem);
Kirill Korotaev4e982312006-10-02 02:18:22 -0700914 shm_info.used_ids = shm_ids(ns).in_use;
Manfred Spraul239521f2014-01-27 17:07:04 -0800915 shm_get_stat(ns, &shm_info.shm_rss, &shm_info.shm_swp);
Kirill Korotaev4e982312006-10-02 02:18:22 -0700916 shm_info.shm_tot = ns->shm_tot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 shm_info.swap_attempts = 0;
918 shm_info.swap_successes = 0;
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700919 err = ipc_get_maxid(&shm_ids(ns));
Davidlohr Buesod9a605e2013-09-11 14:26:24 -0700920 up_read(&shm_ids(ns).rwsem);
WANG Conge8148f72009-01-06 14:42:49 -0800921 if (copy_to_user(buf, &shm_info, sizeof(shm_info))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922 err = -EFAULT;
923 goto out;
924 }
925
926 err = err < 0 ? 0 : err;
927 goto out;
928 }
929 case SHM_STAT:
930 case IPC_STAT:
931 {
932 struct shmid64_ds tbuf;
933 int result;
Nadia Derbey023a5352007-10-18 23:40:51 -0700934
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700935 rcu_read_lock();
Nadia Derbey023a5352007-10-18 23:40:51 -0700936 if (cmd == SHM_STAT) {
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700937 shp = shm_obtain_object(ns, shmid);
Nadia Derbey023a5352007-10-18 23:40:51 -0700938 if (IS_ERR(shp)) {
939 err = PTR_ERR(shp);
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700940 goto out_unlock;
Nadia Derbey023a5352007-10-18 23:40:51 -0700941 }
Nadia Derbey7ca7e562007-10-18 23:40:48 -0700942 result = shp->shm_perm.id;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 } else {
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700944 shp = shm_obtain_object_check(ns, shmid);
Nadia Derbey023a5352007-10-18 23:40:51 -0700945 if (IS_ERR(shp)) {
946 err = PTR_ERR(shp);
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700947 goto out_unlock;
Nadia Derbey023a5352007-10-18 23:40:51 -0700948 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949 result = 0;
950 }
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700951
WANG Conge8148f72009-01-06 14:42:49 -0800952 err = -EACCES;
Serge E. Hallynb0e77592011-03-23 16:43:24 -0700953 if (ipcperms(ns, &shp->shm_perm, S_IRUGO))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 goto out_unlock;
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700955
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956 err = security_shm_shmctl(shp, cmd);
957 if (err)
958 goto out_unlock;
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700959
Nadia Derbey023a5352007-10-18 23:40:51 -0700960 memset(&tbuf, 0, sizeof(tbuf));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961 kernel_to_ipc64_perm(&shp->shm_perm, &tbuf.shm_perm);
962 tbuf.shm_segsz = shp->shm_segsz;
963 tbuf.shm_atime = shp->shm_atim;
964 tbuf.shm_dtime = shp->shm_dtim;
965 tbuf.shm_ctime = shp->shm_ctim;
966 tbuf.shm_cpid = shp->shm_cprid;
967 tbuf.shm_lpid = shp->shm_lprid;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -0800968 tbuf.shm_nattch = shp->shm_nattch;
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700969 rcu_read_unlock();
970
971 if (copy_shmid_to_user(buf, &tbuf, version))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 err = -EFAULT;
973 else
974 err = result;
975 goto out;
976 }
Davidlohr Bueso68eccc12013-09-11 14:26:18 -0700977 default:
978 return -EINVAL;
979 }
980
981out_unlock:
Davidlohr Buesoc97cb9c2013-09-11 14:26:20 -0700982 rcu_read_unlock();
Davidlohr Bueso68eccc12013-09-11 14:26:18 -0700983out:
984 return err;
985}
986
987SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
988{
989 struct shmid_kernel *shp;
990 int err, version;
991 struct ipc_namespace *ns;
992
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -0700993 if (cmd < 0 || shmid < 0)
994 return -EINVAL;
Davidlohr Bueso68eccc12013-09-11 14:26:18 -0700995
996 version = ipc_parse_version(&cmd);
997 ns = current->nsproxy->ipc_ns;
998
999 switch (cmd) {
1000 case IPC_INFO:
1001 case SHM_INFO:
1002 case SHM_STAT:
1003 case IPC_STAT:
1004 return shmctl_nolock(ns, shmid, cmd, version, buf);
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001005 case IPC_RMID:
1006 case IPC_SET:
1007 return shmctl_down(ns, shmid, cmd, buf, version);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008 case SHM_LOCK:
1009 case SHM_UNLOCK:
1010 {
Hugh Dickins850465792012-01-20 14:34:19 -08001011 struct file *shm_file;
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07001012
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001013 rcu_read_lock();
1014 shp = shm_obtain_object_check(ns, shmid);
Nadia Derbey023a5352007-10-18 23:40:51 -07001015 if (IS_ERR(shp)) {
1016 err = PTR_ERR(shp);
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001017 goto out_unlock1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019
Al Viroa33e6752008-12-10 03:40:06 -05001020 audit_ipc_obj(&(shp->shm_perm));
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001021 err = security_shm_shmctl(shp, cmd);
1022 if (err)
1023 goto out_unlock1;
Steve Grubb073115d2006-04-02 17:07:33 -04001024
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001025 ipc_lock_object(&shp->shm_perm);
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08001026
1027 /* check if shm_destroy() is tearing down shp */
1028 if (!ipc_valid_object(&shp->shm_perm)) {
1029 err = -EIDRM;
1030 goto out_unlock0;
1031 }
1032
Serge E. Hallynb0e77592011-03-23 16:43:24 -07001033 if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
Eric W. Biederman1efdb692012-02-07 16:54:11 -08001034 kuid_t euid = current_euid();
Shailesh Pandey63980c82016-12-14 15:06:10 -08001035
Eric W. Biederman1efdb692012-02-07 16:54:11 -08001036 if (!uid_eq(euid, shp->shm_perm.uid) &&
Jesper Nilsson3a726602013-11-21 14:32:08 -08001037 !uid_eq(euid, shp->shm_perm.cuid)) {
1038 err = -EPERM;
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001039 goto out_unlock0;
Jesper Nilsson3a726602013-11-21 14:32:08 -08001040 }
1041 if (cmd == SHM_LOCK && !rlimit(RLIMIT_MEMLOCK)) {
1042 err = -EPERM;
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001043 goto out_unlock0;
Jesper Nilsson3a726602013-11-21 14:32:08 -08001044 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 }
1046
Hugh Dickins850465792012-01-20 14:34:19 -08001047 shm_file = shp->shm_file;
1048 if (is_file_hugepages(shm_file))
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001049 goto out_unlock0;
Hugh Dickins850465792012-01-20 14:34:19 -08001050
1051 if (cmd == SHM_LOCK) {
David Howells86a264a2008-11-14 10:39:18 +11001052 struct user_struct *user = current_user();
Shailesh Pandey63980c82016-12-14 15:06:10 -08001053
Hugh Dickins850465792012-01-20 14:34:19 -08001054 err = shmem_lock(shm_file, 1, user);
1055 if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
1056 shp->shm_perm.mode |= SHM_LOCKED;
1057 shp->mlock_user = user;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 }
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001059 goto out_unlock0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 }
Hugh Dickins850465792012-01-20 14:34:19 -08001061
1062 /* SHM_UNLOCK */
1063 if (!(shp->shm_perm.mode & SHM_LOCKED))
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001064 goto out_unlock0;
Hugh Dickins850465792012-01-20 14:34:19 -08001065 shmem_lock(shm_file, 0, shp->mlock_user);
1066 shp->shm_perm.mode &= ~SHM_LOCKED;
1067 shp->mlock_user = NULL;
1068 get_file(shm_file);
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001069 ipc_unlock_object(&shp->shm_perm);
1070 rcu_read_unlock();
Hugh Dickins24513262012-01-20 14:34:21 -08001071 shmem_unlock_mapping(shm_file->f_mapping);
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001072
Hugh Dickins850465792012-01-20 14:34:19 -08001073 fput(shm_file);
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -07001074 return err;
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001075 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076 default:
Pierre Peiffer8d4cc8b2008-04-29 01:00:47 -07001077 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078 }
1079
Davidlohr Bueso2caacaa2013-09-11 14:26:21 -07001080out_unlock0:
1081 ipc_unlock_object(&shp->shm_perm);
1082out_unlock1:
1083 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084 return err;
1085}
1086
1087/*
1088 * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
1089 *
1090 * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
1091 * "raddr" thing points to kernel space, and there has to be a wrapper around
1092 * this.
1093 */
Will Deacon079a96a2012-07-30 14:42:38 -07001094long do_shmat(int shmid, char __user *shmaddr, int shmflg, ulong *raddr,
1095 unsigned long shmlba)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096{
1097 struct shmid_kernel *shp;
1098 unsigned long addr;
1099 unsigned long size;
Manfred Spraul239521f2014-01-27 17:07:04 -08001100 struct file *file;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001101 int err;
1102 unsigned long flags;
1103 unsigned long prot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104 int acc_mode;
Kirill Korotaev4e982312006-10-02 02:18:22 -07001105 struct ipc_namespace *ns;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001106 struct shm_file_data *sfd;
1107 struct path path;
Al Viroaeb5d722008-09-02 15:28:45 -04001108 fmode_t f_mode;
Michel Lespinasse41badc12013-02-22 16:32:47 -08001109 unsigned long populate = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001111 err = -EINVAL;
1112 if (shmid < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 goto out;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001114 else if ((addr = (ulong)shmaddr)) {
Will Deacon079a96a2012-07-30 14:42:38 -07001115 if (addr & (shmlba - 1)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116 if (shmflg & SHM_RND)
Will Deacon079a96a2012-07-30 14:42:38 -07001117 addr &= ~(shmlba - 1); /* round down */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 else
1119#ifndef __ARCH_FORCE_SHMLBA
1120 if (addr & ~PAGE_MASK)
1121#endif
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001122 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123 }
1124 flags = MAP_SHARED | MAP_FIXED;
1125 } else {
1126 if ((shmflg & SHM_REMAP))
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001127 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128
1129 flags = MAP_SHARED;
1130 }
1131
1132 if (shmflg & SHM_RDONLY) {
1133 prot = PROT_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 acc_mode = S_IRUGO;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001135 f_mode = FMODE_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136 } else {
1137 prot = PROT_READ | PROT_WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 acc_mode = S_IRUGO | S_IWUGO;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001139 f_mode = FMODE_READ | FMODE_WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140 }
1141 if (shmflg & SHM_EXEC) {
1142 prot |= PROT_EXEC;
1143 acc_mode |= S_IXUGO;
1144 }
1145
1146 /*
1147 * We cannot rely on the fs check since SYSV IPC does have an
1148 * additional creator id...
1149 */
Kirill Korotaev4e982312006-10-02 02:18:22 -07001150 ns = current->nsproxy->ipc_ns;
Davidlohr Buesoc2c737a2013-09-11 14:26:23 -07001151 rcu_read_lock();
1152 shp = shm_obtain_object_check(ns, shmid);
Nadia Derbey023a5352007-10-18 23:40:51 -07001153 if (IS_ERR(shp)) {
1154 err = PTR_ERR(shp);
Davidlohr Buesoc2c737a2013-09-11 14:26:23 -07001155 goto out_unlock;
Nadia Derbey023a5352007-10-18 23:40:51 -07001156 }
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001157
1158 err = -EACCES;
Serge E. Hallynb0e77592011-03-23 16:43:24 -07001159 if (ipcperms(ns, &shp->shm_perm, acc_mode))
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001160 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161
1162 err = security_shm_shmat(shp, shmaddr, shmflg);
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001163 if (err)
1164 goto out_unlock;
1165
Davidlohr Buesoc2c737a2013-09-11 14:26:23 -07001166 ipc_lock_object(&shp->shm_perm);
Greg Thelena399b292013-11-21 14:32:00 -08001167
1168 /* check if shm_destroy() is tearing down shp */
Rafael Aquini0f3d2b02014-01-27 17:07:01 -08001169 if (!ipc_valid_object(&shp->shm_perm)) {
Greg Thelena399b292013-11-21 14:32:00 -08001170 ipc_unlock_object(&shp->shm_perm);
1171 err = -EIDRM;
1172 goto out_unlock;
1173 }
1174
Al Viro2c48b9c2009-08-09 00:52:35 +04001175 path = shp->shm_file->f_path;
1176 path_get(&path);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 shp->shm_nattch++;
David Howells75c3cfa2015-03-17 22:26:12 +00001178 size = i_size_read(d_inode(path.dentry));
Davidlohr Buesoc2c737a2013-09-11 14:26:23 -07001179 ipc_unlock_object(&shp->shm_perm);
1180 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001182 err = -ENOMEM;
1183 sfd = kzalloc(sizeof(*sfd), GFP_KERNEL);
Davidlohr Buesof42569b2013-09-11 14:26:22 -07001184 if (!sfd) {
1185 path_put(&path);
1186 goto out_nattch;
1187 }
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001188
Al Viro2c48b9c2009-08-09 00:52:35 +04001189 file = alloc_file(&path, f_mode,
1190 is_file_hugepages(shp->shm_file) ?
Al Viroc4caa772009-11-30 08:38:43 -05001191 &shm_file_operations_huge :
1192 &shm_file_operations);
Anatol Pomozov39b65252012-09-12 20:11:55 -07001193 err = PTR_ERR(file);
Davidlohr Buesof42569b2013-09-11 14:26:22 -07001194 if (IS_ERR(file)) {
1195 kfree(sfd);
1196 path_put(&path);
1197 goto out_nattch;
1198 }
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001199
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001200 file->private_data = sfd;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001201 file->f_mapping = shp->shm_file->f_mapping;
Nadia Derbey7ca7e562007-10-18 23:40:48 -07001202 sfd->id = shp->shm_perm.id;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001203 sfd->ns = get_ipc_ns(ns);
1204 sfd->file = shp->shm_file;
1205 sfd->vm_ops = NULL;
1206
Al Viro8b3ec682012-05-30 17:11:23 -04001207 err = security_mmap_file(file, prot, flags);
1208 if (err)
1209 goto out_fput;
1210
Michal Hocko91f4f942016-05-23 16:25:51 -07001211 if (down_write_killable(&current->mm->mmap_sem)) {
1212 err = -EINTR;
1213 goto out_fput;
1214 }
1215
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216 if (addr && !(shmflg & SHM_REMAP)) {
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001217 err = -EINVAL;
Manfred Spraul247a8ce2014-06-06 14:37:38 -07001218 if (addr + size < addr)
1219 goto invalid;
1220
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221 if (find_vma_intersection(current->mm, addr, addr + size))
1222 goto invalid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223 }
Davidlohr Buesof42569b2013-09-11 14:26:22 -07001224
Michel Lespinassebebeb3d2013-02-22 16:32:37 -08001225 addr = do_mmap_pgoff(file, addr, size, prot, flags, 0, &populate);
1226 *raddr = addr;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001227 err = 0;
Michel Lespinassebebeb3d2013-02-22 16:32:37 -08001228 if (IS_ERR_VALUE(addr))
1229 err = (long)addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230invalid:
1231 up_write(&current->mm->mmap_sem);
Michel Lespinassebebeb3d2013-02-22 16:32:37 -08001232 if (populate)
Michel Lespinasse41badc12013-02-22 16:32:47 -08001233 mm_populate(addr, populate);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234
Al Viro8b3ec682012-05-30 17:11:23 -04001235out_fput:
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001236 fput(file);
1237
1238out_nattch:
Davidlohr Buesod9a605e2013-09-11 14:26:24 -07001239 down_write(&shm_ids(ns).rwsem);
Nadia Derbey00c2bf82008-07-25 01:48:03 -07001240 shp = shm_lock(ns, shmid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 shp->shm_nattch--;
Vasiliy Kulikovb34a6b12011-07-26 16:08:48 -07001242 if (shm_may_destroy(ns, shp))
Kirill Korotaev4e982312006-10-02 02:18:22 -07001243 shm_destroy(ns, shp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244 else
1245 shm_unlock(shp);
Davidlohr Buesod9a605e2013-09-11 14:26:24 -07001246 up_write(&shm_ids(ns).rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247 return err;
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001248
1249out_unlock:
Davidlohr Buesoc2c737a2013-09-11 14:26:23 -07001250 rcu_read_unlock();
Davidlohr Buesof42569b2013-09-11 14:26:22 -07001251out:
1252 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253}
1254
Heiko Carstensd5460c92009-01-14 14:14:27 +01001255SYSCALL_DEFINE3(shmat, int, shmid, char __user *, shmaddr, int, shmflg)
Stephen Rothwell7d87e14c2005-05-01 08:59:12 -07001256{
1257 unsigned long ret;
1258 long err;
1259
Will Deacon079a96a2012-07-30 14:42:38 -07001260 err = do_shmat(shmid, shmaddr, shmflg, &ret, SHMLBA);
Stephen Rothwell7d87e14c2005-05-01 08:59:12 -07001261 if (err)
1262 return err;
1263 force_successful_syscall_return();
1264 return (long)ret;
1265}
1266
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267/*
1268 * detach and kill segment if marked destroyed.
1269 * The work is done in shm_close.
1270 */
Heiko Carstensd5460c92009-01-14 14:14:27 +01001271SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272{
1273 struct mm_struct *mm = current->mm;
Mike Frysinger586c7e62009-06-09 16:26:23 -07001274 struct vm_area_struct *vma;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275 unsigned long addr = (unsigned long)shmaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 int retval = -EINVAL;
Mike Frysinger586c7e62009-06-09 16:26:23 -07001277#ifdef CONFIG_MMU
1278 loff_t size = 0;
Dave Hansend3c97902014-12-12 16:58:19 -08001279 struct file *file;
Mike Frysinger586c7e62009-06-09 16:26:23 -07001280 struct vm_area_struct *next;
1281#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282
Hugh Dickinsdf1e2fb2006-03-24 03:18:06 -08001283 if (addr & ~PAGE_MASK)
1284 return retval;
1285
Michal Hocko91f4f942016-05-23 16:25:51 -07001286 if (down_write_killable(&mm->mmap_sem))
1287 return -EINTR;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288
1289 /*
1290 * This function tries to be smart and unmap shm segments that
1291 * were modified by partial mlock or munmap calls:
1292 * - It first determines the size of the shm segment that should be
1293 * unmapped: It searches for a vma that is backed by shm and that
1294 * started at address shmaddr. It records it's size and then unmaps
1295 * it.
1296 * - Then it unmaps all shm vmas that started at shmaddr and that
Dave Hansend3c97902014-12-12 16:58:19 -08001297 * are within the initially determined size and that are from the
1298 * same shm segment from which we determined the size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 * Errors from do_munmap are ignored: the function only fails if
1300 * it's called with invalid parameters or if it's called to unmap
1301 * a part of a vma. Both calls in this function are for full vmas,
1302 * the parameters are directly copied from the vma itself and always
1303 * valid - therefore do_munmap cannot fail. (famous last words?)
1304 */
1305 /*
1306 * If it had been mremap()'d, the starting address would not
1307 * match the usual checks anyway. So assume all vma's are
1308 * above the starting address given.
1309 */
1310 vma = find_vma(mm, addr);
1311
David Howells8feae132009-01-08 12:04:47 +00001312#ifdef CONFIG_MMU
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 while (vma) {
1314 next = vma->vm_next;
1315
1316 /*
1317 * Check if the starting address would match, i.e. it's
1318 * a fragment created by mprotect() and/or munmap(), or it
1319 * otherwise it starts at this address with no hassles.
1320 */
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001321 if ((vma->vm_ops == &shm_vm_ops) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) {
1323
Dave Hansend3c97902014-12-12 16:58:19 -08001324 /*
1325 * Record the file of the shm segment being
1326 * unmapped. With mremap(), someone could place
1327 * page from another segment but with equal offsets
1328 * in the range we are unmapping.
1329 */
1330 file = vma->vm_file;
Dave Hansen07a46ed2014-12-12 16:58:22 -08001331 size = i_size_read(file_inode(vma->vm_file));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001332 do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
1333 /*
1334 * We discovered the size of the shm segment, so
1335 * break out of here and fall through to the next
1336 * loop that uses the size information to stop
1337 * searching for matching vma's.
1338 */
1339 retval = 0;
1340 vma = next;
1341 break;
1342 }
1343 vma = next;
1344 }
1345
1346 /*
1347 * We need look no further than the maximum address a fragment
1348 * could possibly have landed at. Also cast things to loff_t to
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001349 * prevent overflows and make comparisons vs. equal-width types.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350 */
KAMEZAWA Hiroyuki8e367092006-02-10 01:51:12 -08001351 size = PAGE_ALIGN(size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352 while (vma && (loff_t)(vma->vm_end - addr) <= size) {
1353 next = vma->vm_next;
1354
1355 /* finding a matching vma now does not alter retval */
Eric W. Biedermanbc56bba2007-02-20 13:57:53 -08001356 if ((vma->vm_ops == &shm_vm_ops) &&
Dave Hansend3c97902014-12-12 16:58:19 -08001357 ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) &&
1358 (vma->vm_file == file))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
1360 vma = next;
1361 }
1362
Shailesh Pandey63980c82016-12-14 15:06:10 -08001363#else /* CONFIG_MMU */
David Howells8feae132009-01-08 12:04:47 +00001364 /* under NOMMU conditions, the exact address to be destroyed must be
Shailesh Pandey63980c82016-12-14 15:06:10 -08001365 * given
1366 */
Davidlohr Bueso530fcd162013-09-11 14:26:28 -07001367 if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
David Howells8feae132009-01-08 12:04:47 +00001368 do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
1369 retval = 0;
1370 }
1371
1372#endif
1373
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374 up_write(&mm->mmap_sem);
1375 return retval;
1376}
1377
1378#ifdef CONFIG_PROC_FS
Mike Waychison19b49462005-09-06 15:17:10 -07001379static int sysvipc_shm_proc_show(struct seq_file *s, void *it)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380{
Eric W. Biederman1efdb692012-02-07 16:54:11 -08001381 struct user_namespace *user_ns = seq_user_ns(s);
Mike Waychison19b49462005-09-06 15:17:10 -07001382 struct shmid_kernel *shp = it;
Helge Dellerb7952182010-10-27 15:34:16 -07001383 unsigned long rss = 0, swp = 0;
1384
1385 shm_add_rss_swap(shp, &rss, &swp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386
Paul Menage6c826812008-06-12 15:21:49 -07001387#if BITS_PER_LONG <= 32
1388#define SIZE_SPEC "%10lu"
1389#else
1390#define SIZE_SPEC "%21lu"
1391#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392
Joe Perches7f032d62015-04-15 16:17:54 -07001393 seq_printf(s,
1394 "%10d %10d %4o " SIZE_SPEC " %5u %5u "
1395 "%5lu %5u %5u %5u %5u %10lu %10lu %10lu "
1396 SIZE_SPEC " " SIZE_SPEC "\n",
1397 shp->shm_perm.key,
1398 shp->shm_perm.id,
1399 shp->shm_perm.mode,
1400 shp->shm_segsz,
1401 shp->shm_cprid,
1402 shp->shm_lprid,
1403 shp->shm_nattch,
1404 from_kuid_munged(user_ns, shp->shm_perm.uid),
1405 from_kgid_munged(user_ns, shp->shm_perm.gid),
1406 from_kuid_munged(user_ns, shp->shm_perm.cuid),
1407 from_kgid_munged(user_ns, shp->shm_perm.cgid),
1408 shp->shm_atim,
1409 shp->shm_dtim,
1410 shp->shm_ctim,
1411 rss * PAGE_SIZE,
1412 swp * PAGE_SIZE);
1413
1414 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415}
1416#endif