blob: 755fd7e23ed4b6ba285ee7c3e053e182a17fa4f0 [file] [log] [blame]
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001/*
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002 * Derived from "arch/i386/kernel/process.c"
3 * Copyright (C) 1995 Linus Torvalds
4 *
5 * Updated and modified by Cort Dougan (cort@cs.nmt.edu) and
6 * Paul Mackerras (paulus@cs.anu.edu.au)
7 *
8 * PowerPC version
9 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
10 *
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version
14 * 2 of the License, or (at your option) any later version.
15 */
16
Paul Mackerras14cf11a2005-09-26 16:04:21 +100017#include <linux/errno.h>
18#include <linux/sched.h>
Ingo Molnarb17b0152017-02-08 18:51:35 +010019#include <linux/sched/debug.h>
Ingo Molnar29930022017-02-08 18:51:36 +010020#include <linux/sched/task.h>
Ingo Molnar68db0cf2017-02-08 18:51:37 +010021#include <linux/sched/task_stack.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100022#include <linux/kernel.h>
23#include <linux/mm.h>
24#include <linux/smp.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100025#include <linux/stddef.h>
26#include <linux/unistd.h>
27#include <linux/ptrace.h>
28#include <linux/slab.h>
29#include <linux/user.h>
30#include <linux/elf.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100031#include <linux/prctl.h>
32#include <linux/init_task.h>
Paul Gortmaker4b16f8e2011-07-22 18:24:23 -040033#include <linux/export.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100034#include <linux/kallsyms.h>
35#include <linux/mqueue.h>
36#include <linux/hardirq.h>
Paul Mackerras06d67d52005-10-10 22:29:05 +100037#include <linux/utsname.h>
Steven Rostedt6794c782009-02-09 21:10:27 -080038#include <linux/ftrace.h>
Martin Schwidefsky79741dd2008-12-31 15:11:38 +010039#include <linux/kernel_stat.h>
Anton Blanchardd8390882009-02-22 01:50:03 +000040#include <linux/personality.h>
41#include <linux/random.h>
K.Prasad5aae8a52010-06-15 11:35:19 +053042#include <linux/hw_breakpoint.h>
Anton Blanchard7b051f62014-10-13 20:27:15 +110043#include <linux/uaccess.h>
Daniel Axtens7f92bc52016-01-06 11:45:51 +110044#include <linux/elf-randomize.h>
Ram Pai06bb53b2018-01-18 17:50:31 -080045#include <linux/pkeys.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100046
47#include <asm/pgtable.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100048#include <asm/io.h>
49#include <asm/processor.h>
50#include <asm/mmu.h>
51#include <asm/prom.h>
Michael Ellerman76032de2005-11-07 13:12:03 +110052#include <asm/machdep.h>
Paul Mackerrasc6622f62006-02-24 10:06:59 +110053#include <asm/time.h>
David Howellsae3a1972012-03-28 18:30:02 +010054#include <asm/runlatch.h>
Arnd Bergmanna7f31842006-03-23 00:00:08 +010055#include <asm/syscalls.h>
David Howellsae3a1972012-03-28 18:30:02 +010056#include <asm/switch_to.h>
Michael Neulingfb096922013-02-13 16:21:37 +000057#include <asm/tm.h>
David Howellsae3a1972012-03-28 18:30:02 +010058#include <asm/debug.h>
Paul Mackerras06d67d52005-10-10 22:29:05 +100059#ifdef CONFIG_PPC64
60#include <asm/firmware.h>
Madhavan Srinivasanc2e480b2017-12-20 09:25:42 +053061#include <asm/hw_irq.h>
Paul Mackerras06d67d52005-10-10 22:29:05 +100062#endif
Anton Blanchard7cedd602014-02-04 16:08:51 +110063#include <asm/code-patching.h>
Daniel Axtens7f92bc52016-01-06 11:45:51 +110064#include <asm/exec.h>
Michael Ellerman5d31a962016-03-24 22:04:04 +110065#include <asm/livepatch.h>
Kevin Haob92a2262016-07-23 14:42:40 +053066#include <asm/cpu_has_feature.h>
Daniel Axtens0545d542016-09-06 15:32:43 +100067#include <asm/asm-prototypes.h>
Michael Ellerman5d31a962016-03-24 22:04:04 +110068
Luis Machadod6a61bf2008-07-24 02:10:41 +100069#include <linux/kprobes.h>
70#include <linux/kdebug.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100071
Michael Neuling8b3c34c2013-02-13 16:21:32 +000072/* Transactional Memory debug */
73#ifdef TM_DEBUG_SW
74#define TM_DEBUG(x...) printk(KERN_INFO x)
75#else
76#define TM_DEBUG(x...) do { } while(0)
77#endif
78
Paul Mackerras14cf11a2005-09-26 16:04:21 +100079extern unsigned long _get_SP(void);
80
Paul Mackerrasd31626f2014-01-13 15:56:29 +110081#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Michael Ellerman54820532017-10-12 21:17:18 +110082/*
83 * Are we running in "Suspend disabled" mode? If so we have to block any
84 * sigreturn that would get us into suspended state, and we also warn in some
85 * other paths that we should never reach with suspend disabled.
86 */
87bool tm_suspend_disabled __ro_after_init = false;
88
Anton Blanchardb86fd2b2015-10-29 11:43:58 +110089static void check_if_tm_restore_required(struct task_struct *tsk)
Paul Mackerrasd31626f2014-01-13 15:56:29 +110090{
91 /*
92 * If we are saving the current thread's registers, and the
93 * thread is in a transactional state, set the TIF_RESTORE_TM
94 * bit so that we know to restore the registers before
95 * returning to userspace.
96 */
97 if (tsk == current && tsk->thread.regs &&
98 MSR_TM_ACTIVE(tsk->thread.regs->msr) &&
99 !test_thread_flag(TIF_RESTORE_TM)) {
Anshuman Khandual829023d2015-07-06 16:24:10 +0530100 tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr;
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100101 set_thread_flag(TIF_RESTORE_TM);
102 }
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100103}
Cyril Burdc16b552016-09-23 16:18:08 +1000104
105static inline bool msr_tm_active(unsigned long msr)
106{
107 return MSR_TM_ACTIVE(msr);
108}
Cyril Bura7771172017-11-02 14:09:03 +1100109
110static bool tm_active_with_fp(struct task_struct *tsk)
111{
112 return msr_tm_active(tsk->thread.regs->msr) &&
113 (tsk->thread.ckpt_regs.msr & MSR_FP);
114}
115
116static bool tm_active_with_altivec(struct task_struct *tsk)
117{
118 return msr_tm_active(tsk->thread.regs->msr) &&
119 (tsk->thread.ckpt_regs.msr & MSR_VEC);
120}
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100121#else
Cyril Burdc16b552016-09-23 16:18:08 +1000122static inline bool msr_tm_active(unsigned long msr) { return false; }
Anton Blanchardb86fd2b2015-10-29 11:43:58 +1100123static inline void check_if_tm_restore_required(struct task_struct *tsk) { }
Cyril Bura7771172017-11-02 14:09:03 +1100124static inline bool tm_active_with_fp(struct task_struct *tsk) { return false; }
125static inline bool tm_active_with_altivec(struct task_struct *tsk) { return false; }
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100126#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
127
Anton Blanchard3eb5d582015-10-29 11:44:06 +1100128bool strict_msr_control;
129EXPORT_SYMBOL(strict_msr_control);
130
131static int __init enable_strict_msr_control(char *str)
132{
133 strict_msr_control = true;
134 pr_info("Enabling strict facility control\n");
135
136 return 0;
137}
138early_param("ppc_strict_facility_enable", enable_strict_msr_control);
139
Cyril Bur3cee0702016-09-23 16:18:10 +1000140unsigned long msr_check_and_set(unsigned long bits)
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100141{
142 unsigned long oldmsr = mfmsr();
143 unsigned long newmsr;
144
145 newmsr = oldmsr | bits;
146
147#ifdef CONFIG_VSX
148 if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
149 newmsr |= MSR_VSX;
150#endif
151
152 if (oldmsr != newmsr)
153 mtmsr_isync(newmsr);
Cyril Bur3cee0702016-09-23 16:18:10 +1000154
155 return newmsr;
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100156}
157
Anton Blanchard3eb5d582015-10-29 11:44:06 +1100158void __msr_check_and_clear(unsigned long bits)
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100159{
160 unsigned long oldmsr = mfmsr();
161 unsigned long newmsr;
162
163 newmsr = oldmsr & ~bits;
164
165#ifdef CONFIG_VSX
166 if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
167 newmsr &= ~MSR_VSX;
168#endif
169
170 if (oldmsr != newmsr)
171 mtmsr_isync(newmsr);
172}
Anton Blanchard3eb5d582015-10-29 11:44:06 +1100173EXPORT_SYMBOL(__msr_check_and_clear);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100174
Kevin Hao037f0ee2013-07-14 17:02:05 +0800175#ifdef CONFIG_PPC_FPU
Cyril Bur87924682016-02-29 17:53:49 +1100176void __giveup_fpu(struct task_struct *tsk)
177{
Anton Blanchard8eb98032016-05-29 22:03:50 +1000178 unsigned long msr;
179
Cyril Bur87924682016-02-29 17:53:49 +1100180 save_fpu(tsk);
Anton Blanchard8eb98032016-05-29 22:03:50 +1000181 msr = tsk->thread.regs->msr;
182 msr &= ~MSR_FP;
Cyril Bur87924682016-02-29 17:53:49 +1100183#ifdef CONFIG_VSX
184 if (cpu_has_feature(CPU_FTR_VSX))
Anton Blanchard8eb98032016-05-29 22:03:50 +1000185 msr &= ~MSR_VSX;
Cyril Bur87924682016-02-29 17:53:49 +1100186#endif
Anton Blanchard8eb98032016-05-29 22:03:50 +1000187 tsk->thread.regs->msr = msr;
Cyril Bur87924682016-02-29 17:53:49 +1100188}
189
Anton Blanchard98da5812015-10-29 11:44:01 +1100190void giveup_fpu(struct task_struct *tsk)
191{
Anton Blanchard98da5812015-10-29 11:44:01 +1100192 check_if_tm_restore_required(tsk);
193
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100194 msr_check_and_set(MSR_FP);
Anton Blanchard98da5812015-10-29 11:44:01 +1100195 __giveup_fpu(tsk);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100196 msr_check_and_clear(MSR_FP);
Anton Blanchard98da5812015-10-29 11:44:01 +1100197}
198EXPORT_SYMBOL(giveup_fpu);
199
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000200/*
201 * Make sure the floating-point register state in the
202 * the thread_struct is up to date for task tsk.
203 */
204void flush_fp_to_thread(struct task_struct *tsk)
205{
206 if (tsk->thread.regs) {
207 /*
208 * We need to disable preemption here because if we didn't,
209 * another process could get scheduled after the regs->msr
210 * test but before we have finished saving the FP registers
211 * to the thread_struct. That process could take over the
212 * FPU, and then when we get scheduled again we would store
213 * bogus values for the remaining FP registers.
214 */
215 preempt_disable();
216 if (tsk->thread.regs->msr & MSR_FP) {
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000217 /*
218 * This should only ever be called for current or
219 * for a stopped child process. Since we save away
Anton Blanchardaf1bbc32015-10-29 11:43:57 +1100220 * the FP register state on context switch,
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000221 * there is something wrong if a stopped child appears
222 * to still have its FP state in the CPU registers.
223 */
224 BUG_ON(tsk != current);
Anton Blanchardb86fd2b2015-10-29 11:43:58 +1100225 giveup_fpu(tsk);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000226 }
227 preempt_enable();
228 }
229}
Paul Mackerrasde56a942011-06-29 00:21:34 +0000230EXPORT_SYMBOL_GPL(flush_fp_to_thread);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000231
232void enable_kernel_fp(void)
233{
Cyril Bure909fb82016-09-23 16:18:11 +1000234 unsigned long cpumsr;
235
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000236 WARN_ON(preemptible());
237
Cyril Bure909fb82016-09-23 16:18:11 +1000238 cpumsr = msr_check_and_set(MSR_FP);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100239
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100240 if (current->thread.regs && (current->thread.regs->msr & MSR_FP)) {
241 check_if_tm_restore_required(current);
Cyril Bure909fb82016-09-23 16:18:11 +1000242 /*
243 * If a thread has already been reclaimed then the
244 * checkpointed registers are on the CPU but have definitely
245 * been saved by the reclaim code. Don't need to and *cannot*
246 * giveup as this would save to the 'live' structure not the
247 * checkpointed structure.
248 */
249 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
250 return;
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100251 __giveup_fpu(current);
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100252 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000253}
254EXPORT_SYMBOL(enable_kernel_fp);
Cyril Bur70fe3d92016-02-29 17:53:47 +1100255
Benjamin Herrenschmidt6a303832017-08-16 16:01:15 +1000256static int restore_fp(struct task_struct *tsk)
257{
Cyril Bura7771172017-11-02 14:09:03 +1100258 if (tsk->thread.load_fp || tm_active_with_fp(tsk)) {
Cyril Bur70fe3d92016-02-29 17:53:47 +1100259 load_fp_state(&current->thread.fp_state);
260 current->thread.load_fp++;
261 return 1;
262 }
263 return 0;
264}
265#else
266static int restore_fp(struct task_struct *tsk) { return 0; }
Anton Blanchardd1e1cf22015-10-29 11:44:11 +1100267#endif /* CONFIG_PPC_FPU */
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000268
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000269#ifdef CONFIG_ALTIVEC
Cyril Bur70fe3d92016-02-29 17:53:47 +1100270#define loadvec(thr) ((thr).load_vec)
271
Cyril Bur6f515d82016-02-29 17:53:50 +1100272static void __giveup_altivec(struct task_struct *tsk)
273{
Anton Blanchard8eb98032016-05-29 22:03:50 +1000274 unsigned long msr;
275
Cyril Bur6f515d82016-02-29 17:53:50 +1100276 save_altivec(tsk);
Anton Blanchard8eb98032016-05-29 22:03:50 +1000277 msr = tsk->thread.regs->msr;
278 msr &= ~MSR_VEC;
Cyril Bur6f515d82016-02-29 17:53:50 +1100279#ifdef CONFIG_VSX
280 if (cpu_has_feature(CPU_FTR_VSX))
Anton Blanchard8eb98032016-05-29 22:03:50 +1000281 msr &= ~MSR_VSX;
Cyril Bur6f515d82016-02-29 17:53:50 +1100282#endif
Anton Blanchard8eb98032016-05-29 22:03:50 +1000283 tsk->thread.regs->msr = msr;
Cyril Bur6f515d82016-02-29 17:53:50 +1100284}
285
Anton Blanchard98da5812015-10-29 11:44:01 +1100286void giveup_altivec(struct task_struct *tsk)
287{
Anton Blanchard98da5812015-10-29 11:44:01 +1100288 check_if_tm_restore_required(tsk);
289
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100290 msr_check_and_set(MSR_VEC);
Anton Blanchard98da5812015-10-29 11:44:01 +1100291 __giveup_altivec(tsk);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100292 msr_check_and_clear(MSR_VEC);
Anton Blanchard98da5812015-10-29 11:44:01 +1100293}
294EXPORT_SYMBOL(giveup_altivec);
295
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000296void enable_kernel_altivec(void)
297{
Cyril Bure909fb82016-09-23 16:18:11 +1000298 unsigned long cpumsr;
299
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000300 WARN_ON(preemptible());
301
Cyril Bure909fb82016-09-23 16:18:11 +1000302 cpumsr = msr_check_and_set(MSR_VEC);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100303
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100304 if (current->thread.regs && (current->thread.regs->msr & MSR_VEC)) {
305 check_if_tm_restore_required(current);
Cyril Bure909fb82016-09-23 16:18:11 +1000306 /*
307 * If a thread has already been reclaimed then the
308 * checkpointed registers are on the CPU but have definitely
309 * been saved by the reclaim code. Don't need to and *cannot*
310 * giveup as this would save to the 'live' structure not the
311 * checkpointed structure.
312 */
313 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
314 return;
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100315 __giveup_altivec(current);
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100316 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000317}
318EXPORT_SYMBOL(enable_kernel_altivec);
319
320/*
321 * Make sure the VMX/Altivec register state in the
322 * the thread_struct is up to date for task tsk.
323 */
324void flush_altivec_to_thread(struct task_struct *tsk)
325{
326 if (tsk->thread.regs) {
327 preempt_disable();
328 if (tsk->thread.regs->msr & MSR_VEC) {
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000329 BUG_ON(tsk != current);
Anton Blanchardb86fd2b2015-10-29 11:43:58 +1100330 giveup_altivec(tsk);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000331 }
332 preempt_enable();
333 }
334}
Paul Mackerrasde56a942011-06-29 00:21:34 +0000335EXPORT_SYMBOL_GPL(flush_altivec_to_thread);
Cyril Bur70fe3d92016-02-29 17:53:47 +1100336
337static int restore_altivec(struct task_struct *tsk)
338{
Cyril Burdc16b552016-09-23 16:18:08 +1000339 if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
Cyril Bura7771172017-11-02 14:09:03 +1100340 (tsk->thread.load_vec || tm_active_with_altivec(tsk))) {
Cyril Bur70fe3d92016-02-29 17:53:47 +1100341 load_vr_state(&tsk->thread.vr_state);
342 tsk->thread.used_vr = 1;
343 tsk->thread.load_vec++;
344
345 return 1;
346 }
347 return 0;
348}
349#else
350#define loadvec(thr) 0
351static inline int restore_altivec(struct task_struct *tsk) { return 0; }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000352#endif /* CONFIG_ALTIVEC */
353
Michael Neulingce48b212008-06-25 14:07:18 +1000354#ifdef CONFIG_VSX
Cyril Burbf6a4d52016-02-29 17:53:51 +1100355static void __giveup_vsx(struct task_struct *tsk)
Anton Blancharda7d623d2015-10-29 11:44:02 +1100356{
Benjamin Herrenschmidtdc801082017-08-16 16:01:17 +1000357 unsigned long msr = tsk->thread.regs->msr;
358
359 /*
360 * We should never be ssetting MSR_VSX without also setting
361 * MSR_FP and MSR_VEC
362 */
363 WARN_ON((msr & MSR_VSX) && !((msr & MSR_FP) && (msr & MSR_VEC)));
364
365 /* __giveup_fpu will clear MSR_VSX */
366 if (msr & MSR_FP)
Anton Blancharda7d623d2015-10-29 11:44:02 +1100367 __giveup_fpu(tsk);
Benjamin Herrenschmidtdc801082017-08-16 16:01:17 +1000368 if (msr & MSR_VEC)
Anton Blancharda7d623d2015-10-29 11:44:02 +1100369 __giveup_altivec(tsk);
Cyril Burbf6a4d52016-02-29 17:53:51 +1100370}
371
372static void giveup_vsx(struct task_struct *tsk)
373{
374 check_if_tm_restore_required(tsk);
375
376 msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
Anton Blancharda7d623d2015-10-29 11:44:02 +1100377 __giveup_vsx(tsk);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100378 msr_check_and_clear(MSR_FP|MSR_VEC|MSR_VSX);
Anton Blancharda7d623d2015-10-29 11:44:02 +1100379}
Cyril Burbf6a4d52016-02-29 17:53:51 +1100380
Michael Neulingce48b212008-06-25 14:07:18 +1000381void enable_kernel_vsx(void)
382{
Cyril Bure909fb82016-09-23 16:18:11 +1000383 unsigned long cpumsr;
384
Michael Neulingce48b212008-06-25 14:07:18 +1000385 WARN_ON(preemptible());
386
Cyril Bure909fb82016-09-23 16:18:11 +1000387 cpumsr = msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100388
Benjamin Herrenschmidt5a69aec2017-08-16 16:01:14 +1000389 if (current->thread.regs &&
390 (current->thread.regs->msr & (MSR_VSX|MSR_VEC|MSR_FP))) {
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100391 check_if_tm_restore_required(current);
Cyril Bure909fb82016-09-23 16:18:11 +1000392 /*
393 * If a thread has already been reclaimed then the
394 * checkpointed registers are on the CPU but have definitely
395 * been saved by the reclaim code. Don't need to and *cannot*
396 * giveup as this would save to the 'live' structure not the
397 * checkpointed structure.
398 */
399 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
400 return;
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100401 __giveup_vsx(current);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100402 }
Michael Neulingce48b212008-06-25 14:07:18 +1000403}
404EXPORT_SYMBOL(enable_kernel_vsx);
Michael Neulingce48b212008-06-25 14:07:18 +1000405
406void flush_vsx_to_thread(struct task_struct *tsk)
407{
408 if (tsk->thread.regs) {
409 preempt_disable();
Benjamin Herrenschmidt5a69aec2017-08-16 16:01:14 +1000410 if (tsk->thread.regs->msr & (MSR_VSX|MSR_VEC|MSR_FP)) {
Michael Neulingce48b212008-06-25 14:07:18 +1000411 BUG_ON(tsk != current);
Michael Neulingce48b212008-06-25 14:07:18 +1000412 giveup_vsx(tsk);
413 }
414 preempt_enable();
415 }
416}
Paul Mackerrasde56a942011-06-29 00:21:34 +0000417EXPORT_SYMBOL_GPL(flush_vsx_to_thread);
Cyril Bur70fe3d92016-02-29 17:53:47 +1100418
419static int restore_vsx(struct task_struct *tsk)
420{
421 if (cpu_has_feature(CPU_FTR_VSX)) {
422 tsk->thread.used_vsr = 1;
423 return 1;
424 }
425
426 return 0;
427}
428#else
429static inline int restore_vsx(struct task_struct *tsk) { return 0; }
Michael Neulingce48b212008-06-25 14:07:18 +1000430#endif /* CONFIG_VSX */
431
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000432#ifdef CONFIG_SPE
Anton Blanchard98da5812015-10-29 11:44:01 +1100433void giveup_spe(struct task_struct *tsk)
434{
Anton Blanchard98da5812015-10-29 11:44:01 +1100435 check_if_tm_restore_required(tsk);
436
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100437 msr_check_and_set(MSR_SPE);
Anton Blanchard98da5812015-10-29 11:44:01 +1100438 __giveup_spe(tsk);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100439 msr_check_and_clear(MSR_SPE);
Anton Blanchard98da5812015-10-29 11:44:01 +1100440}
441EXPORT_SYMBOL(giveup_spe);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000442
443void enable_kernel_spe(void)
444{
445 WARN_ON(preemptible());
446
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100447 msr_check_and_set(MSR_SPE);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100448
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100449 if (current->thread.regs && (current->thread.regs->msr & MSR_SPE)) {
450 check_if_tm_restore_required(current);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100451 __giveup_spe(current);
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100452 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000453}
454EXPORT_SYMBOL(enable_kernel_spe);
455
456void flush_spe_to_thread(struct task_struct *tsk)
457{
458 if (tsk->thread.regs) {
459 preempt_disable();
460 if (tsk->thread.regs->msr & MSR_SPE) {
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000461 BUG_ON(tsk != current);
yu liu685659e2011-06-14 18:34:25 -0500462 tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
Kumar Gala0ee6c152007-08-28 21:15:53 -0500463 giveup_spe(tsk);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000464 }
465 preempt_enable();
466 }
467}
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000468#endif /* CONFIG_SPE */
469
Anton Blanchardc2085052015-10-29 11:44:08 +1100470static unsigned long msr_all_available;
471
472static int __init init_msr_all_available(void)
473{
474#ifdef CONFIG_PPC_FPU
475 msr_all_available |= MSR_FP;
476#endif
477#ifdef CONFIG_ALTIVEC
478 if (cpu_has_feature(CPU_FTR_ALTIVEC))
479 msr_all_available |= MSR_VEC;
480#endif
481#ifdef CONFIG_VSX
482 if (cpu_has_feature(CPU_FTR_VSX))
483 msr_all_available |= MSR_VSX;
484#endif
485#ifdef CONFIG_SPE
486 if (cpu_has_feature(CPU_FTR_SPE))
487 msr_all_available |= MSR_SPE;
488#endif
489
490 return 0;
491}
492early_initcall(init_msr_all_available);
493
494void giveup_all(struct task_struct *tsk)
495{
496 unsigned long usermsr;
497
498 if (!tsk->thread.regs)
499 return;
500
501 usermsr = tsk->thread.regs->msr;
502
503 if ((usermsr & msr_all_available) == 0)
504 return;
505
506 msr_check_and_set(msr_all_available);
Cyril Burb0f16b42016-09-23 16:18:09 +1000507 check_if_tm_restore_required(tsk);
Anton Blanchardc2085052015-10-29 11:44:08 +1100508
Benjamin Herrenschmidt96c79b62017-08-16 16:01:18 +1000509 WARN_ON((usermsr & MSR_VSX) && !((usermsr & MSR_FP) && (usermsr & MSR_VEC)));
510
Anton Blanchardc2085052015-10-29 11:44:08 +1100511#ifdef CONFIG_PPC_FPU
512 if (usermsr & MSR_FP)
513 __giveup_fpu(tsk);
514#endif
515#ifdef CONFIG_ALTIVEC
516 if (usermsr & MSR_VEC)
517 __giveup_altivec(tsk);
518#endif
Anton Blanchardc2085052015-10-29 11:44:08 +1100519#ifdef CONFIG_SPE
520 if (usermsr & MSR_SPE)
521 __giveup_spe(tsk);
522#endif
523
524 msr_check_and_clear(msr_all_available);
525}
526EXPORT_SYMBOL(giveup_all);
527
Cyril Bur70fe3d92016-02-29 17:53:47 +1100528void restore_math(struct pt_regs *regs)
529{
530 unsigned long msr;
531
Cyril Burdc16b552016-09-23 16:18:08 +1000532 if (!msr_tm_active(regs->msr) &&
533 !current->thread.load_fp && !loadvec(current->thread))
Cyril Bur70fe3d92016-02-29 17:53:47 +1100534 return;
535
536 msr = regs->msr;
537 msr_check_and_set(msr_all_available);
538
539 /*
540 * Only reload if the bit is not set in the user MSR, the bit BEING set
541 * indicates that the registers are hot
542 */
543 if ((!(msr & MSR_FP)) && restore_fp(current))
544 msr |= MSR_FP | current->thread.fpexc_mode;
545
546 if ((!(msr & MSR_VEC)) && restore_altivec(current))
547 msr |= MSR_VEC;
548
549 if ((msr & (MSR_FP | MSR_VEC)) == (MSR_FP | MSR_VEC) &&
550 restore_vsx(current)) {
551 msr |= MSR_VSX;
552 }
553
554 msr_check_and_clear(msr_all_available);
555
556 regs->msr = msr;
557}
558
Cyril Burde2a20a2016-02-29 17:53:48 +1100559void save_all(struct task_struct *tsk)
560{
561 unsigned long usermsr;
562
563 if (!tsk->thread.regs)
564 return;
565
566 usermsr = tsk->thread.regs->msr;
567
568 if ((usermsr & msr_all_available) == 0)
569 return;
570
571 msr_check_and_set(msr_all_available);
572
Benjamin Herrenschmidt96c79b62017-08-16 16:01:18 +1000573 WARN_ON((usermsr & MSR_VSX) && !((usermsr & MSR_FP) && (usermsr & MSR_VEC)));
Cyril Burde2a20a2016-02-29 17:53:48 +1100574
Benjamin Herrenschmidt96c79b62017-08-16 16:01:18 +1000575 if (usermsr & MSR_FP)
576 save_fpu(tsk);
577
578 if (usermsr & MSR_VEC)
579 save_altivec(tsk);
Cyril Burde2a20a2016-02-29 17:53:48 +1100580
581 if (usermsr & MSR_SPE)
582 __giveup_spe(tsk);
583
584 msr_check_and_clear(msr_all_available);
585}
586
Anton Blanchard579e6332015-10-29 11:44:09 +1100587void flush_all_to_thread(struct task_struct *tsk)
588{
589 if (tsk->thread.regs) {
590 preempt_disable();
591 BUG_ON(tsk != current);
Cyril Burde2a20a2016-02-29 17:53:48 +1100592 save_all(tsk);
Anton Blanchard579e6332015-10-29 11:44:09 +1100593
594#ifdef CONFIG_SPE
595 if (tsk->thread.regs->msr & MSR_SPE)
596 tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
597#endif
598
599 preempt_enable();
600 }
601}
602EXPORT_SYMBOL(flush_all_to_thread);
603
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000604#ifdef CONFIG_PPC_ADV_DEBUG_REGS
605void do_send_trap(struct pt_regs *regs, unsigned long address,
606 unsigned long error_code, int signal_code, int breakpt)
607{
608 siginfo_t info;
609
Ananth N Mavinakayanahalli41ab5262012-08-23 21:27:09 +0000610 current->thread.trap_nr = signal_code;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000611 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
612 11, SIGSEGV) == NOTIFY_STOP)
613 return;
614
615 /* Deliver the signal to userspace */
616 info.si_signo = SIGTRAP;
617 info.si_errno = breakpt; /* breakpoint or watchpoint id */
618 info.si_code = signal_code;
619 info.si_addr = (void __user *)address;
620 force_sig_info(SIGTRAP, &info, current);
621}
622#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
Michael Neuling9422de32012-12-20 14:06:44 +0000623void do_break (struct pt_regs *regs, unsigned long address,
Luis Machadod6a61bf2008-07-24 02:10:41 +1000624 unsigned long error_code)
625{
626 siginfo_t info;
627
Ananth N Mavinakayanahalli41ab5262012-08-23 21:27:09 +0000628 current->thread.trap_nr = TRAP_HWBKPT;
Luis Machadod6a61bf2008-07-24 02:10:41 +1000629 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
630 11, SIGSEGV) == NOTIFY_STOP)
631 return;
632
Michael Neuling9422de32012-12-20 14:06:44 +0000633 if (debugger_break_match(regs))
Luis Machadod6a61bf2008-07-24 02:10:41 +1000634 return;
635
Michael Neuling9422de32012-12-20 14:06:44 +0000636 /* Clear the breakpoint */
637 hw_breakpoint_disable();
Luis Machadod6a61bf2008-07-24 02:10:41 +1000638
639 /* Deliver the signal to userspace */
640 info.si_signo = SIGTRAP;
641 info.si_errno = 0;
642 info.si_code = TRAP_HWBKPT;
643 info.si_addr = (void __user *)address;
644 force_sig_info(SIGTRAP, &info, current);
645}
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000646#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
Luis Machadod6a61bf2008-07-24 02:10:41 +1000647
Michael Neuling9422de32012-12-20 14:06:44 +0000648static DEFINE_PER_CPU(struct arch_hw_breakpoint, current_brk);
Michael Ellermana2ceff52008-03-28 19:11:48 +1100649
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000650#ifdef CONFIG_PPC_ADV_DEBUG_REGS
651/*
652 * Set the debug registers back to their default "safe" values.
653 */
654static void set_debug_reg_defaults(struct thread_struct *thread)
655{
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530656 thread->debug.iac1 = thread->debug.iac2 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000657#if CONFIG_PPC_ADV_DEBUG_IACS > 2
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530658 thread->debug.iac3 = thread->debug.iac4 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000659#endif
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530660 thread->debug.dac1 = thread->debug.dac2 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000661#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530662 thread->debug.dvc1 = thread->debug.dvc2 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000663#endif
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530664 thread->debug.dbcr0 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000665#ifdef CONFIG_BOOKE
666 /*
667 * Force User/Supervisor bits to b11 (user-only MSR[PR]=1)
668 */
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530669 thread->debug.dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US |
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000670 DBCR1_IAC3US | DBCR1_IAC4US;
671 /*
672 * Force Data Address Compare User/Supervisor bits to be User-only
673 * (0b11 MSR[PR]=1) and set all other bits in DBCR2 register to be 0.
674 */
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530675 thread->debug.dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000676#else
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530677 thread->debug.dbcr1 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000678#endif
679}
680
Scott Woodf5f97212013-11-22 15:52:29 -0600681static void prime_debug_regs(struct debug_reg *debug)
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000682{
Scott Wood6cecf762013-05-13 14:14:53 +0000683 /*
684 * We could have inherited MSR_DE from userspace, since
685 * it doesn't get cleared on exception entry. Make sure
686 * MSR_DE is clear before we enable any debug events.
687 */
688 mtmsr(mfmsr() & ~MSR_DE);
689
Scott Woodf5f97212013-11-22 15:52:29 -0600690 mtspr(SPRN_IAC1, debug->iac1);
691 mtspr(SPRN_IAC2, debug->iac2);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000692#if CONFIG_PPC_ADV_DEBUG_IACS > 2
Scott Woodf5f97212013-11-22 15:52:29 -0600693 mtspr(SPRN_IAC3, debug->iac3);
694 mtspr(SPRN_IAC4, debug->iac4);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000695#endif
Scott Woodf5f97212013-11-22 15:52:29 -0600696 mtspr(SPRN_DAC1, debug->dac1);
697 mtspr(SPRN_DAC2, debug->dac2);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000698#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
Scott Woodf5f97212013-11-22 15:52:29 -0600699 mtspr(SPRN_DVC1, debug->dvc1);
700 mtspr(SPRN_DVC2, debug->dvc2);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000701#endif
Scott Woodf5f97212013-11-22 15:52:29 -0600702 mtspr(SPRN_DBCR0, debug->dbcr0);
703 mtspr(SPRN_DBCR1, debug->dbcr1);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000704#ifdef CONFIG_BOOKE
Scott Woodf5f97212013-11-22 15:52:29 -0600705 mtspr(SPRN_DBCR2, debug->dbcr2);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000706#endif
707}
708/*
709 * Unless neither the old or new thread are making use of the
710 * debug registers, set the debug registers from the values
711 * stored in the new thread.
712 */
Scott Woodf5f97212013-11-22 15:52:29 -0600713void switch_booke_debug_regs(struct debug_reg *new_debug)
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000714{
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530715 if ((current->thread.debug.dbcr0 & DBCR0_IDM)
Scott Woodf5f97212013-11-22 15:52:29 -0600716 || (new_debug->dbcr0 & DBCR0_IDM))
717 prime_debug_regs(new_debug);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000718}
Bharat Bhushan3743c9b2013-07-04 12:27:44 +0530719EXPORT_SYMBOL_GPL(switch_booke_debug_regs);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000720#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
K.Prasade0780b72011-02-10 04:44:35 +0000721#ifndef CONFIG_HAVE_HW_BREAKPOINT
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000722static void set_debug_reg_defaults(struct thread_struct *thread)
723{
Michael Neuling9422de32012-12-20 14:06:44 +0000724 thread->hw_brk.address = 0;
725 thread->hw_brk.type = 0;
Michael Neulingb9818c32013-01-10 14:25:34 +0000726 set_breakpoint(&thread->hw_brk);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000727}
K.Prasade0780b72011-02-10 04:44:35 +0000728#endif /* !CONFIG_HAVE_HW_BREAKPOINT */
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000729#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
730
Dave Kleikamp172ae2e2010-02-08 11:50:57 +0000731#ifdef CONFIG_PPC_ADV_DEBUG_REGS
Michael Neuling9422de32012-12-20 14:06:44 +0000732static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
733{
Benjamin Herrenschmidtc6c9eac2009-09-08 14:16:58 +0000734 mtspr(SPRN_DAC1, dabr);
Dave Kleikamp221c1852010-03-05 10:43:24 +0000735#ifdef CONFIG_PPC_47x
736 isync();
737#endif
Michael Neuling9422de32012-12-20 14:06:44 +0000738 return 0;
739}
Benjamin Herrenschmidtc6c9eac2009-09-08 14:16:58 +0000740#elif defined(CONFIG_PPC_BOOK3S)
Michael Neuling9422de32012-12-20 14:06:44 +0000741static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
742{
Michael Ellermancab0af92005-11-03 15:30:49 +1100743 mtspr(SPRN_DABR, dabr);
Michael Neuling82a9f162013-05-16 20:27:31 +0000744 if (cpu_has_feature(CPU_FTR_DABRX))
745 mtspr(SPRN_DABRX, dabrx);
Michael Ellermancab0af92005-11-03 15:30:49 +1100746 return 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000747}
Christophe Leroy4ad86222016-11-29 09:52:15 +0100748#elif defined(CONFIG_PPC_8xx)
749static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
750{
751 unsigned long addr = dabr & ~HW_BRK_TYPE_DABR;
752 unsigned long lctrl1 = 0x90000000; /* compare type: equal on E & F */
753 unsigned long lctrl2 = 0x8e000002; /* watchpoint 1 on cmp E | F */
754
755 if ((dabr & HW_BRK_TYPE_RDWR) == HW_BRK_TYPE_READ)
756 lctrl1 |= 0xa0000;
757 else if ((dabr & HW_BRK_TYPE_RDWR) == HW_BRK_TYPE_WRITE)
758 lctrl1 |= 0xf0000;
759 else if ((dabr & HW_BRK_TYPE_RDWR) == 0)
760 lctrl2 = 0;
761
762 mtspr(SPRN_LCTRL2, 0);
763 mtspr(SPRN_CMPE, addr);
764 mtspr(SPRN_CMPF, addr + 4);
765 mtspr(SPRN_LCTRL1, lctrl1);
766 mtspr(SPRN_LCTRL2, lctrl2);
767
768 return 0;
769}
Michael Neuling9422de32012-12-20 14:06:44 +0000770#else
771static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
772{
773 return -EINVAL;
774}
775#endif
776
777static inline int set_dabr(struct arch_hw_breakpoint *brk)
778{
779 unsigned long dabr, dabrx;
780
781 dabr = brk->address | (brk->type & HW_BRK_TYPE_DABR);
782 dabrx = ((brk->type >> 3) & 0x7);
783
784 if (ppc_md.set_dabr)
785 return ppc_md.set_dabr(dabr, dabrx);
786
787 return __set_dabr(dabr, dabrx);
788}
789
Michael Neulingbf99de32012-12-20 14:06:45 +0000790static inline int set_dawr(struct arch_hw_breakpoint *brk)
791{
Michael Neuling05d694e2013-01-24 15:02:58 +0000792 unsigned long dawr, dawrx, mrd;
Michael Neulingbf99de32012-12-20 14:06:45 +0000793
794 dawr = brk->address;
795
796 dawrx = (brk->type & (HW_BRK_TYPE_READ | HW_BRK_TYPE_WRITE)) \
797 << (63 - 58); //* read/write bits */
798 dawrx |= ((brk->type & (HW_BRK_TYPE_TRANSLATE)) >> 2) \
799 << (63 - 59); //* translate */
800 dawrx |= (brk->type & (HW_BRK_TYPE_PRIV_ALL)) \
801 >> 3; //* PRIM bits */
Michael Neuling05d694e2013-01-24 15:02:58 +0000802 /* dawr length is stored in field MDR bits 48:53. Matches range in
803 doublewords (64 bits) baised by -1 eg. 0b000000=1DW and
804 0b111111=64DW.
805 brk->len is in bytes.
806 This aligns up to double word size, shifts and does the bias.
807 */
808 mrd = ((brk->len + 7) >> 3) - 1;
809 dawrx |= (mrd & 0x3f) << (63 - 53);
Michael Neulingbf99de32012-12-20 14:06:45 +0000810
811 if (ppc_md.set_dawr)
812 return ppc_md.set_dawr(dawr, dawrx);
813 mtspr(SPRN_DAWR, dawr);
814 mtspr(SPRN_DAWRX, dawrx);
815 return 0;
816}
817
Paul Gortmaker21f58502014-04-29 15:25:17 -0400818void __set_breakpoint(struct arch_hw_breakpoint *brk)
Michael Neuling9422de32012-12-20 14:06:44 +0000819{
Christoph Lameter69111ba2014-10-21 15:23:25 -0500820 memcpy(this_cpu_ptr(&current_brk), brk, sizeof(*brk));
Michael Neuling9422de32012-12-20 14:06:44 +0000821
Michael Neulingbf99de32012-12-20 14:06:45 +0000822 if (cpu_has_feature(CPU_FTR_DAWR))
Paul Gortmaker04c32a52014-04-29 15:25:16 -0400823 set_dawr(brk);
824 else
825 set_dabr(brk);
Michael Neuling9422de32012-12-20 14:06:44 +0000826}
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000827
Paul Gortmaker21f58502014-04-29 15:25:17 -0400828void set_breakpoint(struct arch_hw_breakpoint *brk)
829{
830 preempt_disable();
831 __set_breakpoint(brk);
832 preempt_enable();
833}
834
Paul Mackerras06d67d52005-10-10 22:29:05 +1000835#ifdef CONFIG_PPC64
836DEFINE_PER_CPU(struct cpu_usage, cpu_usage_array);
Paul Mackerras06d67d52005-10-10 22:29:05 +1000837#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000838
Michael Neuling9422de32012-12-20 14:06:44 +0000839static inline bool hw_brk_match(struct arch_hw_breakpoint *a,
840 struct arch_hw_breakpoint *b)
841{
842 if (a->address != b->address)
843 return false;
844 if (a->type != b->type)
845 return false;
846 if (a->len != b->len)
847 return false;
848 return true;
849}
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100850
Michael Neulingfb096922013-02-13 16:21:37 +0000851#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Cyril Bur5d176f72016-09-14 18:02:16 +1000852
853static inline bool tm_enabled(struct task_struct *tsk)
854{
855 return tsk && tsk->thread.regs && (tsk->thread.regs->msr & MSR_TM);
856}
857
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100858static void tm_reclaim_thread(struct thread_struct *thr,
859 struct thread_info *ti, uint8_t cause)
860{
Michael Neuling7f821fc2015-11-19 15:44:45 +1100861 /*
862 * Use the current MSR TM suspended bit to track if we have
863 * checkpointed state outstanding.
864 * On signal delivery, we'd normally reclaim the checkpointed
865 * state to obtain stack pointer (see:get_tm_stackpointer()).
866 * This will then directly return to userspace without going
867 * through __switch_to(). However, if the stack frame is bad,
868 * we need to exit this thread which calls __switch_to() which
869 * will again attempt to reclaim the already saved tm state.
870 * Hence we need to check that we've not already reclaimed
871 * this state.
872 * We do this using the current MSR, rather tracking it in
873 * some specific thread_struct bit, as it has the additional
Michael Ellerman027dfac2016-06-01 16:34:37 +1000874 * benefit of checking for a potential TM bad thing exception.
Michael Neuling7f821fc2015-11-19 15:44:45 +1100875 */
876 if (!MSR_TM_SUSPENDED(mfmsr()))
877 return;
878
Cyril Bur91381b92017-11-02 14:09:04 +1100879 giveup_all(container_of(thr, struct task_struct, thread));
880
Cyril Bureb5c3f12017-11-02 14:09:05 +1100881 tm_reclaim(thr, cause);
882
Michael Neulingf48e91e2017-05-08 17:16:26 +1000883 /*
884 * If we are in a transaction and FP is off then we can't have
885 * used FP inside that transaction. Hence the checkpointed
886 * state is the same as the live state. We need to copy the
887 * live state to the checkpointed state so that when the
888 * transaction is restored, the checkpointed state is correct
889 * and the aborted transaction sees the correct state. We use
890 * ckpt_regs.msr here as that's what tm_reclaim will use to
891 * determine if it's going to write the checkpointed state or
892 * not. So either this will write the checkpointed registers,
893 * or reclaim will. Similarly for VMX.
894 */
895 if ((thr->ckpt_regs.msr & MSR_FP) == 0)
896 memcpy(&thr->ckfp_state, &thr->fp_state,
897 sizeof(struct thread_fp_state));
898 if ((thr->ckpt_regs.msr & MSR_VEC) == 0)
899 memcpy(&thr->ckvr_state, &thr->vr_state,
900 sizeof(struct thread_vr_state));
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100901}
902
903void tm_reclaim_current(uint8_t cause)
904{
905 tm_enable();
906 tm_reclaim_thread(&current->thread, current_thread_info(), cause);
907}
908
Michael Neulingfb096922013-02-13 16:21:37 +0000909static inline void tm_reclaim_task(struct task_struct *tsk)
910{
911 /* We have to work out if we're switching from/to a task that's in the
912 * middle of a transaction.
913 *
914 * In switching we need to maintain a 2nd register state as
915 * oldtask->thread.ckpt_regs. We tm_reclaim(oldproc); this saves the
Cyril Bur000ec282016-09-23 16:18:25 +1000916 * checkpointed (tbegin) state in ckpt_regs, ckfp_state and
917 * ckvr_state
Michael Neulingfb096922013-02-13 16:21:37 +0000918 *
919 * We also context switch (save) TFHAR/TEXASR/TFIAR in here.
920 */
921 struct thread_struct *thr = &tsk->thread;
922
923 if (!thr->regs)
924 return;
925
926 if (!MSR_TM_ACTIVE(thr->regs->msr))
927 goto out_and_saveregs;
928
Michael Neuling92fb8692017-10-12 21:17:19 +1100929 WARN_ON(tm_suspend_disabled);
930
Michael Neulingfb096922013-02-13 16:21:37 +0000931 TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, "
932 "ccr=%lx, msr=%lx, trap=%lx)\n",
933 tsk->pid, thr->regs->nip,
934 thr->regs->ccr, thr->regs->msr,
935 thr->regs->trap);
936
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100937 tm_reclaim_thread(thr, task_thread_info(tsk), TM_CAUSE_RESCHED);
Michael Neulingfb096922013-02-13 16:21:37 +0000938
939 TM_DEBUG("--- tm_reclaim on pid %d complete\n",
940 tsk->pid);
941
942out_and_saveregs:
943 /* Always save the regs here, even if a transaction's not active.
944 * This context-switches a thread's TM info SPRs. We do it here to
945 * be consistent with the restore path (in recheckpoint) which
946 * cannot happen later in _switch().
947 */
948 tm_save_sprs(thr);
949}
950
Cyril Bureb5c3f12017-11-02 14:09:05 +1100951extern void __tm_recheckpoint(struct thread_struct *thread);
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100952
Cyril Bureb5c3f12017-11-02 14:09:05 +1100953void tm_recheckpoint(struct thread_struct *thread)
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100954{
955 unsigned long flags;
956
Cyril Bur5d176f72016-09-14 18:02:16 +1000957 if (!(thread->regs->msr & MSR_TM))
958 return;
959
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100960 /* We really can't be interrupted here as the TEXASR registers can't
961 * change and later in the trecheckpoint code, we have a userspace R1.
962 * So let's hard disable over this region.
963 */
964 local_irq_save(flags);
965 hard_irq_disable();
966
967 /* The TM SPRs are restored here, so that TEXASR.FS can be set
968 * before the trecheckpoint and no explosion occurs.
969 */
970 tm_restore_sprs(thread);
971
Cyril Bureb5c3f12017-11-02 14:09:05 +1100972 __tm_recheckpoint(thread);
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100973
974 local_irq_restore(flags);
975}
976
Michael Neulingbc2a9402013-02-13 16:21:40 +0000977static inline void tm_recheckpoint_new_task(struct task_struct *new)
Michael Neulingfb096922013-02-13 16:21:37 +0000978{
Michael Neulingfb096922013-02-13 16:21:37 +0000979 if (!cpu_has_feature(CPU_FTR_TM))
980 return;
981
982 /* Recheckpoint the registers of the thread we're about to switch to.
983 *
984 * If the task was using FP, we non-lazily reload both the original and
985 * the speculative FP register states. This is because the kernel
986 * doesn't see if/when a TM rollback occurs, so if we take an FP
Cyril Burdc310662016-09-23 16:18:24 +1000987 * unavailable later, we are unable to determine which set of FP regs
Michael Neulingfb096922013-02-13 16:21:37 +0000988 * need to be restored.
989 */
Cyril Bur5d176f72016-09-14 18:02:16 +1000990 if (!tm_enabled(new))
Michael Neulingfb096922013-02-13 16:21:37 +0000991 return;
992
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100993 if (!MSR_TM_ACTIVE(new->thread.regs->msr)){
994 tm_restore_sprs(&new->thread);
Michael Neulingfb096922013-02-13 16:21:37 +0000995 return;
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100996 }
Michael Neulingfb096922013-02-13 16:21:37 +0000997 /* Recheckpoint to restore original checkpointed register state. */
Cyril Bureb5c3f12017-11-02 14:09:05 +1100998 TM_DEBUG("*** tm_recheckpoint of pid %d (new->msr 0x%lx)\n",
999 new->pid, new->thread.regs->msr);
Michael Neulingfb096922013-02-13 16:21:37 +00001000
Cyril Bureb5c3f12017-11-02 14:09:05 +11001001 tm_recheckpoint(&new->thread);
Michael Neulingfb096922013-02-13 16:21:37 +00001002
Cyril Burdc310662016-09-23 16:18:24 +10001003 /*
1004 * The checkpointed state has been restored but the live state has
1005 * not, ensure all the math functionality is turned off to trigger
1006 * restore_math() to reload.
1007 */
1008 new->thread.regs->msr &= ~(MSR_FP | MSR_VEC | MSR_VSX);
Michael Neulingfb096922013-02-13 16:21:37 +00001009
1010 TM_DEBUG("*** tm_recheckpoint of pid %d complete "
1011 "(kernel msr 0x%lx)\n",
1012 new->pid, mfmsr());
1013}
1014
Cyril Burdc310662016-09-23 16:18:24 +10001015static inline void __switch_to_tm(struct task_struct *prev,
1016 struct task_struct *new)
Michael Neulingfb096922013-02-13 16:21:37 +00001017{
1018 if (cpu_has_feature(CPU_FTR_TM)) {
Cyril Bur5d176f72016-09-14 18:02:16 +10001019 if (tm_enabled(prev) || tm_enabled(new))
1020 tm_enable();
1021
1022 if (tm_enabled(prev)) {
1023 prev->thread.load_tm++;
1024 tm_reclaim_task(prev);
1025 if (!MSR_TM_ACTIVE(prev->thread.regs->msr) && prev->thread.load_tm == 0)
1026 prev->thread.regs->msr &= ~MSR_TM;
1027 }
1028
Cyril Burdc310662016-09-23 16:18:24 +10001029 tm_recheckpoint_new_task(new);
Michael Neulingfb096922013-02-13 16:21:37 +00001030 }
1031}
Paul Mackerrasd31626f2014-01-13 15:56:29 +11001032
1033/*
1034 * This is called if we are on the way out to userspace and the
1035 * TIF_RESTORE_TM flag is set. It checks if we need to reload
1036 * FP and/or vector state and does so if necessary.
1037 * If userspace is inside a transaction (whether active or
1038 * suspended) and FP/VMX/VSX instructions have ever been enabled
1039 * inside that transaction, then we have to keep them enabled
1040 * and keep the FP/VMX/VSX state loaded while ever the transaction
1041 * continues. The reason is that if we didn't, and subsequently
1042 * got a FP/VMX/VSX unavailable interrupt inside a transaction,
1043 * we don't know whether it's the same transaction, and thus we
1044 * don't know which of the checkpointed state and the transactional
1045 * state to use.
1046 */
1047void restore_tm_state(struct pt_regs *regs)
1048{
1049 unsigned long msr_diff;
1050
Cyril Burdc310662016-09-23 16:18:24 +10001051 /*
1052 * This is the only moment we should clear TIF_RESTORE_TM as
1053 * it is here that ckpt_regs.msr and pt_regs.msr become the same
1054 * again, anything else could lead to an incorrect ckpt_msr being
1055 * saved and therefore incorrect signal contexts.
1056 */
Paul Mackerrasd31626f2014-01-13 15:56:29 +11001057 clear_thread_flag(TIF_RESTORE_TM);
1058 if (!MSR_TM_ACTIVE(regs->msr))
1059 return;
1060
Anshuman Khandual829023d2015-07-06 16:24:10 +05301061 msr_diff = current->thread.ckpt_regs.msr & ~regs->msr;
Paul Mackerrasd31626f2014-01-13 15:56:29 +11001062 msr_diff &= MSR_FP | MSR_VEC | MSR_VSX;
Cyril Bur70fe3d92016-02-29 17:53:47 +11001063
Cyril Burdc16b552016-09-23 16:18:08 +10001064 /* Ensure that restore_math() will restore */
1065 if (msr_diff & MSR_FP)
1066 current->thread.load_fp = 1;
Valentin Rothberg39715bf2016-10-05 07:57:26 +02001067#ifdef CONFIG_ALTIVEC
Cyril Burdc16b552016-09-23 16:18:08 +10001068 if (cpu_has_feature(CPU_FTR_ALTIVEC) && msr_diff & MSR_VEC)
1069 current->thread.load_vec = 1;
1070#endif
Cyril Bur70fe3d92016-02-29 17:53:47 +11001071 restore_math(regs);
1072
Paul Mackerrasd31626f2014-01-13 15:56:29 +11001073 regs->msr |= msr_diff;
1074}
1075
Michael Neulingfb096922013-02-13 16:21:37 +00001076#else
1077#define tm_recheckpoint_new_task(new)
Cyril Burdc310662016-09-23 16:18:24 +10001078#define __switch_to_tm(prev, new)
Michael Neulingfb096922013-02-13 16:21:37 +00001079#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
Michael Neuling9422de32012-12-20 14:06:44 +00001080
Anton Blanchard152d5232015-10-29 11:43:55 +11001081static inline void save_sprs(struct thread_struct *t)
1082{
1083#ifdef CONFIG_ALTIVEC
Oliver O'Halloran01d7c2a22016-03-08 09:08:47 +11001084 if (cpu_has_feature(CPU_FTR_ALTIVEC))
Anton Blanchard152d5232015-10-29 11:43:55 +11001085 t->vrsave = mfspr(SPRN_VRSAVE);
1086#endif
1087#ifdef CONFIG_PPC_BOOK3S_64
1088 if (cpu_has_feature(CPU_FTR_DSCR))
1089 t->dscr = mfspr(SPRN_DSCR);
1090
1091 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
1092 t->bescr = mfspr(SPRN_BESCR);
1093 t->ebbhr = mfspr(SPRN_EBBHR);
1094 t->ebbrr = mfspr(SPRN_EBBRR);
1095
1096 t->fscr = mfspr(SPRN_FSCR);
1097
1098 /*
1099 * Note that the TAR is not available for use in the kernel.
1100 * (To provide this, the TAR should be backed up/restored on
1101 * exception entry/exit instead, and be in pt_regs. FIXME,
1102 * this should be in pt_regs anyway (for debug).)
1103 */
1104 t->tar = mfspr(SPRN_TAR);
1105 }
1106#endif
Ram Pai06bb53b2018-01-18 17:50:31 -08001107
1108 thread_pkey_regs_save(t);
Anton Blanchard152d5232015-10-29 11:43:55 +11001109}
1110
1111static inline void restore_sprs(struct thread_struct *old_thread,
1112 struct thread_struct *new_thread)
1113{
1114#ifdef CONFIG_ALTIVEC
1115 if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
1116 old_thread->vrsave != new_thread->vrsave)
1117 mtspr(SPRN_VRSAVE, new_thread->vrsave);
1118#endif
1119#ifdef CONFIG_PPC_BOOK3S_64
1120 if (cpu_has_feature(CPU_FTR_DSCR)) {
1121 u64 dscr = get_paca()->dscr_default;
Michael Neulingb57bd2d2016-06-09 12:31:08 +10001122 if (new_thread->dscr_inherit)
Anton Blanchard152d5232015-10-29 11:43:55 +11001123 dscr = new_thread->dscr;
Anton Blanchard152d5232015-10-29 11:43:55 +11001124
1125 if (old_thread->dscr != dscr)
1126 mtspr(SPRN_DSCR, dscr);
Anton Blanchard152d5232015-10-29 11:43:55 +11001127 }
1128
1129 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
1130 if (old_thread->bescr != new_thread->bescr)
1131 mtspr(SPRN_BESCR, new_thread->bescr);
1132 if (old_thread->ebbhr != new_thread->ebbhr)
1133 mtspr(SPRN_EBBHR, new_thread->ebbhr);
1134 if (old_thread->ebbrr != new_thread->ebbrr)
1135 mtspr(SPRN_EBBRR, new_thread->ebbrr);
1136
Michael Neulingb57bd2d2016-06-09 12:31:08 +10001137 if (old_thread->fscr != new_thread->fscr)
1138 mtspr(SPRN_FSCR, new_thread->fscr);
1139
Anton Blanchard152d5232015-10-29 11:43:55 +11001140 if (old_thread->tar != new_thread->tar)
1141 mtspr(SPRN_TAR, new_thread->tar);
1142 }
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001143
1144 if (cpu_has_feature(CPU_FTR_ARCH_300) &&
1145 old_thread->tidr != new_thread->tidr)
1146 mtspr(SPRN_TIDR, new_thread->tidr);
Anton Blanchard152d5232015-10-29 11:43:55 +11001147#endif
Ram Pai06bb53b2018-01-18 17:50:31 -08001148
1149 thread_pkey_regs_restore(new_thread, old_thread);
Anton Blanchard152d5232015-10-29 11:43:55 +11001150}
1151
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001152#ifdef CONFIG_PPC_BOOK3S_64
1153#define CP_SIZE 128
1154static const u8 dummy_copy_buffer[CP_SIZE] __attribute__((aligned(CP_SIZE)));
1155#endif
1156
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001157struct task_struct *__switch_to(struct task_struct *prev,
1158 struct task_struct *new)
1159{
1160 struct thread_struct *new_thread, *old_thread;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001161 struct task_struct *last;
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001162#ifdef CONFIG_PPC_BOOK3S_64
1163 struct ppc64_tlb_batch *batch;
1164#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001165
Anton Blanchard152d5232015-10-29 11:43:55 +11001166 new_thread = &new->thread;
1167 old_thread = &current->thread;
1168
Michael Neuling7ba5fef2013-10-02 17:15:14 +10001169 WARN_ON(!irqs_disabled());
1170
Paul Mackerras06d67d52005-10-10 22:29:05 +10001171#ifdef CONFIG_PPC64
1172 /*
1173 * Collect processor utilization data per process
1174 */
1175 if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
Christoph Lameter69111ba2014-10-21 15:23:25 -05001176 struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001177 long unsigned start_tb, current_tb;
1178 start_tb = old_thread->start_tb;
1179 cu->current_tb = current_tb = mfspr(SPRN_PURR);
1180 old_thread->accum_tb += (current_tb - start_tb);
1181 new_thread->start_tb = current_tb;
1182 }
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001183#endif /* CONFIG_PPC64 */
1184
Michael Ellerman4e003742017-10-19 15:08:43 +11001185#ifdef CONFIG_PPC_BOOK3S_64
Christoph Lameter69111ba2014-10-21 15:23:25 -05001186 batch = this_cpu_ptr(&ppc64_tlb_batch);
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001187 if (batch->active) {
1188 current_thread_info()->local_flags |= _TLF_LAZY_MMU;
1189 if (batch->index)
1190 __flush_tlb_pending(batch);
1191 batch->active = 0;
1192 }
Michael Ellerman4e003742017-10-19 15:08:43 +11001193#endif /* CONFIG_PPC_BOOK3S_64 */
Paul Mackerras06d67d52005-10-10 22:29:05 +10001194
Anton Blanchardf3d885c2015-10-29 11:44:10 +11001195#ifdef CONFIG_PPC_ADV_DEBUG_REGS
1196 switch_booke_debug_regs(&new->thread.debug);
1197#else
1198/*
1199 * For PPC_BOOK3S_64, we use the hw-breakpoint interfaces that would
1200 * schedule DABR
1201 */
1202#ifndef CONFIG_HAVE_HW_BREAKPOINT
1203 if (unlikely(!hw_brk_match(this_cpu_ptr(&current_brk), &new->thread.hw_brk)))
1204 __set_breakpoint(&new->thread.hw_brk);
1205#endif /* CONFIG_HAVE_HW_BREAKPOINT */
1206#endif
1207
1208 /*
1209 * We need to save SPRs before treclaim/trecheckpoint as these will
1210 * change a number of them.
1211 */
1212 save_sprs(&prev->thread);
1213
Anton Blanchardf3d885c2015-10-29 11:44:10 +11001214 /* Save FPU, Altivec, VSX and SPE state */
1215 giveup_all(prev);
1216
Cyril Burdc310662016-09-23 16:18:24 +10001217 __switch_to_tm(prev, new);
1218
Nicholas Piggine4c0fc52017-06-09 01:36:06 +10001219 if (!radix_enabled()) {
1220 /*
1221 * We can't take a PMU exception inside _switch() since there
1222 * is a window where the kernel stack SLB and the kernel stack
1223 * are out of sync. Hard disable here.
1224 */
1225 hard_irq_disable();
1226 }
Michael Neulingbc2a9402013-02-13 16:21:40 +00001227
Anton Blanchard20dbe672015-12-10 20:44:39 +11001228 /*
1229 * Call restore_sprs() before calling _switch(). If we move it after
1230 * _switch() then we miss out on calling it for new tasks. The reason
1231 * for this is we manually create a stack frame for new tasks that
1232 * directly returns through ret_from_fork() or
1233 * ret_from_kernel_thread(). See copy_thread() for details.
1234 */
Anton Blanchardf3d885c2015-10-29 11:44:10 +11001235 restore_sprs(old_thread, new_thread);
1236
Anton Blanchard20dbe672015-12-10 20:44:39 +11001237 last = _switch(old_thread, new_thread);
1238
Michael Ellerman4e003742017-10-19 15:08:43 +11001239#ifdef CONFIG_PPC_BOOK3S_64
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001240 if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
1241 current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
Christoph Lameter69111ba2014-10-21 15:23:25 -05001242 batch = this_cpu_ptr(&ppc64_tlb_batch);
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001243 batch->active = 1;
1244 }
Cyril Bur70fe3d92016-02-29 17:53:47 +11001245
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001246 if (current_thread_info()->task->thread.regs) {
Cyril Bur70fe3d92016-02-29 17:53:47 +11001247 restore_math(current_thread_info()->task->thread.regs);
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001248
1249 /*
1250 * The copy-paste buffer can only store into foreign real
1251 * addresses, so unprivileged processes can not see the
1252 * data or use it in any way unless they have foreign real
Sukadev Bhattiprolu9d2a4d72017-11-07 18:23:54 -08001253 * mappings. If the new process has the foreign real address
1254 * mappings, we must issue a cp_abort to clear any state and
1255 * prevent snooping, corruption or a covert channel.
1256 *
1257 * DD1 allows paste into normal system memory so we do an
1258 * unpaired copy, rather than cp_abort, to clear the buffer,
1259 * since cp_abort is quite expensive.
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001260 */
Sukadev Bhattiprolu9d2a4d72017-11-07 18:23:54 -08001261 if (current_thread_info()->task->thread.used_vas) {
1262 asm volatile(PPC_CP_ABORT);
1263 } else if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001264 asm volatile(PPC_COPY(%0, %1)
1265 : : "r"(dummy_copy_buffer), "r"(0));
1266 }
1267 }
Michael Ellerman4e003742017-10-19 15:08:43 +11001268#endif /* CONFIG_PPC_BOOK3S_64 */
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001269
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001270 return last;
1271}
1272
Paul Mackerras06d67d52005-10-10 22:29:05 +10001273static int instructions_to_print = 16;
1274
Paul Mackerras06d67d52005-10-10 22:29:05 +10001275static void show_instructions(struct pt_regs *regs)
1276{
1277 int i;
1278 unsigned long pc = regs->nip - (instructions_to_print * 3 / 4 *
1279 sizeof(int));
1280
1281 printk("Instruction dump:");
1282
1283 for (i = 0; i < instructions_to_print; i++) {
1284 int instr;
1285
1286 if (!(i % 8))
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001287 pr_cont("\n");
Paul Mackerras06d67d52005-10-10 22:29:05 +10001288
Scott Wood0de2d822007-09-28 04:38:55 +10001289#if !defined(CONFIG_BOOKE)
1290 /* If executing with the IMMU off, adjust pc rather
1291 * than print XXXXXXXX.
1292 */
1293 if (!(regs->msr & MSR_IR))
1294 pc = (unsigned long)phys_to_virt(pc);
1295#endif
1296
Anton Blanchard00ae36d2006-10-13 12:17:16 +10001297 if (!__kernel_text_address(pc) ||
Anton Blanchard7b051f62014-10-13 20:27:15 +11001298 probe_kernel_address((unsigned int __user *)pc, instr)) {
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001299 pr_cont("XXXXXXXX ");
Paul Mackerras06d67d52005-10-10 22:29:05 +10001300 } else {
1301 if (regs->nip == pc)
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001302 pr_cont("<%08x> ", instr);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001303 else
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001304 pr_cont("%08x ", instr);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001305 }
1306
1307 pc += sizeof(int);
1308 }
1309
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001310 pr_cont("\n");
Paul Mackerras06d67d52005-10-10 22:29:05 +10001311}
1312
Michael Neuling801c0b22015-11-20 15:15:32 +11001313struct regbit {
Paul Mackerras06d67d52005-10-10 22:29:05 +10001314 unsigned long bit;
1315 const char *name;
Michael Neuling801c0b22015-11-20 15:15:32 +11001316};
1317
1318static struct regbit msr_bits[] = {
Anton Blanchard3bfd0c9c2011-11-24 19:35:57 +00001319#if defined(CONFIG_PPC64) && !defined(CONFIG_BOOKE)
1320 {MSR_SF, "SF"},
1321 {MSR_HV, "HV"},
1322#endif
1323 {MSR_VEC, "VEC"},
1324 {MSR_VSX, "VSX"},
1325#ifdef CONFIG_BOOKE
1326 {MSR_CE, "CE"},
1327#endif
Paul Mackerras06d67d52005-10-10 22:29:05 +10001328 {MSR_EE, "EE"},
1329 {MSR_PR, "PR"},
1330 {MSR_FP, "FP"},
1331 {MSR_ME, "ME"},
Anton Blanchard3bfd0c9c2011-11-24 19:35:57 +00001332#ifdef CONFIG_BOOKE
Kumar Gala1b983262008-11-19 04:39:53 +00001333 {MSR_DE, "DE"},
Anton Blanchard3bfd0c9c2011-11-24 19:35:57 +00001334#else
1335 {MSR_SE, "SE"},
1336 {MSR_BE, "BE"},
1337#endif
Paul Mackerras06d67d52005-10-10 22:29:05 +10001338 {MSR_IR, "IR"},
1339 {MSR_DR, "DR"},
Anton Blanchard3bfd0c9c2011-11-24 19:35:57 +00001340 {MSR_PMM, "PMM"},
1341#ifndef CONFIG_BOOKE
1342 {MSR_RI, "RI"},
1343 {MSR_LE, "LE"},
1344#endif
Paul Mackerras06d67d52005-10-10 22:29:05 +10001345 {0, NULL}
1346};
1347
Michael Neuling801c0b22015-11-20 15:15:32 +11001348static void print_bits(unsigned long val, struct regbit *bits, const char *sep)
Paul Mackerras06d67d52005-10-10 22:29:05 +10001349{
Michael Neuling801c0b22015-11-20 15:15:32 +11001350 const char *s = "";
Paul Mackerras06d67d52005-10-10 22:29:05 +10001351
Paul Mackerras06d67d52005-10-10 22:29:05 +10001352 for (; bits->bit; ++bits)
1353 if (val & bits->bit) {
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001354 pr_cont("%s%s", s, bits->name);
Michael Neuling801c0b22015-11-20 15:15:32 +11001355 s = sep;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001356 }
Michael Neuling801c0b22015-11-20 15:15:32 +11001357}
1358
1359#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1360static struct regbit msr_tm_bits[] = {
1361 {MSR_TS_T, "T"},
1362 {MSR_TS_S, "S"},
1363 {MSR_TM, "E"},
1364 {0, NULL}
1365};
1366
1367static void print_tm_bits(unsigned long val)
1368{
1369/*
1370 * This only prints something if at least one of the TM bit is set.
1371 * Inside the TM[], the output means:
1372 * E: Enabled (bit 32)
1373 * S: Suspended (bit 33)
1374 * T: Transactional (bit 34)
1375 */
1376 if (val & (MSR_TM | MSR_TS_S | MSR_TS_T)) {
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001377 pr_cont(",TM[");
Michael Neuling801c0b22015-11-20 15:15:32 +11001378 print_bits(val, msr_tm_bits, "");
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001379 pr_cont("]");
Michael Neuling801c0b22015-11-20 15:15:32 +11001380 }
1381}
1382#else
1383static void print_tm_bits(unsigned long val) {}
1384#endif
1385
1386static void print_msr_bits(unsigned long val)
1387{
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001388 pr_cont("<");
Michael Neuling801c0b22015-11-20 15:15:32 +11001389 print_bits(val, msr_bits, ",");
1390 print_tm_bits(val);
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001391 pr_cont(">");
Paul Mackerras06d67d52005-10-10 22:29:05 +10001392}
1393
1394#ifdef CONFIG_PPC64
anton@samba.orgf6f7dde2007-03-20 20:38:19 -05001395#define REG "%016lx"
Paul Mackerras06d67d52005-10-10 22:29:05 +10001396#define REGS_PER_LINE 4
1397#define LAST_VOLATILE 13
1398#else
anton@samba.orgf6f7dde2007-03-20 20:38:19 -05001399#define REG "%08lx"
Paul Mackerras06d67d52005-10-10 22:29:05 +10001400#define REGS_PER_LINE 8
1401#define LAST_VOLATILE 12
1402#endif
1403
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001404void show_regs(struct pt_regs * regs)
1405{
1406 int i, trap;
1407
Tejun Heoa43cb952013-04-30 15:27:17 -07001408 show_regs_print_info(KERN_DEFAULT);
1409
Michael Ellermana6036102017-08-23 23:56:24 +10001410 printk("NIP: "REG" LR: "REG" CTR: "REG"\n",
Paul Mackerras06d67d52005-10-10 22:29:05 +10001411 regs->nip, regs->link, regs->ctr);
1412 printk("REGS: %p TRAP: %04lx %s (%s)\n",
Serge E. Hallyn96b644b2006-10-02 02:18:13 -07001413 regs, regs->trap, print_tainted(), init_utsname()->release);
Michael Ellermana6036102017-08-23 23:56:24 +10001414 printk("MSR: "REG" ", regs->msr);
Michael Neuling801c0b22015-11-20 15:15:32 +11001415 print_msr_bits(regs->msr);
Michael Ellermanf6fc73f2017-08-23 23:56:23 +10001416 pr_cont(" CR: %08lx XER: %08lx\n", regs->ccr, regs->xer);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001417 trap = TRAP(regs);
Benjamin Herrenschmidt2271db22018-01-12 13:28:49 +11001418 if ((TRAP(regs) != 0xc00) && cpu_has_feature(CPU_FTR_CFAR))
Michael Ellerman7dae8652016-11-03 20:45:26 +11001419 pr_cont("CFAR: "REG" ", regs->orig_gpr3);
Anton Blanchardc5400642013-11-15 15:41:19 +11001420 if (trap == 0x200 || trap == 0x300 || trap == 0x600)
Kumar Galaba28c9a2011-10-06 02:53:38 +00001421#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
Michael Ellerman7dae8652016-11-03 20:45:26 +11001422 pr_cont("DEAR: "REG" ESR: "REG" ", regs->dar, regs->dsisr);
Kumar Gala14170782007-07-26 00:46:15 -05001423#else
Michael Ellerman7dae8652016-11-03 20:45:26 +11001424 pr_cont("DAR: "REG" DSISR: %08lx ", regs->dar, regs->dsisr);
Anton Blanchard9db8bcf2013-11-15 15:48:38 +11001425#endif
1426#ifdef CONFIG_PPC64
Michael Ellerman7dae8652016-11-03 20:45:26 +11001427 pr_cont("SOFTE: %ld ", regs->softe);
Anton Blanchard9db8bcf2013-11-15 15:48:38 +11001428#endif
1429#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Anton Blanchard6d888d12013-11-18 13:19:17 +11001430 if (MSR_TM_ACTIVE(regs->msr))
Michael Ellerman7dae8652016-11-03 20:45:26 +11001431 pr_cont("\nPACATMSCRATCH: %016llx ", get_paca()->tm_scratch);
Kumar Gala14170782007-07-26 00:46:15 -05001432#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001433
1434 for (i = 0; i < 32; i++) {
Paul Mackerras06d67d52005-10-10 22:29:05 +10001435 if ((i % REGS_PER_LINE) == 0)
Michael Ellerman7dae8652016-11-03 20:45:26 +11001436 pr_cont("\nGPR%02d: ", i);
1437 pr_cont(REG " ", regs->gpr[i]);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001438 if (i == LAST_VOLATILE && !FULL_REGS(regs))
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001439 break;
1440 }
Michael Ellerman7dae8652016-11-03 20:45:26 +11001441 pr_cont("\n");
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001442#ifdef CONFIG_KALLSYMS
1443 /*
1444 * Lookup NIP late so we have the best change of getting the
1445 * above info out without failing
1446 */
Benjamin Herrenschmidt058c78f2008-07-07 13:44:31 +10001447 printk("NIP ["REG"] %pS\n", regs->nip, (void *)regs->nip);
1448 printk("LR ["REG"] %pS\n", regs->link, (void *)regs->link);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001449#endif
1450 show_stack(current, (unsigned long *) regs->gpr[1]);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001451 if (!user_mode(regs))
1452 show_instructions(regs);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001453}
1454
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001455void flush_thread(void)
1456{
K.Prasade0780b72011-02-10 04:44:35 +00001457#ifdef CONFIG_HAVE_HW_BREAKPOINT
K.Prasad5aae8a52010-06-15 11:35:19 +05301458 flush_ptrace_hw_breakpoint(current);
K.Prasade0780b72011-02-10 04:44:35 +00001459#else /* CONFIG_HAVE_HW_BREAKPOINT */
Dave Kleikamp3bffb652010-02-08 11:51:18 +00001460 set_debug_reg_defaults(&current->thread);
K.Prasade0780b72011-02-10 04:44:35 +00001461#endif /* CONFIG_HAVE_HW_BREAKPOINT */
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001462}
1463
Sukadev Bhattiprolu9d2a4d72017-11-07 18:23:54 -08001464int set_thread_uses_vas(void)
1465{
1466#ifdef CONFIG_PPC_BOOK3S_64
1467 if (!cpu_has_feature(CPU_FTR_ARCH_300))
1468 return -EINVAL;
1469
1470 current->thread.used_vas = 1;
1471
1472 /*
1473 * Even a process that has no foreign real address mapping can use
1474 * an unpaired COPY instruction (to no real effect). Issue CP_ABORT
1475 * to clear any pending COPY and prevent a covert channel.
1476 *
1477 * __switch_to() will issue CP_ABORT on future context switches.
1478 */
1479 asm volatile(PPC_CP_ABORT);
1480
1481#endif /* CONFIG_PPC_BOOK3S_64 */
1482 return 0;
1483}
1484
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001485#ifdef CONFIG_PPC64
1486static DEFINE_SPINLOCK(vas_thread_id_lock);
1487static DEFINE_IDA(vas_thread_ida);
1488
1489/*
1490 * We need to assign a unique thread id to each thread in a process.
1491 *
1492 * This thread id, referred to as TIDR, and separate from the Linux's tgid,
1493 * is intended to be used to direct an ASB_Notify from the hardware to the
1494 * thread, when a suitable event occurs in the system.
1495 *
1496 * One such event is a "paste" instruction in the context of Fast Thread
1497 * Wakeup (aka Core-to-core wake up in the Virtual Accelerator Switchboard
1498 * (VAS) in POWER9.
1499 *
1500 * To get a unique TIDR per process we could simply reuse task_pid_nr() but
1501 * the problem is that task_pid_nr() is not yet available copy_thread() is
1502 * called. Fixing that would require changing more intrusive arch-neutral
1503 * code in code path in copy_process()?.
1504 *
1505 * Further, to assign unique TIDRs within each process, we need an atomic
1506 * field (or an IDR) in task_struct, which again intrudes into the arch-
1507 * neutral code. So try to assign globally unique TIDRs for now.
1508 *
1509 * NOTE: TIDR 0 indicates that the thread does not need a TIDR value.
1510 * For now, only threads that expect to be notified by the VAS
1511 * hardware need a TIDR value and we assign values > 0 for those.
1512 */
1513#define MAX_THREAD_CONTEXT ((1 << 16) - 1)
1514static int assign_thread_tidr(void)
1515{
1516 int index;
1517 int err;
1518
1519again:
1520 if (!ida_pre_get(&vas_thread_ida, GFP_KERNEL))
1521 return -ENOMEM;
1522
1523 spin_lock(&vas_thread_id_lock);
1524 err = ida_get_new_above(&vas_thread_ida, 1, &index);
1525 spin_unlock(&vas_thread_id_lock);
1526
1527 if (err == -EAGAIN)
1528 goto again;
1529 else if (err)
1530 return err;
1531
1532 if (index > MAX_THREAD_CONTEXT) {
1533 spin_lock(&vas_thread_id_lock);
1534 ida_remove(&vas_thread_ida, index);
1535 spin_unlock(&vas_thread_id_lock);
1536 return -ENOMEM;
1537 }
1538
1539 return index;
1540}
1541
1542static void free_thread_tidr(int id)
1543{
1544 spin_lock(&vas_thread_id_lock);
1545 ida_remove(&vas_thread_ida, id);
1546 spin_unlock(&vas_thread_id_lock);
1547}
1548
1549/*
1550 * Clear any TIDR value assigned to this thread.
1551 */
1552void clear_thread_tidr(struct task_struct *t)
1553{
1554 if (!t->thread.tidr)
1555 return;
1556
1557 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
1558 WARN_ON_ONCE(1);
1559 return;
1560 }
1561
1562 mtspr(SPRN_TIDR, 0);
1563 free_thread_tidr(t->thread.tidr);
1564 t->thread.tidr = 0;
1565}
1566
1567void arch_release_task_struct(struct task_struct *t)
1568{
1569 clear_thread_tidr(t);
1570}
1571
1572/*
1573 * Assign a unique TIDR (thread id) for task @t and set it in the thread
1574 * structure. For now, we only support setting TIDR for 'current' task.
1575 */
1576int set_thread_tidr(struct task_struct *t)
1577{
Vaibhav Jainaca7573f2017-11-28 08:23:04 +05301578 int rc;
1579
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001580 if (!cpu_has_feature(CPU_FTR_ARCH_300))
1581 return -EINVAL;
1582
1583 if (t != current)
1584 return -EINVAL;
1585
Vaibhav Jain7e4d4232017-11-24 14:03:38 +05301586 if (t->thread.tidr)
1587 return 0;
1588
Vaibhav Jainaca7573f2017-11-28 08:23:04 +05301589 rc = assign_thread_tidr();
1590 if (rc < 0)
1591 return rc;
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001592
Vaibhav Jainaca7573f2017-11-28 08:23:04 +05301593 t->thread.tidr = rc;
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001594 mtspr(SPRN_TIDR, t->thread.tidr);
1595
1596 return 0;
1597}
Christophe Lombardb1db5512018-01-11 09:55:25 +01001598EXPORT_SYMBOL_GPL(set_thread_tidr);
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001599
1600#endif /* CONFIG_PPC64 */
1601
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001602void
1603release_thread(struct task_struct *t)
1604{
1605}
1606
1607/*
Suresh Siddha55ccf3f2012-05-16 15:03:51 -07001608 * this gets called so that we can store coprocessor state into memory and
1609 * copy the current task into the new thread.
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001610 */
Suresh Siddha55ccf3f2012-05-16 15:03:51 -07001611int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001612{
Anton Blanchard579e6332015-10-29 11:44:09 +11001613 flush_all_to_thread(src);
Michael Neuling621b5062014-03-03 14:21:40 +11001614 /*
1615 * Flush TM state out so we can copy it. __switch_to_tm() does this
1616 * flush but it removes the checkpointed state from the current CPU and
1617 * transitions the CPU out of TM mode. Hence we need to call
1618 * tm_recheckpoint_new_task() (on the same task) to restore the
1619 * checkpointed state back and the TM mode.
Cyril Bur5d176f72016-09-14 18:02:16 +10001620 *
1621 * Can't pass dst because it isn't ready. Doesn't matter, passing
1622 * dst is only important for __switch_to()
Michael Neuling621b5062014-03-03 14:21:40 +11001623 */
Cyril Burdc310662016-09-23 16:18:24 +10001624 __switch_to_tm(src, src);
Michael Ellerman330a1eb2013-06-28 18:15:16 +10001625
Suresh Siddha55ccf3f2012-05-16 15:03:51 -07001626 *dst = *src;
Michael Ellerman330a1eb2013-06-28 18:15:16 +10001627
1628 clear_task_ebb(dst);
1629
Suresh Siddha55ccf3f2012-05-16 15:03:51 -07001630 return 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001631}
1632
Michael Ellermancec15482014-07-10 12:29:21 +10001633static void setup_ksp_vsid(struct task_struct *p, unsigned long sp)
1634{
Michael Ellerman4e003742017-10-19 15:08:43 +11001635#ifdef CONFIG_PPC_BOOK3S_64
Michael Ellermancec15482014-07-10 12:29:21 +10001636 unsigned long sp_vsid;
1637 unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp;
1638
Aneesh Kumar K.Vcaca2852016-04-29 23:26:07 +10001639 if (radix_enabled())
1640 return;
1641
Michael Ellermancec15482014-07-10 12:29:21 +10001642 if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
1643 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_1T)
1644 << SLB_VSID_SHIFT_1T;
1645 else
1646 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_256M)
1647 << SLB_VSID_SHIFT;
1648 sp_vsid |= SLB_VSID_KERNEL | llp;
1649 p->thread.ksp_vsid = sp_vsid;
1650#endif
1651}
1652
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001653/*
1654 * Copy a thread..
1655 */
Alexey Kardashevskiyefcac652011-03-02 15:18:48 +00001656
Alex Dowad6eca8932015-03-13 20:14:46 +02001657/*
1658 * Copy architecture-specific thread state
1659 */
Alexey Dobriyan6f2c55b2009-04-02 16:56:59 -07001660int copy_thread(unsigned long clone_flags, unsigned long usp,
Alex Dowad6eca8932015-03-13 20:14:46 +02001661 unsigned long kthread_arg, struct task_struct *p)
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001662{
1663 struct pt_regs *childregs, *kregs;
1664 extern void ret_from_fork(void);
Al Viro58254e12012-09-12 18:32:42 -04001665 extern void ret_from_kernel_thread(void);
1666 void (*f)(void);
Al Viro0cec6fd2006-01-12 01:06:02 -08001667 unsigned long sp = (unsigned long)task_stack_page(p) + THREAD_SIZE;
Michael Ellerman5d31a962016-03-24 22:04:04 +11001668 struct thread_info *ti = task_thread_info(p);
1669
1670 klp_init_thread_info(ti);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001671
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001672 /* Copy registers */
1673 sp -= sizeof(struct pt_regs);
1674 childregs = (struct pt_regs *) sp;
Al Viroab758192012-10-21 22:33:39 -04001675 if (unlikely(p->flags & PF_KTHREAD)) {
Alex Dowad6eca8932015-03-13 20:14:46 +02001676 /* kernel thread */
Al Viro58254e12012-09-12 18:32:42 -04001677 memset(childregs, 0, sizeof(struct pt_regs));
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001678 childregs->gpr[1] = sp + sizeof(struct pt_regs);
Anton Blanchard7cedd602014-02-04 16:08:51 +11001679 /* function */
1680 if (usp)
1681 childregs->gpr[14] = ppc_function_entry((void *)usp);
Al Viro58254e12012-09-12 18:32:42 -04001682#ifdef CONFIG_PPC64
Al Virob5e2fc12006-01-12 01:06:01 -08001683 clear_tsk_thread_flag(p, TIF_32BIT);
Madhavan Srinivasanc2e480b2017-12-20 09:25:42 +05301684 childregs->softe = IRQS_ENABLED;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001685#endif
Alex Dowad6eca8932015-03-13 20:14:46 +02001686 childregs->gpr[15] = kthread_arg;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001687 p->thread.regs = NULL; /* no user register state */
Al Viro138d1ce2012-10-11 08:41:43 -04001688 ti->flags |= _TIF_RESTOREALL;
Al Viro58254e12012-09-12 18:32:42 -04001689 f = ret_from_kernel_thread;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001690 } else {
Alex Dowad6eca8932015-03-13 20:14:46 +02001691 /* user thread */
Al Viroafa86fc2012-10-22 22:51:14 -04001692 struct pt_regs *regs = current_pt_regs();
Al Viro58254e12012-09-12 18:32:42 -04001693 CHECK_FULL_REGS(regs);
1694 *childregs = *regs;
Al Viroea516b12012-10-21 22:28:43 -04001695 if (usp)
1696 childregs->gpr[1] = usp;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001697 p->thread.regs = childregs;
Al Viro58254e12012-09-12 18:32:42 -04001698 childregs->gpr[3] = 0; /* Result from fork() */
Paul Mackerras06d67d52005-10-10 22:29:05 +10001699 if (clone_flags & CLONE_SETTLS) {
1700#ifdef CONFIG_PPC64
Denis Kirjanov9904b002010-07-29 22:04:39 +00001701 if (!is_32bit_task())
Paul Mackerras06d67d52005-10-10 22:29:05 +10001702 childregs->gpr[13] = childregs->gpr[6];
1703 else
1704#endif
1705 childregs->gpr[2] = childregs->gpr[6];
1706 }
Al Viro58254e12012-09-12 18:32:42 -04001707
1708 f = ret_from_fork;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001709 }
Cyril Burd272f662016-02-29 17:53:46 +11001710 childregs->msr &= ~(MSR_FP|MSR_VEC|MSR_VSX);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001711 sp -= STACK_FRAME_OVERHEAD;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001712
1713 /*
1714 * The way this works is that at some point in the future
1715 * some task will call _switch to switch to the new task.
1716 * That will pop off the stack frame created below and start
1717 * the new task running at ret_from_fork. The new task will
1718 * do some house keeping and then return from the fork or clone
1719 * system call, using the stack frame created above.
1720 */
Li Zhongaf945cf2013-05-06 22:44:41 +00001721 ((unsigned long *)sp)[0] = 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001722 sp -= sizeof(struct pt_regs);
1723 kregs = (struct pt_regs *) sp;
1724 sp -= STACK_FRAME_OVERHEAD;
1725 p->thread.ksp = sp;
Benjamin Herrenschmidtcbc95652013-09-24 15:17:21 +10001726#ifdef CONFIG_PPC32
Kumar Gala85218822008-04-28 16:21:22 +10001727 p->thread.ksp_limit = (unsigned long)task_stack_page(p) +
1728 _ALIGN_UP(sizeof(struct thread_info), 16);
Benjamin Herrenschmidtcbc95652013-09-24 15:17:21 +10001729#endif
Oleg Nesterov28d170ab2013-04-21 06:47:59 +00001730#ifdef CONFIG_HAVE_HW_BREAKPOINT
1731 p->thread.ptrace_bps[0] = NULL;
1732#endif
1733
Paul Mackerras18461962013-09-10 20:21:10 +10001734 p->thread.fp_save_area = NULL;
1735#ifdef CONFIG_ALTIVEC
1736 p->thread.vr_save_area = NULL;
1737#endif
1738
Michael Ellermancec15482014-07-10 12:29:21 +10001739 setup_ksp_vsid(p, sp);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001740
Alexey Kardashevskiyefcac652011-03-02 15:18:48 +00001741#ifdef CONFIG_PPC64
1742 if (cpu_has_feature(CPU_FTR_DSCR)) {
Anton Blanchard1021cb22012-09-03 16:49:47 +00001743 p->thread.dscr_inherit = current->thread.dscr_inherit;
Anton Blancharddb1231dc2015-12-09 20:11:47 +11001744 p->thread.dscr = mfspr(SPRN_DSCR);
Alexey Kardashevskiyefcac652011-03-02 15:18:48 +00001745 }
Haren Myneni92779242012-12-06 21:49:56 +00001746 if (cpu_has_feature(CPU_FTR_HAS_PPR))
1747 p->thread.ppr = INIT_PPR;
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001748
1749 p->thread.tidr = 0;
Alexey Kardashevskiyefcac652011-03-02 15:18:48 +00001750#endif
Anton Blanchard7cedd602014-02-04 16:08:51 +11001751 kregs->nip = ppc_function_entry(f);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001752 return 0;
1753}
1754
1755/*
1756 * Set up a thread for executing a new program
1757 */
Paul Mackerras06d67d52005-10-10 22:29:05 +10001758void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001759{
Michael Ellerman90eac722005-10-21 16:01:33 +10001760#ifdef CONFIG_PPC64
1761 unsigned long load_addr = regs->gpr[2]; /* saved by ELF_PLAT_INIT */
1762#endif
1763
Paul Mackerras06d67d52005-10-10 22:29:05 +10001764 /*
1765 * If we exec out of a kernel thread then thread.regs will not be
1766 * set. Do it now.
1767 */
1768 if (!current->thread.regs) {
Al Viro0cec6fd2006-01-12 01:06:02 -08001769 struct pt_regs *regs = task_stack_page(current) + THREAD_SIZE;
1770 current->thread.regs = regs - 1;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001771 }
1772
Cyril Bur8e96a872016-06-17 14:58:34 +10001773#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1774 /*
1775 * Clear any transactional state, we're exec()ing. The cause is
1776 * not important as there will never be a recheckpoint so it's not
1777 * user visible.
1778 */
1779 if (MSR_TM_SUSPENDED(mfmsr()))
1780 tm_reclaim_current(0);
1781#endif
1782
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001783 memset(regs->gpr, 0, sizeof(regs->gpr));
1784 regs->ctr = 0;
1785 regs->link = 0;
1786 regs->xer = 0;
1787 regs->ccr = 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001788 regs->gpr[1] = sp;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001789
Roland McGrath474f8192007-09-24 16:52:44 -07001790 /*
1791 * We have just cleared all the nonvolatile GPRs, so make
1792 * FULL_REGS(regs) return true. This is necessary to allow
1793 * ptrace to examine the thread immediately after exec.
1794 */
1795 regs->trap &= ~1UL;
1796
Paul Mackerras06d67d52005-10-10 22:29:05 +10001797#ifdef CONFIG_PPC32
1798 regs->mq = 0;
1799 regs->nip = start;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001800 regs->msr = MSR_USER;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001801#else
Denis Kirjanov9904b002010-07-29 22:04:39 +00001802 if (!is_32bit_task()) {
Rusty Russell94af3ab2013-11-20 22:15:02 +11001803 unsigned long entry;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001804
Rusty Russell94af3ab2013-11-20 22:15:02 +11001805 if (is_elf2_task()) {
1806 /* Look ma, no function descriptors! */
1807 entry = start;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001808
Rusty Russell94af3ab2013-11-20 22:15:02 +11001809 /*
1810 * Ulrich says:
1811 * The latest iteration of the ABI requires that when
1812 * calling a function (at its global entry point),
1813 * the caller must ensure r12 holds the entry point
1814 * address (so that the function can quickly
1815 * establish addressability).
1816 */
1817 regs->gpr[12] = start;
1818 /* Make sure that's restored on entry to userspace. */
1819 set_thread_flag(TIF_RESTOREALL);
1820 } else {
1821 unsigned long toc;
1822
1823 /* start is a relocated pointer to the function
1824 * descriptor for the elf _start routine. The first
1825 * entry in the function descriptor is the entry
1826 * address of _start and the second entry is the TOC
1827 * value we need to use.
1828 */
1829 __get_user(entry, (unsigned long __user *)start);
1830 __get_user(toc, (unsigned long __user *)start+1);
1831
1832 /* Check whether the e_entry function descriptor entries
1833 * need to be relocated before we can use them.
1834 */
1835 if (load_addr != 0) {
1836 entry += load_addr;
1837 toc += load_addr;
1838 }
1839 regs->gpr[2] = toc;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001840 }
1841 regs->nip = entry;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001842 regs->msr = MSR_USER64;
Stephen Rothwelld4bf9a72005-10-13 13:40:54 +10001843 } else {
1844 regs->nip = start;
1845 regs->gpr[2] = 0;
1846 regs->msr = MSR_USER32;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001847 }
1848#endif
Michael Neulingce48b212008-06-25 14:07:18 +10001849#ifdef CONFIG_VSX
1850 current->thread.used_vsr = 0;
1851#endif
Breno Leitao11958922017-06-02 18:43:30 -03001852 current->thread.load_fp = 0;
Paul Mackerrasde79f7b2013-09-10 20:20:42 +10001853 memset(&current->thread.fp_state, 0, sizeof(current->thread.fp_state));
Paul Mackerras18461962013-09-10 20:21:10 +10001854 current->thread.fp_save_area = NULL;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001855#ifdef CONFIG_ALTIVEC
Paul Mackerrasde79f7b2013-09-10 20:20:42 +10001856 memset(&current->thread.vr_state, 0, sizeof(current->thread.vr_state));
1857 current->thread.vr_state.vscr.u[3] = 0x00010000; /* Java mode disabled */
Paul Mackerras18461962013-09-10 20:21:10 +10001858 current->thread.vr_save_area = NULL;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001859 current->thread.vrsave = 0;
1860 current->thread.used_vr = 0;
Breno Leitao11958922017-06-02 18:43:30 -03001861 current->thread.load_vec = 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001862#endif /* CONFIG_ALTIVEC */
1863#ifdef CONFIG_SPE
1864 memset(current->thread.evr, 0, sizeof(current->thread.evr));
1865 current->thread.acc = 0;
1866 current->thread.spefscr = 0;
1867 current->thread.used_spe = 0;
1868#endif /* CONFIG_SPE */
Michael Neulingbc2a9402013-02-13 16:21:40 +00001869#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Michael Neulingbc2a9402013-02-13 16:21:40 +00001870 current->thread.tm_tfhar = 0;
1871 current->thread.tm_texasr = 0;
1872 current->thread.tm_tfiar = 0;
Breno Leitao7f22ced2017-06-05 11:40:59 -03001873 current->thread.load_tm = 0;
Michael Neulingbc2a9402013-02-13 16:21:40 +00001874#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
Ram Pai06bb53b2018-01-18 17:50:31 -08001875
1876 thread_pkey_regs_init(&current->thread);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001877}
Anton Blancharde1802b02014-08-20 08:00:02 +10001878EXPORT_SYMBOL(start_thread);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001879
1880#define PR_FP_ALL_EXCEPT (PR_FP_EXC_DIV | PR_FP_EXC_OVF | PR_FP_EXC_UND \
1881 | PR_FP_EXC_RES | PR_FP_EXC_INV)
1882
1883int set_fpexc_mode(struct task_struct *tsk, unsigned int val)
1884{
1885 struct pt_regs *regs = tsk->thread.regs;
1886
1887 /* This is a bit hairy. If we are an SPE enabled processor
1888 * (have embedded fp) we store the IEEE exception enable flags in
1889 * fpexc_mode. fpexc_mode is also used for setting FP exception
1890 * mode (asyn, precise, disabled) for 'Classic' FP. */
1891 if (val & PR_FP_EXC_SW_ENABLE) {
1892#ifdef CONFIG_SPE
Kumar Gala5e14d212007-09-13 01:44:20 -05001893 if (cpu_has_feature(CPU_FTR_SPE)) {
Joseph Myers640e9222013-12-10 23:07:45 +00001894 /*
1895 * When the sticky exception bits are set
1896 * directly by userspace, it must call prctl
1897 * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
1898 * in the existing prctl settings) or
1899 * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
1900 * the bits being set). <fenv.h> functions
1901 * saving and restoring the whole
1902 * floating-point environment need to do so
1903 * anyway to restore the prctl settings from
1904 * the saved environment.
1905 */
1906 tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
Kumar Gala5e14d212007-09-13 01:44:20 -05001907 tsk->thread.fpexc_mode = val &
1908 (PR_FP_EXC_SW_ENABLE | PR_FP_ALL_EXCEPT);
1909 return 0;
1910 } else {
1911 return -EINVAL;
1912 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001913#else
1914 return -EINVAL;
1915#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001916 }
Paul Mackerras06d67d52005-10-10 22:29:05 +10001917
1918 /* on a CONFIG_SPE this does not hurt us. The bits that
1919 * __pack_fe01 use do not overlap with bits used for
1920 * PR_FP_EXC_SW_ENABLE. Additionally, the MSR[FE0,FE1] bits
1921 * on CONFIG_SPE implementations are reserved so writing to
1922 * them does not change anything */
1923 if (val > PR_FP_EXC_PRECISE)
1924 return -EINVAL;
1925 tsk->thread.fpexc_mode = __pack_fe01(val);
1926 if (regs != NULL && (regs->msr & MSR_FP) != 0)
1927 regs->msr = (regs->msr & ~(MSR_FE0|MSR_FE1))
1928 | tsk->thread.fpexc_mode;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001929 return 0;
1930}
1931
1932int get_fpexc_mode(struct task_struct *tsk, unsigned long adr)
1933{
1934 unsigned int val;
1935
1936 if (tsk->thread.fpexc_mode & PR_FP_EXC_SW_ENABLE)
1937#ifdef CONFIG_SPE
Joseph Myers640e9222013-12-10 23:07:45 +00001938 if (cpu_has_feature(CPU_FTR_SPE)) {
1939 /*
1940 * When the sticky exception bits are set
1941 * directly by userspace, it must call prctl
1942 * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
1943 * in the existing prctl settings) or
1944 * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
1945 * the bits being set). <fenv.h> functions
1946 * saving and restoring the whole
1947 * floating-point environment need to do so
1948 * anyway to restore the prctl settings from
1949 * the saved environment.
1950 */
1951 tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
Kumar Gala5e14d212007-09-13 01:44:20 -05001952 val = tsk->thread.fpexc_mode;
Joseph Myers640e9222013-12-10 23:07:45 +00001953 } else
Kumar Gala5e14d212007-09-13 01:44:20 -05001954 return -EINVAL;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001955#else
1956 return -EINVAL;
1957#endif
1958 else
1959 val = __unpack_fe01(tsk->thread.fpexc_mode);
1960 return put_user(val, (unsigned int __user *) adr);
1961}
1962
Paul Mackerrasfab5db92006-06-07 16:14:40 +10001963int set_endian(struct task_struct *tsk, unsigned int val)
1964{
1965 struct pt_regs *regs = tsk->thread.regs;
1966
1967 if ((val == PR_ENDIAN_LITTLE && !cpu_has_feature(CPU_FTR_REAL_LE)) ||
1968 (val == PR_ENDIAN_PPC_LITTLE && !cpu_has_feature(CPU_FTR_PPC_LE)))
1969 return -EINVAL;
1970
1971 if (regs == NULL)
1972 return -EINVAL;
1973
1974 if (val == PR_ENDIAN_BIG)
1975 regs->msr &= ~MSR_LE;
1976 else if (val == PR_ENDIAN_LITTLE || val == PR_ENDIAN_PPC_LITTLE)
1977 regs->msr |= MSR_LE;
1978 else
1979 return -EINVAL;
1980
1981 return 0;
1982}
1983
1984int get_endian(struct task_struct *tsk, unsigned long adr)
1985{
1986 struct pt_regs *regs = tsk->thread.regs;
1987 unsigned int val;
1988
1989 if (!cpu_has_feature(CPU_FTR_PPC_LE) &&
1990 !cpu_has_feature(CPU_FTR_REAL_LE))
1991 return -EINVAL;
1992
1993 if (regs == NULL)
1994 return -EINVAL;
1995
1996 if (regs->msr & MSR_LE) {
1997 if (cpu_has_feature(CPU_FTR_REAL_LE))
1998 val = PR_ENDIAN_LITTLE;
1999 else
2000 val = PR_ENDIAN_PPC_LITTLE;
2001 } else
2002 val = PR_ENDIAN_BIG;
2003
2004 return put_user(val, (unsigned int __user *)adr);
2005}
2006
Paul Mackerrase9370ae2006-06-07 16:15:39 +10002007int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
2008{
2009 tsk->thread.align_ctl = val;
2010 return 0;
2011}
2012
2013int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
2014{
2015 return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr);
2016}
2017
Paul Mackerrasbb72c482007-02-19 11:42:42 +11002018static inline int valid_irq_stack(unsigned long sp, struct task_struct *p,
2019 unsigned long nbytes)
2020{
2021 unsigned long stack_page;
2022 unsigned long cpu = task_cpu(p);
2023
2024 /*
2025 * Avoid crashing if the stack has overflowed and corrupted
2026 * task_cpu(p), which is in the thread_info struct.
2027 */
2028 if (cpu < NR_CPUS && cpu_possible(cpu)) {
2029 stack_page = (unsigned long) hardirq_ctx[cpu];
2030 if (sp >= stack_page + sizeof(struct thread_struct)
2031 && sp <= stack_page + THREAD_SIZE - nbytes)
2032 return 1;
2033
2034 stack_page = (unsigned long) softirq_ctx[cpu];
2035 if (sp >= stack_page + sizeof(struct thread_struct)
2036 && sp <= stack_page + THREAD_SIZE - nbytes)
2037 return 1;
2038 }
2039 return 0;
2040}
2041
Anton Blanchard2f251942006-03-27 11:46:18 +11002042int validate_sp(unsigned long sp, struct task_struct *p,
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002043 unsigned long nbytes)
2044{
Al Viro0cec6fd2006-01-12 01:06:02 -08002045 unsigned long stack_page = (unsigned long)task_stack_page(p);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002046
2047 if (sp >= stack_page + sizeof(struct thread_struct)
2048 && sp <= stack_page + THREAD_SIZE - nbytes)
2049 return 1;
2050
Paul Mackerrasbb72c482007-02-19 11:42:42 +11002051 return valid_irq_stack(sp, p, nbytes);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002052}
2053
Anton Blanchard2f251942006-03-27 11:46:18 +11002054EXPORT_SYMBOL(validate_sp);
2055
Paul Mackerras06d67d52005-10-10 22:29:05 +10002056unsigned long get_wchan(struct task_struct *p)
2057{
2058 unsigned long ip, sp;
2059 int count = 0;
2060
2061 if (!p || p == current || p->state == TASK_RUNNING)
2062 return 0;
2063
2064 sp = p->thread.ksp;
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002065 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
Paul Mackerras06d67d52005-10-10 22:29:05 +10002066 return 0;
2067
2068 do {
2069 sp = *(unsigned long *)sp;
Kautuk Consul4ca360f2016-04-19 15:48:21 +05302070 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD) ||
2071 p->state == TASK_RUNNING)
Paul Mackerras06d67d52005-10-10 22:29:05 +10002072 return 0;
2073 if (count > 0) {
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002074 ip = ((unsigned long *)sp)[STACK_FRAME_LR_SAVE];
Paul Mackerras06d67d52005-10-10 22:29:05 +10002075 if (!in_sched_functions(ip))
2076 return ip;
2077 }
2078 } while (count++ < 16);
2079 return 0;
2080}
Paul Mackerras06d67d52005-10-10 22:29:05 +10002081
Johannes Bergc4d04be2008-11-20 03:24:07 +00002082static int kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002083
2084void show_stack(struct task_struct *tsk, unsigned long *stack)
2085{
Paul Mackerras06d67d52005-10-10 22:29:05 +10002086 unsigned long sp, ip, lr, newsp;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002087 int count = 0;
Paul Mackerras06d67d52005-10-10 22:29:05 +10002088 int firstframe = 1;
Steven Rostedt6794c782009-02-09 21:10:27 -08002089#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2090 int curr_frame = current->curr_ret_stack;
2091 extern void return_to_handler(void);
Steven Rostedt9135c3c2009-09-15 08:20:15 -07002092 unsigned long rth = (unsigned long)return_to_handler;
Steven Rostedt6794c782009-02-09 21:10:27 -08002093#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002094
2095 sp = (unsigned long) stack;
2096 if (tsk == NULL)
2097 tsk = current;
2098 if (sp == 0) {
2099 if (tsk == current)
Anton Blanchardacf620e2014-10-13 19:41:39 +11002100 sp = current_stack_pointer();
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002101 else
2102 sp = tsk->thread.ksp;
2103 }
2104
Paul Mackerras06d67d52005-10-10 22:29:05 +10002105 lr = 0;
2106 printk("Call Trace:\n");
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002107 do {
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002108 if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
Paul Mackerras06d67d52005-10-10 22:29:05 +10002109 return;
2110
2111 stack = (unsigned long *) sp;
2112 newsp = stack[0];
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002113 ip = stack[STACK_FRAME_LR_SAVE];
Paul Mackerras06d67d52005-10-10 22:29:05 +10002114 if (!firstframe || ip != lr) {
Benjamin Herrenschmidt058c78f2008-07-07 13:44:31 +10002115 printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip);
Steven Rostedt6794c782009-02-09 21:10:27 -08002116#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Anton Blanchard7d56c652014-09-17 17:07:03 +10002117 if ((ip == rth) && curr_frame >= 0) {
Michael Ellerman9a1f4902016-11-02 22:20:46 +11002118 pr_cont(" (%pS)",
Steven Rostedt6794c782009-02-09 21:10:27 -08002119 (void *)current->ret_stack[curr_frame].ret);
2120 curr_frame--;
2121 }
2122#endif
Paul Mackerras06d67d52005-10-10 22:29:05 +10002123 if (firstframe)
Michael Ellerman9a1f4902016-11-02 22:20:46 +11002124 pr_cont(" (unreliable)");
2125 pr_cont("\n");
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002126 }
Paul Mackerras06d67d52005-10-10 22:29:05 +10002127 firstframe = 0;
2128
2129 /*
2130 * See if this is an exception frame.
2131 * We look for the "regshere" marker in the current frame.
2132 */
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002133 if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE)
2134 && stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
Paul Mackerras06d67d52005-10-10 22:29:05 +10002135 struct pt_regs *regs = (struct pt_regs *)
2136 (sp + STACK_FRAME_OVERHEAD);
Paul Mackerras06d67d52005-10-10 22:29:05 +10002137 lr = regs->link;
Paul Mackerras9be9be22014-06-12 16:53:08 +10002138 printk("--- interrupt: %lx at %pS\n LR = %pS\n",
Benjamin Herrenschmidt058c78f2008-07-07 13:44:31 +10002139 regs->trap, (void *)regs->nip, (void *)lr);
Paul Mackerras06d67d52005-10-10 22:29:05 +10002140 firstframe = 1;
2141 }
2142
2143 sp = newsp;
2144 } while (count++ < kstack_depth_to_print);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002145}
Paul Mackerras06d67d52005-10-10 22:29:05 +10002146
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002147#ifdef CONFIG_PPC64
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002148/* Called with hard IRQs off */
Michael Ellerman0e377392013-06-13 21:04:56 +10002149void notrace __ppc64_runlatch_on(void)
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002150{
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002151 struct thread_info *ti = current_thread_info();
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002152
Nicholas Piggind1d0d5f2017-08-12 02:39:07 +10002153 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
2154 /*
2155 * Least significant bit (RUN) is the only writable bit of
2156 * the CTRL register, so we can avoid mfspr. 2.06 is not the
2157 * earliest ISA where this is the case, but it's convenient.
2158 */
2159 mtspr(SPRN_CTRLT, CTRL_RUNLATCH);
2160 } else {
2161 unsigned long ctrl;
2162
2163 /*
2164 * Some architectures (e.g., Cell) have writable fields other
2165 * than RUN, so do the read-modify-write.
2166 */
2167 ctrl = mfspr(SPRN_CTRLF);
2168 ctrl |= CTRL_RUNLATCH;
2169 mtspr(SPRN_CTRLT, ctrl);
2170 }
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002171
Benjamin Herrenschmidtfae2e0f2012-04-11 10:42:15 +10002172 ti->local_flags |= _TLF_RUNLATCH;
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002173}
2174
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002175/* Called with hard IRQs off */
Michael Ellerman0e377392013-06-13 21:04:56 +10002176void notrace __ppc64_runlatch_off(void)
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002177{
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002178 struct thread_info *ti = current_thread_info();
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002179
Benjamin Herrenschmidtfae2e0f2012-04-11 10:42:15 +10002180 ti->local_flags &= ~_TLF_RUNLATCH;
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002181
Nicholas Piggind1d0d5f2017-08-12 02:39:07 +10002182 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
2183 mtspr(SPRN_CTRLT, 0);
2184 } else {
2185 unsigned long ctrl;
2186
2187 ctrl = mfspr(SPRN_CTRLF);
2188 ctrl &= ~CTRL_RUNLATCH;
2189 mtspr(SPRN_CTRLT, ctrl);
2190 }
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002191}
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002192#endif /* CONFIG_PPC64 */
Benjamin Herrenschmidtf6a61682008-04-18 16:56:17 +10002193
Anton Blanchardd8390882009-02-22 01:50:03 +00002194unsigned long arch_align_stack(unsigned long sp)
2195{
2196 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
2197 sp -= get_random_int() & ~PAGE_MASK;
2198 return sp & ~0xf;
2199}
Anton Blanchard912f9ee2009-02-22 01:50:04 +00002200
2201static inline unsigned long brk_rnd(void)
2202{
2203 unsigned long rnd = 0;
2204
2205 /* 8MB for 32bit, 1GB for 64bit */
2206 if (is_32bit_task())
Daniel Cashman5ef11c32016-02-26 15:19:37 -08002207 rnd = (get_random_long() % (1UL<<(23-PAGE_SHIFT)));
Anton Blanchard912f9ee2009-02-22 01:50:04 +00002208 else
Daniel Cashman5ef11c32016-02-26 15:19:37 -08002209 rnd = (get_random_long() % (1UL<<(30-PAGE_SHIFT)));
Anton Blanchard912f9ee2009-02-22 01:50:04 +00002210
2211 return rnd << PAGE_SHIFT;
2212}
2213
2214unsigned long arch_randomize_brk(struct mm_struct *mm)
2215{
Anton Blanchard8bbde7a2009-09-21 16:52:35 +00002216 unsigned long base = mm->brk;
2217 unsigned long ret;
2218
Michael Ellerman4e003742017-10-19 15:08:43 +11002219#ifdef CONFIG_PPC_BOOK3S_64
Anton Blanchard8bbde7a2009-09-21 16:52:35 +00002220 /*
2221 * If we are using 1TB segments and we are allowed to randomise
2222 * the heap, we can put it above 1TB so it is backed by a 1TB
2223 * segment. Otherwise the heap will be in the bottom 1TB
2224 * which always uses 256MB segments and this may result in a
Aneesh Kumar K.Vcaca2852016-04-29 23:26:07 +10002225 * performance penalty. We don't need to worry about radix. For
2226 * radix, mmu_highuser_ssize remains unchanged from 256MB.
Anton Blanchard8bbde7a2009-09-21 16:52:35 +00002227 */
2228 if (!is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T))
2229 base = max_t(unsigned long, mm->brk, 1UL << SID_SHIFT_1T);
2230#endif
2231
2232 ret = PAGE_ALIGN(base + brk_rnd());
Anton Blanchard912f9ee2009-02-22 01:50:04 +00002233
2234 if (ret < mm->brk)
2235 return mm->brk;
2236
2237 return ret;
2238}
Anton Blanchard501cb162009-02-22 01:50:07 +00002239