blob: 0ae69b9390ce84ee7734f99fda4518b536007d79 [file] [log] [blame]
Dean Nelson94bd2702008-07-29 22:34:05 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
Robin Holta374c572009-04-13 14:40:18 -07006 * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved.
Dean Nelson94bd2702008-07-29 22:34:05 -07007 */
8
9/*
10 * Cross Partition Communication (XPC) sn2-based functions.
11 *
12 * Architecture specific implementation of common functions.
13 *
14 */
15
Dean Nelsone17d4162008-07-29 22:34:06 -070016#include <linux/delay.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090017#include <linux/slab.h>
Dean Nelson94bd2702008-07-29 22:34:05 -070018#include <asm/uncached.h>
Dean Nelson261f3b42008-07-29 22:34:16 -070019#include <asm/sn/mspec.h>
Dean Nelson94bd2702008-07-29 22:34:05 -070020#include <asm/sn/sn_sal.h>
21#include "xpc.h"
22
Dean Nelsonee6665e2008-07-29 22:34:13 -070023/*
24 * Define the number of u64s required to represent all the C-brick nasids
25 * as a bitmap. The cross-partition kernel modules deal only with
26 * C-brick nasids, thus the need for bitmaps which don't account for
27 * odd-numbered (non C-brick) nasids.
28 */
29#define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2)
30#define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
31#define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
32
33/*
34 * Memory for XPC's amo variables is allocated by the MSPEC driver. These
35 * pages are located in the lowest granule. The lowest granule uses 4k pages
36 * for cached references and an alternate TLB handler to never provide a
37 * cacheable mapping for the entire region. This will prevent speculative
38 * reading of cached copies of our lines from being issued which will cause
39 * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64
40 * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of
41 * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify
42 * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote
43 * partitions (i.e., XPCs) consider themselves currently engaged with the
44 * local XPC and 1 amo variable to request partition deactivation.
45 */
46#define XPC_NOTIFY_IRQ_AMOS_SN2 0
47#define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \
48 XP_MAX_NPARTITIONS_SN2)
49#define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \
50 XP_NASID_MASK_WORDS_SN2)
51#define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
52
53/*
54 * Buffer used to store a local copy of portions of a remote partition's
55 * reserved page (either its header and part_nasids mask, or its vars).
56 */
Dean Nelsonee6665e2008-07-29 22:34:13 -070057static void *xpc_remote_copy_buffer_base_sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -070058static char *xpc_remote_copy_buffer_sn2;
Dean Nelsonee6665e2008-07-29 22:34:13 -070059
Dean Nelson8e85c232008-07-29 22:34:13 -070060static struct xpc_vars_sn2 *xpc_vars_sn2;
61static struct xpc_vars_part_sn2 *xpc_vars_part_sn2;
Dean Nelson94bd2702008-07-29 22:34:05 -070062
Dean Nelson5b8669d2008-07-29 22:34:18 -070063static int
Robin Holta7665b02009-04-13 14:40:19 -070064xpc_setup_partitions_sn2(void)
Dean Nelson5b8669d2008-07-29 22:34:18 -070065{
66 /* nothing needs to be done */
67 return 0;
68}
69
Jack Steiner6f2584f2009-04-02 16:59:10 -070070static void
Robin Holta7665b02009-04-13 14:40:19 -070071xpc_teardown_partitions_sn2(void)
Jack Steiner6f2584f2009-04-02 16:59:10 -070072{
73 /* nothing needs to be done */
74}
75
Dean Nelson6e41017a2008-07-29 22:34:09 -070076/* SH_IPI_ACCESS shub register value on startup */
Dean Nelson8e85c232008-07-29 22:34:13 -070077static u64 xpc_sh1_IPI_access_sn2;
78static u64 xpc_sh2_IPI_access0_sn2;
79static u64 xpc_sh2_IPI_access1_sn2;
80static u64 xpc_sh2_IPI_access2_sn2;
81static u64 xpc_sh2_IPI_access3_sn2;
Dean Nelson6e41017a2008-07-29 22:34:09 -070082
83/*
84 * Change protections to allow IPI operations.
85 */
86static void
87xpc_allow_IPI_ops_sn2(void)
88{
89 int node;
90 int nasid;
91
Dean Nelsonea57f802008-07-29 22:34:14 -070092 /* !!! The following should get moved into SAL. */
Dean Nelson6e41017a2008-07-29 22:34:09 -070093 if (is_shub2()) {
Dean Nelson8e85c232008-07-29 22:34:13 -070094 xpc_sh2_IPI_access0_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -070095 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0));
Dean Nelson8e85c232008-07-29 22:34:13 -070096 xpc_sh2_IPI_access1_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -070097 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1));
Dean Nelson8e85c232008-07-29 22:34:13 -070098 xpc_sh2_IPI_access2_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -070099 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2));
Dean Nelson8e85c232008-07-29 22:34:13 -0700100 xpc_sh2_IPI_access3_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -0700101 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3));
102
103 for_each_online_node(node) {
104 nasid = cnodeid_to_nasid(node);
105 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
106 -1UL);
107 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
108 -1UL);
109 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
110 -1UL);
111 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
112 -1UL);
113 }
114 } else {
Dean Nelson8e85c232008-07-29 22:34:13 -0700115 xpc_sh1_IPI_access_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -0700116 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS));
117
118 for_each_online_node(node) {
119 nasid = cnodeid_to_nasid(node);
120 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
121 -1UL);
122 }
123 }
124}
125
126/*
127 * Restrict protections to disallow IPI operations.
128 */
129static void
130xpc_disallow_IPI_ops_sn2(void)
131{
132 int node;
133 int nasid;
134
Dean Nelsonea57f802008-07-29 22:34:14 -0700135 /* !!! The following should get moved into SAL. */
Dean Nelson6e41017a2008-07-29 22:34:09 -0700136 if (is_shub2()) {
137 for_each_online_node(node) {
138 nasid = cnodeid_to_nasid(node);
139 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
Dean Nelson8e85c232008-07-29 22:34:13 -0700140 xpc_sh2_IPI_access0_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700141 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
Dean Nelson8e85c232008-07-29 22:34:13 -0700142 xpc_sh2_IPI_access1_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700143 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
Dean Nelson8e85c232008-07-29 22:34:13 -0700144 xpc_sh2_IPI_access2_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700145 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
Dean Nelson8e85c232008-07-29 22:34:13 -0700146 xpc_sh2_IPI_access3_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700147 }
148 } else {
149 for_each_online_node(node) {
150 nasid = cnodeid_to_nasid(node);
151 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
Dean Nelson8e85c232008-07-29 22:34:13 -0700152 xpc_sh1_IPI_access_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700153 }
154 }
155}
156
Dean Nelson33ba3c72008-07-29 22:34:07 -0700157/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700158 * The following set of functions are used for the sending and receiving of
159 * IRQs (also known as IPIs). There are two flavors of IRQs, one that is
160 * associated with partition activity (SGI_XPC_ACTIVATE) and the other that
161 * is associated with channel activity (SGI_XPC_NOTIFY).
Dean Nelson33ba3c72008-07-29 22:34:07 -0700162 */
163
164static u64
Dean Nelsonc39838c2008-07-29 22:34:11 -0700165xpc_receive_IRQ_amo_sn2(struct amo *amo)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700166{
167 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR);
168}
169
170static enum xp_retval
Dean Nelsonc39838c2008-07-29 22:34:11 -0700171xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid,
172 int vector)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700173{
174 int ret = 0;
175 unsigned long irq_flags;
176
177 local_irq_save(irq_flags);
178
179 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag);
180 sn_send_IPI_phys(nasid, phys_cpuid, vector, 0);
181
182 /*
183 * We must always use the nofault function regardless of whether we
184 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
185 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700186 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700187 */
188 ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable),
189 xp_nofault_PIOR_target));
190
191 local_irq_restore(irq_flags);
192
Dean Nelson261f3b42008-07-29 22:34:16 -0700193 return (ret == 0) ? xpSuccess : xpPioReadError;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700194}
195
Dean Nelsonc39838c2008-07-29 22:34:11 -0700196static struct amo *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700197xpc_init_IRQ_amo_sn2(int index)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700198{
Dean Nelson8e85c232008-07-29 22:34:13 -0700199 struct amo *amo = xpc_vars_sn2->amos_page + index;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700200
Dean Nelsonc39838c2008-07-29 22:34:11 -0700201 (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700202 return amo;
203}
204
205/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700206 * Functions associated with SGI_XPC_ACTIVATE IRQ.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700207 */
208
209/*
Dean Nelson6e41017a2008-07-29 22:34:09 -0700210 * Notify the heartbeat check thread that an activate IRQ has been received.
211 */
212static irqreturn_t
213xpc_handle_activate_IRQ_sn2(int irq, void *dev_id)
214{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700215 unsigned long irq_flags;
216
217 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
218 xpc_activate_IRQ_rcvd++;
219 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
220
Dean Nelson6e41017a2008-07-29 22:34:09 -0700221 wake_up_interruptible(&xpc_activate_IRQ_wq);
222 return IRQ_HANDLED;
223}
224
225/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700226 * Flag the appropriate amo variable and send an IRQ to the specified node.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700227 */
228static void
Dean Nelsona812dcc2008-07-29 22:34:16 -0700229xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid,
230 int to_nasid, int to_phys_cpuid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700231{
Dean Nelsonc39838c2008-07-29 22:34:11 -0700232 struct amo *amos = (struct amo *)__va(amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700233 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700234 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700235
Dean Nelson04de7412008-07-29 22:34:14 -0700236 (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)],
237 BIT_MASK(from_nasid / 2), to_nasid,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700238 to_phys_cpuid, SGI_XPC_ACTIVATE);
239}
240
241static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700242xpc_send_local_activate_IRQ_sn2(int from_nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700243{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700244 unsigned long irq_flags;
Dean Nelson8e85c232008-07-29 22:34:13 -0700245 struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700246 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700247 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700248
249 /* fake the sending and receipt of an activate IRQ from remote nasid */
Dean Nelson04de7412008-07-29 22:34:14 -0700250 FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable),
251 FETCHOP_OR, BIT_MASK(from_nasid / 2));
252
Dean Nelson5b8669d2008-07-29 22:34:18 -0700253 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
254 xpc_activate_IRQ_rcvd++;
255 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
256
Dean Nelson6e41017a2008-07-29 22:34:09 -0700257 wake_up_interruptible(&xpc_activate_IRQ_wq);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700258}
259
Dean Nelson33ba3c72008-07-29 22:34:07 -0700260/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700261 * Functions associated with SGI_XPC_NOTIFY IRQ.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700262 */
263
264/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700265 * Check to see if any chctl flags were sent from the specified partition.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700266 */
267static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700268xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700269{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700270 union xpc_channel_ctl_flags chctl;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700271 unsigned long irq_flags;
272
Dean Nelson7fb5e592008-07-29 22:34:10 -0700273 chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2.
274 local_chctl_amo_va);
275 if (chctl.all_flags == 0)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700276 return;
277
Dean Nelson7fb5e592008-07-29 22:34:10 -0700278 spin_lock_irqsave(&part->chctl_lock, irq_flags);
279 part->chctl.all_flags |= chctl.all_flags;
280 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700281
Dean Nelson7fb5e592008-07-29 22:34:10 -0700282 dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags="
Randy Dunlapf509e342009-09-04 17:02:59 -0700283 "0x%llx\n", XPC_PARTID(part), chctl.all_flags);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700284
285 xpc_wakeup_channel_mgr(part);
286}
287
288/*
289 * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified
290 * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more
Dean Nelsonc39838c2008-07-29 22:34:11 -0700291 * than one partition, we use an amo structure per partition to indicate
Dean Nelson7fb5e592008-07-29 22:34:10 -0700292 * whether a partition has sent an IRQ or not. If it has, then wake up the
Dean Nelsona47d5da2008-07-29 22:34:09 -0700293 * associated kthread to handle it.
294 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700295 * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC
Dean Nelsona47d5da2008-07-29 22:34:09 -0700296 * running on other partitions.
297 *
298 * Noteworthy Arguments:
299 *
300 * irq - Interrupt ReQuest number. NOT USED.
301 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700302 * dev_id - partid of IRQ's potential sender.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700303 */
304static irqreturn_t
305xpc_handle_notify_IRQ_sn2(int irq, void *dev_id)
306{
307 short partid = (short)(u64)dev_id;
308 struct xpc_partition *part = &xpc_partitions[partid];
309
Dean Nelson261f3b42008-07-29 22:34:16 -0700310 DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700311
312 if (xpc_part_ref(part)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700313 xpc_check_for_sent_chctl_flags_sn2(part);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700314
315 xpc_part_deref(part);
316 }
317 return IRQ_HANDLED;
318}
319
320/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700321 * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor
322 * because the write to their associated amo variable completed after the IRQ
Dean Nelsona47d5da2008-07-29 22:34:09 -0700323 * was received.
324 */
325static void
Kees Cook25b42fa2017-10-22 14:49:58 -0700326xpc_check_for_dropped_notify_IRQ_sn2(struct timer_list *t)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700327{
Kees Cook25b42fa2017-10-22 14:49:58 -0700328 struct xpc_partition *part =
329 from_timer(part, t, sn.sn2.dropped_notify_IRQ_timer);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700330
331 if (xpc_part_ref(part)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700332 xpc_check_for_sent_chctl_flags_sn2(part);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700333
Kees Cook25b42fa2017-10-22 14:49:58 -0700334 t->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
335 add_timer(t);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700336 xpc_part_deref(part);
337 }
338}
339
340/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700341 * Send a notify IRQ to the remote partition that is associated with the
Dean Nelson33ba3c72008-07-29 22:34:07 -0700342 * specified channel.
343 */
344static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700345xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
346 char *chctl_flag_string, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700347{
348 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -0700349 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700350 union xpc_channel_ctl_flags chctl = { 0 };
Dean Nelson33ba3c72008-07-29 22:34:07 -0700351 enum xp_retval ret;
352
Dean Nelson83469b52008-07-29 22:34:18 -0700353 if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700354 chctl.flags[ch->number] = chctl_flag;
355 ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va,
356 chctl.all_flags,
357 part_sn2->notify_IRQ_nasid,
358 part_sn2->notify_IRQ_phys_cpuid,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700359 SGI_XPC_NOTIFY);
360 dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n",
Dean Nelson7fb5e592008-07-29 22:34:10 -0700361 chctl_flag_string, ch->partid, ch->number, ret);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700362 if (unlikely(ret != xpSuccess)) {
363 if (irq_flags != NULL)
364 spin_unlock_irqrestore(&ch->lock, *irq_flags);
365 XPC_DEACTIVATE_PARTITION(part, ret);
366 if (irq_flags != NULL)
367 spin_lock_irqsave(&ch->lock, *irq_flags);
368 }
369 }
370}
371
Dean Nelson7fb5e592008-07-29 22:34:10 -0700372#define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \
373 xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700374
375/*
376 * Make it look like the remote partition, which is associated with the
Dean Nelson7fb5e592008-07-29 22:34:10 -0700377 * specified channel, sent us a notify IRQ. This faked IRQ will be handled
378 * by xpc_check_for_dropped_notify_IRQ_sn2().
Dean Nelson33ba3c72008-07-29 22:34:07 -0700379 */
380static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700381xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
382 char *chctl_flag_string)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700383{
384 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelson7fb5e592008-07-29 22:34:10 -0700385 union xpc_channel_ctl_flags chctl = { 0 };
Dean Nelson33ba3c72008-07-29 22:34:07 -0700386
Dean Nelson7fb5e592008-07-29 22:34:10 -0700387 chctl.flags[ch->number] = chctl_flag;
388 FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va->
389 variable), FETCHOP_OR, chctl.all_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700390 dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n",
Dean Nelson7fb5e592008-07-29 22:34:10 -0700391 chctl_flag_string, ch->partid, ch->number);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700392}
393
Dean Nelson7fb5e592008-07-29 22:34:10 -0700394#define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \
395 xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700396
397static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700398xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch,
399 unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700400{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700401 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700402
403 args->reason = ch->reason;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700404 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700405}
406
407static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700408xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700409{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700410 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700411}
412
413static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700414xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700415{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700416 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700417
Dean Nelsonbd3e64c2008-07-29 22:34:19 -0700418 args->entry_size = ch->entry_size;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700419 args->local_nentries = ch->local_nentries;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700420 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700421}
422
423static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700424xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700425{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700426 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700427
428 args->remote_nentries = ch->remote_nentries;
429 args->local_nentries = ch->local_nentries;
Dean Nelson5b8669d2008-07-29 22:34:18 -0700430 args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue);
Dean Nelson7fb5e592008-07-29 22:34:10 -0700431 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700432}
433
434static void
Robin Holtefdd06e2009-04-13 14:40:19 -0700435xpc_send_chctl_opencomplete_sn2(struct xpc_channel *ch,
436 unsigned long *irq_flags)
437{
438 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENCOMPLETE, irq_flags);
439}
440
441static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700442xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700443{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700444 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700445}
446
447static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700448xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700449{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700450 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700451}
452
Jack Steiner6f2584f2009-04-02 16:59:10 -0700453static enum xp_retval
Dean Nelson5b8669d2008-07-29 22:34:18 -0700454xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
455 unsigned long msgqueue_pa)
456{
457 ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
Jack Steiner6f2584f2009-04-02 16:59:10 -0700458 return xpSuccess;
Dean Nelson5b8669d2008-07-29 22:34:18 -0700459}
460
Dean Nelson33ba3c72008-07-29 22:34:07 -0700461/*
462 * This next set of functions are used to keep track of when a partition is
463 * potentially engaged in accessing memory belonging to another partition.
464 */
465
466static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700467xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700468{
469 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700470 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700471 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700472 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700473
474 local_irq_save(irq_flags);
475
Dean Nelsonc39838c2008-07-29 22:34:11 -0700476 /* set bit corresponding to our partid in remote partition's amo */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700477 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
Dean Nelson04de7412008-07-29 22:34:14 -0700478 BIT(sn_partition_id));
479
Dean Nelson33ba3c72008-07-29 22:34:07 -0700480 /*
481 * We must always use the nofault function regardless of whether we
482 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
483 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700484 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700485 */
486 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
487 variable),
488 xp_nofault_PIOR_target));
489
490 local_irq_restore(irq_flags);
491}
492
493static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700494xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700495{
Dean Nelsona47d5da2008-07-29 22:34:09 -0700496 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700497 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700498 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700499 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700500 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700501
502 local_irq_save(irq_flags);
503
Dean Nelsonc39838c2008-07-29 22:34:11 -0700504 /* clear bit corresponding to our partid in remote partition's amo */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700505 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
Dean Nelson04de7412008-07-29 22:34:14 -0700506 ~BIT(sn_partition_id));
507
Dean Nelson33ba3c72008-07-29 22:34:07 -0700508 /*
509 * We must always use the nofault function regardless of whether we
510 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
511 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700512 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700513 */
514 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
515 variable),
516 xp_nofault_PIOR_target));
517
518 local_irq_restore(irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700519
Dean Nelson33ba3c72008-07-29 22:34:07 -0700520 /*
Dean Nelsona47d5da2008-07-29 22:34:09 -0700521 * Send activate IRQ to get other side to see that we've cleared our
Dean Nelsonc39838c2008-07-29 22:34:11 -0700522 * bit in their engaged partitions amo.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700523 */
Dean Nelson7fb5e592008-07-29 22:34:10 -0700524 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -0700525 cnodeid_to_nasid(0),
526 part_sn2->activate_IRQ_nasid,
527 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700528}
529
Dean Nelson5b8669d2008-07-29 22:34:18 -0700530static void
531xpc_assume_partition_disengaged_sn2(short partid)
532{
533 struct amo *amo = xpc_vars_sn2->amos_page +
534 XPC_ENGAGED_PARTITIONS_AMO_SN2;
535
536 /* clear bit(s) based on partid mask in our partition's amo */
537 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
538 ~BIT(partid));
539}
540
Dean Nelsona47d5da2008-07-29 22:34:09 -0700541static int
542xpc_partition_engaged_sn2(short partid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700543{
Dean Nelson8e85c232008-07-29 22:34:13 -0700544 struct amo *amo = xpc_vars_sn2->amos_page +
545 XPC_ENGAGED_PARTITIONS_AMO_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700546
Dean Nelsonc39838c2008-07-29 22:34:11 -0700547 /* our partition's amo variable ANDed with partid mask */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700548 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
Dean Nelson04de7412008-07-29 22:34:14 -0700549 BIT(partid)) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700550}
551
Dean Nelsona47d5da2008-07-29 22:34:09 -0700552static int
553xpc_any_partition_engaged_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700554{
Dean Nelson8e85c232008-07-29 22:34:13 -0700555 struct amo *amo = xpc_vars_sn2->amos_page +
556 XPC_ENGAGED_PARTITIONS_AMO_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700557
Dean Nelsonc39838c2008-07-29 22:34:11 -0700558 /* our partition's amo variable */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700559 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700560}
561
Dean Nelson6e41017a2008-07-29 22:34:09 -0700562/* original protection values for each node */
563static u64 xpc_prot_vec_sn2[MAX_NUMNODES];
564
565/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700566 * Change protections to allow amo operations on non-Shub 1.1 systems.
Dean Nelson6e41017a2008-07-29 22:34:09 -0700567 */
568static enum xp_retval
Dean Nelsonc39838c2008-07-29 22:34:11 -0700569xpc_allow_amo_ops_sn2(struct amo *amos_page)
Dean Nelson6e41017a2008-07-29 22:34:09 -0700570{
Dean Nelson6c1c3252008-11-05 17:27:22 -0600571 enum xp_retval ret = xpSuccess;
Dean Nelson6e41017a2008-07-29 22:34:09 -0700572
573 /*
574 * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST
575 * collides with memory operations. On those systems we call
Dean Nelsonc39838c2008-07-29 22:34:11 -0700576 * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead.
Dean Nelson6e41017a2008-07-29 22:34:09 -0700577 */
Dean Nelson6c1c3252008-11-05 17:27:22 -0600578 if (!enable_shub_wars_1_1())
579 ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE);
580
581 return ret;
Dean Nelson6e41017a2008-07-29 22:34:09 -0700582}
583
584/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700585 * Change protections to allow amo operations on Shub 1.1 systems.
Dean Nelson6e41017a2008-07-29 22:34:09 -0700586 */
587static void
Dean Nelsonc39838c2008-07-29 22:34:11 -0700588xpc_allow_amo_ops_shub_wars_1_1_sn2(void)
Dean Nelson6e41017a2008-07-29 22:34:09 -0700589{
590 int node;
591 int nasid;
592
593 if (!enable_shub_wars_1_1())
594 return;
595
596 for_each_online_node(node) {
597 nasid = cnodeid_to_nasid(node);
598 /* save current protection values */
599 xpc_prot_vec_sn2[node] =
600 (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid,
601 SH1_MD_DQLP_MMR_DIR_PRIVEC0));
602 /* open up everything */
603 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
604 SH1_MD_DQLP_MMR_DIR_PRIVEC0),
605 -1UL);
606 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
607 SH1_MD_DQRP_MMR_DIR_PRIVEC0),
608 -1UL);
609 }
610}
611
Dean Nelson94bd2702008-07-29 22:34:05 -0700612static enum xp_retval
Dean Nelsona812dcc2008-07-29 22:34:16 -0700613xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa,
Dean Nelson261f3b42008-07-29 22:34:16 -0700614 size_t *len)
615{
616 s64 status;
617 enum xp_retval ret;
618
Randy Dunlapf509e342009-09-04 17:02:59 -0700619 status = sn_partition_reserved_page_pa((u64)buf, cookie,
620 (u64 *)rp_pa, (u64 *)len);
Dean Nelson261f3b42008-07-29 22:34:16 -0700621 if (status == SALRET_OK)
622 ret = xpSuccess;
623 else if (status == SALRET_MORE_PASSES)
624 ret = xpNeedMoreInfo;
625 else
626 ret = xpSalError;
627
628 return ret;
629}
630
631
Dean Nelson5b8669d2008-07-29 22:34:18 -0700632static int
Robin Holta7665b02009-04-13 14:40:19 -0700633xpc_setup_rsvd_page_sn2(struct xpc_rsvd_page *rp)
Dean Nelson94bd2702008-07-29 22:34:05 -0700634{
Dean Nelsonc39838c2008-07-29 22:34:11 -0700635 struct amo *amos_page;
Dean Nelson94bd2702008-07-29 22:34:05 -0700636 int i;
637 int ret;
638
Dean Nelson8e85c232008-07-29 22:34:13 -0700639 xpc_vars_sn2 = XPC_RP_VARS(rp);
Dean Nelson94bd2702008-07-29 22:34:05 -0700640
Robin Holta374c572009-04-13 14:40:18 -0700641 rp->sn.sn2.vars_pa = xp_pa(xpc_vars_sn2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700642
Dean Nelsone17d4162008-07-29 22:34:06 -0700643 /* vars_part array follows immediately after vars */
Dean Nelson8e85c232008-07-29 22:34:13 -0700644 xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) +
645 XPC_RP_VARS_SIZE);
Dean Nelsone17d4162008-07-29 22:34:06 -0700646
Dean Nelson94bd2702008-07-29 22:34:05 -0700647 /*
Dean Nelson8e85c232008-07-29 22:34:13 -0700648 * Before clearing xpc_vars_sn2, see if a page of amos had been
649 * previously allocated. If not we'll need to allocate one and set
650 * permissions so that cross-partition amos are allowed.
Dean Nelson94bd2702008-07-29 22:34:05 -0700651 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700652 * The allocated amo page needs MCA reporting to remain disabled after
Dean Nelson94bd2702008-07-29 22:34:05 -0700653 * XPC has unloaded. To make this work, we keep a copy of the pointer
Dean Nelson8e85c232008-07-29 22:34:13 -0700654 * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure,
Dean Nelson94bd2702008-07-29 22:34:05 -0700655 * which is pointed to by the reserved page, and re-use that saved copy
Dean Nelsonc39838c2008-07-29 22:34:11 -0700656 * on subsequent loads of XPC. This amo page is never freed, and its
Dean Nelson94bd2702008-07-29 22:34:05 -0700657 * memory protections are never restricted.
658 */
Dean Nelson8e85c232008-07-29 22:34:13 -0700659 amos_page = xpc_vars_sn2->amos_page;
Dean Nelson94bd2702008-07-29 22:34:05 -0700660 if (amos_page == NULL) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700661 amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1));
Dean Nelson94bd2702008-07-29 22:34:05 -0700662 if (amos_page == NULL) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700663 dev_err(xpc_part, "can't allocate page of amos\n");
Dean Nelson5b8669d2008-07-29 22:34:18 -0700664 return -ENOMEM;
Dean Nelson94bd2702008-07-29 22:34:05 -0700665 }
666
667 /*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700668 * Open up amo-R/W to cpu. This is done on Shub 1.1 systems
669 * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called.
Dean Nelson94bd2702008-07-29 22:34:05 -0700670 */
Dean Nelsonc39838c2008-07-29 22:34:11 -0700671 ret = xpc_allow_amo_ops_sn2(amos_page);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700672 if (ret != xpSuccess) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700673 dev_err(xpc_part, "can't allow amo operations\n");
Dean Nelson6e41017a2008-07-29 22:34:09 -0700674 uncached_free_page(__IA64_UNCACHED_OFFSET |
675 TO_PHYS((u64)amos_page), 1);
Dean Nelson5b8669d2008-07-29 22:34:18 -0700676 return -EPERM;
Dean Nelson94bd2702008-07-29 22:34:05 -0700677 }
678 }
679
Dean Nelson8e85c232008-07-29 22:34:13 -0700680 /* clear xpc_vars_sn2 */
681 memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2));
Dean Nelson94bd2702008-07-29 22:34:05 -0700682
Dean Nelson8e85c232008-07-29 22:34:13 -0700683 xpc_vars_sn2->version = XPC_V_VERSION;
684 xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0);
685 xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0);
Dean Nelsona812dcc2008-07-29 22:34:16 -0700686 xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2);
Dean Nelson8e85c232008-07-29 22:34:13 -0700687 xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page);
688 xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */
Dean Nelson94bd2702008-07-29 22:34:05 -0700689
Dean Nelson8e85c232008-07-29 22:34:13 -0700690 /* clear xpc_vars_part_sn2 */
691 memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) *
Dean Nelson261f3b42008-07-29 22:34:16 -0700692 XP_MAX_NPARTITIONS_SN2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700693
Dean Nelsonc39838c2008-07-29 22:34:11 -0700694 /* initialize the activate IRQ related amo variables */
Dean Nelson04de7412008-07-29 22:34:14 -0700695 for (i = 0; i < xpc_nasid_mask_nlongs; i++)
Dean Nelsonee6665e2008-07-29 22:34:13 -0700696 (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i);
Dean Nelson94bd2702008-07-29 22:34:05 -0700697
Dean Nelsonc39838c2008-07-29 22:34:11 -0700698 /* initialize the engaged remote partitions related amo variables */
Dean Nelsonee6665e2008-07-29 22:34:13 -0700699 (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2);
700 (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700701
Dean Nelson5b8669d2008-07-29 22:34:18 -0700702 return 0;
Dean Nelson94bd2702008-07-29 22:34:05 -0700703}
704
Robin Holta374c572009-04-13 14:40:18 -0700705static int
706xpc_hb_allowed_sn2(short partid, void *heartbeating_to_mask)
707{
708 return test_bit(partid, heartbeating_to_mask);
709}
710
711static void
712xpc_allow_hb_sn2(short partid)
713{
714 DBUG_ON(xpc_vars_sn2 == NULL);
715 set_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
716}
717
718static void
719xpc_disallow_hb_sn2(short partid)
720{
721 DBUG_ON(xpc_vars_sn2 == NULL);
722 clear_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
723}
724
725static void
726xpc_disallow_all_hbs_sn2(void)
727{
728 DBUG_ON(xpc_vars_sn2 == NULL);
729 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, xp_max_npartitions);
730}
731
Dean Nelson33ba3c72008-07-29 22:34:07 -0700732static void
733xpc_increment_heartbeat_sn2(void)
734{
Dean Nelson8e85c232008-07-29 22:34:13 -0700735 xpc_vars_sn2->heartbeat++;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700736}
737
738static void
739xpc_offline_heartbeat_sn2(void)
740{
741 xpc_increment_heartbeat_sn2();
Dean Nelson8e85c232008-07-29 22:34:13 -0700742 xpc_vars_sn2->heartbeat_offline = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700743}
744
745static void
746xpc_online_heartbeat_sn2(void)
747{
748 xpc_increment_heartbeat_sn2();
Dean Nelson8e85c232008-07-29 22:34:13 -0700749 xpc_vars_sn2->heartbeat_offline = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700750}
751
752static void
753xpc_heartbeat_init_sn2(void)
754{
Dean Nelson8e85c232008-07-29 22:34:13 -0700755 DBUG_ON(xpc_vars_sn2 == NULL);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700756
Dean Nelson8e85c232008-07-29 22:34:13 -0700757 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700758 xpc_online_heartbeat_sn2();
759}
760
761static void
762xpc_heartbeat_exit_sn2(void)
763{
764 xpc_offline_heartbeat_sn2();
765}
766
Dean Nelson61deb862008-07-29 22:34:17 -0700767static enum xp_retval
768xpc_get_remote_heartbeat_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700769{
770 struct xpc_vars_sn2 *remote_vars;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700771 enum xp_retval ret;
772
Dean Nelsonee6665e2008-07-29 22:34:13 -0700773 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700774
Dean Nelson61deb862008-07-29 22:34:17 -0700775 /* pull the remote vars structure that contains the heartbeat */
776 ret = xp_remote_memcpy(xp_pa(remote_vars),
777 part->sn.sn2.remote_vars_pa,
778 XPC_RP_VARS_SIZE);
779 if (ret != xpSuccess)
780 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700781
Randy Dunlapf509e342009-09-04 17:02:59 -0700782 dev_dbg(xpc_part, "partid=%d, heartbeat=%lld, last_heartbeat=%lld, "
783 "heartbeat_offline=%lld, HB_mask[0]=0x%lx\n", XPC_PARTID(part),
Dean Nelson61deb862008-07-29 22:34:17 -0700784 remote_vars->heartbeat, part->last_heartbeat,
785 remote_vars->heartbeat_offline,
786 remote_vars->heartbeating_to_mask[0]);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700787
Dean Nelson61deb862008-07-29 22:34:17 -0700788 if ((remote_vars->heartbeat == part->last_heartbeat &&
Robin Holta374c572009-04-13 14:40:18 -0700789 !remote_vars->heartbeat_offline) ||
790 !xpc_hb_allowed_sn2(sn_partition_id,
791 remote_vars->heartbeating_to_mask)) {
Dean Nelson61deb862008-07-29 22:34:17 -0700792 ret = xpNoHeartbeat;
793 } else {
Dean Nelson33ba3c72008-07-29 22:34:07 -0700794 part->last_heartbeat = remote_vars->heartbeat;
795 }
Dean Nelson61deb862008-07-29 22:34:17 -0700796
797 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700798}
799
800/*
801 * Get a copy of the remote partition's XPC variables from the reserved page.
802 *
803 * remote_vars points to a buffer that is cacheline aligned for BTE copies and
804 * assumed to be of size XPC_RP_VARS_SIZE.
805 */
806static enum xp_retval
Dean Nelsona812dcc2008-07-29 22:34:16 -0700807xpc_get_remote_vars_sn2(unsigned long remote_vars_pa,
808 struct xpc_vars_sn2 *remote_vars)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700809{
810 enum xp_retval ret;
811
812 if (remote_vars_pa == 0)
813 return xpVarsNotSet;
814
815 /* pull over the cross partition variables */
Dean Nelsona812dcc2008-07-29 22:34:16 -0700816 ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700817 XPC_RP_VARS_SIZE);
818 if (ret != xpSuccess)
819 return ret;
820
821 if (XPC_VERSION_MAJOR(remote_vars->version) !=
822 XPC_VERSION_MAJOR(XPC_V_VERSION)) {
823 return xpBadVersion;
824 }
825
826 return xpSuccess;
827}
828
829static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700830xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
Dean Nelsona812dcc2008-07-29 22:34:16 -0700831 unsigned long remote_rp_pa, int nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700832{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700833 xpc_send_local_activate_IRQ_sn2(nasid);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700834}
835
836static void
837xpc_request_partition_reactivation_sn2(struct xpc_partition *part)
838{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700839 xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700840}
841
842static void
843xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
844{
845 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
846 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700847 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700848 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700849 sizeof(struct amo)));
Dean Nelsona47d5da2008-07-29 22:34:09 -0700850
851 local_irq_save(irq_flags);
852
Dean Nelsonc39838c2008-07-29 22:34:11 -0700853 /* set bit corresponding to our partid in remote partition's amo */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700854 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
Dean Nelson04de7412008-07-29 22:34:14 -0700855 BIT(sn_partition_id));
856
Dean Nelsona47d5da2008-07-29 22:34:09 -0700857 /*
858 * We must always use the nofault function regardless of whether we
859 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
860 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700861 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700862 */
863 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
864 variable),
865 xp_nofault_PIOR_target));
866
867 local_irq_restore(irq_flags);
868
869 /*
870 * Send activate IRQ to get other side to see that we've set our
Dean Nelsonc39838c2008-07-29 22:34:11 -0700871 * bit in their deactivate request amo.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700872 */
Dean Nelson7fb5e592008-07-29 22:34:10 -0700873 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -0700874 cnodeid_to_nasid(0),
875 part_sn2->activate_IRQ_nasid,
876 part_sn2->activate_IRQ_phys_cpuid);
877}
878
879static void
880xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
881{
882 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700883 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700884 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700885 sizeof(struct amo)));
Dean Nelsona47d5da2008-07-29 22:34:09 -0700886
887 local_irq_save(irq_flags);
888
Dean Nelsonc39838c2008-07-29 22:34:11 -0700889 /* clear bit corresponding to our partid in remote partition's amo */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700890 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
Dean Nelson04de7412008-07-29 22:34:14 -0700891 ~BIT(sn_partition_id));
892
Dean Nelsona47d5da2008-07-29 22:34:09 -0700893 /*
894 * We must always use the nofault function regardless of whether we
895 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
896 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700897 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700898 */
899 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
900 variable),
901 xp_nofault_PIOR_target));
902
903 local_irq_restore(irq_flags);
904}
905
906static int
907xpc_partition_deactivation_requested_sn2(short partid)
908{
Dean Nelson8e85c232008-07-29 22:34:13 -0700909 struct amo *amo = xpc_vars_sn2->amos_page +
910 XPC_DEACTIVATE_REQUEST_AMO_SN2;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700911
Dean Nelsonc39838c2008-07-29 22:34:11 -0700912 /* our partition's amo variable ANDed with partid mask */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700913 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
Dean Nelson04de7412008-07-29 22:34:14 -0700914 BIT(partid)) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700915}
916
917/*
918 * Update the remote partition's info.
919 */
920static void
921xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -0700922 unsigned long *remote_rp_ts_jiffies,
Dean Nelsona812dcc2008-07-29 22:34:16 -0700923 unsigned long remote_rp_pa,
924 unsigned long remote_vars_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700925 struct xpc_vars_sn2 *remote_vars)
926{
Dean Nelsona47d5da2008-07-29 22:34:09 -0700927 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
928
Dean Nelson33ba3c72008-07-29 22:34:07 -0700929 part->remote_rp_version = remote_rp_version;
930 dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n",
931 part->remote_rp_version);
932
Dean Nelson81fe7882008-07-29 22:34:15 -0700933 part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies;
934 dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n",
935 part->remote_rp_ts_jiffies);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700936
937 part->remote_rp_pa = remote_rp_pa;
938 dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa);
939
Dean Nelsona47d5da2008-07-29 22:34:09 -0700940 part_sn2->remote_vars_pa = remote_vars_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700941 dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700942 part_sn2->remote_vars_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700943
Dean Nelson158bc692009-01-15 13:50:57 -0800944 part->last_heartbeat = remote_vars->heartbeat - 1;
Randy Dunlapf509e342009-09-04 17:02:59 -0700945 dev_dbg(xpc_part, " last_heartbeat = 0x%016llx\n",
Dean Nelson33ba3c72008-07-29 22:34:07 -0700946 part->last_heartbeat);
947
Dean Nelsona47d5da2008-07-29 22:34:09 -0700948 part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700949 dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700950 part_sn2->remote_vars_part_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700951
Dean Nelsona47d5da2008-07-29 22:34:09 -0700952 part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid;
953 dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n",
954 part_sn2->activate_IRQ_nasid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700955
Dean Nelsona47d5da2008-07-29 22:34:09 -0700956 part_sn2->activate_IRQ_phys_cpuid =
957 remote_vars->activate_IRQ_phys_cpuid;
958 dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n",
959 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700960
Dean Nelsona47d5da2008-07-29 22:34:09 -0700961 part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700962 dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700963 part_sn2->remote_amos_page_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700964
Dean Nelsona47d5da2008-07-29 22:34:09 -0700965 part_sn2->remote_vars_version = remote_vars->version;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700966 dev_dbg(xpc_part, " remote_vars_version = 0x%x\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700967 part_sn2->remote_vars_version);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700968}
969
970/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700971 * Prior code has determined the nasid which generated a activate IRQ.
972 * Inspect that nasid to determine if its partition needs to be activated
973 * or deactivated.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700974 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700975 * A partition is considered "awaiting activation" if our partition
Dean Nelson33ba3c72008-07-29 22:34:07 -0700976 * flags indicate it is not active and it has a heartbeat. A
977 * partition is considered "awaiting deactivation" if our partition
978 * flags indicate it is active but it has no heartbeat or it is not
979 * sending its heartbeat to us.
980 *
981 * To determine the heartbeat, the remote nasid must have a properly
982 * initialized reserved page.
983 */
984static void
Dean Nelson6e41017a2008-07-29 22:34:09 -0700985xpc_identify_activate_IRQ_req_sn2(int nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700986{
987 struct xpc_rsvd_page *remote_rp;
988 struct xpc_vars_sn2 *remote_vars;
Dean Nelsona812dcc2008-07-29 22:34:16 -0700989 unsigned long remote_rp_pa;
990 unsigned long remote_vars_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700991 int remote_rp_version;
992 int reactivate = 0;
Dean Nelson81fe7882008-07-29 22:34:15 -0700993 unsigned long remote_rp_ts_jiffies = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700994 short partid;
995 struct xpc_partition *part;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700996 struct xpc_partition_sn2 *part_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700997 enum xp_retval ret;
998
999 /* pull over the reserved page structure */
1000
Dean Nelsonee6665e2008-07-29 22:34:13 -07001001 remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001002
1003 ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa);
1004 if (ret != xpSuccess) {
1005 dev_warn(xpc_part, "unable to get reserved page from nasid %d, "
1006 "which sent interrupt, reason=%d\n", nasid, ret);
1007 return;
1008 }
1009
Robin Holta374c572009-04-13 14:40:18 -07001010 remote_vars_pa = remote_rp->sn.sn2.vars_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001011 remote_rp_version = remote_rp->version;
Dean Nelson81fe7882008-07-29 22:34:15 -07001012 remote_rp_ts_jiffies = remote_rp->ts_jiffies;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001013
1014 partid = remote_rp->SAL_partid;
1015 part = &xpc_partitions[partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -07001016 part_sn2 = &part->sn.sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001017
1018 /* pull over the cross partition variables */
1019
Dean Nelsonee6665e2008-07-29 22:34:13 -07001020 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001021
1022 ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
1023 if (ret != xpSuccess) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001024 dev_warn(xpc_part, "unable to get XPC variables from nasid %d, "
1025 "which sent interrupt, reason=%d\n", nasid, ret);
1026
1027 XPC_DEACTIVATE_PARTITION(part, ret);
1028 return;
1029 }
1030
Dean Nelson6e41017a2008-07-29 22:34:09 -07001031 part->activate_IRQ_rcvd++;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001032
1033 dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = "
Randy Dunlapf509e342009-09-04 17:02:59 -07001034 "%lld:0x%lx\n", (int)nasid, (int)partid,
1035 part->activate_IRQ_rcvd,
Dean Nelson33ba3c72008-07-29 22:34:07 -07001036 remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]);
1037
1038 if (xpc_partition_disengaged(part) &&
Dean Nelson83469b52008-07-29 22:34:18 -07001039 part->act_state == XPC_P_AS_INACTIVE) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001040
1041 xpc_update_partition_info_sn2(part, remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -07001042 &remote_rp_ts_jiffies,
1043 remote_rp_pa, remote_vars_pa,
1044 remote_vars);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001045
Dean Nelsona47d5da2008-07-29 22:34:09 -07001046 if (xpc_partition_deactivation_requested_sn2(partid)) {
1047 /*
1048 * Other side is waiting on us to deactivate even though
1049 * we already have.
1050 */
1051 return;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001052 }
1053
1054 xpc_activate_partition(part);
1055 return;
1056 }
1057
1058 DBUG_ON(part->remote_rp_version == 0);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001059 DBUG_ON(part_sn2->remote_vars_version == 0);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001060
Dean Nelson81fe7882008-07-29 22:34:15 -07001061 if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001062
Dean Nelsona47d5da2008-07-29 22:34:09 -07001063 /* the other side rebooted */
Dean Nelson33ba3c72008-07-29 22:34:07 -07001064
Dean Nelsona47d5da2008-07-29 22:34:09 -07001065 DBUG_ON(xpc_partition_engaged_sn2(partid));
1066 DBUG_ON(xpc_partition_deactivation_requested_sn2(partid));
Dean Nelson33ba3c72008-07-29 22:34:07 -07001067
1068 xpc_update_partition_info_sn2(part, remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -07001069 &remote_rp_ts_jiffies,
1070 remote_rp_pa, remote_vars_pa,
1071 remote_vars);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001072 reactivate = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001073 }
1074
Dean Nelsona47d5da2008-07-29 22:34:09 -07001075 if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001076 /* still waiting on other side to disengage from us */
1077 return;
1078 }
1079
Dean Nelsona47d5da2008-07-29 22:34:09 -07001080 if (reactivate)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001081 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001082 else if (xpc_partition_deactivation_requested_sn2(partid))
Dean Nelson33ba3c72008-07-29 22:34:07 -07001083 XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001084}
1085
1086/*
Dean Nelsonc39838c2008-07-29 22:34:11 -07001087 * Loop through the activation amo variables and process any bits
Dean Nelson33ba3c72008-07-29 22:34:07 -07001088 * which are set. Each bit indicates a nasid sending a partition
1089 * activation or deactivation request.
1090 *
1091 * Return #of IRQs detected.
1092 */
1093int
Dean Nelson6e41017a2008-07-29 22:34:09 -07001094xpc_identify_activate_IRQ_sender_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001095{
Dean Nelson04de7412008-07-29 22:34:14 -07001096 int l;
1097 int b;
1098 unsigned long nasid_mask_long;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001099 u64 nasid; /* remote nasid */
1100 int n_IRQs_detected = 0;
Dean Nelsonc39838c2008-07-29 22:34:11 -07001101 struct amo *act_amos;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001102
Dean Nelson8e85c232008-07-29 22:34:13 -07001103 act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001104
Dean Nelson04de7412008-07-29 22:34:14 -07001105 /* scan through activate amo variables looking for non-zero entries */
1106 for (l = 0; l < xpc_nasid_mask_nlongs; l++) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001107
1108 if (xpc_exiting)
1109 break;
1110
Dean Nelson04de7412008-07-29 22:34:14 -07001111 nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]);
1112
1113 b = find_first_bit(&nasid_mask_long, BITS_PER_LONG);
1114 if (b >= BITS_PER_LONG) {
1115 /* no IRQs from nasids in this amo variable */
Dean Nelson33ba3c72008-07-29 22:34:07 -07001116 continue;
1117 }
1118
Dean Nelson04de7412008-07-29 22:34:14 -07001119 dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l,
1120 nasid_mask_long);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001121
1122 /*
1123 * If this nasid has been added to the machine since
1124 * our partition was reset, this will retain the
1125 * remote nasid in our reserved pages machine mask.
1126 * This is used in the event of module reload.
1127 */
Dean Nelson04de7412008-07-29 22:34:14 -07001128 xpc_mach_nasids[l] |= nasid_mask_long;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001129
1130 /* locate the nasid(s) which sent interrupts */
1131
Dean Nelson04de7412008-07-29 22:34:14 -07001132 do {
1133 n_IRQs_detected++;
1134 nasid = (l * BITS_PER_LONG + b) * 2;
Randy Dunlapf509e342009-09-04 17:02:59 -07001135 dev_dbg(xpc_part, "interrupt from nasid %lld\n", nasid);
Dean Nelson04de7412008-07-29 22:34:14 -07001136 xpc_identify_activate_IRQ_req_sn2(nasid);
1137
1138 b = find_next_bit(&nasid_mask_long, BITS_PER_LONG,
1139 b + 1);
1140 } while (b < BITS_PER_LONG);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001141 }
1142 return n_IRQs_detected;
1143}
1144
1145static void
Dean Nelson5b8669d2008-07-29 22:34:18 -07001146xpc_process_activate_IRQ_rcvd_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001147{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001148 unsigned long irq_flags;
1149 int n_IRQs_expected;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001150 int n_IRQs_detected;
1151
Dean Nelson5b8669d2008-07-29 22:34:18 -07001152 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1153 n_IRQs_expected = xpc_activate_IRQ_rcvd;
1154 xpc_activate_IRQ_rcvd = 0;
1155 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1156
Dean Nelson6e41017a2008-07-29 22:34:09 -07001157 n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2();
Dean Nelson33ba3c72008-07-29 22:34:07 -07001158 if (n_IRQs_detected < n_IRQs_expected) {
Dean Nelsonc39838c2008-07-29 22:34:11 -07001159 /* retry once to help avoid missing amo */
Dean Nelson6e41017a2008-07-29 22:34:09 -07001160 (void)xpc_identify_activate_IRQ_sender_sn2();
Dean Nelson33ba3c72008-07-29 22:34:07 -07001161 }
1162}
1163
Dean Nelsone17d4162008-07-29 22:34:06 -07001164/*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001165 * Setup the channel structures that are sn2 specific.
Dean Nelsone17d4162008-07-29 22:34:06 -07001166 */
1167static enum xp_retval
Robin Holta7665b02009-04-13 14:40:19 -07001168xpc_setup_ch_structures_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001169{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001170 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001171 struct xpc_channel_sn2 *ch_sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001172 enum xp_retval retval;
1173 int ret;
1174 int cpuid;
1175 int ch_number;
Dean Nelsone17d4162008-07-29 22:34:06 -07001176 struct timer_list *timer;
1177 short partid = XPC_PARTID(part);
1178
Dean Nelsone17d4162008-07-29 22:34:06 -07001179 /* allocate all the required GET/PUT values */
1180
Dean Nelson185c3a12008-07-29 22:34:11 -07001181 part_sn2->local_GPs =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001182 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1183 &part_sn2->local_GPs_base);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001184 if (part_sn2->local_GPs == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001185 dev_err(xpc_chan, "can't get memory for local get/put "
1186 "values\n");
Dean Nelson5b8669d2008-07-29 22:34:18 -07001187 return xpNoMemory;
Dean Nelsone17d4162008-07-29 22:34:06 -07001188 }
1189
Dean Nelson185c3a12008-07-29 22:34:11 -07001190 part_sn2->remote_GPs =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001191 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1192 &part_sn2->remote_GPs_base);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001193 if (part_sn2->remote_GPs == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001194 dev_err(xpc_chan, "can't get memory for remote get/put "
1195 "values\n");
1196 retval = xpNoMemory;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001197 goto out_1;
Dean Nelsone17d4162008-07-29 22:34:06 -07001198 }
1199
Dean Nelsona47d5da2008-07-29 22:34:09 -07001200 part_sn2->remote_GPs_pa = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001201
1202 /* allocate all the required open and close args */
1203
Dean Nelson5b8669d2008-07-29 22:34:18 -07001204 part_sn2->local_openclose_args =
1205 xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE,
1206 GFP_KERNEL, &part_sn2->
1207 local_openclose_args_base);
1208 if (part_sn2->local_openclose_args == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001209 dev_err(xpc_chan, "can't get memory for local connect args\n");
1210 retval = xpNoMemory;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001211 goto out_2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001212 }
1213
Dean Nelsona47d5da2008-07-29 22:34:09 -07001214 part_sn2->remote_openclose_args_pa = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001215
Dean Nelson7fb5e592008-07-29 22:34:10 -07001216 part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001217
Dean Nelson7fb5e592008-07-29 22:34:10 -07001218 part_sn2->notify_IRQ_nasid = 0;
1219 part_sn2->notify_IRQ_phys_cpuid = 0;
1220 part_sn2->remote_chctl_amo_va = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001221
Dean Nelson7fb5e592008-07-29 22:34:10 -07001222 sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001223 ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2,
Dean Nelson7fb5e592008-07-29 22:34:10 -07001224 IRQF_SHARED, part_sn2->notify_IRQ_owner,
Dean Nelsona47d5da2008-07-29 22:34:09 -07001225 (void *)(u64)partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001226 if (ret != 0) {
1227 dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
1228 "errno=%d\n", -ret);
1229 retval = xpLackOfResources;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001230 goto out_3;
Dean Nelsone17d4162008-07-29 22:34:06 -07001231 }
1232
Dean Nelson7fb5e592008-07-29 22:34:10 -07001233 /* Setup a timer to check for dropped notify IRQs */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001234 timer = &part_sn2->dropped_notify_IRQ_timer;
Kees Cook25b42fa2017-10-22 14:49:58 -07001235 timer_setup(timer, xpc_check_for_dropped_notify_IRQ_sn2, 0);
Dean Nelson7fb5e592008-07-29 22:34:10 -07001236 timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001237 add_timer(timer);
1238
Dean Nelsone17d4162008-07-29 22:34:06 -07001239 for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
Dean Nelson5b8669d2008-07-29 22:34:18 -07001240 ch_sn2 = &part->channels[ch_number].sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001241
Dean Nelson5b8669d2008-07-29 22:34:18 -07001242 ch_sn2->local_GP = &part_sn2->local_GPs[ch_number];
1243 ch_sn2->local_openclose_args =
1244 &part_sn2->local_openclose_args[ch_number];
Dean Nelsone17d4162008-07-29 22:34:06 -07001245
Dean Nelson5b8669d2008-07-29 22:34:18 -07001246 mutex_init(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001247 }
1248
1249 /*
Dean Nelsone17d4162008-07-29 22:34:06 -07001250 * Setup the per partition specific variables required by the
1251 * remote partition to establish channel connections with us.
1252 *
1253 * The setting of the magic # indicates that these per partition
1254 * specific variables are ready to be used.
1255 */
Dean Nelsona812dcc2008-07-29 22:34:16 -07001256 xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs);
Dean Nelson8e85c232008-07-29 22:34:13 -07001257 xpc_vars_part_sn2[partid].openclose_args_pa =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001258 xp_pa(part_sn2->local_openclose_args);
Dean Nelson8e85c232008-07-29 22:34:13 -07001259 xpc_vars_part_sn2[partid].chctl_amo_pa =
Dean Nelsona812dcc2008-07-29 22:34:16 -07001260 xp_pa(part_sn2->local_chctl_amo_va);
Dean Nelsone17d4162008-07-29 22:34:06 -07001261 cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */
Dean Nelson8e85c232008-07-29 22:34:13 -07001262 xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid);
1263 xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid =
1264 cpu_physical_id(cpuid);
1265 xpc_vars_part_sn2[partid].nchannels = part->nchannels;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001266 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001267
1268 return xpSuccess;
1269
Dean Nelson5b8669d2008-07-29 22:34:18 -07001270 /* setup of ch structures failed */
Dean Nelsone17d4162008-07-29 22:34:06 -07001271out_3:
Dean Nelson5b8669d2008-07-29 22:34:18 -07001272 kfree(part_sn2->local_openclose_args_base);
1273 part_sn2->local_openclose_args = NULL;
1274out_2:
Dean Nelsona47d5da2008-07-29 22:34:09 -07001275 kfree(part_sn2->remote_GPs_base);
1276 part_sn2->remote_GPs = NULL;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001277out_1:
Dean Nelsona47d5da2008-07-29 22:34:09 -07001278 kfree(part_sn2->local_GPs_base);
1279 part_sn2->local_GPs = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001280 return retval;
1281}
1282
1283/*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001284 * Teardown the channel structures that are sn2 specific.
Dean Nelsone17d4162008-07-29 22:34:06 -07001285 */
1286static void
Robin Holta7665b02009-04-13 14:40:19 -07001287xpc_teardown_ch_structures_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001288{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001289 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001290 short partid = XPC_PARTID(part);
1291
1292 /*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001293 * Indicate that the variables specific to the remote partition are no
1294 * longer available for its use.
Dean Nelsone17d4162008-07-29 22:34:06 -07001295 */
Dean Nelson8e85c232008-07-29 22:34:13 -07001296 xpc_vars_part_sn2[partid].magic = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001297
Dean Nelsone17d4162008-07-29 22:34:06 -07001298 /* in case we've still got outstanding timers registered... */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001299 del_timer_sync(&part_sn2->dropped_notify_IRQ_timer);
Dean Nelson5b8669d2008-07-29 22:34:18 -07001300 free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001301
Dean Nelson5b8669d2008-07-29 22:34:18 -07001302 kfree(part_sn2->local_openclose_args_base);
1303 part_sn2->local_openclose_args = NULL;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001304 kfree(part_sn2->remote_GPs_base);
1305 part_sn2->remote_GPs = NULL;
1306 kfree(part_sn2->local_GPs_base);
1307 part_sn2->local_GPs = NULL;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001308 part_sn2->local_chctl_amo_va = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001309}
1310
1311/*
1312 * Create a wrapper that hides the underlying mechanism for pulling a cacheline
1313 * (or multiple cachelines) from a remote partition.
1314 *
Dean Nelsona812dcc2008-07-29 22:34:16 -07001315 * src_pa must be a cacheline aligned physical address on the remote partition.
Dean Nelsone17d4162008-07-29 22:34:06 -07001316 * dst must be a cacheline aligned virtual address on this partition.
1317 * cnt must be cacheline sized
1318 */
Dean Nelsonea57f802008-07-29 22:34:14 -07001319/* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */
Dean Nelsone17d4162008-07-29 22:34:06 -07001320static enum xp_retval
1321xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001322 const unsigned long src_pa, size_t cnt)
Dean Nelsone17d4162008-07-29 22:34:06 -07001323{
1324 enum xp_retval ret;
1325
Dean Nelsona812dcc2008-07-29 22:34:16 -07001326 DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa));
1327 DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst));
Dean Nelsone17d4162008-07-29 22:34:06 -07001328 DBUG_ON(cnt != L1_CACHE_ALIGN(cnt));
1329
Dean Nelson83469b52008-07-29 22:34:18 -07001330 if (part->act_state == XPC_P_AS_DEACTIVATING)
Dean Nelsone17d4162008-07-29 22:34:06 -07001331 return part->reason;
1332
Dean Nelsona812dcc2008-07-29 22:34:16 -07001333 ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt);
Dean Nelsone17d4162008-07-29 22:34:06 -07001334 if (ret != xpSuccess) {
1335 dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed,"
1336 " ret=%d\n", XPC_PARTID(part), ret);
1337 }
1338 return ret;
1339}
1340
1341/*
1342 * Pull the remote per partition specific variables from the specified
1343 * partition.
1344 */
1345static enum xp_retval
1346xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
1347{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001348 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001349 u8 buffer[L1_CACHE_BYTES * 2];
1350 struct xpc_vars_part_sn2 *pulled_entry_cacheline =
1351 (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer);
1352 struct xpc_vars_part_sn2 *pulled_entry;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001353 unsigned long remote_entry_cacheline_pa;
1354 unsigned long remote_entry_pa;
Dean Nelsone17d4162008-07-29 22:34:06 -07001355 short partid = XPC_PARTID(part);
1356 enum xp_retval ret;
1357
1358 /* pull the cacheline that contains the variables we're interested in */
1359
Dean Nelsona47d5da2008-07-29 22:34:09 -07001360 DBUG_ON(part_sn2->remote_vars_part_pa !=
1361 L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa));
Dean Nelsone17d4162008-07-29 22:34:06 -07001362 DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2);
1363
Dean Nelsona47d5da2008-07-29 22:34:09 -07001364 remote_entry_pa = part_sn2->remote_vars_part_pa +
Dean Nelsone17d4162008-07-29 22:34:06 -07001365 sn_partition_id * sizeof(struct xpc_vars_part_sn2);
1366
1367 remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
1368
1369 pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline
1370 + (remote_entry_pa &
1371 (L1_CACHE_BYTES - 1)));
1372
1373 ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001374 remote_entry_cacheline_pa,
Dean Nelsone17d4162008-07-29 22:34:06 -07001375 L1_CACHE_BYTES);
1376 if (ret != xpSuccess) {
1377 dev_dbg(xpc_chan, "failed to pull XPC vars_part from "
1378 "partition %d, ret=%d\n", partid, ret);
1379 return ret;
1380 }
1381
1382 /* see if they've been set up yet */
1383
Dean Nelson5b8669d2008-07-29 22:34:18 -07001384 if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 &&
1385 pulled_entry->magic != XPC_VP_MAGIC2_SN2) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001386
1387 if (pulled_entry->magic != 0) {
1388 dev_dbg(xpc_chan, "partition %d's XPC vars_part for "
Randy Dunlapf509e342009-09-04 17:02:59 -07001389 "partition %d has bad magic value (=0x%llx)\n",
Dean Nelsone17d4162008-07-29 22:34:06 -07001390 partid, sn_partition_id, pulled_entry->magic);
1391 return xpBadMagic;
1392 }
1393
1394 /* they've not been initialized yet */
1395 return xpRetry;
1396 }
1397
Dean Nelson5b8669d2008-07-29 22:34:18 -07001398 if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001399
1400 /* validate the variables */
1401
1402 if (pulled_entry->GPs_pa == 0 ||
1403 pulled_entry->openclose_args_pa == 0 ||
Dean Nelson7fb5e592008-07-29 22:34:10 -07001404 pulled_entry->chctl_amo_pa == 0) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001405
1406 dev_err(xpc_chan, "partition %d's XPC vars_part for "
1407 "partition %d are not valid\n", partid,
1408 sn_partition_id);
1409 return xpInvalidAddress;
1410 }
1411
1412 /* the variables we imported look to be valid */
1413
Dean Nelsona47d5da2008-07-29 22:34:09 -07001414 part_sn2->remote_GPs_pa = pulled_entry->GPs_pa;
1415 part_sn2->remote_openclose_args_pa =
Dean Nelsone17d4162008-07-29 22:34:06 -07001416 pulled_entry->openclose_args_pa;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001417 part_sn2->remote_chctl_amo_va =
Dean Nelsonc39838c2008-07-29 22:34:11 -07001418 (struct amo *)__va(pulled_entry->chctl_amo_pa);
Dean Nelson7fb5e592008-07-29 22:34:10 -07001419 part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid;
1420 part_sn2->notify_IRQ_phys_cpuid =
1421 pulled_entry->notify_IRQ_phys_cpuid;
Dean Nelsone17d4162008-07-29 22:34:06 -07001422
1423 if (part->nchannels > pulled_entry->nchannels)
1424 part->nchannels = pulled_entry->nchannels;
1425
1426 /* let the other side know that we've pulled their variables */
1427
Dean Nelson5b8669d2008-07-29 22:34:18 -07001428 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001429 }
1430
Dean Nelson5b8669d2008-07-29 22:34:18 -07001431 if (pulled_entry->magic == XPC_VP_MAGIC1_SN2)
Dean Nelsone17d4162008-07-29 22:34:06 -07001432 return xpRetry;
1433
1434 return xpSuccess;
1435}
1436
1437/*
1438 * Establish first contact with the remote partititon. This involves pulling
1439 * the XPC per partition variables from the remote partition and waiting for
1440 * the remote partition to pull ours.
1441 */
1442static enum xp_retval
1443xpc_make_first_contact_sn2(struct xpc_partition *part)
1444{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001445 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001446 enum xp_retval ret;
1447
Dean Nelson33ba3c72008-07-29 22:34:07 -07001448 /*
Dean Nelsonc39838c2008-07-29 22:34:11 -07001449 * Register the remote partition's amos with SAL so it can handle
Dean Nelson33ba3c72008-07-29 22:34:07 -07001450 * and cleanup errors within that address range should the remote
1451 * partition go down. We don't unregister this range because it is
1452 * difficult to tell when outstanding writes to the remote partition
1453 * are finished and thus when it is safe to unregister. This should
1454 * not result in wasted space in the SAL xp_addr_region table because
1455 * we should get the same page for remote_amos_page_pa after module
1456 * reloads and system reboots.
1457 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001458 if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -07001459 PAGE_SIZE, 1) < 0) {
1460 dev_warn(xpc_part, "xpc_activating(%d) failed to register "
1461 "xp_addr region\n", XPC_PARTID(part));
1462
1463 ret = xpPhysAddrRegFailed;
1464 XPC_DEACTIVATE_PARTITION(part, ret);
1465 return ret;
1466 }
1467
Dean Nelsona47d5da2008-07-29 22:34:09 -07001468 /*
1469 * Send activate IRQ to get other side to activate if they've not
1470 * already begun to do so.
1471 */
Dean Nelson7fb5e592008-07-29 22:34:10 -07001472 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -07001473 cnodeid_to_nasid(0),
1474 part_sn2->activate_IRQ_nasid,
1475 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001476
Dean Nelsone17d4162008-07-29 22:34:06 -07001477 while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) {
1478 if (ret != xpRetry) {
1479 XPC_DEACTIVATE_PARTITION(part, ret);
1480 return ret;
1481 }
1482
1483 dev_dbg(xpc_part, "waiting to make first contact with "
1484 "partition %d\n", XPC_PARTID(part));
1485
1486 /* wait a 1/4 of a second or so */
1487 (void)msleep_interruptible(250);
1488
Dean Nelson83469b52008-07-29 22:34:18 -07001489 if (part->act_state == XPC_P_AS_DEACTIVATING)
Dean Nelsone17d4162008-07-29 22:34:06 -07001490 return part->reason;
1491 }
1492
1493 return xpSuccess;
1494}
1495
1496/*
Dean Nelson7fb5e592008-07-29 22:34:10 -07001497 * Get the chctl flags and pull the openclose args and/or remote GPs as needed.
Dean Nelsone17d4162008-07-29 22:34:06 -07001498 */
1499static u64
Dean Nelson7fb5e592008-07-29 22:34:10 -07001500xpc_get_chctl_all_flags_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001501{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001502 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001503 unsigned long irq_flags;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001504 union xpc_channel_ctl_flags chctl;
Dean Nelsone17d4162008-07-29 22:34:06 -07001505 enum xp_retval ret;
1506
1507 /*
Dean Nelson7fb5e592008-07-29 22:34:10 -07001508 * See if there are any chctl flags to be handled.
Dean Nelsone17d4162008-07-29 22:34:06 -07001509 */
1510
Dean Nelson7fb5e592008-07-29 22:34:10 -07001511 spin_lock_irqsave(&part->chctl_lock, irq_flags);
1512 chctl = part->chctl;
1513 if (chctl.all_flags != 0)
1514 part->chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001515
Dean Nelson7fb5e592008-07-29 22:34:10 -07001516 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
Dean Nelsone17d4162008-07-29 22:34:06 -07001517
Dean Nelson7fb5e592008-07-29 22:34:10 -07001518 if (xpc_any_openclose_chctl_flags_set(&chctl)) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001519 ret = xpc_pull_remote_cachelines_sn2(part, part->
1520 remote_openclose_args,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001521 part_sn2->
Dean Nelsone17d4162008-07-29 22:34:06 -07001522 remote_openclose_args_pa,
1523 XPC_OPENCLOSE_ARGS_SIZE);
1524 if (ret != xpSuccess) {
1525 XPC_DEACTIVATE_PARTITION(part, ret);
1526
1527 dev_dbg(xpc_chan, "failed to pull openclose args from "
1528 "partition %d, ret=%d\n", XPC_PARTID(part),
1529 ret);
1530
Dean Nelson7fb5e592008-07-29 22:34:10 -07001531 /* don't bother processing chctl flags anymore */
1532 chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001533 }
1534 }
1535
Dean Nelson7fb5e592008-07-29 22:34:10 -07001536 if (xpc_any_msg_chctl_flags_set(&chctl)) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001537 ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001538 part_sn2->remote_GPs_pa,
Dean Nelsone17d4162008-07-29 22:34:06 -07001539 XPC_GP_SIZE);
1540 if (ret != xpSuccess) {
1541 XPC_DEACTIVATE_PARTITION(part, ret);
1542
1543 dev_dbg(xpc_chan, "failed to pull GPs from partition "
1544 "%d, ret=%d\n", XPC_PARTID(part), ret);
1545
Dean Nelson7fb5e592008-07-29 22:34:10 -07001546 /* don't bother processing chctl flags anymore */
1547 chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001548 }
1549 }
1550
Dean Nelson7fb5e592008-07-29 22:34:10 -07001551 return chctl.all_flags;
Dean Nelsone17d4162008-07-29 22:34:06 -07001552}
1553
Dean Nelsona47d5da2008-07-29 22:34:09 -07001554/*
Dean Nelson185c3a12008-07-29 22:34:11 -07001555 * Allocate the local message queue and the notify queue.
1556 */
1557static enum xp_retval
1558xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch)
1559{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001560 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001561 unsigned long irq_flags;
1562 int nentries;
1563 size_t nbytes;
1564
1565 for (nentries = ch->local_nentries; nentries > 0; nentries--) {
1566
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001567 nbytes = nentries * ch->entry_size;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001568 ch_sn2->local_msgqueue =
1569 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL,
1570 &ch_sn2->local_msgqueue_base);
1571 if (ch_sn2->local_msgqueue == NULL)
Dean Nelson185c3a12008-07-29 22:34:11 -07001572 continue;
1573
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001574 nbytes = nentries * sizeof(struct xpc_notify_sn2);
Dean Nelson5b8669d2008-07-29 22:34:18 -07001575 ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL);
1576 if (ch_sn2->notify_queue == NULL) {
1577 kfree(ch_sn2->local_msgqueue_base);
1578 ch_sn2->local_msgqueue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001579 continue;
1580 }
1581
1582 spin_lock_irqsave(&ch->lock, irq_flags);
1583 if (nentries < ch->local_nentries) {
1584 dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, "
1585 "partid=%d, channel=%d\n", nentries,
1586 ch->local_nentries, ch->partid, ch->number);
1587
1588 ch->local_nentries = nentries;
1589 }
1590 spin_unlock_irqrestore(&ch->lock, irq_flags);
1591 return xpSuccess;
1592 }
1593
1594 dev_dbg(xpc_chan, "can't get memory for local message queue and notify "
1595 "queue, partid=%d, channel=%d\n", ch->partid, ch->number);
1596 return xpNoMemory;
1597}
1598
1599/*
1600 * Allocate the cached remote message queue.
1601 */
1602static enum xp_retval
1603xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch)
1604{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001605 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001606 unsigned long irq_flags;
1607 int nentries;
1608 size_t nbytes;
1609
1610 DBUG_ON(ch->remote_nentries <= 0);
1611
1612 for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
1613
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001614 nbytes = nentries * ch->entry_size;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001615 ch_sn2->remote_msgqueue =
1616 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2->
1617 remote_msgqueue_base);
1618 if (ch_sn2->remote_msgqueue == NULL)
Dean Nelson185c3a12008-07-29 22:34:11 -07001619 continue;
1620
1621 spin_lock_irqsave(&ch->lock, irq_flags);
1622 if (nentries < ch->remote_nentries) {
1623 dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, "
1624 "partid=%d, channel=%d\n", nentries,
1625 ch->remote_nentries, ch->partid, ch->number);
1626
1627 ch->remote_nentries = nentries;
1628 }
1629 spin_unlock_irqrestore(&ch->lock, irq_flags);
1630 return xpSuccess;
1631 }
1632
1633 dev_dbg(xpc_chan, "can't get memory for cached remote message queue, "
1634 "partid=%d, channel=%d\n", ch->partid, ch->number);
1635 return xpNoMemory;
1636}
1637
1638/*
1639 * Allocate message queues and other stuff associated with a channel.
1640 *
1641 * Note: Assumes all of the channel sizes are filled in.
1642 */
1643static enum xp_retval
Dean Nelson5b8669d2008-07-29 22:34:18 -07001644xpc_setup_msg_structures_sn2(struct xpc_channel *ch)
Dean Nelson185c3a12008-07-29 22:34:11 -07001645{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001646 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001647 enum xp_retval ret;
1648
1649 DBUG_ON(ch->flags & XPC_C_SETUP);
1650
1651 ret = xpc_allocate_local_msgqueue_sn2(ch);
1652 if (ret == xpSuccess) {
1653
1654 ret = xpc_allocate_remote_msgqueue_sn2(ch);
1655 if (ret != xpSuccess) {
Dean Nelson5b8669d2008-07-29 22:34:18 -07001656 kfree(ch_sn2->local_msgqueue_base);
1657 ch_sn2->local_msgqueue = NULL;
1658 kfree(ch_sn2->notify_queue);
1659 ch_sn2->notify_queue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001660 }
1661 }
1662 return ret;
1663}
1664
1665/*
1666 * Free up message queues and other stuff that were allocated for the specified
1667 * channel.
Dean Nelson185c3a12008-07-29 22:34:11 -07001668 */
1669static void
Dean Nelson5b8669d2008-07-29 22:34:18 -07001670xpc_teardown_msg_structures_sn2(struct xpc_channel *ch)
Dean Nelson185c3a12008-07-29 22:34:11 -07001671{
1672 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1673
Lance Royf2199622018-10-02 22:38:49 -07001674 lockdep_assert_held(&ch->lock);
Dean Nelson185c3a12008-07-29 22:34:11 -07001675
Dean Nelson5b8669d2008-07-29 22:34:18 -07001676 ch_sn2->remote_msgqueue_pa = 0;
Dean Nelson185c3a12008-07-29 22:34:11 -07001677
1678 ch_sn2->local_GP->get = 0;
1679 ch_sn2->local_GP->put = 0;
1680 ch_sn2->remote_GP.get = 0;
1681 ch_sn2->remote_GP.put = 0;
1682 ch_sn2->w_local_GP.get = 0;
1683 ch_sn2->w_local_GP.put = 0;
1684 ch_sn2->w_remote_GP.get = 0;
1685 ch_sn2->w_remote_GP.put = 0;
1686 ch_sn2->next_msg_to_pull = 0;
1687
1688 if (ch->flags & XPC_C_SETUP) {
1689 dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n",
1690 ch->flags, ch->partid, ch->number);
1691
Dean Nelson5b8669d2008-07-29 22:34:18 -07001692 kfree(ch_sn2->local_msgqueue_base);
1693 ch_sn2->local_msgqueue = NULL;
1694 kfree(ch_sn2->remote_msgqueue_base);
1695 ch_sn2->remote_msgqueue = NULL;
1696 kfree(ch_sn2->notify_queue);
1697 ch_sn2->notify_queue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001698 }
1699}
1700
1701/*
Dean Nelsona47d5da2008-07-29 22:34:09 -07001702 * Notify those who wanted to be notified upon delivery of their message.
1703 */
1704static void
1705xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put)
1706{
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001707 struct xpc_notify_sn2 *notify;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001708 u8 notify_type;
1709 s64 get = ch->sn.sn2.w_remote_GP.get - 1;
1710
1711 while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
1712
Dean Nelson5b8669d2008-07-29 22:34:18 -07001713 notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries];
Dean Nelsona47d5da2008-07-29 22:34:09 -07001714
1715 /*
1716 * See if the notify entry indicates it was associated with
1717 * a message who's sender wants to be notified. It is possible
1718 * that it is, but someone else is doing or has done the
1719 * notification.
1720 */
1721 notify_type = notify->type;
1722 if (notify_type == 0 ||
1723 cmpxchg(&notify->type, notify_type, 0) != notify_type) {
1724 continue;
1725 }
1726
1727 DBUG_ON(notify_type != XPC_N_CALL);
1728
1729 atomic_dec(&ch->n_to_notify);
1730
1731 if (notify->func != NULL) {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001732 dev_dbg(xpc_chan, "notify->func() called, notify=0x%p "
Randy Dunlapf509e342009-09-04 17:02:59 -07001733 "msg_number=%lld partid=%d channel=%d\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -07001734 (void *)notify, get, ch->partid, ch->number);
1735
1736 notify->func(reason, ch->partid, ch->number,
1737 notify->key);
1738
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001739 dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p"
Randy Dunlapf509e342009-09-04 17:02:59 -07001740 " msg_number=%lld partid=%d channel=%d\n",
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001741 (void *)notify, get, ch->partid, ch->number);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001742 }
1743 }
1744}
1745
1746static void
1747xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch)
1748{
1749 xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put);
1750}
1751
1752/*
1753 * Clear some of the msg flags in the local message queue.
1754 */
1755static inline void
1756xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch)
1757{
1758 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001759 struct xpc_msg_sn2 *msg;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001760 s64 get;
1761
1762 get = ch_sn2->w_remote_GP.get;
1763 do {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001764 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
1765 (get % ch->local_nentries) *
1766 ch->entry_size);
Robin Holt252523e2009-01-29 14:25:07 -08001767 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsona47d5da2008-07-29 22:34:09 -07001768 msg->flags = 0;
1769 } while (++get < ch_sn2->remote_GP.get);
1770}
1771
1772/*
1773 * Clear some of the msg flags in the remote message queue.
1774 */
1775static inline void
1776xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
1777{
1778 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001779 struct xpc_msg_sn2 *msg;
Robin Holt6e873ec2009-04-02 16:59:14 -07001780 s64 put, remote_nentries = ch->remote_nentries;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001781
Robin Holt252523e2009-01-29 14:25:07 -08001782 /* flags are zeroed when the buffer is allocated */
Robin Holt6e873ec2009-04-02 16:59:14 -07001783 if (ch_sn2->remote_GP.put < remote_nentries)
Robin Holt252523e2009-01-29 14:25:07 -08001784 return;
1785
Robin Holt6e873ec2009-04-02 16:59:14 -07001786 put = max(ch_sn2->w_remote_GP.put, remote_nentries);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001787 do {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001788 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
Robin Holt6e873ec2009-04-02 16:59:14 -07001789 (put % remote_nentries) *
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001790 ch->entry_size);
Robin Holt252523e2009-01-29 14:25:07 -08001791 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
1792 DBUG_ON(!(msg->flags & XPC_M_SN2_DONE));
Robin Holt6e873ec2009-04-02 16:59:14 -07001793 DBUG_ON(msg->number != put - remote_nentries);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001794 msg->flags = 0;
1795 } while (++put < ch_sn2->remote_GP.put);
1796}
1797
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001798static int
1799xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch)
1800{
1801 return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get;
1802}
1803
Dean Nelsona47d5da2008-07-29 22:34:09 -07001804static void
Dean Nelson7fb5e592008-07-29 22:34:10 -07001805xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number)
Dean Nelsona47d5da2008-07-29 22:34:09 -07001806{
1807 struct xpc_channel *ch = &part->channels[ch_number];
1808 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001809 int npayloads_sent;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001810
1811 ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number];
1812
1813 /* See what, if anything, has changed for each connected channel */
1814
1815 xpc_msgqueue_ref(ch);
1816
1817 if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get &&
1818 ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) {
1819 /* nothing changed since GPs were last pulled */
1820 xpc_msgqueue_deref(ch);
1821 return;
1822 }
1823
1824 if (!(ch->flags & XPC_C_CONNECTED)) {
1825 xpc_msgqueue_deref(ch);
1826 return;
1827 }
1828
1829 /*
1830 * First check to see if messages recently sent by us have been
1831 * received by the other side. (The remote GET value will have
1832 * changed since we last looked at it.)
1833 */
1834
1835 if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) {
1836
1837 /*
1838 * We need to notify any senders that want to be notified
1839 * that their sent messages have been received by their
1840 * intended recipients. We need to do this before updating
1841 * w_remote_GP.get so that we don't allocate the same message
1842 * queue entries prematurely (see xpc_allocate_msg()).
1843 */
1844 if (atomic_read(&ch->n_to_notify) > 0) {
1845 /*
1846 * Notify senders that messages sent have been
1847 * received and delivered by the other side.
1848 */
1849 xpc_notify_senders_sn2(ch, xpMsgDelivered,
1850 ch_sn2->remote_GP.get);
1851 }
1852
1853 /*
1854 * Clear msg->flags in previously sent messages, so that
1855 * they're ready for xpc_allocate_msg().
1856 */
1857 xpc_clear_local_msgqueue_flags_sn2(ch);
1858
1859 ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get;
1860
Randy Dunlapf509e342009-09-04 17:02:59 -07001861 dev_dbg(xpc_chan, "w_remote_GP.get changed to %lld, partid=%d, "
Dean Nelsona47d5da2008-07-29 22:34:09 -07001862 "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid,
1863 ch->number);
1864
1865 /*
1866 * If anyone was waiting for message queue entries to become
1867 * available, wake them up.
1868 */
1869 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
1870 wake_up(&ch->msg_allocate_wq);
1871 }
1872
1873 /*
1874 * Now check for newly sent messages by the other side. (The remote
1875 * PUT value will have changed since we last looked at it.)
1876 */
1877
1878 if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) {
1879 /*
1880 * Clear msg->flags in previously received messages, so that
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001881 * they're ready for xpc_get_deliverable_payload_sn2().
Dean Nelsona47d5da2008-07-29 22:34:09 -07001882 */
1883 xpc_clear_remote_msgqueue_flags_sn2(ch);
1884
Robin Holt69b3bb62009-01-29 14:25:06 -08001885 smp_wmb(); /* ensure flags have been cleared before bte_copy */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001886 ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put;
1887
Randy Dunlapf509e342009-09-04 17:02:59 -07001888 dev_dbg(xpc_chan, "w_remote_GP.put changed to %lld, partid=%d, "
Dean Nelsona47d5da2008-07-29 22:34:09 -07001889 "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid,
1890 ch->number);
1891
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001892 npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch);
1893 if (npayloads_sent > 0) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001894 dev_dbg(xpc_chan, "msgs waiting to be copied and "
1895 "delivered=%d, partid=%d, channel=%d\n",
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001896 npayloads_sent, ch->partid, ch->number);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001897
1898 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001899 xpc_activate_kthreads(ch, npayloads_sent);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001900 }
1901 }
1902
1903 xpc_msgqueue_deref(ch);
1904}
1905
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001906static struct xpc_msg_sn2 *
Dean Nelsone17d4162008-07-29 22:34:06 -07001907xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
1908{
1909 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -07001910 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001911 unsigned long remote_msg_pa;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001912 struct xpc_msg_sn2 *msg;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001913 u32 msg_index;
1914 u32 nmsgs;
Dean Nelsone17d4162008-07-29 22:34:06 -07001915 u64 msg_offset;
1916 enum xp_retval ret;
1917
Dean Nelsona47d5da2008-07-29 22:34:09 -07001918 if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001919 /* we were interrupted by a signal */
1920 return NULL;
1921 }
1922
Dean Nelsona47d5da2008-07-29 22:34:09 -07001923 while (get >= ch_sn2->next_msg_to_pull) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001924
1925 /* pull as many messages as are ready and able to be pulled */
1926
Dean Nelsona47d5da2008-07-29 22:34:09 -07001927 msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries;
Dean Nelsone17d4162008-07-29 22:34:06 -07001928
Dean Nelsona47d5da2008-07-29 22:34:09 -07001929 DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put);
1930 nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull;
Dean Nelsone17d4162008-07-29 22:34:06 -07001931 if (msg_index + nmsgs > ch->remote_nentries) {
1932 /* ignore the ones that wrap the msg queue for now */
1933 nmsgs = ch->remote_nentries - msg_index;
1934 }
1935
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001936 msg_offset = msg_index * ch->entry_size;
1937 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
Dean Nelson5b8669d2008-07-29 22:34:18 -07001938 msg_offset);
1939 remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset;
Dean Nelsone17d4162008-07-29 22:34:06 -07001940
Dean Nelsona812dcc2008-07-29 22:34:16 -07001941 ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa,
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001942 nmsgs * ch->entry_size);
Dean Nelsone17d4162008-07-29 22:34:06 -07001943 if (ret != xpSuccess) {
1944
1945 dev_dbg(xpc_chan, "failed to pull %d msgs starting with"
Randy Dunlapf509e342009-09-04 17:02:59 -07001946 " msg %lld from partition %d, channel=%d, "
Dean Nelsona47d5da2008-07-29 22:34:09 -07001947 "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull,
Dean Nelsone17d4162008-07-29 22:34:06 -07001948 ch->partid, ch->number, ret);
1949
1950 XPC_DEACTIVATE_PARTITION(part, ret);
1951
Dean Nelsona47d5da2008-07-29 22:34:09 -07001952 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001953 return NULL;
1954 }
1955
Dean Nelsona47d5da2008-07-29 22:34:09 -07001956 ch_sn2->next_msg_to_pull += nmsgs;
Dean Nelsone17d4162008-07-29 22:34:06 -07001957 }
1958
Dean Nelsona47d5da2008-07-29 22:34:09 -07001959 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001960
1961 /* return the message we were looking for */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001962 msg_offset = (get % ch->remote_nentries) * ch->entry_size;
1963 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset);
Dean Nelsone17d4162008-07-29 22:34:06 -07001964
1965 return msg;
1966}
1967
1968/*
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001969 * Get the next deliverable message's payload.
Dean Nelsone17d4162008-07-29 22:34:06 -07001970 */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001971static void *
1972xpc_get_deliverable_payload_sn2(struct xpc_channel *ch)
Dean Nelsone17d4162008-07-29 22:34:06 -07001973{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001974 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001975 struct xpc_msg_sn2 *msg;
1976 void *payload = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001977 s64 get;
1978
1979 do {
1980 if (ch->flags & XPC_C_DISCONNECTING)
1981 break;
1982
Dean Nelsona47d5da2008-07-29 22:34:09 -07001983 get = ch_sn2->w_local_GP.get;
Robin Holt69b3bb62009-01-29 14:25:06 -08001984 smp_rmb(); /* guarantee that .get loads before .put */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001985 if (get == ch_sn2->w_remote_GP.put)
Dean Nelsone17d4162008-07-29 22:34:06 -07001986 break;
1987
1988 /* There are messages waiting to be pulled and delivered.
1989 * We need to try to secure one for ourselves. We'll do this
1990 * by trying to increment w_local_GP.get and hope that no one
1991 * else beats us to it. If they do, we'll we'll simply have
1992 * to try again for the next one.
1993 */
1994
Dean Nelsona47d5da2008-07-29 22:34:09 -07001995 if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001996 /* we got the entry referenced by get */
1997
Randy Dunlapf509e342009-09-04 17:02:59 -07001998 dev_dbg(xpc_chan, "w_local_GP.get changed to %lld, "
Dean Nelsone17d4162008-07-29 22:34:06 -07001999 "partid=%d, channel=%d\n", get + 1,
2000 ch->partid, ch->number);
2001
2002 /* pull the message from the remote partition */
2003
2004 msg = xpc_pull_remote_msg_sn2(ch, get);
2005
Robin Holt17e21612009-01-29 14:25:07 -08002006 if (msg != NULL) {
2007 DBUG_ON(msg->number != get);
2008 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
2009 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsone17d4162008-07-29 22:34:06 -07002010
Robin Holt17e21612009-01-29 14:25:07 -08002011 payload = &msg->payload;
2012 }
Dean Nelsone17d4162008-07-29 22:34:06 -07002013 break;
2014 }
2015
2016 } while (1);
2017
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002018 return payload;
Dean Nelsone17d4162008-07-29 22:34:06 -07002019}
2020
Dean Nelson33ba3c72008-07-29 22:34:07 -07002021/*
2022 * Now we actually send the messages that are ready to be sent by advancing
Dean Nelson7fb5e592008-07-29 22:34:10 -07002023 * the local message queue's Put value and then send a chctl msgrequest to the
2024 * recipient partition.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002025 */
2026static void
2027xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
2028{
Dean Nelsona47d5da2008-07-29 22:34:09 -07002029 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002030 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002031 s64 put = initial_put + 1;
Dean Nelson7fb5e592008-07-29 22:34:10 -07002032 int send_msgrequest = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002033
2034 while (1) {
2035
2036 while (1) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07002037 if (put == ch_sn2->w_local_GP.put)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002038 break;
2039
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002040 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2041 local_msgqueue + (put %
2042 ch->local_nentries) *
2043 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002044
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002045 if (!(msg->flags & XPC_M_SN2_READY))
Dean Nelson33ba3c72008-07-29 22:34:07 -07002046 break;
2047
2048 put++;
2049 }
2050
2051 if (put == initial_put) {
2052 /* nothing's changed */
2053 break;
2054 }
2055
Dean Nelsona47d5da2008-07-29 22:34:09 -07002056 if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002057 initial_put) {
2058 /* someone else beat us to it */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002059 DBUG_ON(ch_sn2->local_GP->put < initial_put);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002060 break;
2061 }
2062
2063 /* we just set the new value of local_GP->put */
2064
Randy Dunlapf509e342009-09-04 17:02:59 -07002065 dev_dbg(xpc_chan, "local_GP->put changed to %lld, partid=%d, "
Dean Nelson33ba3c72008-07-29 22:34:07 -07002066 "channel=%d\n", put, ch->partid, ch->number);
2067
Dean Nelson7fb5e592008-07-29 22:34:10 -07002068 send_msgrequest = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002069
2070 /*
2071 * We need to ensure that the message referenced by
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002072 * local_GP->put is not XPC_M_SN2_READY or that local_GP->put
Dean Nelson33ba3c72008-07-29 22:34:07 -07002073 * equals w_local_GP.put, so we'll go have a look.
2074 */
2075 initial_put = put;
2076 }
2077
Dean Nelson7fb5e592008-07-29 22:34:10 -07002078 if (send_msgrequest)
2079 xpc_send_chctl_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002080}
2081
2082/*
2083 * Allocate an entry for a message from the message queue associated with the
2084 * specified channel.
2085 */
2086static enum xp_retval
2087xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002088 struct xpc_msg_sn2 **address_of_msg)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002089{
Dean Nelsona47d5da2008-07-29 22:34:09 -07002090 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002091 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002092 enum xp_retval ret;
2093 s64 put;
2094
Dean Nelson33ba3c72008-07-29 22:34:07 -07002095 /*
2096 * Get the next available message entry from the local message queue.
2097 * If none are available, we'll make sure that we grab the latest
2098 * GP values.
2099 */
2100 ret = xpTimeout;
2101
2102 while (1) {
2103
Dean Nelsona47d5da2008-07-29 22:34:09 -07002104 put = ch_sn2->w_local_GP.put;
Robin Holt69b3bb62009-01-29 14:25:06 -08002105 smp_rmb(); /* guarantee that .put loads before .get */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002106 if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07002107
2108 /* There are available message entries. We need to try
2109 * to secure one for ourselves. We'll do this by trying
2110 * to increment w_local_GP.put as long as someone else
2111 * doesn't beat us to it. If they do, we'll have to
2112 * try again.
2113 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002114 if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) ==
2115 put) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07002116 /* we got the entry referenced by put */
2117 break;
2118 }
2119 continue; /* try again */
2120 }
2121
2122 /*
2123 * There aren't any available msg entries at this time.
2124 *
2125 * In waiting for a message entry to become available,
Dean Nelson7fb5e592008-07-29 22:34:10 -07002126 * we set a timeout in case the other side is not sending
2127 * completion interrupts. This lets us fake a notify IRQ
2128 * that will cause the notify IRQ handler to fetch the latest
2129 * GP values as if an interrupt was sent by the other side.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002130 */
2131 if (ret == xpTimeout)
Dean Nelson7fb5e592008-07-29 22:34:10 -07002132 xpc_send_chctl_local_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002133
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002134 if (flags & XPC_NOWAIT)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002135 return xpNoWait;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002136
2137 ret = xpc_allocate_msg_wait(ch);
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002138 if (ret != xpInterrupted && ret != xpTimeout)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002139 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002140 }
2141
2142 /* get the message's address and initialize it */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002143 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
2144 (put % ch->local_nentries) *
2145 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002146
2147 DBUG_ON(msg->flags != 0);
2148 msg->number = put;
2149
Randy Dunlapf509e342009-09-04 17:02:59 -07002150 dev_dbg(xpc_chan, "w_local_GP.put changed to %lld; msg=0x%p, "
2151 "msg_number=%lld, partid=%d, channel=%d\n", put + 1,
Dean Nelson33ba3c72008-07-29 22:34:07 -07002152 (void *)msg, msg->number, ch->partid, ch->number);
2153
2154 *address_of_msg = msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002155 return xpSuccess;
2156}
2157
2158/*
2159 * Common code that does the actual sending of the message by advancing the
Dean Nelson7fb5e592008-07-29 22:34:10 -07002160 * local message queue's Put value and sends a chctl msgrequest to the
2161 * partition the message is being sent to.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002162 */
2163static enum xp_retval
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002164xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload,
2165 u16 payload_size, u8 notify_type, xpc_notify_func func,
2166 void *key)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002167{
2168 enum xp_retval ret = xpSuccess;
Dean Nelson5b8669d2008-07-29 22:34:18 -07002169 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002170 struct xpc_msg_sn2 *msg = msg;
2171 struct xpc_notify_sn2 *notify = notify;
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002172 s64 msg_number;
2173 s64 put;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002174
2175 DBUG_ON(notify_type == XPC_N_CALL && func == NULL);
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002176
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002177 if (XPC_MSG_SIZE(payload_size) > ch->entry_size)
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002178 return xpPayloadTooBig;
2179
2180 xpc_msgqueue_ref(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002181
2182 if (ch->flags & XPC_C_DISCONNECTING) {
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002183 ret = ch->reason;
2184 goto out_1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002185 }
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002186 if (!(ch->flags & XPC_C_CONNECTED)) {
2187 ret = xpNotConnected;
2188 goto out_1;
2189 }
2190
2191 ret = xpc_allocate_msg_sn2(ch, flags, &msg);
2192 if (ret != xpSuccess)
2193 goto out_1;
2194
2195 msg_number = msg->number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002196
2197 if (notify_type != 0) {
2198 /*
2199 * Tell the remote side to send an ACK interrupt when the
2200 * message has been delivered.
2201 */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002202 msg->flags |= XPC_M_SN2_INTERRUPT;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002203
2204 atomic_inc(&ch->n_to_notify);
2205
Dean Nelson5b8669d2008-07-29 22:34:18 -07002206 notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries];
Dean Nelson33ba3c72008-07-29 22:34:07 -07002207 notify->func = func;
2208 notify->key = key;
2209 notify->type = notify_type;
2210
Dean Nelsonea57f802008-07-29 22:34:14 -07002211 /* ??? Is a mb() needed here? */
Dean Nelson33ba3c72008-07-29 22:34:07 -07002212
2213 if (ch->flags & XPC_C_DISCONNECTING) {
2214 /*
2215 * An error occurred between our last error check and
2216 * this one. We will try to clear the type field from
2217 * the notify entry. If we succeed then
2218 * xpc_disconnect_channel() didn't already process
2219 * the notify entry.
2220 */
2221 if (cmpxchg(&notify->type, notify_type, 0) ==
2222 notify_type) {
2223 atomic_dec(&ch->n_to_notify);
2224 ret = ch->reason;
2225 }
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002226 goto out_1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002227 }
2228 }
2229
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002230 memcpy(&msg->payload, payload, payload_size);
2231
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002232 msg->flags |= XPC_M_SN2_READY;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002233
2234 /*
2235 * The preceding store of msg->flags must occur before the following
Dean Nelsona47d5da2008-07-29 22:34:09 -07002236 * load of local_GP->put.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002237 */
Robin Holt69b3bb62009-01-29 14:25:06 -08002238 smp_mb();
Dean Nelson33ba3c72008-07-29 22:34:07 -07002239
2240 /* see if the message is next in line to be sent, if so send it */
2241
Dean Nelson5b8669d2008-07-29 22:34:18 -07002242 put = ch_sn2->local_GP->put;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002243 if (put == msg_number)
2244 xpc_send_msgs_sn2(ch, put);
2245
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002246out_1:
Dean Nelson33ba3c72008-07-29 22:34:07 -07002247 xpc_msgqueue_deref(ch);
2248 return ret;
2249}
2250
2251/*
2252 * Now we actually acknowledge the messages that have been delivered and ack'd
2253 * by advancing the cached remote message queue's Get value and if requested
Dean Nelson7fb5e592008-07-29 22:34:10 -07002254 * send a chctl msgrequest to the message sender's partition.
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002255 *
2256 * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition
2257 * that sent the message.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002258 */
2259static void
2260xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
2261{
Dean Nelsona47d5da2008-07-29 22:34:09 -07002262 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002263 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002264 s64 get = initial_get + 1;
Dean Nelson7fb5e592008-07-29 22:34:10 -07002265 int send_msgrequest = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002266
2267 while (1) {
2268
2269 while (1) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07002270 if (get == ch_sn2->w_local_GP.get)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002271 break;
2272
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002273 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2274 remote_msgqueue + (get %
2275 ch->remote_nentries) *
2276 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002277
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002278 if (!(msg->flags & XPC_M_SN2_DONE))
Dean Nelson33ba3c72008-07-29 22:34:07 -07002279 break;
2280
2281 msg_flags |= msg->flags;
2282 get++;
2283 }
2284
2285 if (get == initial_get) {
2286 /* nothing's changed */
2287 break;
2288 }
2289
Dean Nelsona47d5da2008-07-29 22:34:09 -07002290 if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002291 initial_get) {
2292 /* someone else beat us to it */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002293 DBUG_ON(ch_sn2->local_GP->get <= initial_get);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002294 break;
2295 }
2296
2297 /* we just set the new value of local_GP->get */
2298
Randy Dunlapf509e342009-09-04 17:02:59 -07002299 dev_dbg(xpc_chan, "local_GP->get changed to %lld, partid=%d, "
Dean Nelson33ba3c72008-07-29 22:34:07 -07002300 "channel=%d\n", get, ch->partid, ch->number);
2301
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002302 send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002303
2304 /*
2305 * We need to ensure that the message referenced by
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002306 * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get
Dean Nelson33ba3c72008-07-29 22:34:07 -07002307 * equals w_local_GP.get, so we'll go have a look.
2308 */
2309 initial_get = get;
2310 }
2311
Dean Nelson7fb5e592008-07-29 22:34:10 -07002312 if (send_msgrequest)
2313 xpc_send_chctl_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002314}
2315
2316static void
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002317xpc_received_payload_sn2(struct xpc_channel *ch, void *payload)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002318{
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002319 struct xpc_msg_sn2 *msg;
2320 s64 msg_number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002321 s64 get;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002322
2323 msg = container_of(payload, struct xpc_msg_sn2, payload);
2324 msg_number = msg->number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002325
Randy Dunlapf509e342009-09-04 17:02:59 -07002326 dev_dbg(xpc_chan, "msg=0x%p, msg_number=%lld, partid=%d, channel=%d\n",
Dean Nelson33ba3c72008-07-29 22:34:07 -07002327 (void *)msg, msg_number, ch->partid, ch->number);
2328
Robin Holt252523e2009-01-29 14:25:07 -08002329 DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002330 msg_number % ch->remote_nentries);
Robin Holt252523e2009-01-29 14:25:07 -08002331 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002332 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002333
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002334 msg->flags |= XPC_M_SN2_DONE;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002335
2336 /*
2337 * The preceding store of msg->flags must occur before the following
Dean Nelsona47d5da2008-07-29 22:34:09 -07002338 * load of local_GP->get.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002339 */
Robin Holt69b3bb62009-01-29 14:25:06 -08002340 smp_mb();
Dean Nelson33ba3c72008-07-29 22:34:07 -07002341
2342 /*
2343 * See if this message is next in line to be acknowledged as having
2344 * been delivered.
2345 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002346 get = ch->sn.sn2.local_GP->get;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002347 if (get == msg_number)
2348 xpc_acknowledge_msgs_sn2(ch, get, msg->flags);
2349}
2350
Robin Holta7665b02009-04-13 14:40:19 -07002351static struct xpc_arch_operations xpc_arch_ops_sn2 = {
2352 .setup_partitions = xpc_setup_partitions_sn2,
2353 .teardown_partitions = xpc_teardown_partitions_sn2,
2354 .process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2,
2355 .get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2,
2356 .setup_rsvd_page = xpc_setup_rsvd_page_sn2,
2357
2358 .allow_hb = xpc_allow_hb_sn2,
2359 .disallow_hb = xpc_disallow_hb_sn2,
2360 .disallow_all_hbs = xpc_disallow_all_hbs_sn2,
2361 .increment_heartbeat = xpc_increment_heartbeat_sn2,
2362 .offline_heartbeat = xpc_offline_heartbeat_sn2,
2363 .online_heartbeat = xpc_online_heartbeat_sn2,
2364 .heartbeat_init = xpc_heartbeat_init_sn2,
2365 .heartbeat_exit = xpc_heartbeat_exit_sn2,
2366 .get_remote_heartbeat = xpc_get_remote_heartbeat_sn2,
2367
2368 .request_partition_activation =
2369 xpc_request_partition_activation_sn2,
2370 .request_partition_reactivation =
2371 xpc_request_partition_reactivation_sn2,
2372 .request_partition_deactivation =
2373 xpc_request_partition_deactivation_sn2,
2374 .cancel_partition_deactivation_request =
2375 xpc_cancel_partition_deactivation_request_sn2,
2376
2377 .setup_ch_structures = xpc_setup_ch_structures_sn2,
2378 .teardown_ch_structures = xpc_teardown_ch_structures_sn2,
2379
2380 .make_first_contact = xpc_make_first_contact_sn2,
2381
2382 .get_chctl_all_flags = xpc_get_chctl_all_flags_sn2,
2383 .send_chctl_closerequest = xpc_send_chctl_closerequest_sn2,
2384 .send_chctl_closereply = xpc_send_chctl_closereply_sn2,
2385 .send_chctl_openrequest = xpc_send_chctl_openrequest_sn2,
2386 .send_chctl_openreply = xpc_send_chctl_openreply_sn2,
2387 .send_chctl_opencomplete = xpc_send_chctl_opencomplete_sn2,
2388 .process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2,
2389
2390 .save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2,
2391
2392 .setup_msg_structures = xpc_setup_msg_structures_sn2,
2393 .teardown_msg_structures = xpc_teardown_msg_structures_sn2,
2394
2395 .indicate_partition_engaged = xpc_indicate_partition_engaged_sn2,
2396 .indicate_partition_disengaged = xpc_indicate_partition_disengaged_sn2,
2397 .partition_engaged = xpc_partition_engaged_sn2,
2398 .any_partition_engaged = xpc_any_partition_engaged_sn2,
2399 .assume_partition_disengaged = xpc_assume_partition_disengaged_sn2,
2400
2401 .n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2,
2402 .send_payload = xpc_send_payload_sn2,
2403 .get_deliverable_payload = xpc_get_deliverable_payload_sn2,
2404 .received_payload = xpc_received_payload_sn2,
2405 .notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2,
2406};
2407
Dean Nelson6e41017a2008-07-29 22:34:09 -07002408int
Dean Nelson94bd2702008-07-29 22:34:05 -07002409xpc_init_sn2(void)
2410{
Dean Nelson6e41017a2008-07-29 22:34:09 -07002411 int ret;
Dean Nelsonee6665e2008-07-29 22:34:13 -07002412 size_t buf_size;
Dean Nelson6e41017a2008-07-29 22:34:09 -07002413
Robin Holta7665b02009-04-13 14:40:19 -07002414 xpc_arch_ops = xpc_arch_ops_sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002415
2416 if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) {
2417 dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is "
2418 "larger than %d\n", XPC_MSG_HDR_MAX_SIZE);
2419 return -E2BIG;
2420 }
Dean Nelson6e41017a2008-07-29 22:34:09 -07002421
Dean Nelsonee6665e2008-07-29 22:34:13 -07002422 buf_size = max(XPC_RP_VARS_SIZE,
2423 XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2);
2424 xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size,
2425 GFP_KERNEL,
2426 &xpc_remote_copy_buffer_base_sn2);
2427 if (xpc_remote_copy_buffer_sn2 == NULL) {
2428 dev_err(xpc_part, "can't get memory for remote copy buffer\n");
2429 return -ENOMEM;
2430 }
2431
Dean Nelsonc39838c2008-07-29 22:34:11 -07002432 /* open up protections for IPI and [potentially] amo operations */
Dean Nelson6e41017a2008-07-29 22:34:09 -07002433 xpc_allow_IPI_ops_sn2();
Dean Nelsonc39838c2008-07-29 22:34:11 -07002434 xpc_allow_amo_ops_shub_wars_1_1_sn2();
Dean Nelson6e41017a2008-07-29 22:34:09 -07002435
2436 /*
2437 * This is safe to do before the xpc_hb_checker thread has started
2438 * because the handler releases a wait queue. If an interrupt is
2439 * received before the thread is waiting, it will not go to sleep,
2440 * but rather immediately process the interrupt.
2441 */
2442 ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0,
2443 "xpc hb", NULL);
2444 if (ret != 0) {
2445 dev_err(xpc_part, "can't register ACTIVATE IRQ handler, "
2446 "errno=%d\n", -ret);
2447 xpc_disallow_IPI_ops_sn2();
Dean Nelsonee6665e2008-07-29 22:34:13 -07002448 kfree(xpc_remote_copy_buffer_base_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -07002449 }
2450 return ret;
Dean Nelson94bd2702008-07-29 22:34:05 -07002451}
2452
2453void
2454xpc_exit_sn2(void)
2455{
Dean Nelson6e41017a2008-07-29 22:34:09 -07002456 free_irq(SGI_XPC_ACTIVATE, NULL);
2457 xpc_disallow_IPI_ops_sn2();
Dean Nelsonee6665e2008-07-29 22:34:13 -07002458 kfree(xpc_remote_copy_buffer_base_sn2);
Dean Nelson94bd2702008-07-29 22:34:05 -07002459}