Commit | Line | Data |
---|---|---|
94bd2708 DN |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. | |
7 | */ | |
8 | ||
9 | /* | |
10 | * Cross Partition Communication (XPC) sn2-based functions. | |
11 | * | |
12 | * Architecture specific implementation of common functions. | |
13 | * | |
14 | */ | |
15 | ||
e17d416b | 16 | #include <linux/delay.h> |
94bd2708 | 17 | #include <asm/uncached.h> |
261f3b49 | 18 | #include <asm/sn/mspec.h> |
94bd2708 DN |
19 | #include <asm/sn/sn_sal.h> |
20 | #include "xpc.h" | |
21 | ||
ee6665e3 DN |
22 | /* |
23 | * Define the number of u64s required to represent all the C-brick nasids | |
24 | * as a bitmap. The cross-partition kernel modules deal only with | |
25 | * C-brick nasids, thus the need for bitmaps which don't account for | |
26 | * odd-numbered (non C-brick) nasids. | |
27 | */ | |
28 | #define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2) | |
29 | #define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8) | |
30 | #define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64) | |
31 | ||
32 | /* | |
33 | * Memory for XPC's amo variables is allocated by the MSPEC driver. These | |
34 | * pages are located in the lowest granule. The lowest granule uses 4k pages | |
35 | * for cached references and an alternate TLB handler to never provide a | |
36 | * cacheable mapping for the entire region. This will prevent speculative | |
37 | * reading of cached copies of our lines from being issued which will cause | |
38 | * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64 | |
39 | * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of | |
40 | * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify | |
41 | * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote | |
42 | * partitions (i.e., XPCs) consider themselves currently engaged with the | |
43 | * local XPC and 1 amo variable to request partition deactivation. | |
44 | */ | |
45 | #define XPC_NOTIFY_IRQ_AMOS_SN2 0 | |
46 | #define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \ | |
47 | XP_MAX_NPARTITIONS_SN2) | |
48 | #define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \ | |
49 | XP_NASID_MASK_WORDS_SN2) | |
50 | #define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1) | |
51 | ||
52 | /* | |
53 | * Buffer used to store a local copy of portions of a remote partition's | |
54 | * reserved page (either its header and part_nasids mask, or its vars). | |
55 | */ | |
ee6665e3 | 56 | static void *xpc_remote_copy_buffer_base_sn2; |
5b8669df | 57 | static char *xpc_remote_copy_buffer_sn2; |
ee6665e3 | 58 | |
8e85c23e DN |
59 | static struct xpc_vars_sn2 *xpc_vars_sn2; |
60 | static struct xpc_vars_part_sn2 *xpc_vars_part_sn2; | |
94bd2708 | 61 | |
5b8669df DN |
62 | static int |
63 | xpc_setup_partitions_sn_sn2(void) | |
64 | { | |
65 | /* nothing needs to be done */ | |
66 | return 0; | |
67 | } | |
68 | ||
6e41017a | 69 | /* SH_IPI_ACCESS shub register value on startup */ |
8e85c23e DN |
70 | static u64 xpc_sh1_IPI_access_sn2; |
71 | static u64 xpc_sh2_IPI_access0_sn2; | |
72 | static u64 xpc_sh2_IPI_access1_sn2; | |
73 | static u64 xpc_sh2_IPI_access2_sn2; | |
74 | static u64 xpc_sh2_IPI_access3_sn2; | |
6e41017a DN |
75 | |
76 | /* | |
77 | * Change protections to allow IPI operations. | |
78 | */ | |
79 | static void | |
80 | xpc_allow_IPI_ops_sn2(void) | |
81 | { | |
82 | int node; | |
83 | int nasid; | |
84 | ||
ea57f80c | 85 | /* !!! The following should get moved into SAL. */ |
6e41017a | 86 | if (is_shub2()) { |
8e85c23e | 87 | xpc_sh2_IPI_access0_sn2 = |
6e41017a | 88 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0)); |
8e85c23e | 89 | xpc_sh2_IPI_access1_sn2 = |
6e41017a | 90 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1)); |
8e85c23e | 91 | xpc_sh2_IPI_access2_sn2 = |
6e41017a | 92 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2)); |
8e85c23e | 93 | xpc_sh2_IPI_access3_sn2 = |
6e41017a DN |
94 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3)); |
95 | ||
96 | for_each_online_node(node) { | |
97 | nasid = cnodeid_to_nasid(node); | |
98 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
99 | -1UL); | |
100 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), | |
101 | -1UL); | |
102 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), | |
103 | -1UL); | |
104 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), | |
105 | -1UL); | |
106 | } | |
107 | } else { | |
8e85c23e | 108 | xpc_sh1_IPI_access_sn2 = |
6e41017a DN |
109 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS)); |
110 | ||
111 | for_each_online_node(node) { | |
112 | nasid = cnodeid_to_nasid(node); | |
113 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
114 | -1UL); | |
115 | } | |
116 | } | |
117 | } | |
118 | ||
119 | /* | |
120 | * Restrict protections to disallow IPI operations. | |
121 | */ | |
122 | static void | |
123 | xpc_disallow_IPI_ops_sn2(void) | |
124 | { | |
125 | int node; | |
126 | int nasid; | |
127 | ||
ea57f80c | 128 | /* !!! The following should get moved into SAL. */ |
6e41017a DN |
129 | if (is_shub2()) { |
130 | for_each_online_node(node) { | |
131 | nasid = cnodeid_to_nasid(node); | |
132 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
8e85c23e | 133 | xpc_sh2_IPI_access0_sn2); |
6e41017a | 134 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), |
8e85c23e | 135 | xpc_sh2_IPI_access1_sn2); |
6e41017a | 136 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), |
8e85c23e | 137 | xpc_sh2_IPI_access2_sn2); |
6e41017a | 138 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), |
8e85c23e | 139 | xpc_sh2_IPI_access3_sn2); |
6e41017a DN |
140 | } |
141 | } else { | |
142 | for_each_online_node(node) { | |
143 | nasid = cnodeid_to_nasid(node); | |
144 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
8e85c23e | 145 | xpc_sh1_IPI_access_sn2); |
6e41017a DN |
146 | } |
147 | } | |
148 | } | |
149 | ||
33ba3c77 | 150 | /* |
7fb5e59d DN |
151 | * The following set of functions are used for the sending and receiving of |
152 | * IRQs (also known as IPIs). There are two flavors of IRQs, one that is | |
153 | * associated with partition activity (SGI_XPC_ACTIVATE) and the other that | |
154 | * is associated with channel activity (SGI_XPC_NOTIFY). | |
33ba3c77 DN |
155 | */ |
156 | ||
157 | static u64 | |
c39838ce | 158 | xpc_receive_IRQ_amo_sn2(struct amo *amo) |
33ba3c77 DN |
159 | { |
160 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR); | |
161 | } | |
162 | ||
163 | static enum xp_retval | |
c39838ce DN |
164 | xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid, |
165 | int vector) | |
33ba3c77 DN |
166 | { |
167 | int ret = 0; | |
168 | unsigned long irq_flags; | |
169 | ||
170 | local_irq_save(irq_flags); | |
171 | ||
172 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag); | |
173 | sn_send_IPI_phys(nasid, phys_cpuid, vector, 0); | |
174 | ||
175 | /* | |
176 | * We must always use the nofault function regardless of whether we | |
177 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
178 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 179 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
180 | */ |
181 | ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable), | |
182 | xp_nofault_PIOR_target)); | |
183 | ||
184 | local_irq_restore(irq_flags); | |
185 | ||
261f3b49 | 186 | return (ret == 0) ? xpSuccess : xpPioReadError; |
33ba3c77 DN |
187 | } |
188 | ||
c39838ce | 189 | static struct amo * |
7fb5e59d | 190 | xpc_init_IRQ_amo_sn2(int index) |
33ba3c77 | 191 | { |
8e85c23e | 192 | struct amo *amo = xpc_vars_sn2->amos_page + index; |
33ba3c77 | 193 | |
c39838ce | 194 | (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */ |
33ba3c77 DN |
195 | return amo; |
196 | } | |
197 | ||
198 | /* | |
7fb5e59d | 199 | * Functions associated with SGI_XPC_ACTIVATE IRQ. |
33ba3c77 DN |
200 | */ |
201 | ||
6e41017a DN |
202 | /* |
203 | * Notify the heartbeat check thread that an activate IRQ has been received. | |
204 | */ | |
205 | static irqreturn_t | |
206 | xpc_handle_activate_IRQ_sn2(int irq, void *dev_id) | |
207 | { | |
5b8669df DN |
208 | unsigned long irq_flags; |
209 | ||
210 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
211 | xpc_activate_IRQ_rcvd++; | |
212 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
213 | ||
6e41017a DN |
214 | wake_up_interruptible(&xpc_activate_IRQ_wq); |
215 | return IRQ_HANDLED; | |
216 | } | |
217 | ||
33ba3c77 | 218 | /* |
c39838ce | 219 | * Flag the appropriate amo variable and send an IRQ to the specified node. |
33ba3c77 DN |
220 | */ |
221 | static void | |
a812dcc3 DN |
222 | xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid, |
223 | int to_nasid, int to_phys_cpuid) | |
33ba3c77 | 224 | { |
c39838ce | 225 | struct amo *amos = (struct amo *)__va(amos_page_pa + |
ee6665e3 | 226 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 227 | sizeof(struct amo))); |
33ba3c77 | 228 | |
04de7418 DN |
229 | (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)], |
230 | BIT_MASK(from_nasid / 2), to_nasid, | |
33ba3c77 DN |
231 | to_phys_cpuid, SGI_XPC_ACTIVATE); |
232 | } | |
233 | ||
234 | static void | |
7fb5e59d | 235 | xpc_send_local_activate_IRQ_sn2(int from_nasid) |
33ba3c77 | 236 | { |
5b8669df | 237 | unsigned long irq_flags; |
8e85c23e | 238 | struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa + |
ee6665e3 | 239 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 240 | sizeof(struct amo))); |
33ba3c77 DN |
241 | |
242 | /* fake the sending and receipt of an activate IRQ from remote nasid */ | |
04de7418 DN |
243 | FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable), |
244 | FETCHOP_OR, BIT_MASK(from_nasid / 2)); | |
245 | ||
5b8669df DN |
246 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); |
247 | xpc_activate_IRQ_rcvd++; | |
248 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
249 | ||
6e41017a | 250 | wake_up_interruptible(&xpc_activate_IRQ_wq); |
33ba3c77 DN |
251 | } |
252 | ||
a47d5dac | 253 | /* |
7fb5e59d | 254 | * Functions associated with SGI_XPC_NOTIFY IRQ. |
a47d5dac | 255 | */ |
33ba3c77 | 256 | |
a47d5dac | 257 | /* |
7fb5e59d | 258 | * Check to see if any chctl flags were sent from the specified partition. |
a47d5dac | 259 | */ |
33ba3c77 | 260 | static void |
7fb5e59d | 261 | xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part) |
33ba3c77 | 262 | { |
7fb5e59d | 263 | union xpc_channel_ctl_flags chctl; |
a47d5dac | 264 | unsigned long irq_flags; |
33ba3c77 | 265 | |
7fb5e59d DN |
266 | chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2. |
267 | local_chctl_amo_va); | |
268 | if (chctl.all_flags == 0) | |
a47d5dac DN |
269 | return; |
270 | ||
7fb5e59d DN |
271 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
272 | part->chctl.all_flags |= chctl.all_flags; | |
273 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); | |
a47d5dac | 274 | |
7fb5e59d DN |
275 | dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags=" |
276 | "0x%lx\n", XPC_PARTID(part), chctl.all_flags); | |
a47d5dac DN |
277 | |
278 | xpc_wakeup_channel_mgr(part); | |
33ba3c77 DN |
279 | } |
280 | ||
a47d5dac DN |
281 | /* |
282 | * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified | |
283 | * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more | |
c39838ce | 284 | * than one partition, we use an amo structure per partition to indicate |
7fb5e59d | 285 | * whether a partition has sent an IRQ or not. If it has, then wake up the |
a47d5dac DN |
286 | * associated kthread to handle it. |
287 | * | |
7fb5e59d | 288 | * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC |
a47d5dac DN |
289 | * running on other partitions. |
290 | * | |
291 | * Noteworthy Arguments: | |
292 | * | |
293 | * irq - Interrupt ReQuest number. NOT USED. | |
294 | * | |
7fb5e59d | 295 | * dev_id - partid of IRQ's potential sender. |
a47d5dac DN |
296 | */ |
297 | static irqreturn_t | |
298 | xpc_handle_notify_IRQ_sn2(int irq, void *dev_id) | |
33ba3c77 | 299 | { |
a47d5dac DN |
300 | short partid = (short)(u64)dev_id; |
301 | struct xpc_partition *part = &xpc_partitions[partid]; | |
302 | ||
261f3b49 | 303 | DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2); |
a47d5dac DN |
304 | |
305 | if (xpc_part_ref(part)) { | |
7fb5e59d | 306 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
307 | |
308 | xpc_part_deref(part); | |
309 | } | |
310 | return IRQ_HANDLED; | |
33ba3c77 DN |
311 | } |
312 | ||
313 | /* | |
7fb5e59d DN |
314 | * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor |
315 | * because the write to their associated amo variable completed after the IRQ | |
a47d5dac | 316 | * was received. |
33ba3c77 | 317 | */ |
a47d5dac | 318 | static void |
7fb5e59d | 319 | xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part) |
a47d5dac DN |
320 | { |
321 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
322 | ||
323 | if (xpc_part_ref(part)) { | |
7fb5e59d | 324 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
325 | |
326 | part_sn2->dropped_notify_IRQ_timer.expires = jiffies + | |
7fb5e59d | 327 | XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
a47d5dac DN |
328 | add_timer(&part_sn2->dropped_notify_IRQ_timer); |
329 | xpc_part_deref(part); | |
330 | } | |
331 | } | |
33ba3c77 DN |
332 | |
333 | /* | |
7fb5e59d | 334 | * Send a notify IRQ to the remote partition that is associated with the |
33ba3c77 DN |
335 | * specified channel. |
336 | */ | |
337 | static void | |
7fb5e59d DN |
338 | xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
339 | char *chctl_flag_string, unsigned long *irq_flags) | |
33ba3c77 DN |
340 | { |
341 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 342 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
7fb5e59d | 343 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 DN |
344 | enum xp_retval ret; |
345 | ||
83469b55 | 346 | if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) { |
7fb5e59d DN |
347 | chctl.flags[ch->number] = chctl_flag; |
348 | ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va, | |
349 | chctl.all_flags, | |
350 | part_sn2->notify_IRQ_nasid, | |
351 | part_sn2->notify_IRQ_phys_cpuid, | |
33ba3c77 DN |
352 | SGI_XPC_NOTIFY); |
353 | dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n", | |
7fb5e59d | 354 | chctl_flag_string, ch->partid, ch->number, ret); |
33ba3c77 DN |
355 | if (unlikely(ret != xpSuccess)) { |
356 | if (irq_flags != NULL) | |
357 | spin_unlock_irqrestore(&ch->lock, *irq_flags); | |
358 | XPC_DEACTIVATE_PARTITION(part, ret); | |
359 | if (irq_flags != NULL) | |
360 | spin_lock_irqsave(&ch->lock, *irq_flags); | |
361 | } | |
362 | } | |
363 | } | |
364 | ||
7fb5e59d DN |
365 | #define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \ |
366 | xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f) | |
33ba3c77 DN |
367 | |
368 | /* | |
369 | * Make it look like the remote partition, which is associated with the | |
7fb5e59d DN |
370 | * specified channel, sent us a notify IRQ. This faked IRQ will be handled |
371 | * by xpc_check_for_dropped_notify_IRQ_sn2(). | |
33ba3c77 DN |
372 | */ |
373 | static void | |
7fb5e59d DN |
374 | xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
375 | char *chctl_flag_string) | |
33ba3c77 DN |
376 | { |
377 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
7fb5e59d | 378 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 | 379 | |
7fb5e59d DN |
380 | chctl.flags[ch->number] = chctl_flag; |
381 | FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va-> | |
382 | variable), FETCHOP_OR, chctl.all_flags); | |
33ba3c77 | 383 | dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n", |
7fb5e59d | 384 | chctl_flag_string, ch->partid, ch->number); |
33ba3c77 DN |
385 | } |
386 | ||
7fb5e59d DN |
387 | #define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \ |
388 | xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f) | |
33ba3c77 DN |
389 | |
390 | static void | |
7fb5e59d DN |
391 | xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch, |
392 | unsigned long *irq_flags) | |
33ba3c77 | 393 | { |
5b8669df | 394 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 DN |
395 | |
396 | args->reason = ch->reason; | |
7fb5e59d | 397 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags); |
33ba3c77 DN |
398 | } |
399 | ||
400 | static void | |
7fb5e59d | 401 | xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 402 | { |
7fb5e59d | 403 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags); |
33ba3c77 DN |
404 | } |
405 | ||
406 | static void | |
7fb5e59d | 407 | xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 408 | { |
5b8669df | 409 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 | 410 | |
bd3e64c1 | 411 | args->entry_size = ch->entry_size; |
33ba3c77 | 412 | args->local_nentries = ch->local_nentries; |
7fb5e59d | 413 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags); |
33ba3c77 DN |
414 | } |
415 | ||
416 | static void | |
7fb5e59d | 417 | xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 418 | { |
5b8669df | 419 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 DN |
420 | |
421 | args->remote_nentries = ch->remote_nentries; | |
422 | args->local_nentries = ch->local_nentries; | |
5b8669df | 423 | args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue); |
7fb5e59d | 424 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags); |
33ba3c77 DN |
425 | } |
426 | ||
427 | static void | |
7fb5e59d | 428 | xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 429 | { |
7fb5e59d | 430 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL); |
33ba3c77 DN |
431 | } |
432 | ||
433 | static void | |
7fb5e59d | 434 | xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 435 | { |
7fb5e59d | 436 | XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST); |
33ba3c77 DN |
437 | } |
438 | ||
5b8669df DN |
439 | static void |
440 | xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch, | |
441 | unsigned long msgqueue_pa) | |
442 | { | |
443 | ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa; | |
444 | } | |
445 | ||
33ba3c77 DN |
446 | /* |
447 | * This next set of functions are used to keep track of when a partition is | |
448 | * potentially engaged in accessing memory belonging to another partition. | |
449 | */ | |
450 | ||
451 | static void | |
a47d5dac | 452 | xpc_indicate_partition_engaged_sn2(struct xpc_partition *part) |
33ba3c77 DN |
453 | { |
454 | unsigned long irq_flags; | |
c39838ce | 455 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 456 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 457 | sizeof(struct amo))); |
33ba3c77 DN |
458 | |
459 | local_irq_save(irq_flags); | |
460 | ||
c39838ce | 461 | /* set bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 462 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
463 | BIT(sn_partition_id)); |
464 | ||
33ba3c77 DN |
465 | /* |
466 | * We must always use the nofault function regardless of whether we | |
467 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
468 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 469 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
470 | */ |
471 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
472 | variable), | |
473 | xp_nofault_PIOR_target)); | |
474 | ||
475 | local_irq_restore(irq_flags); | |
476 | } | |
477 | ||
478 | static void | |
a47d5dac | 479 | xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part) |
33ba3c77 | 480 | { |
a47d5dac | 481 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
33ba3c77 | 482 | unsigned long irq_flags; |
c39838ce | 483 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 484 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 485 | sizeof(struct amo))); |
33ba3c77 DN |
486 | |
487 | local_irq_save(irq_flags); | |
488 | ||
c39838ce | 489 | /* clear bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 490 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
491 | ~BIT(sn_partition_id)); |
492 | ||
33ba3c77 DN |
493 | /* |
494 | * We must always use the nofault function regardless of whether we | |
495 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
496 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 497 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
498 | */ |
499 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
500 | variable), | |
501 | xp_nofault_PIOR_target)); | |
502 | ||
503 | local_irq_restore(irq_flags); | |
33ba3c77 | 504 | |
33ba3c77 | 505 | /* |
a47d5dac | 506 | * Send activate IRQ to get other side to see that we've cleared our |
c39838ce | 507 | * bit in their engaged partitions amo. |
33ba3c77 | 508 | */ |
7fb5e59d | 509 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
510 | cnodeid_to_nasid(0), |
511 | part_sn2->activate_IRQ_nasid, | |
512 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 DN |
513 | } |
514 | ||
5b8669df DN |
515 | static void |
516 | xpc_assume_partition_disengaged_sn2(short partid) | |
517 | { | |
518 | struct amo *amo = xpc_vars_sn2->amos_page + | |
519 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
520 | ||
521 | /* clear bit(s) based on partid mask in our partition's amo */ | |
522 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, | |
523 | ~BIT(partid)); | |
524 | } | |
525 | ||
a47d5dac DN |
526 | static int |
527 | xpc_partition_engaged_sn2(short partid) | |
33ba3c77 | 528 | { |
8e85c23e DN |
529 | struct amo *amo = xpc_vars_sn2->amos_page + |
530 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 531 | |
c39838ce | 532 | /* our partition's amo variable ANDed with partid mask */ |
33ba3c77 | 533 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 534 | BIT(partid)) != 0; |
33ba3c77 DN |
535 | } |
536 | ||
a47d5dac DN |
537 | static int |
538 | xpc_any_partition_engaged_sn2(void) | |
33ba3c77 | 539 | { |
8e85c23e DN |
540 | struct amo *amo = xpc_vars_sn2->amos_page + |
541 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 542 | |
c39838ce | 543 | /* our partition's amo variable */ |
a47d5dac | 544 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0; |
33ba3c77 DN |
545 | } |
546 | ||
6e41017a DN |
547 | /* original protection values for each node */ |
548 | static u64 xpc_prot_vec_sn2[MAX_NUMNODES]; | |
549 | ||
550 | /* | |
c39838ce | 551 | * Change protections to allow amo operations on non-Shub 1.1 systems. |
6e41017a DN |
552 | */ |
553 | static enum xp_retval | |
c39838ce | 554 | xpc_allow_amo_ops_sn2(struct amo *amos_page) |
6e41017a | 555 | { |
6c1c325d | 556 | enum xp_retval ret = xpSuccess; |
6e41017a DN |
557 | |
558 | /* | |
559 | * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST | |
560 | * collides with memory operations. On those systems we call | |
c39838ce | 561 | * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead. |
6e41017a | 562 | */ |
6c1c325d DN |
563 | if (!enable_shub_wars_1_1()) |
564 | ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE); | |
565 | ||
566 | return ret; | |
6e41017a DN |
567 | } |
568 | ||
569 | /* | |
c39838ce | 570 | * Change protections to allow amo operations on Shub 1.1 systems. |
6e41017a DN |
571 | */ |
572 | static void | |
c39838ce | 573 | xpc_allow_amo_ops_shub_wars_1_1_sn2(void) |
6e41017a DN |
574 | { |
575 | int node; | |
576 | int nasid; | |
577 | ||
578 | if (!enable_shub_wars_1_1()) | |
579 | return; | |
580 | ||
581 | for_each_online_node(node) { | |
582 | nasid = cnodeid_to_nasid(node); | |
583 | /* save current protection values */ | |
584 | xpc_prot_vec_sn2[node] = | |
585 | (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid, | |
586 | SH1_MD_DQLP_MMR_DIR_PRIVEC0)); | |
587 | /* open up everything */ | |
588 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
589 | SH1_MD_DQLP_MMR_DIR_PRIVEC0), | |
590 | -1UL); | |
591 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
592 | SH1_MD_DQRP_MMR_DIR_PRIVEC0), | |
593 | -1UL); | |
594 | } | |
595 | } | |
596 | ||
261f3b49 | 597 | static enum xp_retval |
a812dcc3 | 598 | xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa, |
261f3b49 DN |
599 | size_t *len) |
600 | { | |
601 | s64 status; | |
602 | enum xp_retval ret; | |
603 | ||
a812dcc3 | 604 | status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len); |
261f3b49 DN |
605 | if (status == SALRET_OK) |
606 | ret = xpSuccess; | |
607 | else if (status == SALRET_MORE_PASSES) | |
608 | ret = xpNeedMoreInfo; | |
609 | else | |
610 | ret = xpSalError; | |
611 | ||
612 | return ret; | |
613 | } | |
614 | ||
615 | ||
5b8669df DN |
616 | static int |
617 | xpc_setup_rsvd_page_sn_sn2(struct xpc_rsvd_page *rp) | |
94bd2708 | 618 | { |
c39838ce | 619 | struct amo *amos_page; |
94bd2708 DN |
620 | int i; |
621 | int ret; | |
622 | ||
8e85c23e | 623 | xpc_vars_sn2 = XPC_RP_VARS(rp); |
94bd2708 | 624 | |
a812dcc3 | 625 | rp->sn.vars_pa = xp_pa(xpc_vars_sn2); |
94bd2708 | 626 | |
e17d416b | 627 | /* vars_part array follows immediately after vars */ |
8e85c23e DN |
628 | xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) + |
629 | XPC_RP_VARS_SIZE); | |
e17d416b | 630 | |
94bd2708 | 631 | /* |
8e85c23e DN |
632 | * Before clearing xpc_vars_sn2, see if a page of amos had been |
633 | * previously allocated. If not we'll need to allocate one and set | |
634 | * permissions so that cross-partition amos are allowed. | |
94bd2708 | 635 | * |
c39838ce | 636 | * The allocated amo page needs MCA reporting to remain disabled after |
94bd2708 | 637 | * XPC has unloaded. To make this work, we keep a copy of the pointer |
8e85c23e | 638 | * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure, |
94bd2708 | 639 | * which is pointed to by the reserved page, and re-use that saved copy |
c39838ce | 640 | * on subsequent loads of XPC. This amo page is never freed, and its |
94bd2708 DN |
641 | * memory protections are never restricted. |
642 | */ | |
8e85c23e | 643 | amos_page = xpc_vars_sn2->amos_page; |
94bd2708 | 644 | if (amos_page == NULL) { |
c39838ce | 645 | amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1)); |
94bd2708 | 646 | if (amos_page == NULL) { |
c39838ce | 647 | dev_err(xpc_part, "can't allocate page of amos\n"); |
5b8669df | 648 | return -ENOMEM; |
94bd2708 DN |
649 | } |
650 | ||
651 | /* | |
c39838ce DN |
652 | * Open up amo-R/W to cpu. This is done on Shub 1.1 systems |
653 | * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called. | |
94bd2708 | 654 | */ |
c39838ce | 655 | ret = xpc_allow_amo_ops_sn2(amos_page); |
6e41017a | 656 | if (ret != xpSuccess) { |
c39838ce | 657 | dev_err(xpc_part, "can't allow amo operations\n"); |
6e41017a DN |
658 | uncached_free_page(__IA64_UNCACHED_OFFSET | |
659 | TO_PHYS((u64)amos_page), 1); | |
5b8669df | 660 | return -EPERM; |
94bd2708 DN |
661 | } |
662 | } | |
663 | ||
8e85c23e DN |
664 | /* clear xpc_vars_sn2 */ |
665 | memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2)); | |
94bd2708 | 666 | |
8e85c23e DN |
667 | xpc_vars_sn2->version = XPC_V_VERSION; |
668 | xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0); | |
669 | xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0); | |
a812dcc3 | 670 | xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2); |
8e85c23e DN |
671 | xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page); |
672 | xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */ | |
94bd2708 | 673 | |
8e85c23e DN |
674 | /* clear xpc_vars_part_sn2 */ |
675 | memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) * | |
261f3b49 | 676 | XP_MAX_NPARTITIONS_SN2); |
94bd2708 | 677 | |
c39838ce | 678 | /* initialize the activate IRQ related amo variables */ |
04de7418 | 679 | for (i = 0; i < xpc_nasid_mask_nlongs; i++) |
ee6665e3 | 680 | (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i); |
94bd2708 | 681 | |
c39838ce | 682 | /* initialize the engaged remote partitions related amo variables */ |
ee6665e3 DN |
683 | (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2); |
684 | (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2); | |
33ba3c77 | 685 | |
5b8669df | 686 | return 0; |
33ba3c77 DN |
687 | } |
688 | ||
689 | static void | |
690 | xpc_increment_heartbeat_sn2(void) | |
691 | { | |
8e85c23e | 692 | xpc_vars_sn2->heartbeat++; |
33ba3c77 DN |
693 | } |
694 | ||
695 | static void | |
696 | xpc_offline_heartbeat_sn2(void) | |
697 | { | |
698 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 699 | xpc_vars_sn2->heartbeat_offline = 1; |
33ba3c77 DN |
700 | } |
701 | ||
702 | static void | |
703 | xpc_online_heartbeat_sn2(void) | |
704 | { | |
705 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 706 | xpc_vars_sn2->heartbeat_offline = 0; |
33ba3c77 DN |
707 | } |
708 | ||
709 | static void | |
710 | xpc_heartbeat_init_sn2(void) | |
711 | { | |
8e85c23e | 712 | DBUG_ON(xpc_vars_sn2 == NULL); |
33ba3c77 | 713 | |
8e85c23e DN |
714 | bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2); |
715 | xpc_heartbeating_to_mask = &xpc_vars_sn2->heartbeating_to_mask[0]; | |
33ba3c77 DN |
716 | xpc_online_heartbeat_sn2(); |
717 | } | |
718 | ||
719 | static void | |
720 | xpc_heartbeat_exit_sn2(void) | |
721 | { | |
722 | xpc_offline_heartbeat_sn2(); | |
723 | } | |
724 | ||
61deb86e DN |
725 | static enum xp_retval |
726 | xpc_get_remote_heartbeat_sn2(struct xpc_partition *part) | |
33ba3c77 DN |
727 | { |
728 | struct xpc_vars_sn2 *remote_vars; | |
33ba3c77 DN |
729 | enum xp_retval ret; |
730 | ||
ee6665e3 | 731 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 | 732 | |
61deb86e DN |
733 | /* pull the remote vars structure that contains the heartbeat */ |
734 | ret = xp_remote_memcpy(xp_pa(remote_vars), | |
735 | part->sn.sn2.remote_vars_pa, | |
736 | XPC_RP_VARS_SIZE); | |
737 | if (ret != xpSuccess) | |
738 | return ret; | |
33ba3c77 | 739 | |
61deb86e DN |
740 | dev_dbg(xpc_part, "partid=%d, heartbeat=%ld, last_heartbeat=%ld, " |
741 | "heartbeat_offline=%ld, HB_mask[0]=0x%lx\n", XPC_PARTID(part), | |
742 | remote_vars->heartbeat, part->last_heartbeat, | |
743 | remote_vars->heartbeat_offline, | |
744 | remote_vars->heartbeating_to_mask[0]); | |
745 | ||
746 | if ((remote_vars->heartbeat == part->last_heartbeat && | |
747 | remote_vars->heartbeat_offline == 0) || | |
748 | !xpc_hb_allowed(sn_partition_id, | |
749 | &remote_vars->heartbeating_to_mask)) { | |
750 | ret = xpNoHeartbeat; | |
751 | } else { | |
33ba3c77 DN |
752 | part->last_heartbeat = remote_vars->heartbeat; |
753 | } | |
61deb86e DN |
754 | |
755 | return ret; | |
33ba3c77 DN |
756 | } |
757 | ||
758 | /* | |
759 | * Get a copy of the remote partition's XPC variables from the reserved page. | |
760 | * | |
761 | * remote_vars points to a buffer that is cacheline aligned for BTE copies and | |
762 | * assumed to be of size XPC_RP_VARS_SIZE. | |
763 | */ | |
764 | static enum xp_retval | |
a812dcc3 DN |
765 | xpc_get_remote_vars_sn2(unsigned long remote_vars_pa, |
766 | struct xpc_vars_sn2 *remote_vars) | |
33ba3c77 DN |
767 | { |
768 | enum xp_retval ret; | |
769 | ||
770 | if (remote_vars_pa == 0) | |
771 | return xpVarsNotSet; | |
772 | ||
773 | /* pull over the cross partition variables */ | |
a812dcc3 | 774 | ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa, |
33ba3c77 DN |
775 | XPC_RP_VARS_SIZE); |
776 | if (ret != xpSuccess) | |
777 | return ret; | |
778 | ||
779 | if (XPC_VERSION_MAJOR(remote_vars->version) != | |
780 | XPC_VERSION_MAJOR(XPC_V_VERSION)) { | |
781 | return xpBadVersion; | |
782 | } | |
94bd2708 DN |
783 | |
784 | return xpSuccess; | |
785 | } | |
786 | ||
33ba3c77 | 787 | static void |
a47d5dac | 788 | xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp, |
a812dcc3 | 789 | unsigned long remote_rp_pa, int nasid) |
33ba3c77 | 790 | { |
7fb5e59d | 791 | xpc_send_local_activate_IRQ_sn2(nasid); |
a47d5dac DN |
792 | } |
793 | ||
794 | static void | |
795 | xpc_request_partition_reactivation_sn2(struct xpc_partition *part) | |
796 | { | |
7fb5e59d | 797 | xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid); |
a47d5dac DN |
798 | } |
799 | ||
800 | static void | |
801 | xpc_request_partition_deactivation_sn2(struct xpc_partition *part) | |
802 | { | |
803 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
804 | unsigned long irq_flags; | |
c39838ce | 805 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 806 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 807 | sizeof(struct amo))); |
a47d5dac DN |
808 | |
809 | local_irq_save(irq_flags); | |
810 | ||
c39838ce | 811 | /* set bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 812 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
813 | BIT(sn_partition_id)); |
814 | ||
a47d5dac DN |
815 | /* |
816 | * We must always use the nofault function regardless of whether we | |
817 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
818 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 819 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
820 | */ |
821 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
822 | variable), | |
823 | xp_nofault_PIOR_target)); | |
824 | ||
825 | local_irq_restore(irq_flags); | |
826 | ||
827 | /* | |
828 | * Send activate IRQ to get other side to see that we've set our | |
c39838ce | 829 | * bit in their deactivate request amo. |
a47d5dac | 830 | */ |
7fb5e59d | 831 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
832 | cnodeid_to_nasid(0), |
833 | part_sn2->activate_IRQ_nasid, | |
834 | part_sn2->activate_IRQ_phys_cpuid); | |
835 | } | |
836 | ||
837 | static void | |
838 | xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part) | |
839 | { | |
840 | unsigned long irq_flags; | |
c39838ce | 841 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 842 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 843 | sizeof(struct amo))); |
a47d5dac DN |
844 | |
845 | local_irq_save(irq_flags); | |
846 | ||
c39838ce | 847 | /* clear bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 848 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
849 | ~BIT(sn_partition_id)); |
850 | ||
a47d5dac DN |
851 | /* |
852 | * We must always use the nofault function regardless of whether we | |
853 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
854 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 855 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
856 | */ |
857 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
858 | variable), | |
859 | xp_nofault_PIOR_target)); | |
860 | ||
861 | local_irq_restore(irq_flags); | |
862 | } | |
863 | ||
864 | static int | |
865 | xpc_partition_deactivation_requested_sn2(short partid) | |
866 | { | |
8e85c23e DN |
867 | struct amo *amo = xpc_vars_sn2->amos_page + |
868 | XPC_DEACTIVATE_REQUEST_AMO_SN2; | |
a47d5dac | 869 | |
c39838ce | 870 | /* our partition's amo variable ANDed with partid mask */ |
a47d5dac | 871 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 872 | BIT(partid)) != 0; |
33ba3c77 DN |
873 | } |
874 | ||
875 | /* | |
876 | * Update the remote partition's info. | |
877 | */ | |
878 | static void | |
879 | xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version, | |
81fe7883 | 880 | unsigned long *remote_rp_ts_jiffies, |
a812dcc3 DN |
881 | unsigned long remote_rp_pa, |
882 | unsigned long remote_vars_pa, | |
33ba3c77 DN |
883 | struct xpc_vars_sn2 *remote_vars) |
884 | { | |
a47d5dac DN |
885 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
886 | ||
33ba3c77 DN |
887 | part->remote_rp_version = remote_rp_version; |
888 | dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n", | |
889 | part->remote_rp_version); | |
890 | ||
81fe7883 DN |
891 | part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies; |
892 | dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n", | |
893 | part->remote_rp_ts_jiffies); | |
33ba3c77 DN |
894 | |
895 | part->remote_rp_pa = remote_rp_pa; | |
896 | dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa); | |
897 | ||
a47d5dac | 898 | part_sn2->remote_vars_pa = remote_vars_pa; |
33ba3c77 | 899 | dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n", |
a47d5dac | 900 | part_sn2->remote_vars_pa); |
33ba3c77 | 901 | |
158bc69e | 902 | part->last_heartbeat = remote_vars->heartbeat - 1; |
33ba3c77 DN |
903 | dev_dbg(xpc_part, " last_heartbeat = 0x%016lx\n", |
904 | part->last_heartbeat); | |
905 | ||
a47d5dac | 906 | part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa; |
33ba3c77 | 907 | dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n", |
a47d5dac | 908 | part_sn2->remote_vars_part_pa); |
33ba3c77 | 909 | |
a47d5dac DN |
910 | part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid; |
911 | dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n", | |
912 | part_sn2->activate_IRQ_nasid); | |
33ba3c77 | 913 | |
a47d5dac DN |
914 | part_sn2->activate_IRQ_phys_cpuid = |
915 | remote_vars->activate_IRQ_phys_cpuid; | |
916 | dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n", | |
917 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 918 | |
a47d5dac | 919 | part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa; |
33ba3c77 | 920 | dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n", |
a47d5dac | 921 | part_sn2->remote_amos_page_pa); |
33ba3c77 | 922 | |
a47d5dac | 923 | part_sn2->remote_vars_version = remote_vars->version; |
33ba3c77 | 924 | dev_dbg(xpc_part, " remote_vars_version = 0x%x\n", |
a47d5dac | 925 | part_sn2->remote_vars_version); |
33ba3c77 DN |
926 | } |
927 | ||
928 | /* | |
7fb5e59d DN |
929 | * Prior code has determined the nasid which generated a activate IRQ. |
930 | * Inspect that nasid to determine if its partition needs to be activated | |
931 | * or deactivated. | |
33ba3c77 | 932 | * |
7fb5e59d | 933 | * A partition is considered "awaiting activation" if our partition |
33ba3c77 DN |
934 | * flags indicate it is not active and it has a heartbeat. A |
935 | * partition is considered "awaiting deactivation" if our partition | |
936 | * flags indicate it is active but it has no heartbeat or it is not | |
937 | * sending its heartbeat to us. | |
938 | * | |
939 | * To determine the heartbeat, the remote nasid must have a properly | |
940 | * initialized reserved page. | |
941 | */ | |
942 | static void | |
6e41017a | 943 | xpc_identify_activate_IRQ_req_sn2(int nasid) |
33ba3c77 DN |
944 | { |
945 | struct xpc_rsvd_page *remote_rp; | |
946 | struct xpc_vars_sn2 *remote_vars; | |
a812dcc3 DN |
947 | unsigned long remote_rp_pa; |
948 | unsigned long remote_vars_pa; | |
33ba3c77 DN |
949 | int remote_rp_version; |
950 | int reactivate = 0; | |
81fe7883 | 951 | unsigned long remote_rp_ts_jiffies = 0; |
33ba3c77 DN |
952 | short partid; |
953 | struct xpc_partition *part; | |
a47d5dac | 954 | struct xpc_partition_sn2 *part_sn2; |
33ba3c77 DN |
955 | enum xp_retval ret; |
956 | ||
957 | /* pull over the reserved page structure */ | |
958 | ||
ee6665e3 | 959 | remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
960 | |
961 | ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa); | |
962 | if (ret != xpSuccess) { | |
963 | dev_warn(xpc_part, "unable to get reserved page from nasid %d, " | |
964 | "which sent interrupt, reason=%d\n", nasid, ret); | |
965 | return; | |
966 | } | |
967 | ||
968 | remote_vars_pa = remote_rp->sn.vars_pa; | |
969 | remote_rp_version = remote_rp->version; | |
81fe7883 | 970 | remote_rp_ts_jiffies = remote_rp->ts_jiffies; |
33ba3c77 DN |
971 | |
972 | partid = remote_rp->SAL_partid; | |
973 | part = &xpc_partitions[partid]; | |
a47d5dac | 974 | part_sn2 = &part->sn.sn2; |
33ba3c77 DN |
975 | |
976 | /* pull over the cross partition variables */ | |
977 | ||
ee6665e3 | 978 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
979 | |
980 | ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars); | |
981 | if (ret != xpSuccess) { | |
33ba3c77 DN |
982 | dev_warn(xpc_part, "unable to get XPC variables from nasid %d, " |
983 | "which sent interrupt, reason=%d\n", nasid, ret); | |
984 | ||
985 | XPC_DEACTIVATE_PARTITION(part, ret); | |
986 | return; | |
987 | } | |
988 | ||
6e41017a | 989 | part->activate_IRQ_rcvd++; |
33ba3c77 DN |
990 | |
991 | dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = " | |
6e41017a | 992 | "%ld:0x%lx\n", (int)nasid, (int)partid, part->activate_IRQ_rcvd, |
33ba3c77 DN |
993 | remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]); |
994 | ||
995 | if (xpc_partition_disengaged(part) && | |
83469b55 | 996 | part->act_state == XPC_P_AS_INACTIVE) { |
33ba3c77 DN |
997 | |
998 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
999 | &remote_rp_ts_jiffies, |
1000 | remote_rp_pa, remote_vars_pa, | |
1001 | remote_vars); | |
33ba3c77 | 1002 | |
a47d5dac DN |
1003 | if (xpc_partition_deactivation_requested_sn2(partid)) { |
1004 | /* | |
1005 | * Other side is waiting on us to deactivate even though | |
1006 | * we already have. | |
1007 | */ | |
1008 | return; | |
33ba3c77 DN |
1009 | } |
1010 | ||
1011 | xpc_activate_partition(part); | |
1012 | return; | |
1013 | } | |
1014 | ||
1015 | DBUG_ON(part->remote_rp_version == 0); | |
a47d5dac | 1016 | DBUG_ON(part_sn2->remote_vars_version == 0); |
33ba3c77 | 1017 | |
81fe7883 | 1018 | if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) { |
33ba3c77 | 1019 | |
a47d5dac | 1020 | /* the other side rebooted */ |
33ba3c77 | 1021 | |
a47d5dac DN |
1022 | DBUG_ON(xpc_partition_engaged_sn2(partid)); |
1023 | DBUG_ON(xpc_partition_deactivation_requested_sn2(partid)); | |
33ba3c77 DN |
1024 | |
1025 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
1026 | &remote_rp_ts_jiffies, |
1027 | remote_rp_pa, remote_vars_pa, | |
1028 | remote_vars); | |
33ba3c77 | 1029 | reactivate = 1; |
33ba3c77 DN |
1030 | } |
1031 | ||
a47d5dac | 1032 | if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) { |
33ba3c77 DN |
1033 | /* still waiting on other side to disengage from us */ |
1034 | return; | |
1035 | } | |
1036 | ||
a47d5dac | 1037 | if (reactivate) |
33ba3c77 | 1038 | XPC_DEACTIVATE_PARTITION(part, xpReactivating); |
a47d5dac | 1039 | else if (xpc_partition_deactivation_requested_sn2(partid)) |
33ba3c77 | 1040 | XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown); |
33ba3c77 DN |
1041 | } |
1042 | ||
1043 | /* | |
c39838ce | 1044 | * Loop through the activation amo variables and process any bits |
33ba3c77 DN |
1045 | * which are set. Each bit indicates a nasid sending a partition |
1046 | * activation or deactivation request. | |
1047 | * | |
1048 | * Return #of IRQs detected. | |
1049 | */ | |
1050 | int | |
6e41017a | 1051 | xpc_identify_activate_IRQ_sender_sn2(void) |
33ba3c77 | 1052 | { |
04de7418 DN |
1053 | int l; |
1054 | int b; | |
1055 | unsigned long nasid_mask_long; | |
33ba3c77 DN |
1056 | u64 nasid; /* remote nasid */ |
1057 | int n_IRQs_detected = 0; | |
c39838ce | 1058 | struct amo *act_amos; |
33ba3c77 | 1059 | |
8e85c23e | 1060 | act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2; |
33ba3c77 | 1061 | |
04de7418 DN |
1062 | /* scan through activate amo variables looking for non-zero entries */ |
1063 | for (l = 0; l < xpc_nasid_mask_nlongs; l++) { | |
33ba3c77 DN |
1064 | |
1065 | if (xpc_exiting) | |
1066 | break; | |
1067 | ||
04de7418 DN |
1068 | nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]); |
1069 | ||
1070 | b = find_first_bit(&nasid_mask_long, BITS_PER_LONG); | |
1071 | if (b >= BITS_PER_LONG) { | |
1072 | /* no IRQs from nasids in this amo variable */ | |
33ba3c77 DN |
1073 | continue; |
1074 | } | |
1075 | ||
04de7418 DN |
1076 | dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l, |
1077 | nasid_mask_long); | |
33ba3c77 DN |
1078 | |
1079 | /* | |
1080 | * If this nasid has been added to the machine since | |
1081 | * our partition was reset, this will retain the | |
1082 | * remote nasid in our reserved pages machine mask. | |
1083 | * This is used in the event of module reload. | |
1084 | */ | |
04de7418 | 1085 | xpc_mach_nasids[l] |= nasid_mask_long; |
33ba3c77 DN |
1086 | |
1087 | /* locate the nasid(s) which sent interrupts */ | |
1088 | ||
04de7418 DN |
1089 | do { |
1090 | n_IRQs_detected++; | |
1091 | nasid = (l * BITS_PER_LONG + b) * 2; | |
1092 | dev_dbg(xpc_part, "interrupt from nasid %ld\n", nasid); | |
1093 | xpc_identify_activate_IRQ_req_sn2(nasid); | |
1094 | ||
1095 | b = find_next_bit(&nasid_mask_long, BITS_PER_LONG, | |
1096 | b + 1); | |
1097 | } while (b < BITS_PER_LONG); | |
33ba3c77 DN |
1098 | } |
1099 | return n_IRQs_detected; | |
1100 | } | |
1101 | ||
1102 | static void | |
5b8669df | 1103 | xpc_process_activate_IRQ_rcvd_sn2(void) |
33ba3c77 | 1104 | { |
5b8669df DN |
1105 | unsigned long irq_flags; |
1106 | int n_IRQs_expected; | |
33ba3c77 DN |
1107 | int n_IRQs_detected; |
1108 | ||
5b8669df DN |
1109 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); |
1110 | n_IRQs_expected = xpc_activate_IRQ_rcvd; | |
1111 | xpc_activate_IRQ_rcvd = 0; | |
1112 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
1113 | ||
6e41017a | 1114 | n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 | 1115 | if (n_IRQs_detected < n_IRQs_expected) { |
c39838ce | 1116 | /* retry once to help avoid missing amo */ |
6e41017a | 1117 | (void)xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 DN |
1118 | } |
1119 | } | |
1120 | ||
185c3a1b | 1121 | /* |
5b8669df | 1122 | * Setup the channel structures that are sn2 specific. |
e17d416b DN |
1123 | */ |
1124 | static enum xp_retval | |
5b8669df | 1125 | xpc_setup_ch_structures_sn_sn2(struct xpc_partition *part) |
e17d416b | 1126 | { |
a47d5dac | 1127 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
5b8669df | 1128 | struct xpc_channel_sn2 *ch_sn2; |
e17d416b DN |
1129 | enum xp_retval retval; |
1130 | int ret; | |
1131 | int cpuid; | |
1132 | int ch_number; | |
e17d416b DN |
1133 | struct timer_list *timer; |
1134 | short partid = XPC_PARTID(part); | |
1135 | ||
e17d416b DN |
1136 | /* allocate all the required GET/PUT values */ |
1137 | ||
185c3a1b | 1138 | part_sn2->local_GPs = |
5b8669df DN |
1139 | xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, |
1140 | &part_sn2->local_GPs_base); | |
a47d5dac | 1141 | if (part_sn2->local_GPs == NULL) { |
e17d416b DN |
1142 | dev_err(xpc_chan, "can't get memory for local get/put " |
1143 | "values\n"); | |
5b8669df | 1144 | return xpNoMemory; |
e17d416b DN |
1145 | } |
1146 | ||
185c3a1b | 1147 | part_sn2->remote_GPs = |
5b8669df DN |
1148 | xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, |
1149 | &part_sn2->remote_GPs_base); | |
a47d5dac | 1150 | if (part_sn2->remote_GPs == NULL) { |
e17d416b DN |
1151 | dev_err(xpc_chan, "can't get memory for remote get/put " |
1152 | "values\n"); | |
1153 | retval = xpNoMemory; | |
5b8669df | 1154 | goto out_1; |
e17d416b DN |
1155 | } |
1156 | ||
a47d5dac | 1157 | part_sn2->remote_GPs_pa = 0; |
e17d416b DN |
1158 | |
1159 | /* allocate all the required open and close args */ | |
1160 | ||
5b8669df DN |
1161 | part_sn2->local_openclose_args = |
1162 | xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE, | |
1163 | GFP_KERNEL, &part_sn2-> | |
1164 | local_openclose_args_base); | |
1165 | if (part_sn2->local_openclose_args == NULL) { | |
e17d416b DN |
1166 | dev_err(xpc_chan, "can't get memory for local connect args\n"); |
1167 | retval = xpNoMemory; | |
5b8669df | 1168 | goto out_2; |
e17d416b DN |
1169 | } |
1170 | ||
a47d5dac | 1171 | part_sn2->remote_openclose_args_pa = 0; |
e17d416b | 1172 | |
7fb5e59d | 1173 | part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid); |
e17d416b | 1174 | |
7fb5e59d DN |
1175 | part_sn2->notify_IRQ_nasid = 0; |
1176 | part_sn2->notify_IRQ_phys_cpuid = 0; | |
1177 | part_sn2->remote_chctl_amo_va = NULL; | |
e17d416b | 1178 | |
7fb5e59d | 1179 | sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid); |
a47d5dac | 1180 | ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2, |
7fb5e59d | 1181 | IRQF_SHARED, part_sn2->notify_IRQ_owner, |
a47d5dac | 1182 | (void *)(u64)partid); |
e17d416b DN |
1183 | if (ret != 0) { |
1184 | dev_err(xpc_chan, "can't register NOTIFY IRQ handler, " | |
1185 | "errno=%d\n", -ret); | |
1186 | retval = xpLackOfResources; | |
5b8669df | 1187 | goto out_3; |
e17d416b DN |
1188 | } |
1189 | ||
7fb5e59d | 1190 | /* Setup a timer to check for dropped notify IRQs */ |
a47d5dac | 1191 | timer = &part_sn2->dropped_notify_IRQ_timer; |
e17d416b | 1192 | init_timer(timer); |
a47d5dac | 1193 | timer->function = |
7fb5e59d | 1194 | (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2; |
e17d416b | 1195 | timer->data = (unsigned long)part; |
7fb5e59d | 1196 | timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
e17d416b DN |
1197 | add_timer(timer); |
1198 | ||
e17d416b | 1199 | for (ch_number = 0; ch_number < part->nchannels; ch_number++) { |
5b8669df | 1200 | ch_sn2 = &part->channels[ch_number].sn.sn2; |
e17d416b | 1201 | |
5b8669df DN |
1202 | ch_sn2->local_GP = &part_sn2->local_GPs[ch_number]; |
1203 | ch_sn2->local_openclose_args = | |
1204 | &part_sn2->local_openclose_args[ch_number]; | |
e17d416b | 1205 | |
5b8669df | 1206 | mutex_init(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1207 | } |
1208 | ||
e17d416b DN |
1209 | /* |
1210 | * Setup the per partition specific variables required by the | |
1211 | * remote partition to establish channel connections with us. | |
1212 | * | |
1213 | * The setting of the magic # indicates that these per partition | |
1214 | * specific variables are ready to be used. | |
1215 | */ | |
a812dcc3 | 1216 | xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs); |
8e85c23e | 1217 | xpc_vars_part_sn2[partid].openclose_args_pa = |
5b8669df | 1218 | xp_pa(part_sn2->local_openclose_args); |
8e85c23e | 1219 | xpc_vars_part_sn2[partid].chctl_amo_pa = |
a812dcc3 | 1220 | xp_pa(part_sn2->local_chctl_amo_va); |
e17d416b | 1221 | cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */ |
8e85c23e DN |
1222 | xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid); |
1223 | xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid = | |
1224 | cpu_physical_id(cpuid); | |
1225 | xpc_vars_part_sn2[partid].nchannels = part->nchannels; | |
5b8669df | 1226 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2; |
e17d416b DN |
1227 | |
1228 | return xpSuccess; | |
1229 | ||
5b8669df | 1230 | /* setup of ch structures failed */ |
e17d416b | 1231 | out_3: |
5b8669df DN |
1232 | kfree(part_sn2->local_openclose_args_base); |
1233 | part_sn2->local_openclose_args = NULL; | |
1234 | out_2: | |
a47d5dac DN |
1235 | kfree(part_sn2->remote_GPs_base); |
1236 | part_sn2->remote_GPs = NULL; | |
5b8669df | 1237 | out_1: |
a47d5dac DN |
1238 | kfree(part_sn2->local_GPs_base); |
1239 | part_sn2->local_GPs = NULL; | |
e17d416b DN |
1240 | return retval; |
1241 | } | |
1242 | ||
1243 | /* | |
5b8669df | 1244 | * Teardown the channel structures that are sn2 specific. |
e17d416b DN |
1245 | */ |
1246 | static void | |
5b8669df | 1247 | xpc_teardown_ch_structures_sn_sn2(struct xpc_partition *part) |
e17d416b | 1248 | { |
a47d5dac | 1249 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1250 | short partid = XPC_PARTID(part); |
1251 | ||
1252 | /* | |
5b8669df DN |
1253 | * Indicate that the variables specific to the remote partition are no |
1254 | * longer available for its use. | |
e17d416b | 1255 | */ |
8e85c23e | 1256 | xpc_vars_part_sn2[partid].magic = 0; |
e17d416b | 1257 | |
e17d416b | 1258 | /* in case we've still got outstanding timers registered... */ |
a47d5dac | 1259 | del_timer_sync(&part_sn2->dropped_notify_IRQ_timer); |
5b8669df | 1260 | free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid); |
e17d416b | 1261 | |
5b8669df DN |
1262 | kfree(part_sn2->local_openclose_args_base); |
1263 | part_sn2->local_openclose_args = NULL; | |
a47d5dac DN |
1264 | kfree(part_sn2->remote_GPs_base); |
1265 | part_sn2->remote_GPs = NULL; | |
1266 | kfree(part_sn2->local_GPs_base); | |
1267 | part_sn2->local_GPs = NULL; | |
7fb5e59d | 1268 | part_sn2->local_chctl_amo_va = NULL; |
e17d416b DN |
1269 | } |
1270 | ||
1271 | /* | |
1272 | * Create a wrapper that hides the underlying mechanism for pulling a cacheline | |
1273 | * (or multiple cachelines) from a remote partition. | |
1274 | * | |
a812dcc3 | 1275 | * src_pa must be a cacheline aligned physical address on the remote partition. |
e17d416b DN |
1276 | * dst must be a cacheline aligned virtual address on this partition. |
1277 | * cnt must be cacheline sized | |
1278 | */ | |
ea57f80c | 1279 | /* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */ |
e17d416b DN |
1280 | static enum xp_retval |
1281 | xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst, | |
a812dcc3 | 1282 | const unsigned long src_pa, size_t cnt) |
e17d416b DN |
1283 | { |
1284 | enum xp_retval ret; | |
1285 | ||
a812dcc3 DN |
1286 | DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa)); |
1287 | DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst)); | |
e17d416b DN |
1288 | DBUG_ON(cnt != L1_CACHE_ALIGN(cnt)); |
1289 | ||
83469b55 | 1290 | if (part->act_state == XPC_P_AS_DEACTIVATING) |
e17d416b DN |
1291 | return part->reason; |
1292 | ||
a812dcc3 | 1293 | ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt); |
e17d416b DN |
1294 | if (ret != xpSuccess) { |
1295 | dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed," | |
1296 | " ret=%d\n", XPC_PARTID(part), ret); | |
1297 | } | |
1298 | return ret; | |
1299 | } | |
1300 | ||
1301 | /* | |
1302 | * Pull the remote per partition specific variables from the specified | |
1303 | * partition. | |
1304 | */ | |
1305 | static enum xp_retval | |
1306 | xpc_pull_remote_vars_part_sn2(struct xpc_partition *part) | |
1307 | { | |
a47d5dac | 1308 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1309 | u8 buffer[L1_CACHE_BYTES * 2]; |
1310 | struct xpc_vars_part_sn2 *pulled_entry_cacheline = | |
1311 | (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer); | |
1312 | struct xpc_vars_part_sn2 *pulled_entry; | |
a812dcc3 DN |
1313 | unsigned long remote_entry_cacheline_pa; |
1314 | unsigned long remote_entry_pa; | |
e17d416b DN |
1315 | short partid = XPC_PARTID(part); |
1316 | enum xp_retval ret; | |
1317 | ||
1318 | /* pull the cacheline that contains the variables we're interested in */ | |
1319 | ||
a47d5dac DN |
1320 | DBUG_ON(part_sn2->remote_vars_part_pa != |
1321 | L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa)); | |
e17d416b DN |
1322 | DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2); |
1323 | ||
a47d5dac | 1324 | remote_entry_pa = part_sn2->remote_vars_part_pa + |
e17d416b DN |
1325 | sn_partition_id * sizeof(struct xpc_vars_part_sn2); |
1326 | ||
1327 | remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1)); | |
1328 | ||
1329 | pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline | |
1330 | + (remote_entry_pa & | |
1331 | (L1_CACHE_BYTES - 1))); | |
1332 | ||
1333 | ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline, | |
a812dcc3 | 1334 | remote_entry_cacheline_pa, |
e17d416b DN |
1335 | L1_CACHE_BYTES); |
1336 | if (ret != xpSuccess) { | |
1337 | dev_dbg(xpc_chan, "failed to pull XPC vars_part from " | |
1338 | "partition %d, ret=%d\n", partid, ret); | |
1339 | return ret; | |
1340 | } | |
1341 | ||
1342 | /* see if they've been set up yet */ | |
1343 | ||
5b8669df DN |
1344 | if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 && |
1345 | pulled_entry->magic != XPC_VP_MAGIC2_SN2) { | |
e17d416b DN |
1346 | |
1347 | if (pulled_entry->magic != 0) { | |
1348 | dev_dbg(xpc_chan, "partition %d's XPC vars_part for " | |
1349 | "partition %d has bad magic value (=0x%lx)\n", | |
1350 | partid, sn_partition_id, pulled_entry->magic); | |
1351 | return xpBadMagic; | |
1352 | } | |
1353 | ||
1354 | /* they've not been initialized yet */ | |
1355 | return xpRetry; | |
1356 | } | |
1357 | ||
5b8669df | 1358 | if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) { |
e17d416b DN |
1359 | |
1360 | /* validate the variables */ | |
1361 | ||
1362 | if (pulled_entry->GPs_pa == 0 || | |
1363 | pulled_entry->openclose_args_pa == 0 || | |
7fb5e59d | 1364 | pulled_entry->chctl_amo_pa == 0) { |
e17d416b DN |
1365 | |
1366 | dev_err(xpc_chan, "partition %d's XPC vars_part for " | |
1367 | "partition %d are not valid\n", partid, | |
1368 | sn_partition_id); | |
1369 | return xpInvalidAddress; | |
1370 | } | |
1371 | ||
1372 | /* the variables we imported look to be valid */ | |
1373 | ||
a47d5dac DN |
1374 | part_sn2->remote_GPs_pa = pulled_entry->GPs_pa; |
1375 | part_sn2->remote_openclose_args_pa = | |
e17d416b | 1376 | pulled_entry->openclose_args_pa; |
7fb5e59d | 1377 | part_sn2->remote_chctl_amo_va = |
c39838ce | 1378 | (struct amo *)__va(pulled_entry->chctl_amo_pa); |
7fb5e59d DN |
1379 | part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid; |
1380 | part_sn2->notify_IRQ_phys_cpuid = | |
1381 | pulled_entry->notify_IRQ_phys_cpuid; | |
e17d416b DN |
1382 | |
1383 | if (part->nchannels > pulled_entry->nchannels) | |
1384 | part->nchannels = pulled_entry->nchannels; | |
1385 | ||
1386 | /* let the other side know that we've pulled their variables */ | |
1387 | ||
5b8669df | 1388 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2; |
e17d416b DN |
1389 | } |
1390 | ||
5b8669df | 1391 | if (pulled_entry->magic == XPC_VP_MAGIC1_SN2) |
e17d416b DN |
1392 | return xpRetry; |
1393 | ||
1394 | return xpSuccess; | |
1395 | } | |
1396 | ||
1397 | /* | |
1398 | * Establish first contact with the remote partititon. This involves pulling | |
1399 | * the XPC per partition variables from the remote partition and waiting for | |
1400 | * the remote partition to pull ours. | |
1401 | */ | |
1402 | static enum xp_retval | |
1403 | xpc_make_first_contact_sn2(struct xpc_partition *part) | |
1404 | { | |
a47d5dac | 1405 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1406 | enum xp_retval ret; |
1407 | ||
33ba3c77 | 1408 | /* |
c39838ce | 1409 | * Register the remote partition's amos with SAL so it can handle |
33ba3c77 DN |
1410 | * and cleanup errors within that address range should the remote |
1411 | * partition go down. We don't unregister this range because it is | |
1412 | * difficult to tell when outstanding writes to the remote partition | |
1413 | * are finished and thus when it is safe to unregister. This should | |
1414 | * not result in wasted space in the SAL xp_addr_region table because | |
1415 | * we should get the same page for remote_amos_page_pa after module | |
1416 | * reloads and system reboots. | |
1417 | */ | |
a47d5dac | 1418 | if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa, |
33ba3c77 DN |
1419 | PAGE_SIZE, 1) < 0) { |
1420 | dev_warn(xpc_part, "xpc_activating(%d) failed to register " | |
1421 | "xp_addr region\n", XPC_PARTID(part)); | |
1422 | ||
1423 | ret = xpPhysAddrRegFailed; | |
1424 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1425 | return ret; | |
1426 | } | |
1427 | ||
a47d5dac DN |
1428 | /* |
1429 | * Send activate IRQ to get other side to activate if they've not | |
1430 | * already begun to do so. | |
1431 | */ | |
7fb5e59d | 1432 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
1433 | cnodeid_to_nasid(0), |
1434 | part_sn2->activate_IRQ_nasid, | |
1435 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 1436 | |
e17d416b DN |
1437 | while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) { |
1438 | if (ret != xpRetry) { | |
1439 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1440 | return ret; | |
1441 | } | |
1442 | ||
1443 | dev_dbg(xpc_part, "waiting to make first contact with " | |
1444 | "partition %d\n", XPC_PARTID(part)); | |
1445 | ||
1446 | /* wait a 1/4 of a second or so */ | |
1447 | (void)msleep_interruptible(250); | |
1448 | ||
83469b55 | 1449 | if (part->act_state == XPC_P_AS_DEACTIVATING) |
e17d416b DN |
1450 | return part->reason; |
1451 | } | |
1452 | ||
1453 | return xpSuccess; | |
1454 | } | |
1455 | ||
1456 | /* | |
7fb5e59d | 1457 | * Get the chctl flags and pull the openclose args and/or remote GPs as needed. |
e17d416b DN |
1458 | */ |
1459 | static u64 | |
7fb5e59d | 1460 | xpc_get_chctl_all_flags_sn2(struct xpc_partition *part) |
e17d416b | 1461 | { |
a47d5dac | 1462 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b | 1463 | unsigned long irq_flags; |
7fb5e59d | 1464 | union xpc_channel_ctl_flags chctl; |
e17d416b DN |
1465 | enum xp_retval ret; |
1466 | ||
1467 | /* | |
7fb5e59d | 1468 | * See if there are any chctl flags to be handled. |
e17d416b DN |
1469 | */ |
1470 | ||
7fb5e59d DN |
1471 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
1472 | chctl = part->chctl; | |
1473 | if (chctl.all_flags != 0) | |
1474 | part->chctl.all_flags = 0; | |
e17d416b | 1475 | |
7fb5e59d | 1476 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); |
e17d416b | 1477 | |
7fb5e59d | 1478 | if (xpc_any_openclose_chctl_flags_set(&chctl)) { |
a47d5dac DN |
1479 | ret = xpc_pull_remote_cachelines_sn2(part, part-> |
1480 | remote_openclose_args, | |
a812dcc3 | 1481 | part_sn2-> |
e17d416b DN |
1482 | remote_openclose_args_pa, |
1483 | XPC_OPENCLOSE_ARGS_SIZE); | |
1484 | if (ret != xpSuccess) { | |
1485 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1486 | ||
1487 | dev_dbg(xpc_chan, "failed to pull openclose args from " | |
1488 | "partition %d, ret=%d\n", XPC_PARTID(part), | |
1489 | ret); | |
1490 | ||
7fb5e59d DN |
1491 | /* don't bother processing chctl flags anymore */ |
1492 | chctl.all_flags = 0; | |
e17d416b DN |
1493 | } |
1494 | } | |
1495 | ||
7fb5e59d | 1496 | if (xpc_any_msg_chctl_flags_set(&chctl)) { |
a47d5dac | 1497 | ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs, |
a812dcc3 | 1498 | part_sn2->remote_GPs_pa, |
e17d416b DN |
1499 | XPC_GP_SIZE); |
1500 | if (ret != xpSuccess) { | |
1501 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1502 | ||
1503 | dev_dbg(xpc_chan, "failed to pull GPs from partition " | |
1504 | "%d, ret=%d\n", XPC_PARTID(part), ret); | |
1505 | ||
7fb5e59d DN |
1506 | /* don't bother processing chctl flags anymore */ |
1507 | chctl.all_flags = 0; | |
e17d416b DN |
1508 | } |
1509 | } | |
1510 | ||
7fb5e59d | 1511 | return chctl.all_flags; |
e17d416b DN |
1512 | } |
1513 | ||
185c3a1b DN |
1514 | /* |
1515 | * Allocate the local message queue and the notify queue. | |
1516 | */ | |
1517 | static enum xp_retval | |
1518 | xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch) | |
1519 | { | |
5b8669df | 1520 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1521 | unsigned long irq_flags; |
1522 | int nentries; | |
1523 | size_t nbytes; | |
1524 | ||
1525 | for (nentries = ch->local_nentries; nentries > 0; nentries--) { | |
1526 | ||
bd3e64c1 | 1527 | nbytes = nentries * ch->entry_size; |
5b8669df DN |
1528 | ch_sn2->local_msgqueue = |
1529 | xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, | |
1530 | &ch_sn2->local_msgqueue_base); | |
1531 | if (ch_sn2->local_msgqueue == NULL) | |
185c3a1b DN |
1532 | continue; |
1533 | ||
bd3e64c1 | 1534 | nbytes = nentries * sizeof(struct xpc_notify_sn2); |
5b8669df DN |
1535 | ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL); |
1536 | if (ch_sn2->notify_queue == NULL) { | |
1537 | kfree(ch_sn2->local_msgqueue_base); | |
1538 | ch_sn2->local_msgqueue = NULL; | |
185c3a1b DN |
1539 | continue; |
1540 | } | |
1541 | ||
1542 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1543 | if (nentries < ch->local_nentries) { | |
1544 | dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, " | |
1545 | "partid=%d, channel=%d\n", nentries, | |
1546 | ch->local_nentries, ch->partid, ch->number); | |
1547 | ||
1548 | ch->local_nentries = nentries; | |
1549 | } | |
1550 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1551 | return xpSuccess; | |
1552 | } | |
1553 | ||
1554 | dev_dbg(xpc_chan, "can't get memory for local message queue and notify " | |
1555 | "queue, partid=%d, channel=%d\n", ch->partid, ch->number); | |
1556 | return xpNoMemory; | |
1557 | } | |
1558 | ||
1559 | /* | |
1560 | * Allocate the cached remote message queue. | |
1561 | */ | |
1562 | static enum xp_retval | |
1563 | xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch) | |
1564 | { | |
5b8669df | 1565 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1566 | unsigned long irq_flags; |
1567 | int nentries; | |
1568 | size_t nbytes; | |
1569 | ||
1570 | DBUG_ON(ch->remote_nentries <= 0); | |
1571 | ||
1572 | for (nentries = ch->remote_nentries; nentries > 0; nentries--) { | |
1573 | ||
bd3e64c1 | 1574 | nbytes = nentries * ch->entry_size; |
5b8669df DN |
1575 | ch_sn2->remote_msgqueue = |
1576 | xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2-> | |
1577 | remote_msgqueue_base); | |
1578 | if (ch_sn2->remote_msgqueue == NULL) | |
185c3a1b DN |
1579 | continue; |
1580 | ||
1581 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1582 | if (nentries < ch->remote_nentries) { | |
1583 | dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, " | |
1584 | "partid=%d, channel=%d\n", nentries, | |
1585 | ch->remote_nentries, ch->partid, ch->number); | |
1586 | ||
1587 | ch->remote_nentries = nentries; | |
1588 | } | |
1589 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1590 | return xpSuccess; | |
1591 | } | |
1592 | ||
1593 | dev_dbg(xpc_chan, "can't get memory for cached remote message queue, " | |
1594 | "partid=%d, channel=%d\n", ch->partid, ch->number); | |
1595 | return xpNoMemory; | |
1596 | } | |
1597 | ||
1598 | /* | |
1599 | * Allocate message queues and other stuff associated with a channel. | |
1600 | * | |
1601 | * Note: Assumes all of the channel sizes are filled in. | |
1602 | */ | |
1603 | static enum xp_retval | |
5b8669df | 1604 | xpc_setup_msg_structures_sn2(struct xpc_channel *ch) |
185c3a1b | 1605 | { |
5b8669df | 1606 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1607 | enum xp_retval ret; |
1608 | ||
1609 | DBUG_ON(ch->flags & XPC_C_SETUP); | |
1610 | ||
1611 | ret = xpc_allocate_local_msgqueue_sn2(ch); | |
1612 | if (ret == xpSuccess) { | |
1613 | ||
1614 | ret = xpc_allocate_remote_msgqueue_sn2(ch); | |
1615 | if (ret != xpSuccess) { | |
5b8669df DN |
1616 | kfree(ch_sn2->local_msgqueue_base); |
1617 | ch_sn2->local_msgqueue = NULL; | |
1618 | kfree(ch_sn2->notify_queue); | |
1619 | ch_sn2->notify_queue = NULL; | |
185c3a1b DN |
1620 | } |
1621 | } | |
1622 | return ret; | |
1623 | } | |
1624 | ||
1625 | /* | |
1626 | * Free up message queues and other stuff that were allocated for the specified | |
1627 | * channel. | |
185c3a1b DN |
1628 | */ |
1629 | static void | |
5b8669df | 1630 | xpc_teardown_msg_structures_sn2(struct xpc_channel *ch) |
185c3a1b DN |
1631 | { |
1632 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1633 | ||
1634 | DBUG_ON(!spin_is_locked(&ch->lock)); | |
185c3a1b | 1635 | |
5b8669df | 1636 | ch_sn2->remote_msgqueue_pa = 0; |
185c3a1b DN |
1637 | |
1638 | ch_sn2->local_GP->get = 0; | |
1639 | ch_sn2->local_GP->put = 0; | |
1640 | ch_sn2->remote_GP.get = 0; | |
1641 | ch_sn2->remote_GP.put = 0; | |
1642 | ch_sn2->w_local_GP.get = 0; | |
1643 | ch_sn2->w_local_GP.put = 0; | |
1644 | ch_sn2->w_remote_GP.get = 0; | |
1645 | ch_sn2->w_remote_GP.put = 0; | |
1646 | ch_sn2->next_msg_to_pull = 0; | |
1647 | ||
1648 | if (ch->flags & XPC_C_SETUP) { | |
1649 | dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n", | |
1650 | ch->flags, ch->partid, ch->number); | |
1651 | ||
5b8669df DN |
1652 | kfree(ch_sn2->local_msgqueue_base); |
1653 | ch_sn2->local_msgqueue = NULL; | |
1654 | kfree(ch_sn2->remote_msgqueue_base); | |
1655 | ch_sn2->remote_msgqueue = NULL; | |
1656 | kfree(ch_sn2->notify_queue); | |
1657 | ch_sn2->notify_queue = NULL; | |
185c3a1b DN |
1658 | } |
1659 | } | |
1660 | ||
a47d5dac DN |
1661 | /* |
1662 | * Notify those who wanted to be notified upon delivery of their message. | |
1663 | */ | |
1664 | static void | |
1665 | xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put) | |
1666 | { | |
bd3e64c1 | 1667 | struct xpc_notify_sn2 *notify; |
a47d5dac DN |
1668 | u8 notify_type; |
1669 | s64 get = ch->sn.sn2.w_remote_GP.get - 1; | |
1670 | ||
1671 | while (++get < put && atomic_read(&ch->n_to_notify) > 0) { | |
1672 | ||
5b8669df | 1673 | notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries]; |
a47d5dac DN |
1674 | |
1675 | /* | |
1676 | * See if the notify entry indicates it was associated with | |
1677 | * a message who's sender wants to be notified. It is possible | |
1678 | * that it is, but someone else is doing or has done the | |
1679 | * notification. | |
1680 | */ | |
1681 | notify_type = notify->type; | |
1682 | if (notify_type == 0 || | |
1683 | cmpxchg(¬ify->type, notify_type, 0) != notify_type) { | |
1684 | continue; | |
1685 | } | |
1686 | ||
1687 | DBUG_ON(notify_type != XPC_N_CALL); | |
1688 | ||
1689 | atomic_dec(&ch->n_to_notify); | |
1690 | ||
1691 | if (notify->func != NULL) { | |
bd3e64c1 DN |
1692 | dev_dbg(xpc_chan, "notify->func() called, notify=0x%p " |
1693 | "msg_number=%ld partid=%d channel=%d\n", | |
a47d5dac DN |
1694 | (void *)notify, get, ch->partid, ch->number); |
1695 | ||
1696 | notify->func(reason, ch->partid, ch->number, | |
1697 | notify->key); | |
1698 | ||
bd3e64c1 DN |
1699 | dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p" |
1700 | " msg_number=%ld partid=%d channel=%d\n", | |
1701 | (void *)notify, get, ch->partid, ch->number); | |
a47d5dac DN |
1702 | } |
1703 | } | |
1704 | } | |
1705 | ||
1706 | static void | |
1707 | xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch) | |
1708 | { | |
1709 | xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put); | |
1710 | } | |
1711 | ||
1712 | /* | |
1713 | * Clear some of the msg flags in the local message queue. | |
1714 | */ | |
1715 | static inline void | |
1716 | xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1717 | { | |
1718 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1719 | struct xpc_msg_sn2 *msg; |
a47d5dac DN |
1720 | s64 get; |
1721 | ||
1722 | get = ch_sn2->w_remote_GP.get; | |
1723 | do { | |
bd3e64c1 DN |
1724 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + |
1725 | (get % ch->local_nentries) * | |
1726 | ch->entry_size); | |
252523ef | 1727 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
a47d5dac DN |
1728 | msg->flags = 0; |
1729 | } while (++get < ch_sn2->remote_GP.get); | |
1730 | } | |
1731 | ||
1732 | /* | |
1733 | * Clear some of the msg flags in the remote message queue. | |
1734 | */ | |
1735 | static inline void | |
1736 | xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1737 | { | |
1738 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1739 | struct xpc_msg_sn2 *msg; |
a47d5dac DN |
1740 | s64 put; |
1741 | ||
252523ef RH |
1742 | /* flags are zeroed when the buffer is allocated */ |
1743 | if (ch_sn2->remote_GP.put < ch->remote_nentries) | |
1744 | return; | |
1745 | ||
1746 | put = max(ch_sn2->w_remote_GP.put, ch->remote_nentries); | |
a47d5dac | 1747 | do { |
bd3e64c1 DN |
1748 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + |
1749 | (put % ch->remote_nentries) * | |
1750 | ch->entry_size); | |
252523ef RH |
1751 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
1752 | DBUG_ON(!(msg->flags & XPC_M_SN2_DONE)); | |
1753 | DBUG_ON(msg->number != put - ch->remote_nentries); | |
a47d5dac DN |
1754 | msg->flags = 0; |
1755 | } while (++put < ch_sn2->remote_GP.put); | |
1756 | } | |
1757 | ||
bd3e64c1 DN |
1758 | static int |
1759 | xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch) | |
1760 | { | |
1761 | return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get; | |
1762 | } | |
1763 | ||
a47d5dac | 1764 | static void |
7fb5e59d | 1765 | xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number) |
a47d5dac DN |
1766 | { |
1767 | struct xpc_channel *ch = &part->channels[ch_number]; | |
1768 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1769 | int npayloads_sent; |
a47d5dac DN |
1770 | |
1771 | ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number]; | |
1772 | ||
1773 | /* See what, if anything, has changed for each connected channel */ | |
1774 | ||
1775 | xpc_msgqueue_ref(ch); | |
1776 | ||
1777 | if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get && | |
1778 | ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) { | |
1779 | /* nothing changed since GPs were last pulled */ | |
1780 | xpc_msgqueue_deref(ch); | |
1781 | return; | |
1782 | } | |
1783 | ||
1784 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
1785 | xpc_msgqueue_deref(ch); | |
1786 | return; | |
1787 | } | |
1788 | ||
1789 | /* | |
1790 | * First check to see if messages recently sent by us have been | |
1791 | * received by the other side. (The remote GET value will have | |
1792 | * changed since we last looked at it.) | |
1793 | */ | |
1794 | ||
1795 | if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) { | |
1796 | ||
1797 | /* | |
1798 | * We need to notify any senders that want to be notified | |
1799 | * that their sent messages have been received by their | |
1800 | * intended recipients. We need to do this before updating | |
1801 | * w_remote_GP.get so that we don't allocate the same message | |
1802 | * queue entries prematurely (see xpc_allocate_msg()). | |
1803 | */ | |
1804 | if (atomic_read(&ch->n_to_notify) > 0) { | |
1805 | /* | |
1806 | * Notify senders that messages sent have been | |
1807 | * received and delivered by the other side. | |
1808 | */ | |
1809 | xpc_notify_senders_sn2(ch, xpMsgDelivered, | |
1810 | ch_sn2->remote_GP.get); | |
1811 | } | |
1812 | ||
1813 | /* | |
1814 | * Clear msg->flags in previously sent messages, so that | |
1815 | * they're ready for xpc_allocate_msg(). | |
1816 | */ | |
1817 | xpc_clear_local_msgqueue_flags_sn2(ch); | |
1818 | ||
1819 | ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get; | |
1820 | ||
1821 | dev_dbg(xpc_chan, "w_remote_GP.get changed to %ld, partid=%d, " | |
1822 | "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid, | |
1823 | ch->number); | |
1824 | ||
1825 | /* | |
1826 | * If anyone was waiting for message queue entries to become | |
1827 | * available, wake them up. | |
1828 | */ | |
1829 | if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) | |
1830 | wake_up(&ch->msg_allocate_wq); | |
1831 | } | |
1832 | ||
1833 | /* | |
1834 | * Now check for newly sent messages by the other side. (The remote | |
1835 | * PUT value will have changed since we last looked at it.) | |
1836 | */ | |
1837 | ||
1838 | if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) { | |
1839 | /* | |
1840 | * Clear msg->flags in previously received messages, so that | |
bd3e64c1 | 1841 | * they're ready for xpc_get_deliverable_payload_sn2(). |
a47d5dac DN |
1842 | */ |
1843 | xpc_clear_remote_msgqueue_flags_sn2(ch); | |
1844 | ||
69b3bb65 | 1845 | smp_wmb(); /* ensure flags have been cleared before bte_copy */ |
a47d5dac DN |
1846 | ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put; |
1847 | ||
1848 | dev_dbg(xpc_chan, "w_remote_GP.put changed to %ld, partid=%d, " | |
1849 | "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid, | |
1850 | ch->number); | |
1851 | ||
bd3e64c1 DN |
1852 | npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch); |
1853 | if (npayloads_sent > 0) { | |
a47d5dac DN |
1854 | dev_dbg(xpc_chan, "msgs waiting to be copied and " |
1855 | "delivered=%d, partid=%d, channel=%d\n", | |
bd3e64c1 | 1856 | npayloads_sent, ch->partid, ch->number); |
a47d5dac DN |
1857 | |
1858 | if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) | |
bd3e64c1 | 1859 | xpc_activate_kthreads(ch, npayloads_sent); |
a47d5dac DN |
1860 | } |
1861 | } | |
1862 | ||
1863 | xpc_msgqueue_deref(ch); | |
1864 | } | |
1865 | ||
bd3e64c1 | 1866 | static struct xpc_msg_sn2 * |
e17d416b DN |
1867 | xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get) |
1868 | { | |
1869 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 1870 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
a812dcc3 | 1871 | unsigned long remote_msg_pa; |
bd3e64c1 | 1872 | struct xpc_msg_sn2 *msg; |
a812dcc3 DN |
1873 | u32 msg_index; |
1874 | u32 nmsgs; | |
e17d416b DN |
1875 | u64 msg_offset; |
1876 | enum xp_retval ret; | |
1877 | ||
a47d5dac | 1878 | if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) { |
e17d416b DN |
1879 | /* we were interrupted by a signal */ |
1880 | return NULL; | |
1881 | } | |
1882 | ||
a47d5dac | 1883 | while (get >= ch_sn2->next_msg_to_pull) { |
e17d416b DN |
1884 | |
1885 | /* pull as many messages as are ready and able to be pulled */ | |
1886 | ||
a47d5dac | 1887 | msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries; |
e17d416b | 1888 | |
a47d5dac DN |
1889 | DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put); |
1890 | nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull; | |
e17d416b DN |
1891 | if (msg_index + nmsgs > ch->remote_nentries) { |
1892 | /* ignore the ones that wrap the msg queue for now */ | |
1893 | nmsgs = ch->remote_nentries - msg_index; | |
1894 | } | |
1895 | ||
bd3e64c1 DN |
1896 | msg_offset = msg_index * ch->entry_size; |
1897 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + | |
5b8669df DN |
1898 | msg_offset); |
1899 | remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset; | |
e17d416b | 1900 | |
a812dcc3 | 1901 | ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa, |
bd3e64c1 | 1902 | nmsgs * ch->entry_size); |
e17d416b DN |
1903 | if (ret != xpSuccess) { |
1904 | ||
1905 | dev_dbg(xpc_chan, "failed to pull %d msgs starting with" | |
1906 | " msg %ld from partition %d, channel=%d, " | |
a47d5dac | 1907 | "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull, |
e17d416b DN |
1908 | ch->partid, ch->number, ret); |
1909 | ||
1910 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1911 | ||
a47d5dac | 1912 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1913 | return NULL; |
1914 | } | |
1915 | ||
a47d5dac | 1916 | ch_sn2->next_msg_to_pull += nmsgs; |
e17d416b DN |
1917 | } |
1918 | ||
a47d5dac | 1919 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1920 | |
1921 | /* return the message we were looking for */ | |
bd3e64c1 DN |
1922 | msg_offset = (get % ch->remote_nentries) * ch->entry_size; |
1923 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset); | |
e17d416b DN |
1924 | |
1925 | return msg; | |
1926 | } | |
1927 | ||
1928 | /* | |
bd3e64c1 | 1929 | * Get the next deliverable message's payload. |
e17d416b | 1930 | */ |
bd3e64c1 DN |
1931 | static void * |
1932 | xpc_get_deliverable_payload_sn2(struct xpc_channel *ch) | |
e17d416b | 1933 | { |
a47d5dac | 1934 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 DN |
1935 | struct xpc_msg_sn2 *msg; |
1936 | void *payload = NULL; | |
e17d416b DN |
1937 | s64 get; |
1938 | ||
1939 | do { | |
1940 | if (ch->flags & XPC_C_DISCONNECTING) | |
1941 | break; | |
1942 | ||
a47d5dac | 1943 | get = ch_sn2->w_local_GP.get; |
69b3bb65 | 1944 | smp_rmb(); /* guarantee that .get loads before .put */ |
a47d5dac | 1945 | if (get == ch_sn2->w_remote_GP.put) |
e17d416b DN |
1946 | break; |
1947 | ||
1948 | /* There are messages waiting to be pulled and delivered. | |
1949 | * We need to try to secure one for ourselves. We'll do this | |
1950 | * by trying to increment w_local_GP.get and hope that no one | |
1951 | * else beats us to it. If they do, we'll we'll simply have | |
1952 | * to try again for the next one. | |
1953 | */ | |
1954 | ||
a47d5dac | 1955 | if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) { |
e17d416b DN |
1956 | /* we got the entry referenced by get */ |
1957 | ||
1958 | dev_dbg(xpc_chan, "w_local_GP.get changed to %ld, " | |
1959 | "partid=%d, channel=%d\n", get + 1, | |
1960 | ch->partid, ch->number); | |
1961 | ||
1962 | /* pull the message from the remote partition */ | |
1963 | ||
1964 | msg = xpc_pull_remote_msg_sn2(ch, get); | |
1965 | ||
17e21616 RH |
1966 | if (msg != NULL) { |
1967 | DBUG_ON(msg->number != get); | |
1968 | DBUG_ON(msg->flags & XPC_M_SN2_DONE); | |
1969 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); | |
e17d416b | 1970 | |
17e21616 RH |
1971 | payload = &msg->payload; |
1972 | } | |
e17d416b DN |
1973 | break; |
1974 | } | |
1975 | ||
1976 | } while (1); | |
1977 | ||
bd3e64c1 | 1978 | return payload; |
e17d416b DN |
1979 | } |
1980 | ||
33ba3c77 DN |
1981 | /* |
1982 | * Now we actually send the messages that are ready to be sent by advancing | |
7fb5e59d DN |
1983 | * the local message queue's Put value and then send a chctl msgrequest to the |
1984 | * recipient partition. | |
33ba3c77 DN |
1985 | */ |
1986 | static void | |
1987 | xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put) | |
1988 | { | |
a47d5dac | 1989 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 1990 | struct xpc_msg_sn2 *msg; |
33ba3c77 | 1991 | s64 put = initial_put + 1; |
7fb5e59d | 1992 | int send_msgrequest = 0; |
33ba3c77 DN |
1993 | |
1994 | while (1) { | |
1995 | ||
1996 | while (1) { | |
a47d5dac | 1997 | if (put == ch_sn2->w_local_GP.put) |
33ba3c77 DN |
1998 | break; |
1999 | ||
bd3e64c1 DN |
2000 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> |
2001 | local_msgqueue + (put % | |
2002 | ch->local_nentries) * | |
2003 | ch->entry_size); | |
33ba3c77 | 2004 | |
bd3e64c1 | 2005 | if (!(msg->flags & XPC_M_SN2_READY)) |
33ba3c77 DN |
2006 | break; |
2007 | ||
2008 | put++; | |
2009 | } | |
2010 | ||
2011 | if (put == initial_put) { | |
2012 | /* nothing's changed */ | |
2013 | break; | |
2014 | } | |
2015 | ||
a47d5dac | 2016 | if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) != |
33ba3c77 DN |
2017 | initial_put) { |
2018 | /* someone else beat us to it */ | |
a47d5dac | 2019 | DBUG_ON(ch_sn2->local_GP->put < initial_put); |
33ba3c77 DN |
2020 | break; |
2021 | } | |
2022 | ||
2023 | /* we just set the new value of local_GP->put */ | |
2024 | ||
2025 | dev_dbg(xpc_chan, "local_GP->put changed to %ld, partid=%d, " | |
2026 | "channel=%d\n", put, ch->partid, ch->number); | |
2027 | ||
7fb5e59d | 2028 | send_msgrequest = 1; |
33ba3c77 DN |
2029 | |
2030 | /* | |
2031 | * We need to ensure that the message referenced by | |
bd3e64c1 | 2032 | * local_GP->put is not XPC_M_SN2_READY or that local_GP->put |
33ba3c77 DN |
2033 | * equals w_local_GP.put, so we'll go have a look. |
2034 | */ | |
2035 | initial_put = put; | |
2036 | } | |
2037 | ||
7fb5e59d DN |
2038 | if (send_msgrequest) |
2039 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2040 | } |
2041 | ||
2042 | /* | |
2043 | * Allocate an entry for a message from the message queue associated with the | |
2044 | * specified channel. | |
2045 | */ | |
2046 | static enum xp_retval | |
2047 | xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags, | |
bd3e64c1 | 2048 | struct xpc_msg_sn2 **address_of_msg) |
33ba3c77 | 2049 | { |
a47d5dac | 2050 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2051 | struct xpc_msg_sn2 *msg; |
33ba3c77 DN |
2052 | enum xp_retval ret; |
2053 | s64 put; | |
2054 | ||
33ba3c77 DN |
2055 | /* |
2056 | * Get the next available message entry from the local message queue. | |
2057 | * If none are available, we'll make sure that we grab the latest | |
2058 | * GP values. | |
2059 | */ | |
2060 | ret = xpTimeout; | |
2061 | ||
2062 | while (1) { | |
2063 | ||
a47d5dac | 2064 | put = ch_sn2->w_local_GP.put; |
69b3bb65 | 2065 | smp_rmb(); /* guarantee that .put loads before .get */ |
a47d5dac | 2066 | if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) { |
33ba3c77 DN |
2067 | |
2068 | /* There are available message entries. We need to try | |
2069 | * to secure one for ourselves. We'll do this by trying | |
2070 | * to increment w_local_GP.put as long as someone else | |
2071 | * doesn't beat us to it. If they do, we'll have to | |
2072 | * try again. | |
2073 | */ | |
a47d5dac DN |
2074 | if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) == |
2075 | put) { | |
33ba3c77 DN |
2076 | /* we got the entry referenced by put */ |
2077 | break; | |
2078 | } | |
2079 | continue; /* try again */ | |
2080 | } | |
2081 | ||
2082 | /* | |
2083 | * There aren't any available msg entries at this time. | |
2084 | * | |
2085 | * In waiting for a message entry to become available, | |
7fb5e59d DN |
2086 | * we set a timeout in case the other side is not sending |
2087 | * completion interrupts. This lets us fake a notify IRQ | |
2088 | * that will cause the notify IRQ handler to fetch the latest | |
2089 | * GP values as if an interrupt was sent by the other side. | |
33ba3c77 DN |
2090 | */ |
2091 | if (ret == xpTimeout) | |
7fb5e59d | 2092 | xpc_send_chctl_local_msgrequest_sn2(ch); |
33ba3c77 | 2093 | |
97bf1aa1 | 2094 | if (flags & XPC_NOWAIT) |
33ba3c77 | 2095 | return xpNoWait; |
33ba3c77 DN |
2096 | |
2097 | ret = xpc_allocate_msg_wait(ch); | |
97bf1aa1 | 2098 | if (ret != xpInterrupted && ret != xpTimeout) |
33ba3c77 | 2099 | return ret; |
33ba3c77 DN |
2100 | } |
2101 | ||
2102 | /* get the message's address and initialize it */ | |
bd3e64c1 DN |
2103 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + |
2104 | (put % ch->local_nentries) * | |
2105 | ch->entry_size); | |
33ba3c77 DN |
2106 | |
2107 | DBUG_ON(msg->flags != 0); | |
2108 | msg->number = put; | |
2109 | ||
2110 | dev_dbg(xpc_chan, "w_local_GP.put changed to %ld; msg=0x%p, " | |
2111 | "msg_number=%ld, partid=%d, channel=%d\n", put + 1, | |
2112 | (void *)msg, msg->number, ch->partid, ch->number); | |
2113 | ||
2114 | *address_of_msg = msg; | |
33ba3c77 DN |
2115 | return xpSuccess; |
2116 | } | |
2117 | ||
2118 | /* | |
2119 | * Common code that does the actual sending of the message by advancing the | |
7fb5e59d DN |
2120 | * local message queue's Put value and sends a chctl msgrequest to the |
2121 | * partition the message is being sent to. | |
33ba3c77 DN |
2122 | */ |
2123 | static enum xp_retval | |
bd3e64c1 DN |
2124 | xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload, |
2125 | u16 payload_size, u8 notify_type, xpc_notify_func func, | |
2126 | void *key) | |
33ba3c77 DN |
2127 | { |
2128 | enum xp_retval ret = xpSuccess; | |
5b8669df | 2129 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 DN |
2130 | struct xpc_msg_sn2 *msg = msg; |
2131 | struct xpc_notify_sn2 *notify = notify; | |
97bf1aa1 DN |
2132 | s64 msg_number; |
2133 | s64 put; | |
33ba3c77 DN |
2134 | |
2135 | DBUG_ON(notify_type == XPC_N_CALL && func == NULL); | |
97bf1aa1 | 2136 | |
bd3e64c1 | 2137 | if (XPC_MSG_SIZE(payload_size) > ch->entry_size) |
97bf1aa1 DN |
2138 | return xpPayloadTooBig; |
2139 | ||
2140 | xpc_msgqueue_ref(ch); | |
33ba3c77 DN |
2141 | |
2142 | if (ch->flags & XPC_C_DISCONNECTING) { | |
97bf1aa1 DN |
2143 | ret = ch->reason; |
2144 | goto out_1; | |
2145 | } | |
2146 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
2147 | ret = xpNotConnected; | |
2148 | goto out_1; | |
33ba3c77 DN |
2149 | } |
2150 | ||
97bf1aa1 DN |
2151 | ret = xpc_allocate_msg_sn2(ch, flags, &msg); |
2152 | if (ret != xpSuccess) | |
2153 | goto out_1; | |
2154 | ||
2155 | msg_number = msg->number; | |
2156 | ||
33ba3c77 DN |
2157 | if (notify_type != 0) { |
2158 | /* | |
2159 | * Tell the remote side to send an ACK interrupt when the | |
2160 | * message has been delivered. | |
2161 | */ | |
bd3e64c1 | 2162 | msg->flags |= XPC_M_SN2_INTERRUPT; |
33ba3c77 DN |
2163 | |
2164 | atomic_inc(&ch->n_to_notify); | |
2165 | ||
5b8669df | 2166 | notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries]; |
33ba3c77 DN |
2167 | notify->func = func; |
2168 | notify->key = key; | |
2169 | notify->type = notify_type; | |
2170 | ||
ea57f80c | 2171 | /* ??? Is a mb() needed here? */ |
33ba3c77 DN |
2172 | |
2173 | if (ch->flags & XPC_C_DISCONNECTING) { | |
2174 | /* | |
2175 | * An error occurred between our last error check and | |
2176 | * this one. We will try to clear the type field from | |
2177 | * the notify entry. If we succeed then | |
2178 | * xpc_disconnect_channel() didn't already process | |
2179 | * the notify entry. | |
2180 | */ | |
2181 | if (cmpxchg(¬ify->type, notify_type, 0) == | |
2182 | notify_type) { | |
2183 | atomic_dec(&ch->n_to_notify); | |
2184 | ret = ch->reason; | |
2185 | } | |
97bf1aa1 | 2186 | goto out_1; |
33ba3c77 DN |
2187 | } |
2188 | } | |
2189 | ||
97bf1aa1 DN |
2190 | memcpy(&msg->payload, payload, payload_size); |
2191 | ||
bd3e64c1 | 2192 | msg->flags |= XPC_M_SN2_READY; |
33ba3c77 DN |
2193 | |
2194 | /* | |
2195 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2196 | * load of local_GP->put. |
33ba3c77 | 2197 | */ |
69b3bb65 | 2198 | smp_mb(); |
33ba3c77 DN |
2199 | |
2200 | /* see if the message is next in line to be sent, if so send it */ | |
2201 | ||
5b8669df | 2202 | put = ch_sn2->local_GP->put; |
33ba3c77 DN |
2203 | if (put == msg_number) |
2204 | xpc_send_msgs_sn2(ch, put); | |
2205 | ||
97bf1aa1 | 2206 | out_1: |
33ba3c77 DN |
2207 | xpc_msgqueue_deref(ch); |
2208 | return ret; | |
2209 | } | |
2210 | ||
2211 | /* | |
2212 | * Now we actually acknowledge the messages that have been delivered and ack'd | |
2213 | * by advancing the cached remote message queue's Get value and if requested | |
7fb5e59d | 2214 | * send a chctl msgrequest to the message sender's partition. |
bd3e64c1 DN |
2215 | * |
2216 | * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition | |
2217 | * that sent the message. | |
33ba3c77 DN |
2218 | */ |
2219 | static void | |
2220 | xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags) | |
2221 | { | |
a47d5dac | 2222 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2223 | struct xpc_msg_sn2 *msg; |
33ba3c77 | 2224 | s64 get = initial_get + 1; |
7fb5e59d | 2225 | int send_msgrequest = 0; |
33ba3c77 DN |
2226 | |
2227 | while (1) { | |
2228 | ||
2229 | while (1) { | |
a47d5dac | 2230 | if (get == ch_sn2->w_local_GP.get) |
33ba3c77 DN |
2231 | break; |
2232 | ||
bd3e64c1 DN |
2233 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> |
2234 | remote_msgqueue + (get % | |
2235 | ch->remote_nentries) * | |
2236 | ch->entry_size); | |
33ba3c77 | 2237 | |
bd3e64c1 | 2238 | if (!(msg->flags & XPC_M_SN2_DONE)) |
33ba3c77 DN |
2239 | break; |
2240 | ||
2241 | msg_flags |= msg->flags; | |
2242 | get++; | |
2243 | } | |
2244 | ||
2245 | if (get == initial_get) { | |
2246 | /* nothing's changed */ | |
2247 | break; | |
2248 | } | |
2249 | ||
a47d5dac | 2250 | if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) != |
33ba3c77 DN |
2251 | initial_get) { |
2252 | /* someone else beat us to it */ | |
a47d5dac | 2253 | DBUG_ON(ch_sn2->local_GP->get <= initial_get); |
33ba3c77 DN |
2254 | break; |
2255 | } | |
2256 | ||
2257 | /* we just set the new value of local_GP->get */ | |
2258 | ||
2259 | dev_dbg(xpc_chan, "local_GP->get changed to %ld, partid=%d, " | |
2260 | "channel=%d\n", get, ch->partid, ch->number); | |
2261 | ||
bd3e64c1 | 2262 | send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT); |
33ba3c77 DN |
2263 | |
2264 | /* | |
2265 | * We need to ensure that the message referenced by | |
bd3e64c1 | 2266 | * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get |
33ba3c77 DN |
2267 | * equals w_local_GP.get, so we'll go have a look. |
2268 | */ | |
2269 | initial_get = get; | |
2270 | } | |
2271 | ||
7fb5e59d DN |
2272 | if (send_msgrequest) |
2273 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2274 | } |
2275 | ||
2276 | static void | |
bd3e64c1 | 2277 | xpc_received_payload_sn2(struct xpc_channel *ch, void *payload) |
33ba3c77 | 2278 | { |
bd3e64c1 DN |
2279 | struct xpc_msg_sn2 *msg; |
2280 | s64 msg_number; | |
33ba3c77 | 2281 | s64 get; |
bd3e64c1 DN |
2282 | |
2283 | msg = container_of(payload, struct xpc_msg_sn2, payload); | |
2284 | msg_number = msg->number; | |
33ba3c77 DN |
2285 | |
2286 | dev_dbg(xpc_chan, "msg=0x%p, msg_number=%ld, partid=%d, channel=%d\n", | |
2287 | (void *)msg, msg_number, ch->partid, ch->number); | |
2288 | ||
252523ef | 2289 | DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) != |
33ba3c77 | 2290 | msg_number % ch->remote_nentries); |
252523ef | 2291 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
bd3e64c1 | 2292 | DBUG_ON(msg->flags & XPC_M_SN2_DONE); |
33ba3c77 | 2293 | |
bd3e64c1 | 2294 | msg->flags |= XPC_M_SN2_DONE; |
33ba3c77 DN |
2295 | |
2296 | /* | |
2297 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2298 | * load of local_GP->get. |
33ba3c77 | 2299 | */ |
69b3bb65 | 2300 | smp_mb(); |
33ba3c77 DN |
2301 | |
2302 | /* | |
2303 | * See if this message is next in line to be acknowledged as having | |
2304 | * been delivered. | |
2305 | */ | |
a47d5dac | 2306 | get = ch->sn.sn2.local_GP->get; |
33ba3c77 DN |
2307 | if (get == msg_number) |
2308 | xpc_acknowledge_msgs_sn2(ch, get, msg->flags); | |
2309 | } | |
2310 | ||
6e41017a | 2311 | int |
94bd2708 DN |
2312 | xpc_init_sn2(void) |
2313 | { | |
6e41017a | 2314 | int ret; |
ee6665e3 | 2315 | size_t buf_size; |
6e41017a | 2316 | |
5b8669df | 2317 | xpc_setup_partitions_sn = xpc_setup_partitions_sn_sn2; |
261f3b49 | 2318 | xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2; |
5b8669df | 2319 | xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_sn2; |
33ba3c77 DN |
2320 | xpc_increment_heartbeat = xpc_increment_heartbeat_sn2; |
2321 | xpc_offline_heartbeat = xpc_offline_heartbeat_sn2; | |
2322 | xpc_online_heartbeat = xpc_online_heartbeat_sn2; | |
2323 | xpc_heartbeat_init = xpc_heartbeat_init_sn2; | |
2324 | xpc_heartbeat_exit = xpc_heartbeat_exit_sn2; | |
61deb86e | 2325 | xpc_get_remote_heartbeat = xpc_get_remote_heartbeat_sn2; |
33ba3c77 | 2326 | |
a47d5dac DN |
2327 | xpc_request_partition_activation = xpc_request_partition_activation_sn2; |
2328 | xpc_request_partition_reactivation = | |
2329 | xpc_request_partition_reactivation_sn2; | |
2330 | xpc_request_partition_deactivation = | |
2331 | xpc_request_partition_deactivation_sn2; | |
2332 | xpc_cancel_partition_deactivation_request = | |
2333 | xpc_cancel_partition_deactivation_request_sn2; | |
2334 | ||
6e41017a | 2335 | xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2; |
5b8669df DN |
2336 | xpc_setup_ch_structures_sn = xpc_setup_ch_structures_sn_sn2; |
2337 | xpc_teardown_ch_structures_sn = xpc_teardown_ch_structures_sn_sn2; | |
e17d416b | 2338 | xpc_make_first_contact = xpc_make_first_contact_sn2; |
5b8669df | 2339 | |
7fb5e59d | 2340 | xpc_get_chctl_all_flags = xpc_get_chctl_all_flags_sn2; |
5b8669df DN |
2341 | xpc_send_chctl_closerequest = xpc_send_chctl_closerequest_sn2; |
2342 | xpc_send_chctl_closereply = xpc_send_chctl_closereply_sn2; | |
2343 | xpc_send_chctl_openrequest = xpc_send_chctl_openrequest_sn2; | |
2344 | xpc_send_chctl_openreply = xpc_send_chctl_openreply_sn2; | |
2345 | ||
2346 | xpc_save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2; | |
2347 | ||
2348 | xpc_setup_msg_structures = xpc_setup_msg_structures_sn2; | |
2349 | xpc_teardown_msg_structures = xpc_teardown_msg_structures_sn2; | |
2350 | ||
a47d5dac | 2351 | xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2; |
7fb5e59d | 2352 | xpc_process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2; |
bd3e64c1 DN |
2353 | xpc_n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2; |
2354 | xpc_get_deliverable_payload = xpc_get_deliverable_payload_sn2; | |
33ba3c77 | 2355 | |
a47d5dac | 2356 | xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_sn2; |
a47d5dac DN |
2357 | xpc_indicate_partition_disengaged = |
2358 | xpc_indicate_partition_disengaged_sn2; | |
5b8669df DN |
2359 | xpc_partition_engaged = xpc_partition_engaged_sn2; |
2360 | xpc_any_partition_engaged = xpc_any_partition_engaged_sn2; | |
a47d5dac DN |
2361 | xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_sn2; |
2362 | ||
bd3e64c1 DN |
2363 | xpc_send_payload = xpc_send_payload_sn2; |
2364 | xpc_received_payload = xpc_received_payload_sn2; | |
2365 | ||
2366 | if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) { | |
2367 | dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is " | |
2368 | "larger than %d\n", XPC_MSG_HDR_MAX_SIZE); | |
2369 | return -E2BIG; | |
2370 | } | |
6e41017a | 2371 | |
ee6665e3 DN |
2372 | buf_size = max(XPC_RP_VARS_SIZE, |
2373 | XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2); | |
2374 | xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size, | |
2375 | GFP_KERNEL, | |
2376 | &xpc_remote_copy_buffer_base_sn2); | |
2377 | if (xpc_remote_copy_buffer_sn2 == NULL) { | |
2378 | dev_err(xpc_part, "can't get memory for remote copy buffer\n"); | |
2379 | return -ENOMEM; | |
2380 | } | |
2381 | ||
c39838ce | 2382 | /* open up protections for IPI and [potentially] amo operations */ |
6e41017a | 2383 | xpc_allow_IPI_ops_sn2(); |
c39838ce | 2384 | xpc_allow_amo_ops_shub_wars_1_1_sn2(); |
6e41017a DN |
2385 | |
2386 | /* | |
2387 | * This is safe to do before the xpc_hb_checker thread has started | |
2388 | * because the handler releases a wait queue. If an interrupt is | |
2389 | * received before the thread is waiting, it will not go to sleep, | |
2390 | * but rather immediately process the interrupt. | |
2391 | */ | |
2392 | ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0, | |
2393 | "xpc hb", NULL); | |
2394 | if (ret != 0) { | |
2395 | dev_err(xpc_part, "can't register ACTIVATE IRQ handler, " | |
2396 | "errno=%d\n", -ret); | |
2397 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2398 | kfree(xpc_remote_copy_buffer_base_sn2); |
6e41017a DN |
2399 | } |
2400 | return ret; | |
94bd2708 DN |
2401 | } |
2402 | ||
2403 | void | |
2404 | xpc_exit_sn2(void) | |
2405 | { | |
6e41017a DN |
2406 | free_irq(SGI_XPC_ACTIVATE, NULL); |
2407 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2408 | kfree(xpc_remote_copy_buffer_base_sn2); |
94bd2708 | 2409 | } |