projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sgi-xp: define xp_partition_id and xp_region_size
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / misc / sgi-xp / xpc_uv.c
CommitLineData
94bd2708
DN		 1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
7 */
8
9/*
10 * Cross Partition Communication (XPC) uv-based functions.
11 *
12 * Architecture specific implementation of common functions.
13 *
14 */
15
16#include <linux/kernel.h>
5b8669df
DN		 17#include <linux/mm.h>
18#include <linux/interrupt.h>
19#include <linux/delay.h>
20#include <linux/device.h>
2525789b 21#include <linux/err.h>
261f3b49 22#include <asm/uv/uv_hub.h>
2525789b
DN		 23#if defined CONFIG_X86_64
24#include <asm/uv/bios.h>
25#include <asm/uv/uv_irq.h>
26#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
27#include <asm/sn/intr.h>
28#include <asm/sn/sn_sal.h>
29#endif
5b8669df 30#include "../sgi-gru/gru.h"
261f3b49 31#include "../sgi-gru/grukservices.h"
94bd2708
DN		 32#include "xpc.h"
33
5b8669df 34static atomic64_t xpc_heartbeat_uv;
33ba3c77
DN		 35static DECLARE_BITMAP(xpc_heartbeating_to_mask_uv, XP_MAX_NPARTITIONS_UV);
36
5b8669df 37#define XPC_ACTIVATE_MSG_SIZE_UV (1 * GRU_CACHE_LINE_BYTES)
2525789b
DN		 38#define XPC_ACTIVATE_MQ_SIZE_UV (4 * XP_MAX_NPARTITIONS_UV * \
39 XPC_ACTIVATE_MSG_SIZE_UV)
40#define XPC_ACTIVATE_IRQ_NAME "xpc_activate"
5b8669df 41
2525789b
DN		 42#define XPC_NOTIFY_MSG_SIZE_UV (2 * GRU_CACHE_LINE_BYTES)
43#define XPC_NOTIFY_MQ_SIZE_UV (4 * XP_MAX_NPARTITIONS_UV * \
44 XPC_NOTIFY_MSG_SIZE_UV)
45#define XPC_NOTIFY_IRQ_NAME "xpc_notify"
5b8669df 46
2525789b
DN		 47static struct xpc_gru_mq_uv *xpc_activate_mq_uv;
48static struct xpc_gru_mq_uv *xpc_notify_mq_uv;
5b8669df
DN		 49
50static int
51xpc_setup_partitions_sn_uv(void)
52{
53 short partid;
54 struct xpc_partition_uv *part_uv;
55
56 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
57 part_uv = &xpc_partitions[partid].sn.uv;
58
59 spin_lock_init(&part_uv->flags_lock);
60 part_uv->remote_act_state = XPC_P_AS_INACTIVE;
61 }
62 return 0;
63}
64
2525789b
DN		 65static int
66xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name)
67{
68#if defined CONFIG_X86_64
69 mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset);
70 if (mq->irq < 0) {
71 dev_err(xpc_part, "uv_setup_irq() returned error=%d\n",
72 mq->irq);
73 }
74
75#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
76 int mmr_pnode;
77 unsigned long mmr_value;
78
79 if (strcmp(irq_name, XPC_ACTIVATE_IRQ_NAME) == 0)
80 mq->irq = SGI_XPC_ACTIVATE;
81 else if (strcmp(irq_name, XPC_NOTIFY_IRQ_NAME) == 0)
82 mq->irq = SGI_XPC_NOTIFY;
83 else
84 return -EINVAL;
85
86 mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
87 mmr_value = (unsigned long)cpu_physical_id(cpu) << 32 | mq->irq;
88
89 uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mmr_value);
90#else
91 #error not a supported configuration
92#endif
93
94 return 0;
95}
96
97static void
98xpc_release_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq)
99{
100#if defined CONFIG_X86_64
101 uv_teardown_irq(mq->irq, mq->mmr_blade, mq->mmr_offset);
102
103#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
104 int mmr_pnode;
105 unsigned long mmr_value;
106
107 mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
108 mmr_value = 1UL << 16;
109
110 uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mmr_value);
111#else
112 #error not a supported configuration
113#endif
114}
115
116static int
117xpc_gru_mq_watchlist_alloc_uv(struct xpc_gru_mq_uv *mq)
118{
119 int ret;
120
121#if defined CONFIG_X86_64
122 ret = uv_bios_mq_watchlist_alloc(mq->mmr_blade, mq->address, mq->order,
123 &mq->mmr_offset);
124 if (ret < 0) {
125 dev_err(xpc_part, "uv_bios_mq_watchlist_alloc() failed, "
126 "ret=%d\n", ret);
127 return ret;
128 }
129#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
130 ret = sn_mq_watchlist_alloc(mq->mmr_blade, mq->address, mq->order,
131 &mq->mmr_offset);
132 if (ret < 0) {
133 dev_err(xpc_part, "sn_mq_watchlist_alloc() failed, ret=%d\n",
134 ret);
135 return -EBUSY;
136 }
137#else
138 #error not a supported configuration
139#endif
140
141 mq->watchlist_num = ret;
142 return 0;
143}
144
145static void
146xpc_gru_mq_watchlist_free_uv(struct xpc_gru_mq_uv *mq)
147{
148 int ret;
149
150#if defined CONFIG_X86_64
151 ret = uv_bios_mq_watchlist_free(mq->mmr_blade, mq->watchlist_num);
152 BUG_ON(ret != BIOS_STATUS_SUCCESS);
153#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
154 ret = sn_mq_watchlist_free(mq->mmr_blade, mq->watchlist_num);
155 BUG_ON(ret != SALRET_OK);
156#else
157 #error not a supported configuration
158#endif
159}
160
161static struct xpc_gru_mq_uv *
162xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name,
5b8669df
DN		 163 irq_handler_t irq_handler)
164{
2525789b 165 enum xp_retval xp_ret;
5b8669df
DN		 166 int ret;
167 int nid;
2525789b 168 int pg_order;
5b8669df 169 struct page *page;
2525789b
DN		 170 struct xpc_gru_mq_uv *mq;
171
172 mq = kmalloc(sizeof(struct xpc_gru_mq_uv), GFP_KERNEL);
173 if (mq == NULL) {
174 dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
175 "a xpc_gru_mq_uv structure\n");
176 ret = -ENOMEM;
177 goto out_1;
178 }
5b8669df 179
2525789b
DN		 180 pg_order = get_order(mq_size);
181 mq->order = pg_order + PAGE_SHIFT;
182 mq_size = 1UL << mq->order;
183
184 mq->mmr_blade = uv_cpu_to_blade_id(cpu);
185
186 nid = cpu_to_node(cpu);
5b8669df 187 page = alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
2525789b 188 pg_order);
bd3e64c1
DN		 189 if (page == NULL) {
190 dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d "
191 "bytes of memory on nid=%d for GRU mq\n", mq_size, nid);
2525789b
DN		 192 ret = -ENOMEM;
193 goto out_2;
bd3e64c1 194 }
2525789b 195 mq->address = page_address(page);
5b8669df 196
2525789b 197 ret = gru_create_message_queue(mq->address, mq_size);
5b8669df
DN		 198 if (ret != 0) {
199 dev_err(xpc_part, "gru_create_message_queue() returned "
200 "error=%d\n", ret);
2525789b
DN		 201 ret = -EINVAL;
202 goto out_3;
5b8669df
DN		 203 }
204
2525789b
DN		 205 /* enable generation of irq when GRU mq operation occurs to this mq */
206 ret = xpc_gru_mq_watchlist_alloc_uv(mq);
207 if (ret != 0)
208 goto out_3;
5b8669df 209
2525789b
DN		 210 ret = xpc_get_gru_mq_irq_uv(mq, cpu, irq_name);
211 if (ret != 0)
212 goto out_4;
213
214 ret = request_irq(mq->irq, irq_handler, 0, irq_name, NULL);
5b8669df
DN		 215 if (ret != 0) {
216 dev_err(xpc_part, "request_irq(irq=%d) returned error=%d\n",
2525789b
DN		 217 mq->irq, ret);
218 goto out_5;
5b8669df
DN		 219 }
220
2525789b
DN		 221 /* allow other partitions to access this GRU mq */
222 xp_ret = xp_expand_memprotect(xp_pa(mq->address), mq_size);
223 if (xp_ret != xpSuccess) {
224 ret = -EACCES;
225 goto out_6;
226 }
5b8669df
DN		 227
228 return mq;
2525789b
DN		 229
230 /* something went wrong */
231out_6:
232 free_irq(mq->irq, NULL);
233out_5:
234 xpc_release_gru_mq_irq_uv(mq);
235out_4:
236 xpc_gru_mq_watchlist_free_uv(mq);
237out_3:
238 free_pages((unsigned long)mq->address, pg_order);
239out_2:
240 kfree(mq);
241out_1:
242 return ERR_PTR(ret);
5b8669df 243}
94bd2708 244
33ba3c77 245static void
2525789b 246xpc_destroy_gru_mq_uv(struct xpc_gru_mq_uv *mq)
5b8669df 247{
2525789b
DN		 248 unsigned int mq_size;
249 int pg_order;
250 int ret;
251
252 /* disallow other partitions to access GRU mq */
253 mq_size = 1UL << mq->order;
254 ret = xp_restrict_memprotect(xp_pa(mq->address), mq_size);
255 BUG_ON(ret != xpSuccess);
256
257 /* unregister irq handler and release mq irq/vector mapping */
258 free_irq(mq->irq, NULL);
259 xpc_release_gru_mq_irq_uv(mq);
5b8669df 260
2525789b
DN		 261 /* disable generation of irq when GRU mq op occurs to this mq */
262 xpc_gru_mq_watchlist_free_uv(mq);
5b8669df 263
2525789b
DN		 264 pg_order = mq->order - PAGE_SHIFT;
265 free_pages((unsigned long)mq->address, pg_order);
5b8669df 266
2525789b 267 kfree(mq);
5b8669df
DN		 268}
269
270static enum xp_retval
271xpc_send_gru_msg(unsigned long mq_gpa, void *msg, size_t msg_size)
33ba3c77 272{
5b8669df
DN		 273 enum xp_retval xp_ret;
274 int ret;
275
276 while (1) {
277 ret = gru_send_message_gpa(mq_gpa, msg, msg_size);
278 if (ret == MQE_OK) {
279 xp_ret = xpSuccess;
280 break;
281 }
282
283 if (ret == MQE_QUEUE_FULL) {
284 dev_dbg(xpc_chan, "gru_send_message_gpa() returned "
285 "error=MQE_QUEUE_FULL\n");
286 /* !!! handle QLimit reached; delay & try again */
287 /* ??? Do we add a limit to the number of retries? */
288 (void)msleep_interruptible(10);
289 } else if (ret == MQE_CONGESTION) {
290 dev_dbg(xpc_chan, "gru_send_message_gpa() returned "
291 "error=MQE_CONGESTION\n");
292 /* !!! handle LB Overflow; simply try again */
293 /* ??? Do we add a limit to the number of retries? */
294 } else {
295 /* !!! Currently this is MQE_UNEXPECTED_CB_ERR */
296 dev_err(xpc_chan, "gru_send_message_gpa() returned "
297 "error=%d\n", ret);
298 xp_ret = xpGruSendMqError;
299 break;
300 }
301 }
302 return xp_ret;
303}
304
305static void
306xpc_process_activate_IRQ_rcvd_uv(void)
307{
308 unsigned long irq_flags;
309 short partid;
310 struct xpc_partition *part;
311 u8 act_state_req;
312
313 DBUG_ON(xpc_activate_IRQ_rcvd == 0);
314
315 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
316 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
317 part = &xpc_partitions[partid];
318
319 if (part->sn.uv.act_state_req == 0)
320 continue;
321
322 xpc_activate_IRQ_rcvd--;
323 BUG_ON(xpc_activate_IRQ_rcvd < 0);
324
325 act_state_req = part->sn.uv.act_state_req;
326 part->sn.uv.act_state_req = 0;
327 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
328
329 if (act_state_req == XPC_P_ASR_ACTIVATE_UV) {
330 if (part->act_state == XPC_P_AS_INACTIVE)
331 xpc_activate_partition(part);
332 else if (part->act_state == XPC_P_AS_DEACTIVATING)
333 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
334
335 } else if (act_state_req == XPC_P_ASR_REACTIVATE_UV) {
336 if (part->act_state == XPC_P_AS_INACTIVE)
337 xpc_activate_partition(part);
338 else
339 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
340
341 } else if (act_state_req == XPC_P_ASR_DEACTIVATE_UV) {
342 XPC_DEACTIVATE_PARTITION(part, part->sn.uv.reason);
343
344 } else {
345 BUG();
346 }
347
348 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
349 if (xpc_activate_IRQ_rcvd == 0)
350 break;
351 }
352 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
353
354}
355
bd3e64c1
DN		 356static void
357xpc_handle_activate_mq_msg_uv(struct xpc_partition *part,
358 struct xpc_activate_mq_msghdr_uv *msg_hdr,
359 int *wakeup_hb_checker)
5b8669df
DN		 360{
361 unsigned long irq_flags;
bd3e64c1 362 struct xpc_partition_uv *part_uv = &part->sn.uv;
5b8669df 363 struct xpc_openclose_args *args;
5b8669df 364
bd3e64c1 365 part_uv->remote_act_state = msg_hdr->act_state;
5b8669df 366
bd3e64c1
DN		 367 switch (msg_hdr->type) {
368 case XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV:
369 /* syncing of remote_act_state was just done above */
370 break;
5b8669df 371
bd3e64c1
DN		 372 case XPC_ACTIVATE_MQ_MSG_INC_HEARTBEAT_UV: {
373 struct xpc_activate_mq_msg_heartbeat_req_uv *msg;
5b8669df 374
bd3e64c1
DN		 375 msg = container_of(msg_hdr,
376 struct xpc_activate_mq_msg_heartbeat_req_uv,
377 hdr);
378 part_uv->heartbeat = msg->heartbeat;
379 break;
380 }
381 case XPC_ACTIVATE_MQ_MSG_OFFLINE_HEARTBEAT_UV: {
382 struct xpc_activate_mq_msg_heartbeat_req_uv *msg;
383
384 msg = container_of(msg_hdr,
385 struct xpc_activate_mq_msg_heartbeat_req_uv,
386 hdr);
387 part_uv->heartbeat = msg->heartbeat;
388
389 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
390 part_uv->flags |= XPC_P_HEARTBEAT_OFFLINE_UV;
391 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
392 break;
393 }
394 case XPC_ACTIVATE_MQ_MSG_ONLINE_HEARTBEAT_UV: {
395 struct xpc_activate_mq_msg_heartbeat_req_uv *msg;
396
397 msg = container_of(msg_hdr,
398 struct xpc_activate_mq_msg_heartbeat_req_uv,
399 hdr);
400 part_uv->heartbeat = msg->heartbeat;
401
402 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
403 part_uv->flags &= ~XPC_P_HEARTBEAT_OFFLINE_UV;
404 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
405 break;
406 }
407 case XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV: {
408 struct xpc_activate_mq_msg_activate_req_uv *msg;
5b8669df 409
bd3e64c1
DN		 410 /*
411 * ??? Do we deal here with ts_jiffies being different
412 * ??? if act_state != XPC_P_AS_INACTIVE instead of
413 * ??? below?
414 */
415 msg = container_of(msg_hdr, struct
416 xpc_activate_mq_msg_activate_req_uv, hdr);
5b8669df 417
bd3e64c1
DN		 418 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
419 if (part_uv->act_state_req == 0)
420 xpc_activate_IRQ_rcvd++;
421 part_uv->act_state_req = XPC_P_ASR_ACTIVATE_UV;
422 part->remote_rp_pa = msg->rp_gpa; /* !!! _pa is _gpa */
423 part->remote_rp_ts_jiffies = msg_hdr->rp_ts_jiffies;
424 part_uv->remote_activate_mq_gpa = msg->activate_mq_gpa;
425 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
5b8669df 426
bd3e64c1
DN		 427 (*wakeup_hb_checker)++;
428 break;
429 }
430 case XPC_ACTIVATE_MQ_MSG_DEACTIVATE_REQ_UV: {
431 struct xpc_activate_mq_msg_deactivate_req_uv *msg;
5b8669df 432
bd3e64c1
DN		 433 msg = container_of(msg_hdr, struct
434 xpc_activate_mq_msg_deactivate_req_uv, hdr);
5b8669df 435
bd3e64c1
DN		 436 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
437 if (part_uv->act_state_req == 0)
438 xpc_activate_IRQ_rcvd++;
439 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
440 part_uv->reason = msg->reason;
441 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
442
443 (*wakeup_hb_checker)++;
444 return;
445 }
446 case XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREQUEST_UV: {
447 struct xpc_activate_mq_msg_chctl_closerequest_uv *msg;
5b8669df 448
bd3e64c1
DN		 449 msg = container_of(msg_hdr, struct
450 xpc_activate_mq_msg_chctl_closerequest_uv,
451 hdr);
452 args = &part->remote_openclose_args[msg->ch_number];
453 args->reason = msg->reason;
5b8669df 454
bd3e64c1
DN		 455 spin_lock_irqsave(&part->chctl_lock, irq_flags);
456 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREQUEST;
457 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
5b8669df 458
bd3e64c1
DN		 459 xpc_wakeup_channel_mgr(part);
460 break;
461 }
462 case XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREPLY_UV: {
463 struct xpc_activate_mq_msg_chctl_closereply_uv *msg;
5b8669df 464
bd3e64c1
DN		 465 msg = container_of(msg_hdr, struct
466 xpc_activate_mq_msg_chctl_closereply_uv,
467 hdr);
5b8669df 468
bd3e64c1
DN		 469 spin_lock_irqsave(&part->chctl_lock, irq_flags);
470 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREPLY;
471 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
5b8669df 472
bd3e64c1
DN		 473 xpc_wakeup_channel_mgr(part);
474 break;
475 }
476 case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREQUEST_UV: {
477 struct xpc_activate_mq_msg_chctl_openrequest_uv *msg;
478
479 msg = container_of(msg_hdr, struct
480 xpc_activate_mq_msg_chctl_openrequest_uv,
481 hdr);
482 args = &part->remote_openclose_args[msg->ch_number];
483 args->entry_size = msg->entry_size;
484 args->local_nentries = msg->local_nentries;
485
486 spin_lock_irqsave(&part->chctl_lock, irq_flags);
487 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREQUEST;
488 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
489
490 xpc_wakeup_channel_mgr(part);
491 break;
492 }
493 case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV: {
494 struct xpc_activate_mq_msg_chctl_openreply_uv *msg;
495
496 msg = container_of(msg_hdr, struct
497 xpc_activate_mq_msg_chctl_openreply_uv, hdr);
498 args = &part->remote_openclose_args[msg->ch_number];
499 args->remote_nentries = msg->remote_nentries;
500 args->local_nentries = msg->local_nentries;
501 args->local_msgqueue_pa = msg->local_notify_mq_gpa;
502
503 spin_lock_irqsave(&part->chctl_lock, irq_flags);
504 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREPLY;
505 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
506
507 xpc_wakeup_channel_mgr(part);
508 break;
509 }
510 case XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV:
511 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
512 part_uv->flags |= XPC_P_ENGAGED_UV;
513 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
514 break;
515
516 case XPC_ACTIVATE_MQ_MSG_MARK_DISENGAGED_UV:
517 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
518 part_uv->flags &= ~XPC_P_ENGAGED_UV;
519 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
520 break;
521
522 default:
523 dev_err(xpc_part, "received unknown activate_mq msg type=%d "
524 "from partition=%d\n", msg_hdr->type, XPC_PARTID(part));
525
526 /* get hb checker to deactivate from the remote partition */
527 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
528 if (part_uv->act_state_req == 0)
529 xpc_activate_IRQ_rcvd++;
530 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
531 part_uv->reason = xpBadMsgType;
532 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
5b8669df 533
bd3e64c1
DN		 534 (*wakeup_hb_checker)++;
535 return;
536 }
5b8669df 537
bd3e64c1
DN		 538 if (msg_hdr->rp_ts_jiffies != part->remote_rp_ts_jiffies &&
539 part->remote_rp_ts_jiffies != 0) {
540 /*
541 * ??? Does what we do here need to be sensitive to
542 * ??? act_state or remote_act_state?
543 */
544 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
545 if (part_uv->act_state_req == 0)
546 xpc_activate_IRQ_rcvd++;
547 part_uv->act_state_req = XPC_P_ASR_REACTIVATE_UV;
548 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
5b8669df 549
bd3e64c1
DN		 550 (*wakeup_hb_checker)++;
551 }
552}
553
554static irqreturn_t
555xpc_handle_activate_IRQ_uv(int irq, void *dev_id)
556{
557 struct xpc_activate_mq_msghdr_uv *msg_hdr;
558 short partid;
559 struct xpc_partition *part;
560 int wakeup_hb_checker = 0;
561
2525789b
DN		 562 while (1) {
563 msg_hdr = gru_get_next_message(xpc_activate_mq_uv->address);
564 if (msg_hdr == NULL)
565 break;
bd3e64c1
DN		 566
567 partid = msg_hdr->partid;
568 if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) {
569 dev_err(xpc_part, "xpc_handle_activate_IRQ_uv() "
570 "received invalid partid=0x%x in message\n",
571 partid);
572 } else {
573 part = &xpc_partitions[partid];
574 if (xpc_part_ref(part)) {
575 xpc_handle_activate_mq_msg_uv(part, msg_hdr,
576 &wakeup_hb_checker);
577 xpc_part_deref(part);
578 }
5b8669df
DN		 579 }
580
2525789b 581 gru_free_message(xpc_activate_mq_uv->address, msg_hdr);
5b8669df
DN		 582 }
583
584 if (wakeup_hb_checker)
585 wake_up_interruptible(&xpc_activate_IRQ_wq);
586
587 return IRQ_HANDLED;
588}
589
590static enum xp_retval
591xpc_send_activate_IRQ_uv(struct xpc_partition *part, void *msg, size_t msg_size,
592 int msg_type)
593{
594 struct xpc_activate_mq_msghdr_uv *msg_hdr = msg;
595
596 DBUG_ON(msg_size > XPC_ACTIVATE_MSG_SIZE_UV);
597
598 msg_hdr->type = msg_type;
599 msg_hdr->partid = XPC_PARTID(part);
600 msg_hdr->act_state = part->act_state;
601 msg_hdr->rp_ts_jiffies = xpc_rsvd_page->ts_jiffies;
602
603 /* ??? Is holding a spin_lock (ch->lock) during this call a bad idea? */
604 return xpc_send_gru_msg(part->sn.uv.remote_activate_mq_gpa, msg,
605 msg_size);
606}
607
608static void
609xpc_send_activate_IRQ_part_uv(struct xpc_partition *part, void *msg,
610 size_t msg_size, int msg_type)
611{
612 enum xp_retval ret;
613
614 ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type);
615 if (unlikely(ret != xpSuccess))
616 XPC_DEACTIVATE_PARTITION(part, ret);
617}
618
619static void
620xpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags,
621 void *msg, size_t msg_size, int msg_type)
622{
623 struct xpc_partition *part = &xpc_partitions[ch->number];
624 enum xp_retval ret;
625
626 ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type);
627 if (unlikely(ret != xpSuccess)) {
628 if (irq_flags != NULL)
629 spin_unlock_irqrestore(&ch->lock, *irq_flags);
630
631 XPC_DEACTIVATE_PARTITION(part, ret);
632
633 if (irq_flags != NULL)
634 spin_lock_irqsave(&ch->lock, *irq_flags);
635 }
636}
637
638static void
639xpc_send_local_activate_IRQ_uv(struct xpc_partition *part, int act_state_req)
640{
641 unsigned long irq_flags;
642 struct xpc_partition_uv *part_uv = &part->sn.uv;
643
33ba3c77 644 /*
ea57f80c 645 * !!! Make our side think that the remote parition sent an activate
5b8669df 646 * !!! message our way by doing what the activate IRQ handler would
ea57f80c 647 * !!! do had one really been sent.
33ba3c77 648 */
5b8669df
DN		 649
650 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
651 if (part_uv->act_state_req == 0)
652 xpc_activate_IRQ_rcvd++;
653 part_uv->act_state_req = act_state_req;
654 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
655
656 wake_up_interruptible(&xpc_activate_IRQ_wq);
33ba3c77
DN		 657}
658
94bd2708 659static enum xp_retval
5b8669df
DN		 660xpc_get_partition_rsvd_page_pa_uv(void *buf, u64 *cookie, unsigned long *rp_pa,
661 size_t *len)
94bd2708 662{
5b8669df
DN		 663 /* !!! call the UV version of sn_partition_reserved_page_pa() */
664 return xpUnsupported;
665}
666
667static int
668xpc_setup_rsvd_page_sn_uv(struct xpc_rsvd_page *rp)
669{
2525789b 670 rp->sn.activate_mq_gpa = uv_gpa(xpc_activate_mq_uv->address);
5b8669df
DN		 671 return 0;
672}
673
674static void
675xpc_send_heartbeat_uv(int msg_type)
676{
677 short partid;
678 struct xpc_partition *part;
679 struct xpc_activate_mq_msg_heartbeat_req_uv msg;
680
681 /*
682 * !!! On uv we're broadcasting a heartbeat message every 5 seconds.
683 * !!! Whereas on sn2 we're bte_copy'ng the heartbeat info every 20
684 * !!! seconds. This is an increase in numalink traffic.
685 * ??? Is this good?
686 */
687
688 msg.heartbeat = atomic64_inc_return(&xpc_heartbeat_uv);
689
690 partid = find_first_bit(xpc_heartbeating_to_mask_uv,
691 XP_MAX_NPARTITIONS_UV);
692
693 while (partid < XP_MAX_NPARTITIONS_UV) {
694 part = &xpc_partitions[partid];
695
696 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
697 msg_type);
698
699 partid = find_next_bit(xpc_heartbeating_to_mask_uv,
700 XP_MAX_NPARTITIONS_UV, partid + 1);
701 }
94bd2708
DN		 702}
703
33ba3c77
DN		 704static void
705xpc_increment_heartbeat_uv(void)
706{
5b8669df
DN		 707 xpc_send_heartbeat_uv(XPC_ACTIVATE_MQ_MSG_INC_HEARTBEAT_UV);
708}
709
710static void
711xpc_offline_heartbeat_uv(void)
712{
713 xpc_send_heartbeat_uv(XPC_ACTIVATE_MQ_MSG_OFFLINE_HEARTBEAT_UV);
714}
715
716static void
717xpc_online_heartbeat_uv(void)
718{
719 xpc_send_heartbeat_uv(XPC_ACTIVATE_MQ_MSG_ONLINE_HEARTBEAT_UV);
33ba3c77
DN		 720}
721
722static void
723xpc_heartbeat_init_uv(void)
724{
5b8669df 725 atomic64_set(&xpc_heartbeat_uv, 0);
33ba3c77
DN		 726 bitmap_zero(xpc_heartbeating_to_mask_uv, XP_MAX_NPARTITIONS_UV);
727 xpc_heartbeating_to_mask = &xpc_heartbeating_to_mask_uv[0];
728}
729
730static void
731xpc_heartbeat_exit_uv(void)
732{
5b8669df
DN		 733 xpc_send_heartbeat_uv(XPC_ACTIVATE_MQ_MSG_OFFLINE_HEARTBEAT_UV);
734}
735
736static enum xp_retval
737xpc_get_remote_heartbeat_uv(struct xpc_partition *part)
738{
739 struct xpc_partition_uv *part_uv = &part->sn.uv;
740 enum xp_retval ret = xpNoHeartbeat;
741
742 if (part_uv->remote_act_state != XPC_P_AS_INACTIVE &&
743 part_uv->remote_act_state != XPC_P_AS_DEACTIVATING) {
744
745 if (part_uv->heartbeat != part->last_heartbeat ||
746 (part_uv->flags & XPC_P_HEARTBEAT_OFFLINE_UV)) {
747
748 part->last_heartbeat = part_uv->heartbeat;
749 ret = xpSuccess;
750 }
751 }
752 return ret;
33ba3c77
DN		 753}
754
755static void
a47d5dac 756xpc_request_partition_activation_uv(struct xpc_rsvd_page *remote_rp,
5b8669df 757 unsigned long remote_rp_gpa, int nasid)
33ba3c77
DN		 758{
759 short partid = remote_rp->SAL_partid;
760 struct xpc_partition *part = &xpc_partitions[partid];
5b8669df 761 struct xpc_activate_mq_msg_activate_req_uv msg;
33ba3c77 762
5b8669df
DN		 763 part->remote_rp_pa = remote_rp_gpa; /* !!! _pa here is really _gpa */
764 part->remote_rp_ts_jiffies = remote_rp->ts_jiffies;
765 part->sn.uv.remote_activate_mq_gpa = remote_rp->sn.activate_mq_gpa;
766
767 /*
768 * ??? Is it a good idea to make this conditional on what is
769 * ??? potentially stale state information?
770 */
771 if (part->sn.uv.remote_act_state == XPC_P_AS_INACTIVE) {
772 msg.rp_gpa = uv_gpa(xpc_rsvd_page);
773 msg.activate_mq_gpa = xpc_rsvd_page->sn.activate_mq_gpa;
774 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
775 XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV);
776 }
33ba3c77 777
5b8669df
DN		 778 if (part->act_state == XPC_P_AS_INACTIVE)
779 xpc_send_local_activate_IRQ_uv(part, XPC_P_ASR_ACTIVATE_UV);
33ba3c77
DN		 780}
781
a47d5dac
DN		 782static void
783xpc_request_partition_reactivation_uv(struct xpc_partition *part)
784{
5b8669df
DN		 785 xpc_send_local_activate_IRQ_uv(part, XPC_P_ASR_ACTIVATE_UV);
786}
787
788static void
789xpc_request_partition_deactivation_uv(struct xpc_partition *part)
790{
791 struct xpc_activate_mq_msg_deactivate_req_uv msg;
792
793 /*
794 * ??? Is it a good idea to make this conditional on what is
795 * ??? potentially stale state information?
796 */
797 if (part->sn.uv.remote_act_state != XPC_P_AS_DEACTIVATING &&
798 part->sn.uv.remote_act_state != XPC_P_AS_INACTIVE) {
799
800 msg.reason = part->reason;
801 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
802 XPC_ACTIVATE_MQ_MSG_DEACTIVATE_REQ_UV);
803 }
a47d5dac
DN		 804}
805
bd3e64c1
DN		 806static void
807xpc_cancel_partition_deactivation_request_uv(struct xpc_partition *part)
808{
809 /* nothing needs to be done */
810 return;
811}
812
813static void
814xpc_init_fifo_uv(struct xpc_fifo_head_uv *head)
815{
816 head->first = NULL;
817 head->last = NULL;
818 spin_lock_init(&head->lock);
819 head->n_entries = 0;
820}
821
822static void *
823xpc_get_fifo_entry_uv(struct xpc_fifo_head_uv *head)
824{
825 unsigned long irq_flags;
826 struct xpc_fifo_entry_uv *first;
827
828 spin_lock_irqsave(&head->lock, irq_flags);
829 first = head->first;
830 if (head->first != NULL) {
831 head->first = first->next;
832 if (head->first == NULL)
833 head->last = NULL;
834 }
835 head->n_entries++;
836 spin_unlock_irqrestore(&head->lock, irq_flags);
837 first->next = NULL;
838 return first;
839}
840
841static void
842xpc_put_fifo_entry_uv(struct xpc_fifo_head_uv *head,
843 struct xpc_fifo_entry_uv *last)
844{
845 unsigned long irq_flags;
846
847 last->next = NULL;
848 spin_lock_irqsave(&head->lock, irq_flags);
849 if (head->last != NULL)
850 head->last->next = last;
851 else
852 head->first = last;
853 head->last = last;
854 head->n_entries--;
855 BUG_ON(head->n_entries < 0);
856 spin_unlock_irqrestore(&head->lock, irq_flags);
857}
858
859static int
860xpc_n_of_fifo_entries_uv(struct xpc_fifo_head_uv *head)
861{
862 return head->n_entries;
863}
864
e17d416b 865/*
5b8669df 866 * Setup the channel structures that are uv specific.
e17d416b
DN		 867 */
868static enum xp_retval
5b8669df 869xpc_setup_ch_structures_sn_uv(struct xpc_partition *part)
e17d416b 870{
bd3e64c1
DN		 871 struct xpc_channel_uv *ch_uv;
872 int ch_number;
873
874 for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
875 ch_uv = &part->channels[ch_number].sn.uv;
876
877 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
878 xpc_init_fifo_uv(&ch_uv->recv_msg_list);
879 }
880
881 return xpSuccess;
e17d416b
DN		 882}
883
884/*
5b8669df 885 * Teardown the channel structures that are uv specific.
e17d416b
DN		 886 */
887static void
5b8669df 888xpc_teardown_ch_structures_sn_uv(struct xpc_partition *part)
e17d416b 889{
bd3e64c1 890 /* nothing needs to be done */
e17d416b
DN		 891 return;
892}
893
894static enum xp_retval
895xpc_make_first_contact_uv(struct xpc_partition *part)
896{
5b8669df
DN		 897 struct xpc_activate_mq_msg_uv msg;
898
899 /*
900 * We send a sync msg to get the remote partition's remote_act_state
901 * updated to our current act_state which at this point should
902 * be XPC_P_AS_ACTIVATING.
903 */
904 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
905 XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV);
906
907 while (part->sn.uv.remote_act_state != XPC_P_AS_ACTIVATING) {
908
909 dev_dbg(xpc_part, "waiting to make first contact with "
910 "partition %d\n", XPC_PARTID(part));
911
912 /* wait a 1/4 of a second or so */
913 (void)msleep_interruptible(250);
914
915 if (part->act_state == XPC_P_AS_DEACTIVATING)
916 return part->reason;
917 }
918
919 return xpSuccess;
e17d416b
DN		 920}
921
922static u64
7fb5e59d 923xpc_get_chctl_all_flags_uv(struct xpc_partition *part)
e17d416b 924{
5b8669df
DN		 925 unsigned long irq_flags;
926 union xpc_channel_ctl_flags chctl;
927
928 spin_lock_irqsave(&part->chctl_lock, irq_flags);
929 chctl = part->chctl;
930 if (chctl.all_flags != 0)
931 part->chctl.all_flags = 0;
932
933 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
934 return chctl.all_flags;
935}
936
bd3e64c1
DN		 937static enum xp_retval
938xpc_allocate_send_msg_slot_uv(struct xpc_channel *ch)
939{
940 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
941 struct xpc_send_msg_slot_uv *msg_slot;
942 unsigned long irq_flags;
943 int nentries;
944 int entry;
945 size_t nbytes;
946
947 for (nentries = ch->local_nentries; nentries > 0; nentries--) {
948 nbytes = nentries * sizeof(struct xpc_send_msg_slot_uv);
949 ch_uv->send_msg_slots = kzalloc(nbytes, GFP_KERNEL);
950 if (ch_uv->send_msg_slots == NULL)
951 continue;
952
953 for (entry = 0; entry < nentries; entry++) {
954 msg_slot = &ch_uv->send_msg_slots[entry];
955
956 msg_slot->msg_slot_number = entry;
957 xpc_put_fifo_entry_uv(&ch_uv->msg_slot_free_list,
958 &msg_slot->next);
959 }
960
961 spin_lock_irqsave(&ch->lock, irq_flags);
962 if (nentries < ch->local_nentries)
963 ch->local_nentries = nentries;
964 spin_unlock_irqrestore(&ch->lock, irq_flags);
965 return xpSuccess;
966 }
967
968 return xpNoMemory;
969}
970
971static enum xp_retval
972xpc_allocate_recv_msg_slot_uv(struct xpc_channel *ch)
973{
974 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
975 struct xpc_notify_mq_msg_uv *msg_slot;
976 unsigned long irq_flags;
977 int nentries;
978 int entry;
979 size_t nbytes;
980
981 for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
982 nbytes = nentries * ch->entry_size;
983 ch_uv->recv_msg_slots = kzalloc(nbytes, GFP_KERNEL);
984 if (ch_uv->recv_msg_slots == NULL)
985 continue;
986
987 for (entry = 0; entry < nentries; entry++) {
988 msg_slot = ch_uv->recv_msg_slots + entry *
989 ch->entry_size;
990
991 msg_slot->hdr.msg_slot_number = entry;
992 }
993
994 spin_lock_irqsave(&ch->lock, irq_flags);
995 if (nentries < ch->remote_nentries)
996 ch->remote_nentries = nentries;
997 spin_unlock_irqrestore(&ch->lock, irq_flags);
998 return xpSuccess;
999 }
1000
1001 return xpNoMemory;
1002}
1003
1004/*
1005 * Allocate msg_slots associated with the channel.
1006 */
5b8669df
DN		 1007static enum xp_retval
1008xpc_setup_msg_structures_uv(struct xpc_channel *ch)
1009{
bd3e64c1
DN		 1010 static enum xp_retval ret;
1011 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
1012
1013 DBUG_ON(ch->flags & XPC_C_SETUP);
1014
1015 ret = xpc_allocate_send_msg_slot_uv(ch);
1016 if (ret == xpSuccess) {
1017
1018 ret = xpc_allocate_recv_msg_slot_uv(ch);
1019 if (ret != xpSuccess) {
1020 kfree(ch_uv->send_msg_slots);
1021 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
1022 }
1023 }
1024 return ret;
5b8669df
DN		 1025}
1026
bd3e64c1
DN		 1027/*
1028 * Free up msg_slots and clear other stuff that were setup for the specified
1029 * channel.
1030 */
5b8669df
DN		 1031static void
1032xpc_teardown_msg_structures_uv(struct xpc_channel *ch)
1033{
1034 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
1035
bd3e64c1
DN		 1036 DBUG_ON(!spin_is_locked(&ch->lock));
1037
5b8669df
DN		 1038 ch_uv->remote_notify_mq_gpa = 0;
1039
bd3e64c1
DN		 1040 if (ch->flags & XPC_C_SETUP) {
1041 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
1042 kfree(ch_uv->send_msg_slots);
1043 xpc_init_fifo_uv(&ch_uv->recv_msg_list);
1044 kfree(ch_uv->recv_msg_slots);
1045 }
5b8669df
DN		 1046}
1047
1048static void
1049xpc_send_chctl_closerequest_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1050{
1051 struct xpc_activate_mq_msg_chctl_closerequest_uv msg;
1052
1053 msg.ch_number = ch->number;
1054 msg.reason = ch->reason;
1055 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1056 XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREQUEST_UV);
1057}
1058
1059static void
1060xpc_send_chctl_closereply_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1061{
1062 struct xpc_activate_mq_msg_chctl_closereply_uv msg;
1063
1064 msg.ch_number = ch->number;
1065 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1066 XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREPLY_UV);
1067}
1068
1069static void
1070xpc_send_chctl_openrequest_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1071{
1072 struct xpc_activate_mq_msg_chctl_openrequest_uv msg;
1073
1074 msg.ch_number = ch->number;
bd3e64c1 1075 msg.entry_size = ch->entry_size;
5b8669df
DN		 1076 msg.local_nentries = ch->local_nentries;
1077 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1078 XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREQUEST_UV);
1079}
1080
1081static void
1082xpc_send_chctl_openreply_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1083{
1084 struct xpc_activate_mq_msg_chctl_openreply_uv msg;
1085
1086 msg.ch_number = ch->number;
1087 msg.local_nentries = ch->local_nentries;
1088 msg.remote_nentries = ch->remote_nentries;
1089 msg.local_notify_mq_gpa = uv_gpa(xpc_notify_mq_uv);
1090 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1091 XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV);
1092}
1093
bd3e64c1
DN		 1094static void
1095xpc_send_chctl_local_msgrequest_uv(struct xpc_partition *part, int ch_number)
1096{
1097 unsigned long irq_flags;
1098
1099 spin_lock_irqsave(&part->chctl_lock, irq_flags);
1100 part->chctl.flags[ch_number] |= XPC_CHCTL_MSGREQUEST;
1101 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
1102
1103 xpc_wakeup_channel_mgr(part);
1104}
1105
5b8669df
DN		 1106static void
1107xpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch,
1108 unsigned long msgqueue_pa)
1109{
1110 ch->sn.uv.remote_notify_mq_gpa = msgqueue_pa;
1111}
1112
1113static void
1114xpc_indicate_partition_engaged_uv(struct xpc_partition *part)
1115{
1116 struct xpc_activate_mq_msg_uv msg;
1117
1118 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
1119 XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV);
1120}
1121
1122static void
1123xpc_indicate_partition_disengaged_uv(struct xpc_partition *part)
1124{
1125 struct xpc_activate_mq_msg_uv msg;
1126
1127 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
1128 XPC_ACTIVATE_MQ_MSG_MARK_DISENGAGED_UV);
1129}
1130
1131static void
1132xpc_assume_partition_disengaged_uv(short partid)
1133{
1134 struct xpc_partition_uv *part_uv = &xpc_partitions[partid].sn.uv;
1135 unsigned long irq_flags;
1136
1137 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
1138 part_uv->flags &= ~XPC_P_ENGAGED_UV;
1139 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
1140}
1141
1142static int
1143xpc_partition_engaged_uv(short partid)
1144{
1145 return (xpc_partitions[partid].sn.uv.flags & XPC_P_ENGAGED_UV) != 0;
1146}
1147
1148static int
1149xpc_any_partition_engaged_uv(void)
1150{
1151 struct xpc_partition_uv *part_uv;
1152 short partid;
1153
1154 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
1155 part_uv = &xpc_partitions[partid].sn.uv;
1156 if ((part_uv->flags & XPC_P_ENGAGED_UV) != 0)
1157 return 1;
1158 }
1159 return 0;
e17d416b
DN		 1160}
1161
bd3e64c1
DN		 1162static enum xp_retval
1163xpc_allocate_msg_slot_uv(struct xpc_channel *ch, u32 flags,
1164 struct xpc_send_msg_slot_uv **address_of_msg_slot)
1165{
1166 enum xp_retval ret;
1167 struct xpc_send_msg_slot_uv *msg_slot;
1168 struct xpc_fifo_entry_uv *entry;
1169
1170 while (1) {
1171 entry = xpc_get_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list);
1172 if (entry != NULL)
1173 break;
1174
1175 if (flags & XPC_NOWAIT)
1176 return xpNoWait;
1177
1178 ret = xpc_allocate_msg_wait(ch);
1179 if (ret != xpInterrupted && ret != xpTimeout)
1180 return ret;
1181 }
1182
1183 msg_slot = container_of(entry, struct xpc_send_msg_slot_uv, next);
1184 *address_of_msg_slot = msg_slot;
1185 return xpSuccess;
1186}
1187
1188static void
1189xpc_free_msg_slot_uv(struct xpc_channel *ch,
1190 struct xpc_send_msg_slot_uv *msg_slot)
1191{
1192 xpc_put_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list, &msg_slot->next);
1193
1194 /* wakeup anyone waiting for a free msg slot */
1195 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
1196 wake_up(&ch->msg_allocate_wq);
1197}
1198
1199static void
1200xpc_notify_sender_uv(struct xpc_channel *ch,
1201 struct xpc_send_msg_slot_uv *msg_slot,
1202 enum xp_retval reason)
1203{
1204 xpc_notify_func func = msg_slot->func;
1205
1206 if (func != NULL && cmpxchg(&msg_slot->func, func, NULL) == func) {
1207
1208 atomic_dec(&ch->n_to_notify);
1209
1210 dev_dbg(xpc_chan, "msg_slot->func() called, msg_slot=0x%p "
1211 "msg_slot_number=%d partid=%d channel=%d\n", msg_slot,
1212 msg_slot->msg_slot_number, ch->partid, ch->number);
1213
1214 func(reason, ch->partid, ch->number, msg_slot->key);
1215
1216 dev_dbg(xpc_chan, "msg_slot->func() returned, msg_slot=0x%p "
1217 "msg_slot_number=%d partid=%d channel=%d\n", msg_slot,
1218 msg_slot->msg_slot_number, ch->partid, ch->number);
1219 }
1220}
1221
1222static void
1223xpc_handle_notify_mq_ack_uv(struct xpc_channel *ch,
1224 struct xpc_notify_mq_msg_uv *msg)
1225{
1226 struct xpc_send_msg_slot_uv *msg_slot;
1227 int entry = msg->hdr.msg_slot_number % ch->local_nentries;
1228
1229 msg_slot = &ch->sn.uv.send_msg_slots[entry];
1230
1231 BUG_ON(msg_slot->msg_slot_number != msg->hdr.msg_slot_number);
1232 msg_slot->msg_slot_number += ch->local_nentries;
1233
1234 if (msg_slot->func != NULL)
1235 xpc_notify_sender_uv(ch, msg_slot, xpMsgDelivered);
1236
1237 xpc_free_msg_slot_uv(ch, msg_slot);
1238}
1239
1240static void
1241xpc_handle_notify_mq_msg_uv(struct xpc_partition *part,
1242 struct xpc_notify_mq_msg_uv *msg)
1243{
1244 struct xpc_partition_uv *part_uv = &part->sn.uv;
1245 struct xpc_channel *ch;
1246 struct xpc_channel_uv *ch_uv;
1247 struct xpc_notify_mq_msg_uv *msg_slot;
1248 unsigned long irq_flags;
1249 int ch_number = msg->hdr.ch_number;
1250
1251 if (unlikely(ch_number >= part->nchannels)) {
1252 dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received invalid "
1253 "channel number=0x%x in message from partid=%d\n",
1254 ch_number, XPC_PARTID(part));
1255
1256 /* get hb checker to deactivate from the remote partition */
1257 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1258 if (part_uv->act_state_req == 0)
1259 xpc_activate_IRQ_rcvd++;
1260 part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
1261 part_uv->reason = xpBadChannelNumber;
1262 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1263
1264 wake_up_interruptible(&xpc_activate_IRQ_wq);
1265 return;
1266 }
1267
1268 ch = &part->channels[ch_number];
1269 xpc_msgqueue_ref(ch);
1270
1271 if (!(ch->flags & XPC_C_CONNECTED)) {
1272 xpc_msgqueue_deref(ch);
1273 return;
1274 }
1275
1276 /* see if we're really dealing with an ACK for a previously sent msg */
1277 if (msg->hdr.size == 0) {
1278 xpc_handle_notify_mq_ack_uv(ch, msg);
1279 xpc_msgqueue_deref(ch);
1280 return;
1281 }
1282
1283 /* we're dealing with a normal message sent via the notify_mq */
1284 ch_uv = &ch->sn.uv;
1285
1286 msg_slot = (struct xpc_notify_mq_msg_uv *)((u64)ch_uv->recv_msg_slots +
1287 (msg->hdr.msg_slot_number % ch->remote_nentries) *
1288 ch->entry_size);
1289
1290 BUG_ON(msg->hdr.msg_slot_number != msg_slot->hdr.msg_slot_number);
1291 BUG_ON(msg_slot->hdr.size != 0);
1292
1293 memcpy(msg_slot, msg, msg->hdr.size);
1294
1295 xpc_put_fifo_entry_uv(&ch_uv->recv_msg_list, &msg_slot->hdr.u.next);
1296
1297 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) {
1298 /*
1299 * If there is an existing idle kthread get it to deliver
1300 * the payload, otherwise we'll have to get the channel mgr
1301 * for this partition to create a kthread to do the delivery.
1302 */
1303 if (atomic_read(&ch->kthreads_idle) > 0)
1304 wake_up_nr(&ch->idle_wq, 1);
1305 else
1306 xpc_send_chctl_local_msgrequest_uv(part, ch->number);
1307 }
1308 xpc_msgqueue_deref(ch);
1309}
1310
1311static irqreturn_t
1312xpc_handle_notify_IRQ_uv(int irq, void *dev_id)
1313{
1314 struct xpc_notify_mq_msg_uv *msg;
1315 short partid;
1316 struct xpc_partition *part;
1317
1318 while ((msg = gru_get_next_message(xpc_notify_mq_uv)) != NULL) {
1319
1320 partid = msg->hdr.partid;
1321 if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) {
1322 dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received "
1323 "invalid partid=0x%x in message\n", partid);
1324 } else {
1325 part = &xpc_partitions[partid];
1326
1327 if (xpc_part_ref(part)) {
1328 xpc_handle_notify_mq_msg_uv(part, msg);
1329 xpc_part_deref(part);
1330 }
1331 }
1332
1333 gru_free_message(xpc_notify_mq_uv, msg);
1334 }
1335
1336 return IRQ_HANDLED;
1337}
1338
1339static int
1340xpc_n_of_deliverable_payloads_uv(struct xpc_channel *ch)
1341{
1342 return xpc_n_of_fifo_entries_uv(&ch->sn.uv.recv_msg_list);
1343}
1344
1345static void
1346xpc_process_msg_chctl_flags_uv(struct xpc_partition *part, int ch_number)
1347{
1348 struct xpc_channel *ch = &part->channels[ch_number];
1349 int ndeliverable_payloads;
1350
1351 xpc_msgqueue_ref(ch);
1352
1353 ndeliverable_payloads = xpc_n_of_deliverable_payloads_uv(ch);
1354
1355 if (ndeliverable_payloads > 0 &&
1356 (ch->flags & XPC_C_CONNECTED) &&
1357 (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)) {
1358
1359 xpc_activate_kthreads(ch, ndeliverable_payloads);
1360 }
1361
1362 xpc_msgqueue_deref(ch);
1363}
1364
1365static enum xp_retval
1366xpc_send_payload_uv(struct xpc_channel *ch, u32 flags, void *payload,
1367 u16 payload_size, u8 notify_type, xpc_notify_func func,
1368 void *key)
1369{
1370 enum xp_retval ret = xpSuccess;
1371 struct xpc_send_msg_slot_uv *msg_slot = NULL;
1372 struct xpc_notify_mq_msg_uv *msg;
1373 u8 msg_buffer[XPC_NOTIFY_MSG_SIZE_UV];
1374 size_t msg_size;
1375
1376 DBUG_ON(notify_type != XPC_N_CALL);
1377
1378 msg_size = sizeof(struct xpc_notify_mq_msghdr_uv) + payload_size;
1379 if (msg_size > ch->entry_size)
1380 return xpPayloadTooBig;
1381
1382 xpc_msgqueue_ref(ch);
1383
1384 if (ch->flags & XPC_C_DISCONNECTING) {
1385 ret = ch->reason;
1386 goto out_1;
1387 }
1388 if (!(ch->flags & XPC_C_CONNECTED)) {
1389 ret = xpNotConnected;
1390 goto out_1;
1391 }
1392
1393 ret = xpc_allocate_msg_slot_uv(ch, flags, &msg_slot);
1394 if (ret != xpSuccess)
1395 goto out_1;
1396
1397 if (func != NULL) {
1398 atomic_inc(&ch->n_to_notify);
1399
1400 msg_slot->key = key;
1401 wmb(); /* a non-NULL func must hit memory after the key */
1402 msg_slot->func = func;
1403
1404 if (ch->flags & XPC_C_DISCONNECTING) {
1405 ret = ch->reason;
1406 goto out_2;
1407 }
1408 }
1409
1410 msg = (struct xpc_notify_mq_msg_uv *)&msg_buffer;
1411 msg->hdr.partid = xp_partition_id;
1412 msg->hdr.ch_number = ch->number;
1413 msg->hdr.size = msg_size;
1414 msg->hdr.msg_slot_number = msg_slot->msg_slot_number;
1415 memcpy(&msg->payload, payload, payload_size);
1416
1417 ret = xpc_send_gru_msg(ch->sn.uv.remote_notify_mq_gpa, msg, msg_size);
1418 if (ret == xpSuccess)
1419 goto out_1;
1420
1421 XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);
1422out_2:
1423 if (func != NULL) {
1424 /*
1425 * Try to NULL the msg_slot's func field. If we fail, then
1426 * xpc_notify_senders_of_disconnect_uv() beat us to it, in which
1427 * case we need to pretend we succeeded to send the message
1428 * since the user will get a callout for the disconnect error
1429 * by xpc_notify_senders_of_disconnect_uv(), and to also get an
1430 * error returned here will confuse them. Additionally, since
1431 * in this case the channel is being disconnected we don't need
1432 * to put the the msg_slot back on the free list.
1433 */
1434 if (cmpxchg(&msg_slot->func, func, NULL) != func) {
1435 ret = xpSuccess;
1436 goto out_1;
1437 }
1438
1439 msg_slot->key = NULL;
1440 atomic_dec(&ch->n_to_notify);
1441 }
1442 xpc_free_msg_slot_uv(ch, msg_slot);
1443out_1:
1444 xpc_msgqueue_deref(ch);
1445 return ret;
1446}
1447
1448/*
1449 * Tell the callers of xpc_send_notify() that the status of their payloads
1450 * is unknown because the channel is now disconnecting.
1451 *
1452 * We don't worry about putting these msg_slots on the free list since the
1453 * msg_slots themselves are about to be kfree'd.
1454 */
1455static void
1456xpc_notify_senders_of_disconnect_uv(struct xpc_channel *ch)
1457{
1458 struct xpc_send_msg_slot_uv *msg_slot;
1459 int entry;
1460
1461 DBUG_ON(!(ch->flags & XPC_C_DISCONNECTING));
1462
1463 for (entry = 0; entry < ch->local_nentries; entry++) {
1464
1465 if (atomic_read(&ch->n_to_notify) == 0)
1466 break;
1467
1468 msg_slot = &ch->sn.uv.send_msg_slots[entry];
1469 if (msg_slot->func != NULL)
1470 xpc_notify_sender_uv(ch, msg_slot, ch->reason);
1471 }
1472}
1473
1474/*
1475 * Get the next deliverable message's payload.
1476 */
1477static void *
1478xpc_get_deliverable_payload_uv(struct xpc_channel *ch)
1479{
1480 struct xpc_fifo_entry_uv *entry;
1481 struct xpc_notify_mq_msg_uv *msg;
1482 void *payload = NULL;
1483
1484 if (!(ch->flags & XPC_C_DISCONNECTING)) {
1485 entry = xpc_get_fifo_entry_uv(&ch->sn.uv.recv_msg_list);
1486 if (entry != NULL) {
1487 msg = container_of(entry, struct xpc_notify_mq_msg_uv,
1488 hdr.u.next);
1489 payload = &msg->payload;
1490 }
1491 }
1492 return payload;
1493}
1494
1495static void
1496xpc_received_payload_uv(struct xpc_channel *ch, void *payload)
e17d416b 1497{
bd3e64c1
DN		 1498 struct xpc_notify_mq_msg_uv *msg;
1499 enum xp_retval ret;
1500
1501 msg = container_of(payload, struct xpc_notify_mq_msg_uv, payload);
1502
1503 /* return an ACK to the sender of this message */
1504
1505 msg->hdr.partid = xp_partition_id;
1506 msg->hdr.size = 0; /* size of zero indicates this is an ACK */
1507
1508 ret = xpc_send_gru_msg(ch->sn.uv.remote_notify_mq_gpa, msg,
1509 sizeof(struct xpc_notify_mq_msghdr_uv));
1510 if (ret != xpSuccess)
1511 XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);
1512
1513 msg->hdr.msg_slot_number += ch->remote_nentries;
e17d416b
DN		 1514}
1515
5b8669df 1516int
94bd2708
DN		 1517xpc_init_uv(void)
1518{
5b8669df
DN		 1519 xpc_setup_partitions_sn = xpc_setup_partitions_sn_uv;
1520 xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_uv;
1521 xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_uv;
1522 xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_uv;
33ba3c77 1523 xpc_increment_heartbeat = xpc_increment_heartbeat_uv;
5b8669df
DN		 1524 xpc_offline_heartbeat = xpc_offline_heartbeat_uv;
1525 xpc_online_heartbeat = xpc_online_heartbeat_uv;
33ba3c77
DN		 1526 xpc_heartbeat_init = xpc_heartbeat_init_uv;
1527 xpc_heartbeat_exit = xpc_heartbeat_exit_uv;
5b8669df
DN		 1528 xpc_get_remote_heartbeat = xpc_get_remote_heartbeat_uv;
1529
a47d5dac
DN		 1530 xpc_request_partition_activation = xpc_request_partition_activation_uv;
1531 xpc_request_partition_reactivation =
1532 xpc_request_partition_reactivation_uv;
5b8669df
DN		 1533 xpc_request_partition_deactivation =
1534 xpc_request_partition_deactivation_uv;
bd3e64c1
DN		 1535 xpc_cancel_partition_deactivation_request =
1536 xpc_cancel_partition_deactivation_request_uv;
5b8669df
DN		 1537
1538 xpc_setup_ch_structures_sn = xpc_setup_ch_structures_sn_uv;
1539 xpc_teardown_ch_structures_sn = xpc_teardown_ch_structures_sn_uv;
1540
e17d416b 1541 xpc_make_first_contact = xpc_make_first_contact_uv;
5b8669df 1542
7fb5e59d 1543 xpc_get_chctl_all_flags = xpc_get_chctl_all_flags_uv;
5b8669df
DN		 1544 xpc_send_chctl_closerequest = xpc_send_chctl_closerequest_uv;
1545 xpc_send_chctl_closereply = xpc_send_chctl_closereply_uv;
1546 xpc_send_chctl_openrequest = xpc_send_chctl_openrequest_uv;
1547 xpc_send_chctl_openreply = xpc_send_chctl_openreply_uv;
1548
1549 xpc_save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_uv;
1550
1551 xpc_setup_msg_structures = xpc_setup_msg_structures_uv;
1552 xpc_teardown_msg_structures = xpc_teardown_msg_structures_uv;
1553
1554 xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_uv;
1555 xpc_indicate_partition_disengaged =
1556 xpc_indicate_partition_disengaged_uv;
1557 xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_uv;
1558 xpc_partition_engaged = xpc_partition_engaged_uv;
1559 xpc_any_partition_engaged = xpc_any_partition_engaged_uv;
1560
bd3e64c1
DN		 1561 xpc_n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_uv;
1562 xpc_process_msg_chctl_flags = xpc_process_msg_chctl_flags_uv;
1563 xpc_send_payload = xpc_send_payload_uv;
1564 xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_uv;
1565 xpc_get_deliverable_payload = xpc_get_deliverable_payload_uv;
1566 xpc_received_payload = xpc_received_payload_uv;
1567
1568 if (sizeof(struct xpc_notify_mq_msghdr_uv) > XPC_MSG_HDR_MAX_SIZE) {
1569 dev_err(xpc_part, "xpc_notify_mq_msghdr_uv is larger than %d\n",
1570 XPC_MSG_HDR_MAX_SIZE);
1571 return -E2BIG;
1572 }
5b8669df 1573
2525789b
DN		 1574 xpc_activate_mq_uv = xpc_create_gru_mq_uv(XPC_ACTIVATE_MQ_SIZE_UV, 0,
1575 XPC_ACTIVATE_IRQ_NAME,
5b8669df 1576 xpc_handle_activate_IRQ_uv);
2525789b
DN		 1577 if (IS_ERR(xpc_activate_mq_uv))
1578 return PTR_ERR(xpc_activate_mq_uv);
5b8669df 1579
2525789b
DN		 1580 xpc_notify_mq_uv = xpc_create_gru_mq_uv(XPC_NOTIFY_MQ_SIZE_UV, 0,
1581 XPC_NOTIFY_IRQ_NAME,
bd3e64c1 1582 xpc_handle_notify_IRQ_uv);
2525789b
DN		 1583 if (IS_ERR(xpc_notify_mq_uv)) {
1584 xpc_destroy_gru_mq_uv(xpc_activate_mq_uv);
1585 return PTR_ERR(xpc_notify_mq_uv);
bd3e64c1
DN		 1586 }
1587
5b8669df 1588 return 0;
94bd2708
DN		 1589}
1590
1591void
1592xpc_exit_uv(void)
1593{
2525789b
DN		 1594 xpc_destroy_gru_mq_uv(xpc_notify_mq_uv);
1595 xpc_destroy_gru_mq_uv(xpc_activate_mq_uv);
94bd2708 1596}
kernel source for telekom puls (alcatel/mediatek)