1 // SPDX-License-Identifier: GPL-2.0
2 /* ldc.c: Logical Domain Channel link-layer protocol driver.
3 *
4 * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
5 */
6
7 #include <linux/kernel.h>
8 #include <linux/export.h>
9 #include <linux/slab.h>
10 #include <linux/spinlock.h>
11 #include <linux/delay.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/scatterlist.h>
15 #include <linux/interrupt.h>
16 #include <linux/list.h>
17 #include <linux/init.h>
18 #include <linux/bitmap.h>
19 #include <asm/iommu-common.h>
20
21 #include <asm/hypervisor.h>
22 #include <asm/iommu.h>
23 #include <asm/page.h>
24 #include <asm/ldc.h>
25 #include <asm/mdesc.h>
26
27 #define DRV_MODULE_NAME "ldc"
28 #define PFX DRV_MODULE_NAME ": "
29 #define DRV_MODULE_VERSION "1.1"
30 #define DRV_MODULE_RELDATE "July 22, 2008"
31
32 #define COOKIE_PGSZ_CODE 0xf000000000000000ULL
33 #define COOKIE_PGSZ_CODE_SHIFT 60ULL
34
35
36 static char version[] =
37 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
38
39 /* Packet header layout for unreliable and reliable mode frames.
40 * When in RAW mode, packets are simply straight 64-byte payloads
41 * with no headers.
42 */
43 struct ldc_packet {
44 u8 type;
45 #define LDC_CTRL 0x01
46 #define LDC_DATA 0x02
47 #define LDC_ERR 0x10
48
49 u8 stype;
50 #define LDC_INFO 0x01
51 #define LDC_ACK 0x02
52 #define LDC_NACK 0x04
53
54 u8 ctrl;
55 #define LDC_VERS 0x01 /* Link Version */
56 #define LDC_RTS 0x02 /* Request To Send */
57 #define LDC_RTR 0x03 /* Ready To Receive */
58 #define LDC_RDX 0x04 /* Ready for Data eXchange */
59 #define LDC_CTRL_MSK 0x0f
60
61 u8 env;
62 #define LDC_LEN 0x3f
63 #define LDC_FRAG_MASK 0xc0
64 #define LDC_START 0x40
65 #define LDC_STOP 0x80
66
67 u32 seqid;
68
69 union {
70 u8 u_data[LDC_PACKET_SIZE - 8];
71 struct {
72 u32 pad;
73 u32 ackid;
74 u8 r_data[LDC_PACKET_SIZE - 8 - 8];
75 } r;
76 } u;
77 };
78
79 struct ldc_version {
80 u16 major;
81 u16 minor;
82 };
83
84 /* Ordered from largest major to lowest. */
85 static struct ldc_version ver_arr[] = {
86 { .major = 1, .minor = 0 },
87 };
88
89 #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE)
90 #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE)
91
92 struct ldc_channel;
93
94 struct ldc_mode_ops {
95 int (*write)(struct ldc_channel *, const void *, unsigned int);
96 int (*read)(struct ldc_channel *, void *, unsigned int);
97 };
98
99 static const struct ldc_mode_ops raw_ops;
100 static const struct ldc_mode_ops nonraw_ops;
101 static const struct ldc_mode_ops stream_ops;
102
103 int ldom_domaining_enabled;
104
105 struct ldc_iommu {
106 /* Protects ldc_unmap. */
107 spinlock_t lock;
108 struct ldc_mtable_entry *page_table;
109 struct iommu_map_table iommu_map_table;
110 };
111
112 struct ldc_channel {
113 /* Protects all operations that depend upon channel state. */
114 spinlock_t lock;
115
116 unsigned long id;
117
118 u8 *mssbuf;
119 u32 mssbuf_len;
120 u32 mssbuf_off;
121
122 struct ldc_packet *tx_base;
123 unsigned long tx_head;
124 unsigned long tx_tail;
125 unsigned long tx_num_entries;
126 unsigned long tx_ra;
127
128 unsigned long tx_acked;
129
130 struct ldc_packet *rx_base;
131 unsigned long rx_head;
132 unsigned long rx_tail;
133 unsigned long rx_num_entries;
134 unsigned long rx_ra;
135
136 u32 rcv_nxt;
137 u32 snd_nxt;
138
139 unsigned long chan_state;
140
141 struct ldc_channel_config cfg;
142 void *event_arg;
143
144 const struct ldc_mode_ops *mops;
145
146 struct ldc_iommu iommu;
147
148 struct ldc_version ver;
149
150 u8 hs_state;
151 #define LDC_HS_CLOSED 0x00
152 #define LDC_HS_OPEN 0x01
153 #define LDC_HS_GOTVERS 0x02
154 #define LDC_HS_SENTRTR 0x03
155 #define LDC_HS_GOTRTR 0x04
156 #define LDC_HS_COMPLETE 0x10
157
158 u8 flags;
159 #define LDC_FLAG_ALLOCED_QUEUES 0x01
160 #define LDC_FLAG_REGISTERED_QUEUES 0x02
161 #define LDC_FLAG_REGISTERED_IRQS 0x04
162 #define LDC_FLAG_RESET 0x10
163
164 u8 mss;
165 u8 state;
166
167 #define LDC_IRQ_NAME_MAX 32
168 char rx_irq_name[LDC_IRQ_NAME_MAX];
169 char tx_irq_name[LDC_IRQ_NAME_MAX];
170
171 struct hlist_head mh_list;
172
173 struct hlist_node list;
174 };
175
176 #define ldcdbg(TYPE, f, a...) \
177 do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
178 printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
179 } while (0)
180
181 #define LDC_ABORT(lp) ldc_abort((lp), __func__)
182
183 static const char *state_to_str(u8 state)
184 {
185 switch (state) {
186 case LDC_STATE_INVALID:
187 return "INVALID";
188 case LDC_STATE_INIT:
189 return "INIT";
190 case LDC_STATE_BOUND:
191 return "BOUND";
192 case LDC_STATE_READY:
193 return "READY";
194 case LDC_STATE_CONNECTED:
195 return "CONNECTED";
196 default:
197 return "<UNKNOWN>";
198 }
199 }
200
201 static unsigned long __advance(unsigned long off, unsigned long num_entries)
202 {
203 off += LDC_PACKET_SIZE;
204 if (off == (num_entries * LDC_PACKET_SIZE))
205 off = 0;
206
207 return off;
208 }
209
210 static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
211 {
212 return __advance(off, lp->rx_num_entries);
213 }
214
215 static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
216 {
217 return __advance(off, lp->tx_num_entries);
218 }
219
220 static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
221 unsigned long *new_tail)
222 {
223 struct ldc_packet *p;
224 unsigned long t;
225
226 t = tx_advance(lp, lp->tx_tail);
227 if (t == lp->tx_head)
228 return NULL;
229
230 *new_tail = t;
231
232 p = lp->tx_base;
233 return p + (lp->tx_tail / LDC_PACKET_SIZE);
234 }
235
236 /* When we are in reliable or stream mode, have to track the next packet
237 * we haven't gotten an ACK for in the TX queue using tx_acked. We have
238 * to be careful not to stomp over the queue past that point. During
239 * the handshake, we don't have TX data packets pending in the queue
240 * and that's why handshake_get_tx_packet() need not be mindful of
241 * lp->tx_acked.
242 */
243 static unsigned long head_for_data(struct ldc_channel *lp)
244 {
245 if (lp->cfg.mode == LDC_MODE_STREAM)
246 return lp->tx_acked;
247 return lp->tx_head;
248 }
249
250 static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
251 {
252 unsigned long limit, tail, new_tail, diff;
253 unsigned int mss;
254
255 limit = head_for_data(lp);
256 tail = lp->tx_tail;
257 new_tail = tx_advance(lp, tail);
258 if (new_tail == limit)
259 return 0;
260
261 if (limit > new_tail)
262 diff = limit - new_tail;
263 else
264 diff = (limit +
265 ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
266 diff /= LDC_PACKET_SIZE;
267 mss = lp->mss;
268
269 if (diff * mss < size)
270 return 0;
271
272 return 1;
273 }
274
275 static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
276 unsigned long *new_tail)
277 {
278 struct ldc_packet *p;
279 unsigned long h, t;
280
281 h = head_for_data(lp);
282 t = tx_advance(lp, lp->tx_tail);
283 if (t == h)
284 return NULL;
285
286 *new_tail = t;
287
288 p = lp->tx_base;
289 return p + (lp->tx_tail / LDC_PACKET_SIZE);
290 }
291
292 static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
293 {
294 unsigned long orig_tail = lp->tx_tail;
295 int limit = 1000;
296
297 lp->tx_tail = tail;
298 while (limit-- > 0) {
299 unsigned long err;
300
301 err = sun4v_ldc_tx_set_qtail(lp->id, tail);
302 if (!err)
303 return 0;
304
305 if (err != HV_EWOULDBLOCK) {
306 lp->tx_tail = orig_tail;
307 return -EINVAL;
308 }
309 udelay(1);
310 }
311
312 lp->tx_tail = orig_tail;
313 return -EBUSY;
314 }
315
316 /* This just updates the head value in the hypervisor using
317 * a polling loop with a timeout. The caller takes care of
318 * upating software state representing the head change, if any.
319 */
320 static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
321 {
322 int limit = 1000;
323
324 while (limit-- > 0) {
325 unsigned long err;
326
327 err = sun4v_ldc_rx_set_qhead(lp->id, head);
328 if (!err)
329 return 0;
330
331 if (err != HV_EWOULDBLOCK)
332 return -EINVAL;
333
334 udelay(1);
335 }
336
337 return -EBUSY;
338 }
339
340 static int send_tx_packet(struct ldc_channel *lp,
341 struct ldc_packet *p,
342 unsigned long new_tail)
343 {
344 BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));
345
346 return set_tx_tail(lp, new_tail);
347 }
348
349 static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
350 u8 stype, u8 ctrl,
351 void *data, int dlen,
352 unsigned long *new_tail)
353 {
354 struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);
355
356 if (p) {
357 memset(p, 0, sizeof(*p));
358 p->type = LDC_CTRL;
359 p->stype = stype;
360 p->ctrl = ctrl;
361 if (data)
362 memcpy(p->u.u_data, data, dlen);
363 }
364 return p;
365 }
366
367 static int start_handshake(struct ldc_channel *lp)
368 {
369 struct ldc_packet *p;
370 struct ldc_version *ver;
371 unsigned long new_tail;
372
373 ver = &ver_arr[0];
374
375 ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
376 ver->major, ver->minor);
377
378 p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
379 ver, sizeof(*ver), &new_tail);
380 if (p) {
381 int err = send_tx_packet(lp, p, new_tail);
382 if (!err)
383 lp->flags &= ~LDC_FLAG_RESET;
384 return err;
385 }
386 return -EBUSY;
387 }
388
389 static int send_version_nack(struct ldc_channel *lp,
390 u16 major, u16 minor)
391 {
392 struct ldc_packet *p;
393 struct ldc_version ver;
394 unsigned long new_tail;
395
396 ver.major = major;
397 ver.minor = minor;
398
399 p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
400 &ver, sizeof(ver), &new_tail);
401 if (p) {
402 ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
403 ver.major, ver.minor);
404
405 return send_tx_packet(lp, p, new_tail);
406 }
407 return -EBUSY;
408 }
409
410 static int send_version_ack(struct ldc_channel *lp,
411 struct ldc_version *vp)
412 {
413 struct ldc_packet *p;
414 unsigned long new_tail;
415
416 p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
417 vp, sizeof(*vp), &new_tail);
418 if (p) {
419 ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
420 vp->major, vp->minor);
421
422 return send_tx_packet(lp, p, new_tail);
423 }
424 return -EBUSY;
425 }
426
427 static int send_rts(struct ldc_channel *lp)
428 {
429 struct ldc_packet *p;
430 unsigned long new_tail;
431
432 p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
433 &new_tail);
434 if (p) {
435 p->env = lp->cfg.mode;
436 p->seqid = 0;
437 lp->rcv_nxt = 0;
438
439 ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
440 p->env, p->seqid);
441
442 return send_tx_packet(lp, p, new_tail);
443 }
444 return -EBUSY;
445 }
446
447 static int send_rtr(struct ldc_channel *lp)
448 {
449 struct ldc_packet *p;
450 unsigned long new_tail;
451
452 p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
453 &new_tail);
454 if (p) {
455 p->env = lp->cfg.mode;
456 p->seqid = 0;
457
458 ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
459 p->env, p->seqid);
460
461 return send_tx_packet(lp, p, new_tail);
462 }
463 return -EBUSY;
464 }
465
466 static int send_rdx(struct ldc_channel *lp)
467 {
468 struct ldc_packet *p;
469 unsigned long new_tail;
470
471 p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
472 &new_tail);
473 if (p) {
474 p->env = 0;
475 p->seqid = ++lp->snd_nxt;
476 p->u.r.ackid = lp->rcv_nxt;
477
478 ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
479 p->env, p->seqid, p->u.r.ackid);
480
481 return send_tx_packet(lp, p, new_tail);
482 }
483 return -EBUSY;
484 }
485
486 static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
487 {
488 struct ldc_packet *p;
489 unsigned long new_tail;
490 int err;
491
492 p = data_get_tx_packet(lp, &new_tail);
493 if (!p)
494 return -EBUSY;
495 memset(p, 0, sizeof(*p));
496 p->type = data_pkt->type;
497 p->stype = LDC_NACK;
498 p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
499 p->seqid = lp->snd_nxt + 1;
500 p->u.r.ackid = lp->rcv_nxt;
501
502 ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
503 p->type, p->ctrl, p->seqid, p->u.r.ackid);
504
505 err = send_tx_packet(lp, p, new_tail);
506 if (!err)
507 lp->snd_nxt++;
508
509 return err;
510 }
511
512 static int ldc_abort(struct ldc_channel *lp, const char *msg)
513 {
514 unsigned long hv_err;
515
516 ldcdbg(STATE, "ABORT[%s]\n", msg);
517 ldc_print(lp);
518
519 /* We report but do not act upon the hypervisor errors because
520 * there really isn't much we can do if they fail at this point.
521 */
522 hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
523 if (hv_err)
524 printk(KERN_ERR PFX "ldc_abort: "
525 "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
526 lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);
527
528 hv_err = sun4v_ldc_tx_get_state(lp->id,
529 &lp->tx_head,
530 &lp->tx_tail,
531 &lp->chan_state);
532 if (hv_err)
533 printk(KERN_ERR PFX "ldc_abort: "
534 "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
535 lp->id, hv_err);
536
537 hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
538 if (hv_err)
539 printk(KERN_ERR PFX "ldc_abort: "
540 "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
541 lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);
542
543 /* Refetch the RX queue state as well, because we could be invoked
544 * here in the queue processing context.
545 */
546 hv_err = sun4v_ldc_rx_get_state(lp->id,
547 &lp->rx_head,
548 &lp->rx_tail,
549 &lp->chan_state);
550 if (hv_err)
551 printk(KERN_ERR PFX "ldc_abort: "
552 "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
553 lp->id, hv_err);
554
555 return -ECONNRESET;
556 }
557
558 static struct ldc_version *find_by_major(u16 major)
559 {
560 struct ldc_version *ret = NULL;
561 int i;
562
563 for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
564 struct ldc_version *v = &ver_arr[i];
565 if (v->major <= major) {
566 ret = v;
567 break;
568 }
569 }
570 return ret;
571 }
572
573 static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
574 {
575 struct ldc_version *vap;
576 int err;
577
578 ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
579 vp->major, vp->minor);
580
581 if (lp->hs_state == LDC_HS_GOTVERS) {
582 lp->hs_state = LDC_HS_OPEN;
583 memset(&lp->ver, 0, sizeof(lp->ver));
584 }
585
586 vap = find_by_major(vp->major);
587 if (!vap) {
588 err = send_version_nack(lp, 0, 0);
589 } else if (vap->major != vp->major) {
590 err = send_version_nack(lp, vap->major, vap->minor);
591 } else {
592 struct ldc_version ver = *vp;
593 if (ver.minor > vap->minor)
594 ver.minor = vap->minor;
595 err = send_version_ack(lp, &ver);
596 if (!err) {
597 lp->ver = ver;
598 lp->hs_state = LDC_HS_GOTVERS;
599 }
600 }
601 if (err)
602 return LDC_ABORT(lp);
603
604 return 0;
605 }
606
607 static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
608 {
609 ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
610 vp->major, vp->minor);
611
612 if (lp->hs_state == LDC_HS_GOTVERS) {
613 if (lp->ver.major != vp->major ||
614 lp->ver.minor != vp->minor)
615 return LDC_ABORT(lp);
616 } else {
617 lp->ver = *vp;
618 lp->hs_state = LDC_HS_GOTVERS;
619 }
620 if (send_rts(lp))
621 return LDC_ABORT(lp);
622 return 0;
623 }
624
625 static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
626 {
627 struct ldc_version *vap;
628 struct ldc_packet *p;
629 unsigned long new_tail;
630
631 if (vp->major == 0 && vp->minor == 0)
632 return LDC_ABORT(lp);
633
634 vap = find_by_major(vp->major);
635 if (!vap)
636 return LDC_ABORT(lp);
637
638 p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
639 vap, sizeof(*vap),
640 &new_tail);
641 if (!p)
642 return LDC_ABORT(lp);
643
644 return send_tx_packet(lp, p, new_tail);
645 }
646
647 static int process_version(struct ldc_channel *lp,
648 struct ldc_packet *p)
649 {
650 struct ldc_version *vp;
651
652 vp = (struct ldc_version *) p->u.u_data;
653
654 switch (p->stype) {
655 case LDC_INFO:
656 return process_ver_info(lp, vp);
657
658 case LDC_ACK:
659 return process_ver_ack(lp, vp);
660
661 case LDC_NACK:
662 return process_ver_nack(lp, vp);
663
664 default:
665 return LDC_ABORT(lp);
666 }
667 }
668
669 static int process_rts(struct ldc_channel *lp,
670 struct ldc_packet *p)
671 {
672 ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
673 p->stype, p->seqid, p->env);
674
675 if (p->stype != LDC_INFO ||
676 lp->hs_state != LDC_HS_GOTVERS ||
677 p->env != lp->cfg.mode)
678 return LDC_ABORT(lp);
679
680 lp->snd_nxt = p->seqid;
681 lp->rcv_nxt = p->seqid;
682 lp->hs_state = LDC_HS_SENTRTR;
683 if (send_rtr(lp))
684 return LDC_ABORT(lp);
685
686 return 0;
687 }
688
689 static int process_rtr(struct ldc_channel *lp,
690 struct ldc_packet *p)
691 {
692 ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
693 p->stype, p->seqid, p->env);
694
695 if (p->stype != LDC_INFO ||
696 p->env != lp->cfg.mode)
697 return LDC_ABORT(lp);
698
699 lp->snd_nxt = p->seqid;
700 lp->hs_state = LDC_HS_COMPLETE;
701 ldc_set_state(lp, LDC_STATE_CONNECTED);
702 send_rdx(lp);
703
704 return LDC_EVENT_UP;
705 }
706
707 static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
708 {
709 return lp->rcv_nxt + 1 == seqid;
710 }
711
712 static int process_rdx(struct ldc_channel *lp,
713 struct ldc_packet *p)
714 {
715 ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
716 p->stype, p->seqid, p->env, p->u.r.ackid);
717
718 if (p->stype != LDC_INFO ||
719 !(rx_seq_ok(lp, p->seqid)))
720 return LDC_ABORT(lp);
721
722 lp->rcv_nxt = p->seqid;
723
724 lp->hs_state = LDC_HS_COMPLETE;
725 ldc_set_state(lp, LDC_STATE_CONNECTED);
726
727 return LDC_EVENT_UP;
728 }
729
730 static int process_control_frame(struct ldc_channel *lp,
731 struct ldc_packet *p)
732 {
733 switch (p->ctrl) {
734 case LDC_VERS:
735 return process_version(lp, p);
736
737 case LDC_RTS:
738 return process_rts(lp, p);
739
740 case LDC_RTR:
741 return process_rtr(lp, p);
742
743 case LDC_RDX:
744 return process_rdx(lp, p);
745
746 default:
747 return LDC_ABORT(lp);
748 }
749 }
750
751 static int process_error_frame(struct ldc_channel *lp,
752 struct ldc_packet *p)
753 {
754 return LDC_ABORT(lp);
755 }
756
757 static int process_data_ack(struct ldc_channel *lp,
758 struct ldc_packet *ack)
759 {
760 unsigned long head = lp->tx_acked;
761 u32 ackid = ack->u.r.ackid;
762
763 while (1) {
764 struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);
765
766 head = tx_advance(lp, head);
767
768 if (p->seqid == ackid) {
769 lp->tx_acked = head;
770 return 0;
771 }
772 if (head == lp->tx_tail)
773 return LDC_ABORT(lp);
774 }
775
776 return 0;
777 }
778
779 static void send_events(struct ldc_channel *lp, unsigned int event_mask)
780 {
781 if (event_mask & LDC_EVENT_RESET)
782 lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
783 if (event_mask & LDC_EVENT_UP)
784 lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
785 if (event_mask & LDC_EVENT_DATA_READY)
786 lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
787 }
788
789 static irqreturn_t ldc_rx(int irq, void *dev_id)
790 {
791 struct ldc_channel *lp = dev_id;
792 unsigned long orig_state, flags;
793 unsigned int event_mask;
794
795 spin_lock_irqsave(&lp->lock, flags);
796
797 orig_state = lp->chan_state;
798
799 /* We should probably check for hypervisor errors here and
800 * reset the LDC channel if we get one.
801 */
802 sun4v_ldc_rx_get_state(lp->id,
803 &lp->rx_head,
804 &lp->rx_tail,
805 &lp->chan_state);
806
807 ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
808 orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);
809
810 event_mask = 0;
811
812 if (lp->cfg.mode == LDC_MODE_RAW &&
813 lp->chan_state == LDC_CHANNEL_UP) {
814 lp->hs_state = LDC_HS_COMPLETE;
815 ldc_set_state(lp, LDC_STATE_CONNECTED);
816
817 /*
818 * Generate an LDC_EVENT_UP event if the channel
819 * was not already up.
820 */
821 if (orig_state != LDC_CHANNEL_UP) {
822 event_mask |= LDC_EVENT_UP;
823 orig_state = lp->chan_state;
824 }
825 }
826
827 /* If we are in reset state, flush the RX queue and ignore
828 * everything.
829 */
830 if (lp->flags & LDC_FLAG_RESET) {
831 (void) ldc_rx_reset(lp);
832 goto out;
833 }
834
835 /* Once we finish the handshake, we let the ldc_read()
836 * paths do all of the control frame and state management.
837 * Just trigger the callback.
838 */
839 if (lp->hs_state == LDC_HS_COMPLETE) {
840 handshake_complete:
841 if (lp->chan_state != orig_state) {
842 unsigned int event = LDC_EVENT_RESET;
843
844 if (lp->chan_state == LDC_CHANNEL_UP)
845 event = LDC_EVENT_UP;
846
847 event_mask |= event;
848 }
849 if (lp->rx_head != lp->rx_tail)
850 event_mask |= LDC_EVENT_DATA_READY;
851
852 goto out;
853 }
854
855 if (lp->chan_state != orig_state)
856 goto out;
857
858 while (lp->rx_head != lp->rx_tail) {
859 struct ldc_packet *p;
860 unsigned long new;
861 int err;
862
863 p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
864
865 switch (p->type) {
866 case LDC_CTRL:
867 err = process_control_frame(lp, p);
868 if (err > 0)
869 event_mask |= err;
870 break;
871
872 case LDC_DATA:
873 event_mask |= LDC_EVENT_DATA_READY;
874 err = 0;
875 break;
876
877 case LDC_ERR:
878 err = process_error_frame(lp, p);
879 break;
880
881 default:
882 err = LDC_ABORT(lp);
883 break;
884 }
885
886 if (err < 0)
887 break;
888
889 new = lp->rx_head;
890 new += LDC_PACKET_SIZE;
891 if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
892 new = 0;
893 lp->rx_head = new;
894
895 err = __set_rx_head(lp, new);
896 if (err < 0) {
897 (void) LDC_ABORT(lp);
898 break;
899 }
900 if (lp->hs_state == LDC_HS_COMPLETE)
901 goto handshake_complete;
902 }
903
904 out:
905 spin_unlock_irqrestore(&lp->lock, flags);
906
907 send_events(lp, event_mask);
908
909 return IRQ_HANDLED;
910 }
911
912 static irqreturn_t ldc_tx(int irq, void *dev_id)
913 {
914 struct ldc_channel *lp = dev_id;
915 unsigned long flags, orig_state;
916 unsigned int event_mask = 0;
917
918 spin_lock_irqsave(&lp->lock, flags);
919
920 orig_state = lp->chan_state;
921
922 /* We should probably check for hypervisor errors here and
923 * reset the LDC channel if we get one.
924 */
925 sun4v_ldc_tx_get_state(lp->id,
926 &lp->tx_head,
927 &lp->tx_tail,
928 &lp->chan_state);
929
930 ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
931 orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);
932
933 if (lp->cfg.mode == LDC_MODE_RAW &&
934 lp->chan_state == LDC_CHANNEL_UP) {
935 lp->hs_state = LDC_HS_COMPLETE;
936 ldc_set_state(lp, LDC_STATE_CONNECTED);
937
938 /*
939 * Generate an LDC_EVENT_UP event if the channel
940 * was not already up.
941 */
942 if (orig_state != LDC_CHANNEL_UP) {
943 event_mask |= LDC_EVENT_UP;
944 orig_state = lp->chan_state;
945 }
946 }
947
948 spin_unlock_irqrestore(&lp->lock, flags);
949
950 send_events(lp, event_mask);
951
952 return IRQ_HANDLED;
953 }
954
955 /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
956 * XXX that addition and removal from the ldc_channel_list has
957 * XXX atomicity, otherwise the __ldc_channel_exists() check is
958 * XXX totally pointless as another thread can slip into ldc_alloc()
959 * XXX and add a channel with the same ID. There also needs to be
960 * XXX a spinlock for ldc_channel_list.
961 */
962 static HLIST_HEAD(ldc_channel_list);
963
964 static int __ldc_channel_exists(unsigned long id)
965 {
966 struct ldc_channel *lp;
967
968 hlist_for_each_entry(lp, &ldc_channel_list, list) {
969 if (lp->id == id)
970 return 1;
971 }
972 return 0;
973 }
974
975 static int alloc_queue(const char *name, unsigned long num_entries,
976 struct ldc_packet **base, unsigned long *ra)
977 {
978 unsigned long size, order;
979 void *q;
980
981 size = num_entries * LDC_PACKET_SIZE;
982 order = get_order(size);
983
984 q = (void *) __get_free_pages(GFP_KERNEL, order);
985 if (!q) {
986 printk(KERN_ERR PFX "Alloc of %s queue failed with "
987 "size=%lu order=%lu\n", name, size, order);
988 return -ENOMEM;
989 }
990
991 memset(q, 0, PAGE_SIZE << order);
992
993 *base = q;
994 *ra = __pa(q);
995
996 return 0;
997 }
998
999 static void free_queue(unsigned long num_entries, struct ldc_packet *q)
1000 {
1001 unsigned long size, order;
1002
1003 if (!q)
1004 return;
1005
1006 size = num_entries * LDC_PACKET_SIZE;
1007 order = get_order(size);
1008
1009 free_pages((unsigned long)q, order);
1010 }
1011
1012 static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
1013 {
1014 u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
1015 /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
1016
1017 cookie &= ~COOKIE_PGSZ_CODE;
1018
1019 return (cookie >> (13ULL + (szcode * 3ULL)));
1020 }
1021
1022 static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
1023 unsigned long entry, unsigned long npages)
1024 {
1025 struct ldc_mtable_entry *base;
1026 unsigned long i, shift;
1027
1028 shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
1029 base = iommu->page_table + entry;
1030 for (i = 0; i < npages; i++) {
1031 if (base->cookie)
1032 sun4v_ldc_revoke(id, cookie + (i << shift),
1033 base->cookie);
1034 base->mte = 0;
1035 }
1036 }
1037
1038 /* XXX Make this configurable... XXX */
1039 #define LDC_IOTABLE_SIZE (8 * 1024)
1040
1041 static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
1042 {
1043 unsigned long sz, num_tsb_entries, tsbsize, order;
1044 struct ldc_iommu *ldc_iommu = &lp->iommu;
1045 struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
1046 struct ldc_mtable_entry *table;
1047 unsigned long hv_err;
1048 int err;
1049
1050 num_tsb_entries = LDC_IOTABLE_SIZE;
1051 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1052 spin_lock_init(&ldc_iommu->lock);
1053
1054 sz = num_tsb_entries / 8;
1055 sz = (sz + 7UL) & ~7UL;
1056 iommu->map = kzalloc(sz, GFP_KERNEL);
1057 if (!iommu->map) {
1058 printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
1059 return -ENOMEM;
1060 }
1061 iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
1062 NULL, false /* no large pool */,
1063 1 /* npools */,
1064 true /* skip span boundary check */);
1065
1066 order = get_order(tsbsize);
1067
1068 table = (struct ldc_mtable_entry *)
1069 __get_free_pages(GFP_KERNEL, order);
1070 err = -ENOMEM;
1071 if (!table) {
1072 printk(KERN_ERR PFX "Alloc of MTE table failed, "
1073 "size=%lu order=%lu\n", tsbsize, order);
1074 goto out_free_map;
1075 }
1076
1077 memset(table, 0, PAGE_SIZE << order);
1078
1079 ldc_iommu->page_table = table;
1080
1081 hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
1082 num_tsb_entries);
1083 err = -EINVAL;
1084 if (hv_err)
1085 goto out_free_table;
1086
1087 return 0;
1088
1089 out_free_table:
1090 free_pages((unsigned long) table, order);
1091 ldc_iommu->page_table = NULL;
1092
1093 out_free_map:
1094 kfree(iommu->map);
1095 iommu->map = NULL;
1096
1097 return err;
1098 }
1099
1100 static void ldc_iommu_release(struct ldc_channel *lp)
1101 {
1102 struct ldc_iommu *ldc_iommu = &lp->iommu;
1103 struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
1104 unsigned long num_tsb_entries, tsbsize, order;
1105
1106 (void) sun4v_ldc_set_map_table(lp->id, 0, 0);
1107
1108 num_tsb_entries = iommu->poolsize * iommu->nr_pools;
1109 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1110 order = get_order(tsbsize);
1111
1112 free_pages((unsigned long) ldc_iommu->page_table, order);
1113 ldc_iommu->page_table = NULL;
1114
1115 kfree(iommu->map);
1116 iommu->map = NULL;
1117 }
1118
1119 struct ldc_channel *ldc_alloc(unsigned long id,
1120 const struct ldc_channel_config *cfgp,
1121 void *event_arg,
1122 const char *name)
1123 {
1124 struct ldc_channel *lp;
1125 const struct ldc_mode_ops *mops;
1126 unsigned long dummy1, dummy2, hv_err;
1127 u8 mss, *mssbuf;
1128 int err;
1129
1130 err = -ENODEV;
1131 if (!ldom_domaining_enabled)
1132 goto out_err;
1133
1134 err = -EINVAL;
1135 if (!cfgp)
1136 goto out_err;
1137 if (!name)
1138 goto out_err;
1139
1140 switch (cfgp->mode) {
1141 case LDC_MODE_RAW:
1142 mops = &raw_ops;
1143 mss = LDC_PACKET_SIZE;
1144 break;
1145
1146 case LDC_MODE_UNRELIABLE:
1147 mops = &nonraw_ops;
1148 mss = LDC_PACKET_SIZE - 8;
1149 break;
1150
1151 case LDC_MODE_STREAM:
1152 mops = &stream_ops;
1153 mss = LDC_PACKET_SIZE - 8 - 8;
1154 break;
1155
1156 default:
1157 goto out_err;
1158 }
1159
1160 if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
1161 goto out_err;
1162
1163 hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
1164 err = -ENODEV;
1165 if (hv_err == HV_ECHANNEL)
1166 goto out_err;
1167
1168 err = -EEXIST;
1169 if (__ldc_channel_exists(id))
1170 goto out_err;
1171
1172 mssbuf = NULL;
1173
1174 lp = kzalloc(sizeof(*lp), GFP_KERNEL);
1175 err = -ENOMEM;
1176 if (!lp)
1177 goto out_err;
1178
1179 spin_lock_init(&lp->lock);
1180
1181 lp->id = id;
1182
1183 err = ldc_iommu_init(name, lp);
1184 if (err)
1185 goto out_free_ldc;
1186
1187 lp->mops = mops;
1188 lp->mss = mss;
1189
1190 lp->cfg = *cfgp;
1191 if (!lp->cfg.mtu)
1192 lp->cfg.mtu = LDC_DEFAULT_MTU;
1193
1194 if (lp->cfg.mode == LDC_MODE_STREAM) {
1195 mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
1196 if (!mssbuf) {
1197 err = -ENOMEM;
1198 goto out_free_iommu;
1199 }
1200 lp->mssbuf = mssbuf;
1201 }
1202
1203 lp->event_arg = event_arg;
1204
1205 /* XXX allow setting via ldc_channel_config to override defaults
1206 * XXX or use some formula based upon mtu
1207 */
1208 lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
1209 lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
1210
1211 err = alloc_queue("TX", lp->tx_num_entries,
1212 &lp->tx_base, &lp->tx_ra);
1213 if (err)
1214 goto out_free_mssbuf;
1215
1216 err = alloc_queue("RX", lp->rx_num_entries,
1217 &lp->rx_base, &lp->rx_ra);
1218 if (err)
1219 goto out_free_txq;
1220
1221 lp->flags |= LDC_FLAG_ALLOCED_QUEUES;
1222
1223 lp->hs_state = LDC_HS_CLOSED;
1224 ldc_set_state(lp, LDC_STATE_INIT);
1225
1226 INIT_HLIST_NODE(&lp->list);
1227 hlist_add_head(&lp->list, &ldc_channel_list);
1228
1229 INIT_HLIST_HEAD(&lp->mh_list);
1230
1231 snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
1232 snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
1233
1234 err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
1235 lp->rx_irq_name, lp);
1236 if (err)
1237 goto out_free_txq;
1238
1239 err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
1240 lp->tx_irq_name, lp);
1241 if (err) {
1242 free_irq(lp->cfg.rx_irq, lp);
1243 goto out_free_txq;
1244 }
1245
1246 return lp;
1247
1248 out_free_txq:
1249 free_queue(lp->tx_num_entries, lp->tx_base);
1250
1251 out_free_mssbuf:
1252 kfree(mssbuf);
1253
1254 out_free_iommu:
1255 ldc_iommu_release(lp);
1256
1257 out_free_ldc:
1258 kfree(lp);
1259
1260 out_err:
1261 return ERR_PTR(err);
1262 }
1263 EXPORT_SYMBOL(ldc_alloc);
1264
1265 void ldc_unbind(struct ldc_channel *lp)
1266 {
1267 if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
1268 free_irq(lp->cfg.rx_irq, lp);
1269 free_irq(lp->cfg.tx_irq, lp);
1270 lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
1271 }
1272
1273 if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
1274 sun4v_ldc_tx_qconf(lp->id, 0, 0);
1275 sun4v_ldc_rx_qconf(lp->id, 0, 0);
1276 lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
1277 }
1278 if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
1279 free_queue(lp->tx_num_entries, lp->tx_base);
1280 free_queue(lp->rx_num_entries, lp->rx_base);
1281 lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
1282 }
1283
1284 ldc_set_state(lp, LDC_STATE_INIT);
1285 }
1286 EXPORT_SYMBOL(ldc_unbind);
1287
1288 void ldc_free(struct ldc_channel *lp)
1289 {
1290 ldc_unbind(lp);
1291 hlist_del(&lp->list);
1292 kfree(lp->mssbuf);
1293 ldc_iommu_release(lp);
1294
1295 kfree(lp);
1296 }
1297 EXPORT_SYMBOL(ldc_free);
1298
1299 /* Bind the channel. This registers the LDC queues with
1300 * the hypervisor and puts the channel into a pseudo-listening
1301 * state. This does not initiate a handshake, ldc_connect() does
1302 * that.
1303 */
1304 int ldc_bind(struct ldc_channel *lp)
1305 {
1306 unsigned long hv_err, flags;
1307 int err = -EINVAL;
1308
1309 if (lp->state != LDC_STATE_INIT)
1310 return -EINVAL;
1311
1312 spin_lock_irqsave(&lp->lock, flags);
1313
1314 enable_irq(lp->cfg.rx_irq);
1315 enable_irq(lp->cfg.tx_irq);
1316
1317 lp->flags |= LDC_FLAG_REGISTERED_IRQS;
1318
1319 err = -ENODEV;
1320 hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
1321 if (hv_err)
1322 goto out_free_irqs;
1323
1324 hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
1325 if (hv_err)
1326 goto out_free_irqs;
1327
1328 hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
1329 if (hv_err)
1330 goto out_unmap_tx;
1331
1332 hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
1333 if (hv_err)
1334 goto out_unmap_tx;
1335
1336 lp->flags |= LDC_FLAG_REGISTERED_QUEUES;
1337
1338 hv_err = sun4v_ldc_tx_get_state(lp->id,
1339 &lp->tx_head,
1340 &lp->tx_tail,
1341 &lp->chan_state);
1342 err = -EBUSY;
1343 if (hv_err)
1344 goto out_unmap_rx;
1345
1346 lp->tx_acked = lp->tx_head;
1347
1348 lp->hs_state = LDC_HS_OPEN;
1349 ldc_set_state(lp, LDC_STATE_BOUND);
1350
1351 if (lp->cfg.mode == LDC_MODE_RAW) {
1352 /*
1353 * There is no handshake in RAW mode, so handshake
1354 * is completed.
1355 */
1356 lp->hs_state = LDC_HS_COMPLETE;
1357 }
1358
1359 spin_unlock_irqrestore(&lp->lock, flags);
1360
1361 return 0;
1362
1363 out_unmap_rx:
1364 lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
1365 sun4v_ldc_rx_qconf(lp->id, 0, 0);
1366
1367 out_unmap_tx:
1368 sun4v_ldc_tx_qconf(lp->id, 0, 0);
1369
1370 out_free_irqs:
1371 lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
1372 free_irq(lp->cfg.tx_irq, lp);
1373 free_irq(lp->cfg.rx_irq, lp);
1374
1375 spin_unlock_irqrestore(&lp->lock, flags);
1376
1377 return err;
1378 }
1379 EXPORT_SYMBOL(ldc_bind);
1380
1381 int ldc_connect(struct ldc_channel *lp)
1382 {
1383 unsigned long flags;
1384 int err;
1385
1386 if (lp->cfg.mode == LDC_MODE_RAW)
1387 return -EINVAL;
1388
1389 spin_lock_irqsave(&lp->lock, flags);
1390
1391 if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
1392 !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
1393 lp->hs_state != LDC_HS_OPEN)
1394 err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
1395 else
1396 err = start_handshake(lp);
1397
1398 spin_unlock_irqrestore(&lp->lock, flags);
1399
1400 return err;
1401 }
1402 EXPORT_SYMBOL(ldc_connect);
1403
1404 int ldc_disconnect(struct ldc_channel *lp)
1405 {
1406 unsigned long hv_err, flags;
1407 int err;
1408
1409 if (lp->cfg.mode == LDC_MODE_RAW)
1410 return -EINVAL;
1411
1412 if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
1413 !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
1414 return -EINVAL;
1415
1416 spin_lock_irqsave(&lp->lock, flags);
1417
1418 err = -ENODEV;
1419 hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
1420 if (hv_err)
1421 goto out_err;
1422
1423 hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
1424 if (hv_err)
1425 goto out_err;
1426
1427 hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
1428 if (hv_err)
1429 goto out_err;
1430
1431 hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
1432 if (hv_err)
1433 goto out_err;
1434
1435 ldc_set_state(lp, LDC_STATE_BOUND);
1436 lp->hs_state = LDC_HS_OPEN;
1437 lp->flags |= LDC_FLAG_RESET;
1438
1439 spin_unlock_irqrestore(&lp->lock, flags);
1440
1441 return 0;
1442
1443 out_err:
1444 sun4v_ldc_tx_qconf(lp->id, 0, 0);
1445 sun4v_ldc_rx_qconf(lp->id, 0, 0);
1446 free_irq(lp->cfg.tx_irq, lp);
1447 free_irq(lp->cfg.rx_irq, lp);
1448 lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
1449 LDC_FLAG_REGISTERED_QUEUES);
1450 ldc_set_state(lp, LDC_STATE_INIT);
1451
1452 spin_unlock_irqrestore(&lp->lock, flags);
1453
1454 return err;
1455 }
1456 EXPORT_SYMBOL(ldc_disconnect);
1457
1458 int ldc_state(struct ldc_channel *lp)
1459 {
1460 return lp->state;
1461 }
1462 EXPORT_SYMBOL(ldc_state);
1463
1464 void ldc_set_state(struct ldc_channel *lp, u8 state)
1465 {
1466 ldcdbg(STATE, "STATE (%s) --> (%s)\n",
1467 state_to_str(lp->state),
1468 state_to_str(state));
1469
1470 lp->state = state;
1471 }
1472 EXPORT_SYMBOL(ldc_set_state);
1473
1474 int ldc_mode(struct ldc_channel *lp)
1475 {
1476 return lp->cfg.mode;
1477 }
1478 EXPORT_SYMBOL(ldc_mode);
1479
1480 int ldc_rx_reset(struct ldc_channel *lp)
1481 {
1482 return __set_rx_head(lp, lp->rx_tail);
1483 }
1484 EXPORT_SYMBOL(ldc_rx_reset);
1485
1486 void __ldc_print(struct ldc_channel *lp, const char *caller)
1487 {
1488 pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
1489 "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
1490 "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
1491 "\trcv_nxt=%u snd_nxt=%u\n",
1492 caller, lp->id, lp->flags, state_to_str(lp->state),
1493 lp->chan_state, lp->hs_state,
1494 lp->rx_head, lp->rx_tail, lp->rx_num_entries,
1495 lp->tx_head, lp->tx_tail, lp->tx_num_entries,
1496 lp->rcv_nxt, lp->snd_nxt);
1497 }
1498 EXPORT_SYMBOL(__ldc_print);
1499
1500 static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
1501 {
1502 struct ldc_packet *p;
1503 unsigned long new_tail, hv_err;
1504 int err;
1505
1506 hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
1507 &lp->chan_state);
1508 if (unlikely(hv_err))
1509 return -EBUSY;
1510
1511 if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
1512 return LDC_ABORT(lp);
1513
1514 if (size > LDC_PACKET_SIZE)
1515 return -EMSGSIZE;
1516
1517 p = data_get_tx_packet(lp, &new_tail);
1518 if (!p)
1519 return -EAGAIN;
1520
1521 memcpy(p, buf, size);
1522
1523 err = send_tx_packet(lp, p, new_tail);
1524 if (!err)
1525 err = size;
1526
1527 return err;
1528 }
1529
1530 static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
1531 {
1532 struct ldc_packet *p;
1533 unsigned long hv_err, new;
1534 int err;
1535
1536 if (size < LDC_PACKET_SIZE)
1537 return -EINVAL;
1538
1539 hv_err = sun4v_ldc_rx_get_state(lp->id,
1540 &lp->rx_head,
1541 &lp->rx_tail,
1542 &lp->chan_state);
1543 if (hv_err)
1544 return LDC_ABORT(lp);
1545
1546 if (lp->chan_state == LDC_CHANNEL_DOWN ||
1547 lp->chan_state == LDC_CHANNEL_RESETTING)
1548 return -ECONNRESET;
1549
1550 if (lp->rx_head == lp->rx_tail)
1551 return 0;
1552
1553 p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
1554 memcpy(buf, p, LDC_PACKET_SIZE);
1555
1556 new = rx_advance(lp, lp->rx_head);
1557 lp->rx_head = new;
1558
1559 err = __set_rx_head(lp, new);
1560 if (err < 0)
1561 err = -ECONNRESET;
1562 else
1563 err = LDC_PACKET_SIZE;
1564
1565 return err;
1566 }
1567
1568 static const struct ldc_mode_ops raw_ops = {
1569 .write = write_raw,
1570 .read = read_raw,
1571 };
1572
1573 static int write_nonraw(struct ldc_channel *lp, const void *buf,
1574 unsigned int size)
1575 {
1576 unsigned long hv_err, tail;
1577 unsigned int copied;
1578 u32 seq;
1579 int err;
1580
1581 hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
1582 &lp->chan_state);
1583 if (unlikely(hv_err))
1584 return -EBUSY;
1585
1586 if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
1587 return LDC_ABORT(lp);
1588
1589 if (!tx_has_space_for(lp, size))
1590 return -EAGAIN;
1591
1592 seq = lp->snd_nxt;
1593 copied = 0;
1594 tail = lp->tx_tail;
1595 while (copied < size) {
1596 struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
1597 u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
1598 p->u.u_data :
1599 p->u.r.r_data);
1600 int data_len;
1601
1602 p->type = LDC_DATA;
1603 p->stype = LDC_INFO;
1604 p->ctrl = 0;
1605
1606 data_len = size - copied;
1607 if (data_len > lp->mss)
1608 data_len = lp->mss;
1609
1610 BUG_ON(data_len > LDC_LEN);
1611
1612 p->env = (data_len |
1613 (copied == 0 ? LDC_START : 0) |
1614 (data_len == size - copied ? LDC_STOP : 0));
1615
1616 p->seqid = ++seq;
1617
1618 ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
1619 p->type,
1620 p->stype,
1621 p->ctrl,
1622 p->env,
1623 p->seqid);
1624
1625 memcpy(data, buf, data_len);
1626 buf += data_len;
1627 copied += data_len;
1628
1629 tail = tx_advance(lp, tail);
1630 }
1631
1632 err = set_tx_tail(lp, tail);
1633 if (!err) {
1634 lp->snd_nxt = seq;
1635 err = size;
1636 }
1637
1638 return err;
1639 }
1640
1641 static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
1642 struct ldc_packet *first_frag)
1643 {
1644 int err;
1645
1646 if (first_frag)
1647 lp->rcv_nxt = first_frag->seqid - 1;
1648
1649 err = send_data_nack(lp, p);
1650 if (err)
1651 return err;
1652
1653 err = ldc_rx_reset(lp);
1654 if (err < 0)
1655 return LDC_ABORT(lp);
1656
1657 return 0;
1658 }
1659
1660 static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
1661 {
1662 if (p->stype & LDC_ACK) {
1663 int err = process_data_ack(lp, p);
1664 if (err)
1665 return err;
1666 }
1667 if (p->stype & LDC_NACK)
1668 return LDC_ABORT(lp);
1669
1670 return 0;
1671 }
1672
1673 static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
1674 {
1675 unsigned long dummy;
1676 int limit = 1000;
1677
1678 ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
1679 cur_head, lp->rx_head, lp->rx_tail);
1680 while (limit-- > 0) {
1681 unsigned long hv_err;
1682
1683 hv_err = sun4v_ldc_rx_get_state(lp->id,
1684 &dummy,
1685 &lp->rx_tail,
1686 &lp->chan_state);
1687 if (hv_err)
1688 return LDC_ABORT(lp);
1689
1690 if (lp->chan_state == LDC_CHANNEL_DOWN ||
1691 lp->chan_state == LDC_CHANNEL_RESETTING)
1692 return -ECONNRESET;
1693
1694 if (cur_head != lp->rx_tail) {
1695 ldcdbg(DATA, "DATA WAIT DONE "
1696 "head[%lx] tail[%lx] chan_state[%lx]\n",
1697 dummy, lp->rx_tail, lp->chan_state);
1698 return 0;
1699 }
1700
1701 udelay(1);
1702 }
1703 return -EAGAIN;
1704 }
1705
1706 static int rx_set_head(struct ldc_channel *lp, unsigned long head)
1707 {
1708 int err = __set_rx_head(lp, head);
1709
1710 if (err < 0)
1711 return LDC_ABORT(lp);
1712
1713 lp->rx_head = head;
1714 return 0;
1715 }
1716
1717 static void send_data_ack(struct ldc_channel *lp)
1718 {
1719 unsigned long new_tail;
1720 struct ldc_packet *p;
1721
1722 p = data_get_tx_packet(lp, &new_tail);
1723 if (likely(p)) {
1724 int err;
1725
1726 memset(p, 0, sizeof(*p));
1727 p->type = LDC_DATA;
1728 p->stype = LDC_ACK;
1729 p->ctrl = 0;
1730 p->seqid = lp->snd_nxt + 1;
1731 p->u.r.ackid = lp->rcv_nxt;
1732
1733 err = send_tx_packet(lp, p, new_tail);
1734 if (!err)
1735 lp->snd_nxt++;
1736 }
1737 }
1738
1739 static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
1740 {
1741 struct ldc_packet *first_frag;
1742 unsigned long hv_err, new;
1743 int err, copied;
1744
1745 hv_err = sun4v_ldc_rx_get_state(lp->id,
1746 &lp->rx_head,
1747 &lp->rx_tail,
1748 &lp->chan_state);
1749 if (hv_err)
1750 return LDC_ABORT(lp);
1751
1752 if (lp->chan_state == LDC_CHANNEL_DOWN ||
1753 lp->chan_state == LDC_CHANNEL_RESETTING)
1754 return -ECONNRESET;
1755
1756 if (lp->rx_head == lp->rx_tail)
1757 return 0;
1758
1759 first_frag = NULL;
1760 copied = err = 0;
1761 new = lp->rx_head;
1762 while (1) {
1763 struct ldc_packet *p;
1764 int pkt_len;
1765
1766 BUG_ON(new == lp->rx_tail);
1767 p = lp->rx_base + (new / LDC_PACKET_SIZE);
1768
1769 ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
1770 "rcv_nxt[%08x]\n",
1771 p->type,
1772 p->stype,
1773 p->ctrl,
1774 p->env,
1775 p->seqid,
1776 p->u.r.ackid,
1777 lp->rcv_nxt);
1778
1779 if (unlikely(!rx_seq_ok(lp, p->seqid))) {
1780 err = rx_bad_seq(lp, p, first_frag);
1781 copied = 0;
1782 break;
1783 }
1784
1785 if (p->type & LDC_CTRL) {
1786 err = process_control_frame(lp, p);
1787 if (err < 0)
1788 break;
1789 err = 0;
1790 }
1791
1792 lp->rcv_nxt = p->seqid;
1793
1794 /*
1795 * If this is a control-only packet, there is nothing
1796 * else to do but advance the rx queue since the packet
1797 * was already processed above.
1798 */
1799 if (!(p->type & LDC_DATA)) {
1800 new = rx_advance(lp, new);
1801 break;
1802 }
1803 if (p->stype & (LDC_ACK | LDC_NACK)) {
1804 err = data_ack_nack(lp, p);
1805 if (err)
1806 break;
1807 }
1808 if (!(p->stype & LDC_INFO)) {
1809 new = rx_advance(lp, new);
1810 err = rx_set_head(lp, new);
1811 if (err)
1812 break;
1813 goto no_data;
1814 }
1815
1816 pkt_len = p->env & LDC_LEN;
1817
1818 /* Every initial packet starts with the START bit set.
1819 *
1820 * Singleton packets will have both START+STOP set.
1821 *
1822 * Fragments will have START set in the first frame, STOP
1823 * set in the last frame, and neither bit set in middle
1824 * frames of the packet.
1825 *
1826 * Therefore if we are at the beginning of a packet and
1827 * we don't see START, or we are in the middle of a fragmented
1828 * packet and do see START, we are unsynchronized and should
1829 * flush the RX queue.
1830 */
1831 if ((first_frag == NULL && !(p->env & LDC_START)) ||
1832 (first_frag != NULL && (p->env & LDC_START))) {
1833 if (!first_frag)
1834 new = rx_advance(lp, new);
1835
1836 err = rx_set_head(lp, new);
1837 if (err)
1838 break;
1839
1840 if (!first_frag)
1841 goto no_data;
1842 }
1843 if (!first_frag)
1844 first_frag = p;
1845
1846 if (pkt_len > size - copied) {
1847 /* User didn't give us a big enough buffer,
1848 * what to do? This is a pretty serious error.
1849 *
1850 * Since we haven't updated the RX ring head to
1851 * consume any of the packets, signal the error
1852 * to the user and just leave the RX ring alone.
1853 *
1854 * This seems the best behavior because this allows
1855 * a user of the LDC layer to start with a small
1856 * RX buffer for ldc_read() calls and use -EMSGSIZE
1857 * as a cue to enlarge its read buffer.
1858 */
1859 err = -EMSGSIZE;
1860 break;
1861 }
1862
1863 /* Ok, we are gonna eat this one. */
1864 new = rx_advance(lp, new);
1865
1866 memcpy(buf,
1867 (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
1868 p->u.u_data : p->u.r.r_data), pkt_len);
1869 buf += pkt_len;
1870 copied += pkt_len;
1871
1872 if (p->env & LDC_STOP)
1873 break;
1874
1875 no_data:
1876 if (new == lp->rx_tail) {
1877 err = rx_data_wait(lp, new);
1878 if (err)
1879 break;
1880 }
1881 }
1882
1883 if (!err)
1884 err = rx_set_head(lp, new);
1885
1886 if (err && first_frag)
1887 lp->rcv_nxt = first_frag->seqid - 1;
1888
1889 if (!err) {
1890 err = copied;
1891 if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
1892 send_data_ack(lp);
1893 }
1894
1895 return err;
1896 }
1897
1898 static const struct ldc_mode_ops nonraw_ops = {
1899 .write = write_nonraw,
1900 .read = read_nonraw,
1901 };
1902
1903 static int write_stream(struct ldc_channel *lp, const void *buf,
1904 unsigned int size)
1905 {
1906 if (size > lp->cfg.mtu)
1907 size = lp->cfg.mtu;
1908 return write_nonraw(lp, buf, size);
1909 }
1910
1911 static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
1912 {
1913 if (!lp->mssbuf_len) {
1914 int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
1915 if (err < 0)
1916 return err;
1917
1918 lp->mssbuf_len = err;
1919 lp->mssbuf_off = 0;
1920 }
1921
1922 if (size > lp->mssbuf_len)
1923 size = lp->mssbuf_len;
1924 memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);
1925
1926 lp->mssbuf_off += size;
1927 lp->mssbuf_len -= size;
1928
1929 return size;
1930 }
1931
1932 static const struct ldc_mode_ops stream_ops = {
1933 .write = write_stream,
1934 .read = read_stream,
1935 };
1936
1937 int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
1938 {
1939 unsigned long flags;
1940 int err;
1941
1942 if (!buf)
1943 return -EINVAL;
1944
1945 if (!size)
1946 return 0;
1947
1948 spin_lock_irqsave(&lp->lock, flags);
1949
1950 if (lp->hs_state != LDC_HS_COMPLETE)
1951 err = -ENOTCONN;
1952 else
1953 err = lp->mops->write(lp, buf, size);
1954
1955 spin_unlock_irqrestore(&lp->lock, flags);
1956
1957 return err;
1958 }
1959 EXPORT_SYMBOL(ldc_write);
1960
1961 int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
1962 {
1963 unsigned long flags;
1964 int err;
1965
1966 ldcdbg(RX, "%s: entered size=%d\n", __func__, size);
1967
1968 if (!buf)
1969 return -EINVAL;
1970
1971 if (!size)
1972 return 0;
1973
1974 spin_lock_irqsave(&lp->lock, flags);
1975
1976 if (lp->hs_state != LDC_HS_COMPLETE)
1977 err = -ENOTCONN;
1978 else
1979 err = lp->mops->read(lp, buf, size);
1980
1981 spin_unlock_irqrestore(&lp->lock, flags);
1982
1983 ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
1984 lp->cfg.mode, lp->rx_head, lp->rx_tail, err);
1985
1986 return err;
1987 }
1988 EXPORT_SYMBOL(ldc_read);
1989
1990 static u64 pagesize_code(void)
1991 {
1992 switch (PAGE_SIZE) {
1993 default:
1994 case (8ULL * 1024ULL):
1995 return 0;
1996 case (64ULL * 1024ULL):
1997 return 1;
1998 case (512ULL * 1024ULL):
1999 return 2;
2000 case (4ULL * 1024ULL * 1024ULL):
2001 return 3;
2002 case (32ULL * 1024ULL * 1024ULL):
2003 return 4;
2004 case (256ULL * 1024ULL * 1024ULL):
2005 return 5;
2006 }
2007 }
2008
2009 static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
2010 {
2011 return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
2012 (index << PAGE_SHIFT) |
2013 page_offset);
2014 }
2015
2016
2017 static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
2018 unsigned long npages)
2019 {
2020 long entry;
2021
2022 entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
2023 npages, NULL, (unsigned long)-1, 0);
2024 if (unlikely(entry == IOMMU_ERROR_CODE))
2025 return NULL;
2026
2027 return iommu->page_table + entry;
2028 }
2029
2030 static u64 perm_to_mte(unsigned int map_perm)
2031 {
2032 u64 mte_base;
2033
2034 mte_base = pagesize_code();
2035
2036 if (map_perm & LDC_MAP_SHADOW) {
2037 if (map_perm & LDC_MAP_R)
2038 mte_base |= LDC_MTE_COPY_R;
2039 if (map_perm & LDC_MAP_W)
2040 mte_base |= LDC_MTE_COPY_W;
2041 }
2042 if (map_perm & LDC_MAP_DIRECT) {
2043 if (map_perm & LDC_MAP_R)
2044 mte_base |= LDC_MTE_READ;
2045 if (map_perm & LDC_MAP_W)
2046 mte_base |= LDC_MTE_WRITE;
2047 if (map_perm & LDC_MAP_X)
2048 mte_base |= LDC_MTE_EXEC;
2049 }
2050 if (map_perm & LDC_MAP_IO) {
2051 if (map_perm & LDC_MAP_R)
2052 mte_base |= LDC_MTE_IOMMU_R;
2053 if (map_perm & LDC_MAP_W)
2054 mte_base |= LDC_MTE_IOMMU_W;
2055 }
2056
2057 return mte_base;
2058 }
2059
2060 static int pages_in_region(unsigned long base, long len)
2061 {
2062 int count = 0;
2063
2064 do {
2065 unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;
2066
2067 len -= (new - base);
2068 base = new;
2069 count++;
2070 } while (len > 0);
2071
2072 return count;
2073 }
2074
2075 struct cookie_state {
2076 struct ldc_mtable_entry *page_table;
2077 struct ldc_trans_cookie *cookies;
2078 u64 mte_base;
2079 u64 prev_cookie;
2080 u32 pte_idx;
2081 u32 nc;
2082 };
2083
2084 static void fill_cookies(struct cookie_state *sp, unsigned long pa,
2085 unsigned long off, unsigned long len)
2086 {
2087 do {
2088 unsigned long tlen, new = pa + PAGE_SIZE;
2089 u64 this_cookie;
2090
2091 sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;
2092
2093 tlen = PAGE_SIZE;
2094 if (off)
2095 tlen = PAGE_SIZE - off;
2096 if (tlen > len)
2097 tlen = len;
2098
2099 this_cookie = make_cookie(sp->pte_idx,
2100 pagesize_code(), off);
2101
2102 off = 0;
2103
2104 if (this_cookie == sp->prev_cookie) {
2105 sp->cookies[sp->nc - 1].cookie_size += tlen;
2106 } else {
2107 sp->cookies[sp->nc].cookie_addr = this_cookie;
2108 sp->cookies[sp->nc].cookie_size = tlen;
2109 sp->nc++;
2110 }
2111 sp->prev_cookie = this_cookie + tlen;
2112
2113 sp->pte_idx++;
2114
2115 len -= tlen;
2116 pa = new;
2117 } while (len > 0);
2118 }
2119
2120 static int sg_count_one(struct scatterlist *sg)
2121 {
2122 unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
2123 long len = sg->length;
2124
2125 if ((sg->offset | len) & (8UL - 1))
2126 return -EFAULT;
2127
2128 return pages_in_region(base + sg->offset, len);
2129 }
2130
2131 static int sg_count_pages(struct scatterlist *sg, int num_sg)
2132 {
2133 int count;
2134 int i;
2135
2136 count = 0;
2137 for (i = 0; i < num_sg; i++) {
2138 int err = sg_count_one(sg + i);
2139 if (err < 0)
2140 return err;
2141 count += err;
2142 }
2143
2144 return count;
2145 }
2146
2147 int ldc_map_sg(struct ldc_channel *lp,
2148 struct scatterlist *sg, int num_sg,
2149 struct ldc_trans_cookie *cookies, int ncookies,
2150 unsigned int map_perm)
2151 {
2152 unsigned long i, npages;
2153 struct ldc_mtable_entry *base;
2154 struct cookie_state state;
2155 struct ldc_iommu *iommu;
2156 int err;
2157 struct scatterlist *s;
2158
2159 if (map_perm & ~LDC_MAP_ALL)
2160 return -EINVAL;
2161
2162 err = sg_count_pages(sg, num_sg);
2163 if (err < 0)
2164 return err;
2165
2166 npages = err;
2167 if (err > ncookies)
2168 return -EMSGSIZE;
2169
2170 iommu = &lp->iommu;
2171
2172 base = alloc_npages(iommu, npages);
2173
2174 if (!base)
2175 return -ENOMEM;
2176
2177 state.page_table = iommu->page_table;
2178 state.cookies = cookies;
2179 state.mte_base = perm_to_mte(map_perm);
2180 state.prev_cookie = ~(u64)0;
2181 state.pte_idx = (base - iommu->page_table);
2182 state.nc = 0;
2183
2184 for_each_sg(sg, s, num_sg, i) {
2185 fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
2186 s->offset, s->length);
2187 }
2188
2189 return state.nc;
2190 }
2191 EXPORT_SYMBOL(ldc_map_sg);
2192
2193 int ldc_map_single(struct ldc_channel *lp,
2194 void *buf, unsigned int len,
2195 struct ldc_trans_cookie *cookies, int ncookies,
2196 unsigned int map_perm)
2197 {
2198 unsigned long npages, pa;
2199 struct ldc_mtable_entry *base;
2200 struct cookie_state state;
2201 struct ldc_iommu *iommu;
2202
2203 if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
2204 return -EINVAL;
2205
2206 pa = __pa(buf);
2207 if ((pa | len) & (8UL - 1))
2208 return -EFAULT;
2209
2210 npages = pages_in_region(pa, len);
2211
2212 iommu = &lp->iommu;
2213
2214 base = alloc_npages(iommu, npages);
2215
2216 if (!base)
2217 return -ENOMEM;
2218
2219 state.page_table = iommu->page_table;
2220 state.cookies = cookies;
2221 state.mte_base = perm_to_mte(map_perm);
2222 state.prev_cookie = ~(u64)0;
2223 state.pte_idx = (base - iommu->page_table);
2224 state.nc = 0;
2225 fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
2226 BUG_ON(state.nc > ncookies);
2227
2228 return state.nc;
2229 }
2230 EXPORT_SYMBOL(ldc_map_single);
2231
2232
2233 static void free_npages(unsigned long id, struct ldc_iommu *iommu,
2234 u64 cookie, u64 size)
2235 {
2236 unsigned long npages, entry;
2237
2238 npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
2239
2240 entry = ldc_cookie_to_index(cookie, iommu);
2241 ldc_demap(iommu, id, cookie, entry, npages);
2242 iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
2243 }
2244
2245 void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
2246 int ncookies)
2247 {
2248 struct ldc_iommu *iommu = &lp->iommu;
2249 int i;
2250 unsigned long flags;
2251
2252 spin_lock_irqsave(&iommu->lock, flags);
2253 for (i = 0; i < ncookies; i++) {
2254 u64 addr = cookies[i].cookie_addr;
2255 u64 size = cookies[i].cookie_size;
2256
2257 free_npages(lp->id, iommu, addr, size);
2258 }
2259 spin_unlock_irqrestore(&iommu->lock, flags);
2260 }
2261 EXPORT_SYMBOL(ldc_unmap);
2262
2263 int ldc_copy(struct ldc_channel *lp, int copy_dir,
2264 void *buf, unsigned int len, unsigned long offset,
2265 struct ldc_trans_cookie *cookies, int ncookies)
2266 {
2267 unsigned int orig_len;
2268 unsigned long ra;
2269 int i;
2270
2271 if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
2272 printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
2273 lp->id, copy_dir);
2274 return -EINVAL;
2275 }
2276
2277 ra = __pa(buf);
2278 if ((ra | len | offset) & (8UL - 1)) {
2279 printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
2280 "ra[%lx] len[%x] offset[%lx]\n",
2281 lp->id, ra, len, offset);
2282 return -EFAULT;
2283 }
2284
2285 if (lp->hs_state != LDC_HS_COMPLETE ||
2286 (lp->flags & LDC_FLAG_RESET)) {
2287 printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
2288 "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
2289 return -ECONNRESET;
2290 }
2291
2292 orig_len = len;
2293 for (i = 0; i < ncookies; i++) {
2294 unsigned long cookie_raddr = cookies[i].cookie_addr;
2295 unsigned long this_len = cookies[i].cookie_size;
2296 unsigned long actual_len;
2297
2298 if (unlikely(offset)) {
2299 unsigned long this_off = offset;
2300
2301 if (this_off > this_len)
2302 this_off = this_len;
2303
2304 offset -= this_off;
2305 this_len -= this_off;
2306 if (!this_len)
2307 continue;
2308 cookie_raddr += this_off;
2309 }
2310
2311 if (this_len > len)
2312 this_len = len;
2313
2314 while (1) {
2315 unsigned long hv_err;
2316
2317 hv_err = sun4v_ldc_copy(lp->id, copy_dir,
2318 cookie_raddr, ra,
2319 this_len, &actual_len);
2320 if (unlikely(hv_err)) {
2321 printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
2322 "HV error %lu\n",
2323 lp->id, hv_err);
2324 if (lp->hs_state != LDC_HS_COMPLETE ||
2325 (lp->flags & LDC_FLAG_RESET))
2326 return -ECONNRESET;
2327 else
2328 return -EFAULT;
2329 }
2330
2331 cookie_raddr += actual_len;
2332 ra += actual_len;
2333 len -= actual_len;
2334 if (actual_len == this_len)
2335 break;
2336
2337 this_len -= actual_len;
2338 }
2339
2340 if (!len)
2341 break;
2342 }
2343
2344 /* It is caller policy what to do about short copies.
2345 * For example, a networking driver can declare the
2346 * packet a runt and drop it.
2347 */
2348
2349 return orig_len - len;
2350 }
2351 EXPORT_SYMBOL(ldc_copy);
2352
2353 void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
2354 struct ldc_trans_cookie *cookies, int *ncookies,
2355 unsigned int map_perm)
2356 {
2357 void *buf;
2358 int err;
2359
2360 if (len & (8UL - 1))
2361 return ERR_PTR(-EINVAL);
2362
2363 buf = kzalloc(len, GFP_ATOMIC);
2364 if (!buf)
2365 return ERR_PTR(-ENOMEM);
2366
2367 err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
2368 if (err < 0) {
2369 kfree(buf);
2370 return ERR_PTR(err);
2371 }
2372 *ncookies = err;
2373
2374 return buf;
2375 }
2376 EXPORT_SYMBOL(ldc_alloc_exp_dring);
2377
2378 void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
2379 struct ldc_trans_cookie *cookies, int ncookies)
2380 {
2381 ldc_unmap(lp, cookies, ncookies);
2382 kfree(buf);
2383 }
2384 EXPORT_SYMBOL(ldc_free_exp_dring);
2385
2386 static int __init ldc_init(void)
2387 {
2388 unsigned long major, minor;
2389 struct mdesc_handle *hp;
2390 const u64 *v;
2391 int err;
2392 u64 mp;
2393
2394 hp = mdesc_grab();
2395 if (!hp)
2396 return -ENODEV;
2397
2398 mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
2399 err = -ENODEV;
2400 if (mp == MDESC_NODE_NULL)
2401 goto out;
2402
2403 v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
2404 if (!v)
2405 goto out;
2406
2407 major = 1;
2408 minor = 0;
2409 if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
2410 printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
2411 goto out;
2412 }
2413
2414 printk(KERN_INFO "%s", version);
2415
2416 if (!*v) {
2417 printk(KERN_INFO PFX "Domaining disabled.\n");
2418 goto out;
2419 }
2420 ldom_domaining_enabled = 1;
2421 err = 0;
2422
2423 out:
2424 mdesc_release(hp);
2425 return err;
2426 }
2427
2428 core_initcall(ldc_init);
2429
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.