1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * linux/arch/alpha/kernel/smp.c
4 *
5 * 2001-07-09 Phil Ezolt (Phillip.Ezolt@compaq.com)
6 * Renamed modified smp_call_function to smp_call_function_on_cpu()
7 * Created an function that conforms to the old calling convention
8 * of smp_call_function().
9 *
10 * This is helpful for DCPI.
11 *
12 */
13
14 #include <linux/errno.h>
15 #include <linux/kernel.h>
16 #include <linux/kernel_stat.h>
17 #include <linux/module.h>
18 #include <linux/sched/mm.h>
19 #include <linux/mm.h>
20 #include <linux/err.h>
21 #include <linux/threads.h>
22 #include <linux/smp.h>
23 #include <linux/interrupt.h>
24 #include <linux/init.h>
25 #include <linux/delay.h>
26 #include <linux/spinlock.h>
27 #include <linux/irq.h>
28 #include <linux/cache.h>
29 #include <linux/profile.h>
30 #include <linux/bitops.h>
31 #include <linux/cpu.h>
32
33 #include <asm/hwrpb.h>
34 #include <asm/ptrace.h>
35 #include <linux/atomic.h>
36
37 #include <asm/io.h>
38 #include <asm/irq.h>
39 #include <asm/mmu_context.h>
40 #include <asm/tlbflush.h>
41 #include <asm/cacheflush.h>
42
43 #include "proto.h"
44 #include "irq_impl.h"
45
46
47 #define DEBUG_SMP 0
48 #if DEBUG_SMP
49 #define DBGS(args) printk args
50 #else
51 #define DBGS(args)
52 #endif
53
54 /* A collection of per-processor data. */
55 struct cpuinfo_alpha cpu_data[NR_CPUS];
56 EXPORT_SYMBOL(cpu_data);
57
58 /* A collection of single bit ipi messages. */
59 static struct {
60 unsigned long bits ____cacheline_aligned;
61 } ipi_data[NR_CPUS] __cacheline_aligned;
62
63 enum ipi_message_type {
64 IPI_RESCHEDULE,
65 IPI_CALL_FUNC,
66 IPI_CPU_STOP,
67 };
68
69 /* Set to a secondary's cpuid when it comes online. */
70 static int smp_secondary_alive = 0;
71
72 int smp_num_probed; /* Internal processor count */
73 int smp_num_cpus = 1; /* Number that came online. */
74 EXPORT_SYMBOL(smp_num_cpus);
75
76 /*
77 * Called by both boot and secondaries to move global data into
78 * per-processor storage.
79 */
80 static inline void __init
81 smp_store_cpu_info(int cpuid)
82 {
83 cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
84 cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
85 cpu_data[cpuid].need_new_asn = 0;
86 cpu_data[cpuid].asn_lock = 0;
87 }
88
89 /*
90 * Ideally sets up per-cpu profiling hooks. Doesn't do much now...
91 */
92 static inline void __init
93 smp_setup_percpu_timer(int cpuid)
94 {
95 cpu_data[cpuid].prof_counter = 1;
96 cpu_data[cpuid].prof_multiplier = 1;
97 }
98
99 static void __init
100 wait_boot_cpu_to_stop(int cpuid)
101 {
102 unsigned long stop = jiffies + 10*HZ;
103
104 while (time_before(jiffies, stop)) {
105 if (!smp_secondary_alive)
106 return;
107 barrier();
108 }
109
110 printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
111 for (;;)
112 barrier();
113 }
114
115 /*
116 * Where secondaries begin a life of C.
117 */
118 void __init
119 smp_callin(void)
120 {
121 int cpuid = hard_smp_processor_id();
122
123 if (cpu_online(cpuid)) {
124 printk("??, cpu 0x%x already present??\n", cpuid);
125 BUG();
126 }
127 set_cpu_online(cpuid, true);
128
129 /* Turn on machine checks. */
130 wrmces(7);
131
132 /* Set trap vectors. */
133 trap_init();
134
135 /* Set interrupt vector. */
136 wrent(entInt, 0);
137
138 /* Get our local ticker going. */
139 smp_setup_percpu_timer(cpuid);
140 init_clockevent();
141
142 /* Call platform-specific callin, if specified */
143 if (alpha_mv.smp_callin)
144 alpha_mv.smp_callin();
145
146 /* All kernel threads share the same mm context. */
147 mmgrab(&init_mm);
148 current->active_mm = &init_mm;
149
150 /* inform the notifiers about the new cpu */
151 notify_cpu_starting(cpuid);
152
153 /* Must have completely accurate bogos. */
154 local_irq_enable();
155
156 /* Wait boot CPU to stop with irq enabled before running
157 calibrate_delay. */
158 wait_boot_cpu_to_stop(cpuid);
159 mb();
160 calibrate_delay();
161
162 smp_store_cpu_info(cpuid);
163 /* Allow master to continue only after we written loops_per_jiffy. */
164 wmb();
165 smp_secondary_alive = 1;
166
167 DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
168 cpuid, current, current->active_mm));
169
170 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
171 }
172
173 /* Wait until hwrpb->txrdy is clear for cpu. Return -1 on timeout. */
174 static int
175 wait_for_txrdy (unsigned long cpumask)
176 {
177 unsigned long timeout;
178
179 if (!(hwrpb->txrdy & cpumask))
180 return 0;
181
182 timeout = jiffies + 10*HZ;
183 while (time_before(jiffies, timeout)) {
184 if (!(hwrpb->txrdy & cpumask))
185 return 0;
186 udelay(10);
187 barrier();
188 }
189
190 return -1;
191 }
192
193 /*
194 * Send a message to a secondary's console. "START" is one such
195 * interesting message. ;-)
196 */
197 static void
198 send_secondary_console_msg(char *str, int cpuid)
199 {
200 struct percpu_struct *cpu;
201 register char *cp1, *cp2;
202 unsigned long cpumask;
203 size_t len;
204
205 cpu = (struct percpu_struct *)
206 ((char*)hwrpb
207 + hwrpb->processor_offset
208 + cpuid * hwrpb->processor_size);
209
210 cpumask = (1UL << cpuid);
211 if (wait_for_txrdy(cpumask))
212 goto timeout;
213
214 cp2 = str;
215 len = strlen(cp2);
216 *(unsigned int *)&cpu->ipc_buffer[0] = len;
217 cp1 = (char *) &cpu->ipc_buffer[1];
218 memcpy(cp1, cp2, len);
219
220 /* atomic test and set */
221 wmb();
222 set_bit(cpuid, &hwrpb->rxrdy);
223
224 if (wait_for_txrdy(cpumask))
225 goto timeout;
226 return;
227
228 timeout:
229 printk("Processor %x not ready\n", cpuid);
230 }
231
232 /*
233 * A secondary console wants to send a message. Receive it.
234 */
235 static void
236 recv_secondary_console_msg(void)
237 {
238 int mycpu, i, cnt;
239 unsigned long txrdy = hwrpb->txrdy;
240 char *cp1, *cp2, buf[80];
241 struct percpu_struct *cpu;
242
243 DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));
244
245 mycpu = hard_smp_processor_id();
246
247 for (i = 0; i < NR_CPUS; i++) {
248 if (!(txrdy & (1UL << i)))
249 continue;
250
251 DBGS(("recv_secondary_console_msg: "
252 "TXRDY contains CPU %d.\n", i));
253
254 cpu = (struct percpu_struct *)
255 ((char*)hwrpb
256 + hwrpb->processor_offset
257 + i * hwrpb->processor_size);
258
259 DBGS(("recv_secondary_console_msg: on %d from %d"
260 " HALT_REASON 0x%lx FLAGS 0x%lx\n",
261 mycpu, i, cpu->halt_reason, cpu->flags));
262
263 cnt = cpu->ipc_buffer[0] >> 32;
264 if (cnt <= 0 || cnt >= 80)
265 strcpy(buf, "<<< BOGUS MSG >>>");
266 else {
267 cp1 = (char *) &cpu->ipc_buffer[1];
268 cp2 = buf;
269 memcpy(cp2, cp1, cnt);
270 cp2[cnt] = '\0';
271
272 while ((cp2 = strchr(cp2, '\r')) != 0) {
273 *cp2 = ' ';
274 if (cp2[1] == '\n')
275 cp2[1] = ' ';
276 }
277 }
278
279 DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
280 "message is '%s'\n", mycpu, buf));
281 }
282
283 hwrpb->txrdy = 0;
284 }
285
286 /*
287 * Convince the console to have a secondary cpu begin execution.
288 */
289 static int
290 secondary_cpu_start(int cpuid, struct task_struct *idle)
291 {
292 struct percpu_struct *cpu;
293 struct pcb_struct *hwpcb, *ipcb;
294 unsigned long timeout;
295
296 cpu = (struct percpu_struct *)
297 ((char*)hwrpb
298 + hwrpb->processor_offset
299 + cpuid * hwrpb->processor_size);
300 hwpcb = (struct pcb_struct *) cpu->hwpcb;
301 ipcb = &task_thread_info(idle)->pcb;
302
303 /* Initialize the CPU's HWPCB to something just good enough for
304 us to get started. Immediately after starting, we'll swpctx
305 to the target idle task's pcb. Reuse the stack in the mean
306 time. Precalculate the target PCBB. */
307 hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
308 hwpcb->usp = 0;
309 hwpcb->ptbr = ipcb->ptbr;
310 hwpcb->pcc = 0;
311 hwpcb->asn = 0;
312 hwpcb->unique = virt_to_phys(ipcb);
313 hwpcb->flags = ipcb->flags;
314 hwpcb->res1 = hwpcb->res2 = 0;
315
316 #if 0
317 DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
318 hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
319 #endif
320 DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
321 cpuid, idle->state, ipcb->flags));
322
323 /* Setup HWRPB fields that SRM uses to activate secondary CPU */
324 hwrpb->CPU_restart = __smp_callin;
325 hwrpb->CPU_restart_data = (unsigned long) __smp_callin;
326
327 /* Recalculate and update the HWRPB checksum */
328 hwrpb_update_checksum(hwrpb);
329
330 /*
331 * Send a "start" command to the specified processor.
332 */
333
334 /* SRM III 3.4.1.3 */
335 cpu->flags |= 0x22; /* turn on Context Valid and Restart Capable */
336 cpu->flags &= ~1; /* turn off Bootstrap In Progress */
337 wmb();
338
339 send_secondary_console_msg("START\r\n", cpuid);
340
341 /* Wait 10 seconds for an ACK from the console. */
342 timeout = jiffies + 10*HZ;
343 while (time_before(jiffies, timeout)) {
344 if (cpu->flags & 1)
345 goto started;
346 udelay(10);
347 barrier();
348 }
349 printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
350 return -1;
351
352 started:
353 DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
354 return 0;
355 }
356
357 /*
358 * Bring one cpu online.
359 */
360 static int
361 smp_boot_one_cpu(int cpuid, struct task_struct *idle)
362 {
363 unsigned long timeout;
364
365 /* Signal the secondary to wait a moment. */
366 smp_secondary_alive = -1;
367
368 /* Whirrr, whirrr, whirrrrrrrrr... */
369 if (secondary_cpu_start(cpuid, idle))
370 return -1;
371
372 /* Notify the secondary CPU it can run calibrate_delay. */
373 mb();
374 smp_secondary_alive = 0;
375
376 /* We've been acked by the console; wait one second for
377 the task to start up for real. */
378 timeout = jiffies + 1*HZ;
379 while (time_before(jiffies, timeout)) {
380 if (smp_secondary_alive == 1)
381 goto alive;
382 udelay(10);
383 barrier();
384 }
385
386 /* We failed to boot the CPU. */
387
388 printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
389 return -1;
390
391 alive:
392 /* Another "Red Snapper". */
393 return 0;
394 }
395
396 /*
397 * Called from setup_arch. Detect an SMP system and which processors
398 * are present.
399 */
400 void __init
401 setup_smp(void)
402 {
403 struct percpu_struct *cpubase, *cpu;
404 unsigned long i;
405
406 if (boot_cpuid != 0) {
407 printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
408 boot_cpuid);
409 }
410
411 if (hwrpb->nr_processors > 1) {
412 int boot_cpu_palrev;
413
414 DBGS(("setup_smp: nr_processors %ld\n",
415 hwrpb->nr_processors));
416
417 cpubase = (struct percpu_struct *)
418 ((char*)hwrpb + hwrpb->processor_offset);
419 boot_cpu_palrev = cpubase->pal_revision;
420
421 for (i = 0; i < hwrpb->nr_processors; i++) {
422 cpu = (struct percpu_struct *)
423 ((char *)cpubase + i*hwrpb->processor_size);
424 if ((cpu->flags & 0x1cc) == 0x1cc) {
425 smp_num_probed++;
426 set_cpu_possible(i, true);
427 set_cpu_present(i, true);
428 cpu->pal_revision = boot_cpu_palrev;
429 }
430
431 DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
432 i, cpu->flags, cpu->type));
433 DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
434 i, cpu->pal_revision));
435 }
436 } else {
437 smp_num_probed = 1;
438 }
439
440 printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n",
441 smp_num_probed, cpumask_bits(cpu_present_mask)[0]);
442 }
443
444 /*
445 * Called by smp_init prepare the secondaries
446 */
447 void __init
448 smp_prepare_cpus(unsigned int max_cpus)
449 {
450 /* Take care of some initial bookkeeping. */
451 memset(ipi_data, 0, sizeof(ipi_data));
452
453 current_thread_info()->cpu = boot_cpuid;
454
455 smp_store_cpu_info(boot_cpuid);
456 smp_setup_percpu_timer(boot_cpuid);
457
458 /* Nothing to do on a UP box, or when told not to. */
459 if (smp_num_probed == 1 || max_cpus == 0) {
460 init_cpu_possible(cpumask_of(boot_cpuid));
461 init_cpu_present(cpumask_of(boot_cpuid));
462 printk(KERN_INFO "SMP mode deactivated.\n");
463 return;
464 }
465
466 printk(KERN_INFO "SMP starting up secondaries.\n");
467
468 smp_num_cpus = smp_num_probed;
469 }
470
471 int
472 __cpu_up(unsigned int cpu, struct task_struct *tidle)
473 {
474 smp_boot_one_cpu(cpu, tidle);
475
476 return cpu_online(cpu) ? 0 : -ENOSYS;
477 }
478
479 void __init
480 smp_cpus_done(unsigned int max_cpus)
481 {
482 int cpu;
483 unsigned long bogosum = 0;
484
485 for(cpu = 0; cpu < NR_CPUS; cpu++)
486 if (cpu_online(cpu))
487 bogosum += cpu_data[cpu].loops_per_jiffy;
488
489 printk(KERN_INFO "SMP: Total of %d processors activated "
490 "(%lu.%02lu BogoMIPS).\n",
491 num_online_cpus(),
492 (bogosum + 2500) / (500000/HZ),
493 ((bogosum + 2500) / (5000/HZ)) % 100);
494 }
495
496 static void
497 send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
498 {
499 int i;
500
501 mb();
502 for_each_cpu(i, to_whom)
503 set_bit(operation, &ipi_data[i].bits);
504
505 mb();
506 for_each_cpu(i, to_whom)
507 wripir(i);
508 }
509
510 void
511 handle_ipi(struct pt_regs *regs)
512 {
513 int this_cpu = smp_processor_id();
514 unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
515 unsigned long ops;
516
517 #if 0
518 DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
519 this_cpu, *pending_ipis, regs->pc));
520 #endif
521
522 mb(); /* Order interrupt and bit testing. */
523 while ((ops = xchg(pending_ipis, 0)) != 0) {
524 mb(); /* Order bit clearing and data access. */
525 do {
526 unsigned long which;
527
528 which = ops & -ops;
529 ops &= ~which;
530 which = __ffs(which);
531
532 switch (which) {
533 case IPI_RESCHEDULE:
534 scheduler_ipi();
535 break;
536
537 case IPI_CALL_FUNC:
538 generic_smp_call_function_interrupt();
539 break;
540
541 case IPI_CPU_STOP:
542 halt();
543
544 default:
545 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
546 this_cpu, which);
547 break;
548 }
549 } while (ops);
550
551 mb(); /* Order data access and bit testing. */
552 }
553
554 cpu_data[this_cpu].ipi_count++;
555
556 if (hwrpb->txrdy)
557 recv_secondary_console_msg();
558 }
559
560 void
561 arch_smp_send_reschedule(int cpu)
562 {
563 #ifdef DEBUG_IPI_MSG
564 if (cpu == hard_smp_processor_id())
565 printk(KERN_WARNING
566 "smp_send_reschedule: Sending IPI to self.\n");
567 #endif
568 send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
569 }
570
571 void
572 smp_send_stop(void)
573 {
574 cpumask_t to_whom;
575 cpumask_copy(&to_whom, cpu_online_mask);
576 cpumask_clear_cpu(smp_processor_id(), &to_whom);
577 #ifdef DEBUG_IPI_MSG
578 if (hard_smp_processor_id() != boot_cpu_id)
579 printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
580 #endif
581 send_ipi_message(&to_whom, IPI_CPU_STOP);
582 }
583
584 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
585 {
586 send_ipi_message(mask, IPI_CALL_FUNC);
587 }
588
589 void arch_send_call_function_single_ipi(int cpu)
590 {
591 send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
592 }
593
594 static void
595 ipi_imb(void *ignored)
596 {
597 imb();
598 }
599
600 void
601 smp_imb(void)
602 {
603 /* Must wait other processors to flush their icache before continue. */
604 on_each_cpu(ipi_imb, NULL, 1);
605 }
606 EXPORT_SYMBOL(smp_imb);
607
608 static void
609 ipi_flush_tlb_all(void *ignored)
610 {
611 tbia();
612 }
613
614 void
615 flush_tlb_all(void)
616 {
617 /* Although we don't have any data to pass, we do want to
618 synchronize with the other processors. */
619 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
620 }
621
622 #define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
623
624 static void
625 ipi_flush_tlb_mm(void *x)
626 {
627 struct mm_struct *mm = x;
628 if (mm == current->active_mm && !asn_locked())
629 flush_tlb_current(mm);
630 else
631 flush_tlb_other(mm);
632 }
633
634 void
635 flush_tlb_mm(struct mm_struct *mm)
636 {
637 preempt_disable();
638
639 if (mm == current->active_mm) {
640 flush_tlb_current(mm);
641 if (atomic_read(&mm->mm_users) <= 1) {
642 int cpu, this_cpu = smp_processor_id();
643 for (cpu = 0; cpu < NR_CPUS; cpu++) {
644 if (!cpu_online(cpu) || cpu == this_cpu)
645 continue;
646 if (mm->context[cpu])
647 mm->context[cpu] = 0;
648 }
649 preempt_enable();
650 return;
651 }
652 }
653
654 smp_call_function(ipi_flush_tlb_mm, mm, 1);
655
656 preempt_enable();
657 }
658 EXPORT_SYMBOL(flush_tlb_mm);
659
660 struct flush_tlb_page_struct {
661 struct vm_area_struct *vma;
662 struct mm_struct *mm;
663 unsigned long addr;
664 };
665
666 static void
667 ipi_flush_tlb_page(void *x)
668 {
669 struct flush_tlb_page_struct *data = x;
670 struct mm_struct * mm = data->mm;
671
672 if (mm == current->active_mm && !asn_locked())
673 flush_tlb_current_page(mm, data->vma, data->addr);
674 else
675 flush_tlb_other(mm);
676 }
677
678 void
679 flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
680 {
681 struct flush_tlb_page_struct data;
682 struct mm_struct *mm = vma->vm_mm;
683
684 preempt_disable();
685
686 if (mm == current->active_mm) {
687 flush_tlb_current_page(mm, vma, addr);
688 if (atomic_read(&mm->mm_users) <= 1) {
689 int cpu, this_cpu = smp_processor_id();
690 for (cpu = 0; cpu < NR_CPUS; cpu++) {
691 if (!cpu_online(cpu) || cpu == this_cpu)
692 continue;
693 if (mm->context[cpu])
694 mm->context[cpu] = 0;
695 }
696 preempt_enable();
697 return;
698 }
699 }
700
701 data.vma = vma;
702 data.mm = mm;
703 data.addr = addr;
704
705 smp_call_function(ipi_flush_tlb_page, &data, 1);
706
707 preempt_enable();
708 }
709 EXPORT_SYMBOL(flush_tlb_page);
710
711 void
712 flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
713 {
714 /* On the Alpha we always flush the whole user tlb. */
715 flush_tlb_mm(vma->vm_mm);
716 }
717 EXPORT_SYMBOL(flush_tlb_range);
718
719 static void
720 ipi_flush_icache_page(void *x)
721 {
722 struct mm_struct *mm = (struct mm_struct *) x;
723 if (mm == current->active_mm && !asn_locked())
724 __load_new_mm_context(mm);
725 else
726 flush_tlb_other(mm);
727 }
728
729 void
730 flush_icache_user_page(struct vm_area_struct *vma, struct page *page,
731 unsigned long addr, int len)
732 {
733 struct mm_struct *mm = vma->vm_mm;
734
735 if ((vma->vm_flags & VM_EXEC) == 0)
736 return;
737
738 preempt_disable();
739
740 if (mm == current->active_mm) {
741 __load_new_mm_context(mm);
742 if (atomic_read(&mm->mm_users) <= 1) {
743 int cpu, this_cpu = smp_processor_id();
744 for (cpu = 0; cpu < NR_CPUS; cpu++) {
745 if (!cpu_online(cpu) || cpu == this_cpu)
746 continue;
747 if (mm->context[cpu])
748 mm->context[cpu] = 0;
749 }
750 preempt_enable();
751 return;
752 }
753 }
754
755 smp_call_function(ipi_flush_icache_page, mm, 1);
756
757 preempt_enable();
758 }
759
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.