在Linux系統中，對於多核的ARM芯片而言，Bootrom代碼中，CPU0會率先起來，引導Bootloader和Linux內核執行，而其他的核則在上電時Bootrom一般將自身置於WFI或者WFE狀態，並等待CPU0給其發CPU核間中斷（IPI）或事件（一般透過SEV指令）喚醒之。一個典型的啟動過程如下圖：
被CPU0喚醒的CPUn可以在運行過程中進行熱插拔。譬如運行如下命令即可卸載CPU1並且將CPU1上的任務全部遷移到其他CPU：
# echo 0 > /sys/devices/system/cpu/cpu1/online
同樣地，運行如下命令可以再次啟動CPU1:
# echo 1 > /sys/devices/system/cpu/cpu1/online
之後CPU1會主動參與系統中各個CPU之間要運行任務的負載均衡工作。
CPU0喚醒其他 CPU的動作在內核中被封裝為一個smp_operations的結構體，該結構體的成員如下：
83struct smp_operations {
84#ifdef CONFIG_SMP
85 /*
86 * Setup the set of possible CPUs (via set_cpu_possible)
87 */
88 void (*smp_init_cpus)(void);
89 /*
90 * Initialize cpu_possible map, and enable coherency
91 */
92 void (*smp_prepare_cpus)(unsigned int max_cpus);
93
94 /*
95 * Perform platform specific initialisation of the specified CPU.
96 */
97 void (*smp_secondary_init)(unsigned int cpu);
98 /*
99 * Boot a secondary CPU, and assign it the specified idle task.
100 * This also gives us the initial stack to use for this CPU.
101 */
102 int (*smp_boot_secondary)(unsigned int cpu, struct task_struct *idle);
103#ifdef CONFIG_HOTPLUG_CPU
104 int (*cpu_kill)(unsigned int cpu);
105 void (*cpu_die)(unsigned int cpu);
106 int (*cpu_disable)(unsigned int cpu);
107#endif
108#endif
109};
我們從arch/arm/mach-vexpress/v2m.c看到VEXPRESS電路板用到的smp_ops為vexpress_smp_ops：
666DT_MACHINE_START(VEXPRESS_DT, "ARM-Versatile Express")
667 .dt_compat = v2m_dt_match,
668 .smp = smp_ops(vexpress_smp_ops),
669 .map_io = v2m_dt_map_io,
670 .init_early = v2m_dt_init_early,
671 .init_irq = v2m_dt_init_irq,
672 .timer = &v2m_dt_timer,
673 .init_machine = v2m_dt_init,
674 .handle_irq = gic_handle_irq,
675 .restart = v2m_restart,
676MACHINE_END
透過arch/arm/mach-vexpress/platsmp.c的實現代碼可以看出，smp_operations的成員函數smp_init_cpus() 即vexpress_smp_init_cpus()會探測SoC內CPU核的個數，並設置了核間通信的方式為gic_raise_softirq()。可見於vexpress_smp_init_cpus()中調用的vexpress_dt_smp_init_cpus()：
103static void __init vexpress_dt_smp_init_cpus(void)
104{
…
128 for (i = 0; i < ncores; ++i)
129 set_cpu_possible(i, true);
130
131 set_smp_cross_call(gic_raise_softirq);
132}
而smp_operations的成員函數smp_prepare_cpus()即vexpress_smp_prepare_cpus()則會透過v2m_flags_set(virt_to_phys(versatile_secondary_startup))設置其他CPU的啟動地址為versatile_secondary_startup：
179static void __init vexpress_smp_prepare_cpus(unsigned int max_cpus)
180{
181 …
189
190 /*
191 * Write the address of secondary startup into the
192 * system-wide flags register. The boot monitor waits
193 * until it receives a soft interrupt, and then the
194 * secondary CPU branches to this address.
195 */
196 v2m_flags_set(virt_to_phys(versatile_secondary_startup));
197}
注意這部分的具體實現方法是SoC相關的，由芯片的設計以及芯片內部的Bootrom決定。對於VEXPRESS來講，設置方法如下：
139void __init v2m_flags_set(u32 data)
140{
141 writel(~0, v2m_sysreg_base + V2M_SYS_FLAGSCLR);
142 writel(data, v2m_sysreg_base + V2M_SYS_FLAGSSET);
143}
即填充v2m_sysreg_base + V2M_SYS_FLAGSCLR地址為0xFFFFFFFF,將其他CPU初始啟動執行的指令地址填入v2m_sysreg_base + V2M_SYS_FLAGSSET。這2個地址屬於芯片實現時候設定的。填入的CPUn的起始地址都透過virt_to_phys()轉化為物理地址，因為此時CPUn的MMU尚未開啟。
比較關鍵的是smp_operations的成員函數smp_boot_secondary()，它完成最終的CPUn的喚醒工作：
27static void __cpuinit write_pen_release(int val)
28{
29 pen_release = val;
30 smp_wmb();
31 __cpuc_flush_dcache_area((void *)&pen_release, sizeof(pen_release));
32 outer_clean_range(__pa(&pen_release), __pa(&pen_release + 1));
33}
59int __cpuinit versatile_boot_secondary(unsigned int cpu, struct task_struct *idle)
60{
61 unsigned long timeout;
62
63 /*
64 * Set synchronisation state between this boot processor
65 * and the secondary one
66 */
67 spin_lock(&boot_lock);
68
69 /*
70 * This is really belt and braces; we hold unintended secondary
71 * CPUs in the holding pen until we're ready for them. However,
72 * since we haven't sent them a soft interrupt, they shouldn't
73 * be there.
74 */
75 write_pen_release(cpu_logical_map(cpu));
76
77 /*
78 * Send the secondary CPU a soft interrupt, thereby causing
79 * the boot monitor to read the system wide flags register,
80 * and branch to the address found there.
81 */
82 gic_raise_softirq(cpumask_of(cpu), 0);
83
84 timeout = jiffies + (1 * HZ);
85 while (time_before(jiffies, timeout)) {
86 smp_rmb();
87 if (pen_release == -1)
88 break;
89
90 udelay(10);
91 }
92
93 /*
94 * now the secondary core is starting up let it run its
95 * calibrations, then wait for it to finish
96 */
97 spin_unlock(&boot_lock);
98
99 return pen_release != -1 ? -ENOSYS : 0;
100}
上述代碼中高亮的部分首先會將pen_release變量設置為要喚醒的CPU核的CPU號cpu_logical_map(cpu)，而後透過gic_raise_softirq(cpumask_of(cpu), 0)給CPUcpu發起0號IPI，這個時候，CPUcpu核會從前面smp_operations中的smp_prepare_cpus()成員函數即vexpress_smp_prepare_cpus()透過v2m_flags_set()設置的其他CPU核的起始地址versatile_secondary_startup開始執行，如果順利的話，該CPU會將原先為正數的pen_release寫為-1，以便CPU0從等待pen_release成為-1的循環中跳出。
versatile_secondary_startup實現於arch/arm/plat-versatile/headsmp.S，是一段匯編：
21ENTRY(versatile_secondary_startup)
22 mrc p15, 0, r0, c0, c0, 5
23 and r0, r0, #15
24 adr r4, 1f
25 ldmia r4, {r5, r6}
26 sub r4, r4, r5
27 add r6, r6, r4
28pen: ldr r7, [r6]
29 cmp r7, r0
30 bne pen
31
32 /*
33 * we've been released from the holding pen: secondary_stack
34 * should now contain the SVC stack for this core
35 */
36 b secondary_startup
37
38 .align
391: .long .
40 .long pen_release
41ENDPROC(versatile_secondary_startup)
第1段高亮的部分實際上是等待pen_release成為CPU0設置的cpu_logical_map(cpu)，一般直接就成立了。第2段高亮的部分則調用到內核通用的secondary_startup()函數，經過一系列的初始化如MMU等，最終新的被喚醒的CPU將調用到smp_operations的smp_secondary_init()成員函數，對於本例為versatile_secondary_init()：
37void __cpuinit versatile_secondary_init(unsigned int cpu)
38{
39 /*
40 * if any interrupts are already enabled for the primary
41 * core (e.g. timer irq), then they will not have been enabled
42 * for us: do so
43 */
44 gic_secondary_init(0);
45
46 /*
47 * let the primary processor know we're out of the
48 * pen, then head off into the C entry point
49 */
50 write_pen_release(-1);
51
52 /*
53 * Synchronise with the boot thread.
54 */
55 spin_lock(&boot_lock);
56 spin_unlock(&boot_lock);
57}
上述代碼中高亮的那1行會將pen_release寫為-1，於是CPU0還在執行的 versatile_boot_secondary()函數中的如下循環就退出了：
85 while (time_before(jiffies, timeout)) {
86 smp_rmb();
87 if (pen_release == -1)
88 break;
89
90 udelay(10);
91 }
此後CPU0和新喚醒的其他CPU各自狂奔。整個系統在運行過程中會進行實時進程和正常進程的動態負載均衡。
CPU hotplug的實現也是芯片相關的，對於VEXPRESS而言，實現了smp_operations的cpu_die()成員函數即vexpress_cpu_die()。它會在進行CPUn的拔除操作時將CPUn投入低功耗的WFI狀態，相關代碼位於arch/arm/mach-vexpress/hotplug.c：
90void __ref vexpress_cpu_die(unsigned int cpu)
91{
92 int spurious = 0;
93
94 /*
95 * we're ready for shutdown now, so do it
96 */
97 cpu_enter_lowpower();
98 platform_do_lowpower(cpu, &spurious);
99
100 /*
101 * bring this CPU back into the world of cache
102 * coherency, and then restore interrupts
103 */
104 cpu_leave_lowpower();
105
106 if (spurious)
107 pr_warn("CPU%u: %u spurious wakeup calls\n", cpu, spurious);
108}
57static inline void platform_do_lowpower(unsigned int cpu, int *spurious)
58{
59 /*
60 * there is no power-control hardware on this platform, so all
61 * we can do is put the core into WFI; this is safe as the calling
62 * code will have already disabled interrupts
63 */
64 for (;;) {
65 wfi();
66
67 if (pen_release == cpu_logical_map(cpu)) {
68 /*
69 * OK, proper wakeup, we're done
70 */
71 break;
72 }
73
74 /*
75 * Getting here, means that we have come out of WFI without
76 * having been woken up - this shouldn't happen
77 *
78 * Just note it happening - when we're woken, we can report
79 * its occurrence.
80 */
81 (*spurious)++;
82 }
83}
CPUn睡眠於wfi()，之後再次online的時候，又會因為CPU0給它發出的IPI而從wfi()函數返回繼續執行，醒來時CPUn也判決了是否pen_release == cpu_logical_map(cpu)成立，以確定該次醒來確確實實是由CPU0喚醒的一次正常醒來。
本文出自 “宋寶華的博客” 博客，請務必保留此出處http://21cnbao.blog.51cto.com/109393/1143518
