SMP系统的引导

/* Called by boot processor to activate the rest. */
static void __init smp_init(void)
{
	unsigned int cpu;

	/* FIXME: This should be done in userspace --RR */
	for_each_present_cpu(cpu) {
		if (num_online_cpus() >= setup_max_cpus)
			break;
		if (!cpu_online(cpu))
			cpu_up(cpu);//（1）--------
        	//cpu_up到最终调用smp_ops.cpu_up(cpu);
        	//.cpu_up = native_cpu_up是一个回调函数。在arch/x86/kernel/smp.c注册 
	}

	/* Any cleanup work */
	printk(KERN_INFO "Brought up %ld CPUs\n", (long)num_online_cpus());
}

struct smp_ops smp_ops = { 
   …… 
  .smp_cpus_done		= native_smp_cpus_done,
  .cpu_up = native_cpu_up, 
   …… 
} 

int __cpuinit native_cpu_up(unsigned int cpu)
{
    ......
    mtrr_save_state();
    per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;//设置对应CPU的状态
        
    err = do_boot_cpu(apicid, cpu);	//唤醒AP------------
    ......

	while (!cpu_online(cpu)) {//在这里不停的一直等。确认前一个AP唤醒后，再唤醒下一个AP
		cpu_relax();
		......
	}

	return 0;
}

static int __cpuinit do_boot_cpu(int apicid, int cpu)
{
	unsigned long boot_error = 0;
	unsigned long start_ip;
	int timeout;
	struct create_idle c_idle = {
		.cpu	= cpu,
		.done	= COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
	};
	/*  
	 * 完成c_idle.work.func = do_fork_idle
	 */
	INIT_WORK(&c_idle.work, do_fork_idle);
	......
	if (!keventd_up() || current_is_keventd())
        /* 执行do_fork_idle：将init进程使用copy_process复制，并且调用init_idle函数，设置可以运行   
         * 的CPU。fork出一个idel线程，地址空间还是沿用init进程地址空间。
         */
		c_idle.work.func(&c_idle.work);
	else {
		......
	}

	set_idle_for_cpu(cpu, c_idle.idle);
do_rest:
	per_cpu(current_task, cpu) = c_idle.idle;
	......

	/* AP的GDT已经在start_kernel()-->setup_per_cpu_areas()初始化完成，这里只是保存它的基地址
     * 到early_gdt_descr，等后面唤醒时，AP自己设置到GDTR。见startup_32_smp末尾
     */
    early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
    
    //AP初始化完成后，就运行start_secondary函数，见startup_32_smp末尾
	initial_code = (unsigned long)start_secondary;
    
    //为AP设定好执行start_secondary时将要使用的stack，见startup_32_smp末尾
	stack_start.sp = (void *) c_idle.idle->thread.sp;
	
    //real-mode code that AP runs after BSP kicks it（嘻嘻）
    /* 复制trampoline_data到trampoline_end之间的代码(在arch/i386/kernel/trampoline.S中）到
     * trampoline_base处。这里复制到trampoline_base的代码是等下AP唤醒后要执行的代码。所以得通过IPI
     * 的方式告诉AP，trampoline_base对应物理页所在位置。
     * trampoline_base是之前在start_kernel()-->setup_arch()-->smp_alloc_memory():
     *        trampoline_base = (void *) alloc_bootmem_low_pages(PAGE_SIZE)
     * 处申请的页。这里为什么要在低端内存去分配trampoline_base？还记得之前说的 IPI传递给AP只是传递
     * 了一个vector，这个vector只有8位大小，AP自己再<<12，所以AP总共只能寻址1M的物理地址空间。因为
     * AP在唤醒后是处于实模式的。
     * 
     * 所以底下调用virt_to_phys，获取trampoline_base对应物理页的地址start_eip，start_eip是4K对其
     * 的，所以start_eip是形如0xSS000，等下通过IPI发送给AP的是0xSS
     */
    start_ip = setup_trampoline(){
        memcpy(trampoline_base, trampoline_data,
                        trampoline_end - trampoline_data);
        return virt_to_phys(trampoline_base);
    }
	......
    
	/*
	 * Kick the secondary CPU. Use the method in the APIC driver
	 * if it's defined - or use an INIT boot APIC message otherwise:
	 */
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
        /* 这里是重点拉，发送IPI中断。
         * 在这个函数中通过操作APIC_ICR寄存器，BSP向目标AP发送IPI消息，触发目标AP从start_eip地址处，
         * 实模式开始运行。
         */
		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);	

	if (!boot_error) {
		/*
		 * allow APs to start initializing.
		 */
		pr_debug("Before Callout %d.\n", cpu);
        
		cpumask_set_cpu(cpu, cpu_callout_mask);
		pr_debug("After Callout %d.\n", cpu);

		/*
		 * Wait 5s total for a response
		 */
		for (timeout = 0; timeout < 50000; timeout++) {
            /* AP唤醒后会进入start_secondary()-->smp_callin() 设置对应的cpu_callin_mask
             * 所以这里只要检测到cpu_callin_mask被设置了，代表AP激活成功
			 */
			if (cpumask_test_cpu(cpu, cpu_callin_mask))
				break;	/* It has booted */
			udelay(100);
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
			schedule();
		}

		if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
			/* Signal AP that it may continue to boot */
			cpumask_set_cpu(cpu, cpu_may_complete_boot_mask);
			pr_debug("CPU%d: has booted.\n", cpu);//提示对应的AP激活成功
		} else {
			boot_error = 1;
			......可能出了什么问题
		}
	}
	......

	return boot_error;
}

wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
{
    ......
    /* 
    * STARTUP IPI 
    */  

    /* Target chip */  
    /* Boot on the stack */  
    /* Kick the second */  
    apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),  
    phys_apicid); 
    ......
} 

ENTRY(trampoline_data)
r_base = .
	wbinvd			# Needed for NUMA-Q should be harmless for others
	mov	%cs, %ax	# Code and data in the same place
	mov	%ax, %ds

	cli			# We should be safe anyway
	
	/* 这个是设置标识，以便BP知道AP运行到这里了。当前处于实模式，DS段寄存器指向前面的r_base处，此处往
	 * r_base处写入0xA5A5A5A5。BP可以
	 * 通过虚拟地址trampoline_base寻址到r_base来查看是否设置$0xA5A5A5A5，以此来检测AP激活是否成功
	 */
	movl	$0xA5A5A5A5, trampoline_data - r_base
				# write marker for master knows we're running

	/* GDT tables in non default location kernel can be beyond 16MB and
	 * lgdt will not be able to load the address as in real mode default
	 * operand size is 16bit. Use lgdtl instead to force operand size
	 * to 32 bit.
	 */
	
	/* 设置临时idt和gdt，方便后面开启保护模式
	 * 至于为什么这里要减r_base，因为此时的DS段寄存器已经指向r_base
	 * boot_idt_descr - r_base + DS段寄存器<<4 = boot_idt_descr
	 */
	lidtl	boot_idt_descr - r_base	# load idt with 0, 0
	lgdtl	boot_gdt_descr - r_base	# load gdt with whatever is appropriate

	xor	%ax, %ax
	inc	%ax		# protected mode (PE) bit
	lmsw	%ax		# into protected mode 将%ax加载到CR0，进入保护模式
	
	# flush prefetch and jump to startup_32_smp in arch/i386/kernel/head.S
	/* 长跳转至startup_32_smp。此时的__BOOT_CS为0x10，对应GDT的描述符base为0，然后没有开启分页，直接
	 * 访问startup_32_smp物理地址
	 */
	ljmpl	$__BOOT_CS, $(startup_32_smp-__PAGE_OFFSET)

boot_gdt_descr:
	.word	__BOOT_DS + 7			# gdt limit
	.long	boot_gdt - __PAGE_OFFSET	# gdt base 
	/* 由于编译时boot_gdt是加上了__PAGE_OFFSET，而当前还没有开启页表，所以boot_gdt - __PAGE_OFFSET
	 * 后作为物理地址直接使用。
	 */
	
boot_idt_descr:
	.word	0				# idt limit = 0
	.long	0				# idt base = 0L

.globl trampoline_end
trampoline_end:
-------------------------------------boot_gdt来自于arch/x86/kernel/head_32.S
ENTRY(boot_gdt)
	.fill GDT_ENTRY_BOOT_CS,8,0 /* GDT_ENTRY_BOOT_CS为2，这里有两项 */
	.quad 0x00cf9a000000ffff	/* kernel 4GB code at 0x00000000 */
	.quad 0x00cf92000000ffff	/* kernel 4GB data at 0x00000000 */

static void __cpuinit start_secondary(void *unused)
{
	......
	cpu_init();
	preempt_disable();
    
    /* 设定cpu_callin_mask来告诉BP，AP已经启动。BP才能继续运行。	
     * 参考do_boot_cpu：if (cpumask_test_cpu(cpu, cpu_callin_mask)) 
	 */ 
	smp_callin();
	
    /* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	
    ......
	
    //通知BP AP已经启动（BP会在native_cpu_up的while循环里等待）
	set_cpu_online(smp_processor_id(), true);
	......
    //更新AP的状态
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
	......
	cpu_idle();
}