Skip to content

Commit

Permalink
x86/smpboot: Serialize topology updates for secondary bringup
Browse files Browse the repository at this point in the history 
The toplogy update is performed by the AP via smp_callin() after the BSP
has called do_wait_cpu_initialized(), setting the AP's bit in
cpu_callout_mask to allow it to proceed.

In preparation to enable further parallelism of AP bringup, add locking to
serialize the update even if multiple APs are (in future) permitted to
proceed through the next stages of bringup in parallel.

Without such ordering (and with that future extra parallelism), confusion
ensues:

[    1.360149] x86: Booting SMP configuration:
[    1.360221] .... node  #0, CPUs:        #1  #2  #3  #4  #5  torvalds#6  torvalds#7  torvalds#8  torvalds#9 torvalds#10 torvalds#11 torvalds#12 torvalds#13 torvalds#14 torvalds#15 torvalds#16 torvalds#17 torvalds#18 torvalds#19 torvalds#20 torvalds#21 torvalds#22 torvalds#23
[    1.366225] .... node  #1, CPUs:   torvalds#24 torvalds#25 torvalds#26 torvalds#27 torvalds#28 torvalds#29 torvalds#30 torvalds#31 torvalds#32 torvalds#33 torvalds#34 torvalds#35 torvalds#36 torvalds#37 torvalds#38 torvalds#39 torvalds#40 torvalds#41 torvalds#42 torvalds#43 torvalds#44 torvalds#45 torvalds#46 torvalds#47
[    1.370219] .... node  #0, CPUs:   torvalds#48 torvalds#49 torvalds#50 torvalds#51 #52 #53 torvalds#54 torvalds#55 torvalds#56 torvalds#57 #58 torvalds#59 torvalds#60 torvalds#61 torvalds#62 torvalds#63 torvalds#64 torvalds#65 torvalds#66 torvalds#67 torvalds#68 torvalds#69 #70 torvalds#71
[    1.378226] .... node  #1, CPUs:   torvalds#72 torvalds#73 torvalds#74 torvalds#75 torvalds#76 torvalds#77 torvalds#78 torvalds#79 torvalds#80 torvalds#81 torvalds#82 torvalds#83 torvalds#84 torvalds#85 torvalds#86 torvalds#87 torvalds#88 torvalds#89 torvalds#90 torvalds#91 torvalds#92 torvalds#93 torvalds#94 torvalds#95
[    1.382037] Brought 96 CPUs to x86/cpu:kick in 72232606 cycles
[    0.104104] smpboot: CPU 26 Converting physical 0 to logical die 1
[    0.104104] smpboot: CPU 27 Converting physical 1 to logical package 2
[    0.104104] smpboot: CPU 24 Converting physical 1 to logical package 3
[    0.104104] smpboot: CPU 27 Converting physical 0 to logical die 2
[    0.104104] smpboot: CPU 25 Converting physical 1 to logical package 4
[    1.385609] Brought 96 CPUs to x86/cpu:wait-init in 9269218 cycles
[    1.395285] Brought CPUs online in 28930764 cycles
[    1.395469] smp: Brought up 2 nodes, 96 CPUs
[    1.395689] smpboot: Max logical packages: 2
[    1.396222] smpboot: Total of 96 processors activated (576000.00 BogoMIPS)

[Usama Arif: fixed rebase conflict]
Signed-off-by: David Woodhouse <[email protected]>
Signed-off-by: Usama Arif <[email protected]>
Signed-off-by: Paul E. McKenney <[email protected]>
  • Loading branch information
@dwmw2 @paulmckrcu
dwmw2 authored and paulmckrcu committed Feb 15, 2023
1 parent 0fe0195 commit 14b5793
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 41 deletions.
4 changes: 1 addition & 3 deletions arch/x86/include/asm/smp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ struct smp_ops {
};

/* Globals due to paravirt */
extern void set_cpu_sibling_map(int cpu);

#ifdef CONFIG_SMP
extern struct smp_ops smp_ops;

Expand Down Expand Up @@ -137,7 +135,7 @@ void native_send_call_func_single_ipi(int cpu);
void x86_idle_thread_init(unsigned int cpu, struct task_struct *idle);

void smp_store_boot_cpu_info(void);
void smp_store_cpu_info(int id);
void smp_store_cpu_info(int id, bool force_single_core);

asmlinkage __visible void smp_reboot_interrupt(void);
__visible void smp_reschedule_interrupt(struct pt_regs *regs);
Expand Down
2 changes: 0 additions & 2 deletions arch/x86/include/asm/topology.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,6 @@ static inline int topology_max_smt_threads(void)
return __max_smt_threads;
}

int topology_update_package_map(unsigned int apicid, unsigned int cpu);
int topology_update_die_map(unsigned int dieid, unsigned int cpu);
int topology_phys_to_logical_pkg(unsigned int pkg);
int topology_phys_to_logical_die(unsigned int die, unsigned int cpu);
bool topology_is_primary_thread(unsigned int cpu);
Expand Down
6 changes: 2 additions & 4 deletions arch/x86/kernel/cpu/common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1766,7 +1766,7 @@ static void generic_identify(struct cpuinfo_x86 *c)
* Validate that ACPI/mptables have the same information about the
* effective APIC id and update the package map.
*/
static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
static void validate_apic_id(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_SMP
unsigned int apicid, cpu = smp_processor_id();
Expand All @@ -1777,8 +1777,6 @@ static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
cpu, apicid, c->initial_apicid);
}
BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
BUG_ON(topology_update_die_map(c->cpu_die_id, cpu));
#else
c->logical_proc_id = 0;
#endif
Expand Down Expand Up @@ -1969,7 +1967,7 @@ void identify_secondary_cpu(struct cpuinfo_x86 *c)
#ifdef CONFIG_X86_32
enable_sep_cpu();
#endif
validate_apic_and_package_id(c);
validate_apic_id(c);
x86_spec_ctrl_setup_ap();
update_srbds_msr();

Expand Down
73 changes: 44 additions & 29 deletions arch/x86/kernel/smpboot.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,16 +180,12 @@ static void smp_callin(void)
apic_ap_setup();

/*
* Save our processor parameters. Note: this information
* is needed for clock calibration.
*/
smp_store_cpu_info(cpuid);

/*
* Save our processor parameters and update topology.
* Note: this information is needed for clock calibration.
* The topology information must be up to date before
* calibrate_delay() and notify_cpu_starting().
*/
set_cpu_sibling_map(raw_smp_processor_id());
smp_store_cpu_info(cpuid, false);

ap_init_aperfmperf();

Expand Down Expand Up @@ -243,6 +239,12 @@ static void notrace start_secondary(void *unused)
* its bit bit in cpu_callout_mask to release it.
*/
cpu_init_secondary();

/*
* Even though notify_cpu_starting() will do this, it does so too late
* as the AP may already have triggered lockdep splats by then. See
* commit 29368e093 ("x86/smpboot: Move rcu_cpu_starting() earlier").
*/
rcu_cpu_starting(raw_smp_processor_id());
x86_cpuinit.early_percpu_clock_init();

Expand Down Expand Up @@ -351,7 +353,7 @@ EXPORT_SYMBOL(topology_phys_to_logical_die);
* @pkg: The physical package id as retrieved via CPUID
* @cpu: The cpu for which this is updated
*/
int topology_update_package_map(unsigned int pkg, unsigned int cpu)
static int topology_update_package_map(unsigned int pkg, unsigned int cpu)
{
int new;

Expand All @@ -374,7 +376,7 @@ int topology_update_package_map(unsigned int pkg, unsigned int cpu)
* @die: The die id as retrieved via CPUID
* @cpu: The cpu for which this is updated
*/
int topology_update_die_map(unsigned int die, unsigned int cpu)
static int topology_update_die_map(unsigned int die, unsigned int cpu)
{
int new;

Expand Down Expand Up @@ -405,25 +407,7 @@ void __init smp_store_boot_cpu_info(void)
c->initialized = true;
}

/*
* The bootstrap kernel entry code has set these up. Save them for
* a given CPU
*/
void smp_store_cpu_info(int id)
{
struct cpuinfo_x86 *c = &cpu_data(id);

/* Copy boot_cpu_data only on the first bringup */
if (!c->initialized)
*c = boot_cpu_data;
c->cpu_index = id;
/*
* During boot time, CPU0 has this setup already. Save the info when
* bringing up AP or offlined CPU0.
*/
identify_secondary_cpu(c);
c->initialized = true;
}
static arch_spinlock_t topology_lock = __ARCH_SPIN_LOCK_UNLOCKED;

static bool
topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
Expand Down Expand Up @@ -629,7 +613,7 @@ static struct sched_domain_topology_level x86_topology[] = {
*/
static bool x86_has_numa_in_package;

void set_cpu_sibling_map(int cpu)
static void set_cpu_sibling_map(int cpu)
{
bool has_smt = smp_num_siblings > 1;
bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
Expand Down Expand Up @@ -708,6 +692,37 @@ void set_cpu_sibling_map(int cpu)
}
}

/*
* The bootstrap kernel entry code has set these up. Save them for
* a given CPU
*/
void smp_store_cpu_info(int id, bool force_single_core)
{
struct cpuinfo_x86 *c = &cpu_data(id);

/* Copy boot_cpu_data only on the first bringup */
if (!c->initialized)
*c = boot_cpu_data;
c->cpu_index = id;
/*
* During boot time, CPU0 has this setup already. Save the info when
* bringing up AP or offlined CPU0.
*/
identify_secondary_cpu(c);

arch_spin_lock(&topology_lock);
BUG_ON(topology_update_package_map(c->phys_proc_id, id));
BUG_ON(topology_update_die_map(c->cpu_die_id, id));
c->initialized = true;

/* For Xen PV */
if (force_single_core)
c->x86_max_cores = 1;

set_cpu_sibling_map(id);
arch_spin_unlock(&topology_lock);
}

/* maps the cpu to the sched domain representing multi-core */
const struct cpumask *cpu_coregroup_mask(int cpu)
{
Expand Down
4 changes: 1 addition & 3 deletions arch/x86/xen/smp_pv.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,7 @@ static void cpu_bringup(void)
xen_enable_syscall();
}
cpu = smp_processor_id();
smp_store_cpu_info(cpu);
cpu_data(cpu).x86_max_cores = 1;
set_cpu_sibling_map(cpu);
smp_store_cpu_info(cpu, true);

speculative_store_bypass_ht_init();

Expand Down

0 comments on commit 14b5793

Please sign in to comment.