From 81ebe02e396cb96e6055bc76b58ffcf300889199 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 24 Jun 2022 11:36:55 +0200 Subject: [PATCH 1/4] sched: rename TASK_GROUP_UNSPECIFIED to TASK_GROUP_ALL Signed-off-by: Pawel Wieczorkiewicz --- common/sched.c | 4 ++-- include/sched.h | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/common/sched.c b/common/sched.c index 7e6392e8..f2c019d3 100644 --- a/common/sched.c +++ b/common/sched.c @@ -85,7 +85,7 @@ static task_t *create_task(void) { memset(task, 0, sizeof(*task)); task->id = next_tid++; - task->gid = TASK_GROUP_UNSPECIFIED; + task->gid = TASK_GROUP_ALL; task->cpu = INVALID_CPU; set_task_state(task, TASK_STATE_NEW); @@ -215,7 +215,7 @@ void wait_for_task_group(task_group_t group) { list_for_each_entry (task, &tasks, list) { /* When group is unspecified the functions waits for all tasks. */ - if (group != TASK_GROUP_UNSPECIFIED && task->gid != group) + if (group != TASK_GROUP_ALL && task->gid != group) continue; if (get_task_state(task) != TASK_STATE_DONE) { diff --git a/include/sched.h b/include/sched.h index 004a5880..92941c2f 100644 --- a/include/sched.h +++ b/include/sched.h @@ -42,7 +42,7 @@ enum task_state { typedef enum task_state task_state_t; enum task_group { - TASK_GROUP_UNSPECIFIED = 0, + TASK_GROUP_ALL = 0, TASK_GROUP_ACPI, TASK_GROUP_TEST, }; @@ -82,8 +82,6 @@ extern void wait_for_task_group(task_group_t group); static inline void set_task_group(task_t *task, task_group_t gid) { task->gid = gid; } -static inline void wait_for_all_tasks(void) { - wait_for_task_group(TASK_GROUP_UNSPECIFIED); -} +static inline void wait_for_all_tasks(void) { wait_for_task_group(TASK_GROUP_ALL); } #endif /* KTF_SCHED_H */ From 0bce4de2b0a5099d84db1e3b4e15120875e86bf7 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 12:25:29 +0200 Subject: [PATCH 2/4] sched: add tasks types support Currently only KERNEL, USER, INTERRUPT and ACPI_SERVICE are defined. KERNEL type is the default one. Signed-off-by: Pawel Wieczorkiewicz --- common/kernel.c | 2 +- common/sched.c | 8 +++++--- drivers/acpi/acpica/osl.c | 2 +- include/sched.h | 19 ++++++++++++++++++- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/common/kernel.c b/common/kernel.c index 3c0b258a..fcfabac7 100644 --- a/common/kernel.c +++ b/common/kernel.c @@ -62,7 +62,7 @@ void kernel_main(void) { display_multiboot_mmap(); } - tests_task = new_task("tests", test_main, NULL); + tests_task = new_kernel_task("tests", test_main, NULL); schedule_task(tests_task, smp_processor_id()); run_tasks(smp_processor_id()); diff --git a/common/sched.c b/common/sched.c index f2c019d3..c6e81f39 100644 --- a/common/sched.c +++ b/common/sched.c @@ -108,7 +108,8 @@ static void destroy_task(task_t *task) { kfree(task); } -static int prepare_task(task_t *task, const char *name, task_func_t func, void *arg) { +static int prepare_task(task_t *task, const char *name, task_func_t func, void *arg, + task_type_t type) { if (!task) return -EINVAL; @@ -120,6 +121,7 @@ static int prepare_task(task_t *task, const char *name, task_func_t func, void * task->name = name; task->func = func; task->arg = arg; + task->type = type; set_task_state(task, TASK_STATE_READY); return ESUCCESS; } @@ -132,13 +134,13 @@ static void wait_for_task_state(task_t *task, task_state_t state) { cpu_relax(); } -task_t *new_task(const char *name, task_func_t func, void *arg) { +task_t *new_task(const char *name, task_func_t func, void *arg, task_type_t type) { task_t *task = create_task(); if (!task) return NULL; - if (unlikely(prepare_task(task, name, func, arg) != ESUCCESS)) { + if (unlikely(prepare_task(task, name, func, arg, type) != ESUCCESS)) { destroy_task(task); return NULL; } diff --git a/drivers/acpi/acpica/osl.c b/drivers/acpi/acpica/osl.c index 6fd7a1ac..4159277f 100644 --- a/drivers/acpi/acpica/osl.c +++ b/drivers/acpi/acpica/osl.c @@ -369,7 +369,7 @@ ACPI_STATUS AcpiOsExecute(ACPI_EXECUTE_TYPE Type, ACPI_OSD_EXEC_CALLBACK Functio cb.Function = Function; cb.Context = Context; - task = new_task(name, _osd_exec_cb_wrapper, &cb); + task = new_kernel_task(name, _osd_exec_cb_wrapper, &cb); if (!task) return AE_NO_MEMORY; diff --git a/include/sched.h b/include/sched.h index 92941c2f..005f1607 100644 --- a/include/sched.h +++ b/include/sched.h @@ -48,12 +48,21 @@ enum task_group { }; typedef enum task_group task_group_t; +enum task_type { + TASK_TYPE_KERNEL = 0, + TASK_TYPE_USER, + TASK_TYPE_INTERRUPT, + TASK_TYPE_ACPI_SERVICE, +}; +typedef enum task_type task_type_t; + typedef unsigned int tid_t; struct task { list_head_t list; tid_t id; + task_type_t type; task_group_t gid; task_state_t state; @@ -73,10 +82,10 @@ extern void init_tasks(void); extern task_t *get_task_by_id(tid_t id); extern task_t *get_task_by_name(const char *name); extern task_t *get_task_for_cpu(unsigned int cpu); -extern task_t *new_task(const char *name, task_func_t func, void *arg); extern void schedule_task(task_t *task, unsigned int cpu); extern void run_tasks(unsigned int cpu); extern void wait_for_task_group(task_group_t group); +extern task_t *new_task(const char *name, task_func_t func, void *arg, task_type_t type); /* Static declarations */ @@ -84,4 +93,12 @@ static inline void set_task_group(task_t *task, task_group_t gid) { task->gid = static inline void wait_for_all_tasks(void) { wait_for_task_group(TASK_GROUP_ALL); } +static inline task_t *new_kernel_task(const char *name, task_func_t func, void *arg) { + return new_task(name, func, arg, TASK_TYPE_KERNEL); +} + +static inline task_t *new_user_task(const char *name, task_func_t func, void *arg) { + return new_task(name, func, arg, TASK_TYPE_USER); +} + #endif /* KTF_SCHED_H */ From 51622c8ad7887dbd10d1bf1bb8d3d0dc9a4725f8 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 12:29:32 +0200 Subject: [PATCH 3/4] sched, smp: assign tasks to independent, per-CPU task lists Signed-off-by: Pawel Wieczorkiewicz --- common/cpu.c | 2 + common/kernel.c | 8 ++- common/sched.c | 118 ++++++++++++++++++-------------------- drivers/acpi/acpica/osl.c | 8 ++- include/cpu.h | 4 +- include/sched.h | 17 +++--- include/smp/smp.h | 2 - smp/smp.c | 2 +- 8 files changed, 82 insertions(+), 79 deletions(-) diff --git a/common/cpu.c b/common/cpu.c index 166e97bb..798b2fba 100644 --- a/common/cpu.c +++ b/common/cpu.c @@ -42,12 +42,14 @@ static void init_cpu(cpu_t *cpu, unsigned int id, bool is_bsp, bool enabled) { cpu->id = id; cpu->bsp = is_bsp; cpu->enabled = enabled; + cpu->scheduled = false; cpu->done = false; cpu->percpu = get_percpu_page(id); BUG_ON(!cpu->percpu); cpu->lock = SPINLOCK_INIT; + list_init(&cpu->task_queue); } cpu_t *init_cpus(void) { diff --git a/common/kernel.c b/common/kernel.c index fcfabac7..7749ac04 100644 --- a/common/kernel.c +++ b/common/kernel.c @@ -53,6 +53,7 @@ static void __noreturn echo_loop(void) { void kernel_main(void) { task_t *tests_task; + cpu_t *cpu; printk("\nKTF - Kernel Test Framework!\n"); @@ -63,11 +64,12 @@ void kernel_main(void) { } tests_task = new_kernel_task("tests", test_main, NULL); - schedule_task(tests_task, smp_processor_id()); + cpu = get_bsp_cpu(); + schedule_task(tests_task, cpu); - run_tasks(smp_processor_id()); + run_tasks(cpu); - wait_for_all_tasks(); + wait_for_all_cpus(); printk("All tasks done.\n"); diff --git a/common/sched.c b/common/sched.c index c6e81f39..22c019b3 100644 --- a/common/sched.c +++ b/common/sched.c @@ -23,6 +23,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include +#include #include #include #include @@ -36,15 +37,12 @@ #include -static list_head_t tasks; static tid_t next_tid; -static spinlock_t lock = SPINLOCK_INIT; - void init_tasks(void) { printk("Initializing tasks\n"); - list_init(&tasks); + next_tid = 0; } static const char *task_state_names[] = { @@ -86,13 +84,8 @@ static task_t *create_task(void) { memset(task, 0, sizeof(*task)); task->id = next_tid++; task->gid = TASK_GROUP_ALL; - task->cpu = INVALID_CPU; set_task_state(task, TASK_STATE_NEW); - spin_lock(&lock); - list_add(&task->list, &tasks); - spin_unlock(&lock); - return task; } @@ -101,10 +94,9 @@ static void destroy_task(task_t *task) { if (!task) return; - spin_lock(&lock); + spin_lock(&task->cpu->lock); list_unlink(&task->list); - spin_unlock(&lock); - + spin_unlock(&task->cpu->lock); kfree(task); } @@ -113,9 +105,6 @@ static int prepare_task(task_t *task, const char *name, task_func_t func, void * if (!task) return -EINVAL; - if (get_task_by_name(name)) - return -EEXIST; - BUG_ON(get_task_state(task) > TASK_STATE_READY); task->name = name; @@ -148,21 +137,10 @@ task_t *new_task(const char *name, task_func_t func, void *arg, task_type_t type return task; } -task_t *get_task_by_id(tid_t id) { - task_t *task; - - list_for_each_entry (task, &tasks, list) { - if (task->id == id) - return task; - } - - return NULL; -} - -task_t *get_task_by_name(const char *name) { +task_t *get_task_by_name(cpu_t *cpu, const char *name) { task_t *task; - list_for_each_entry (task, &tasks, list) { + list_for_each_entry (task, &cpu->task_queue, list) { if (string_equal(task->name, name)) return task; } @@ -170,29 +148,26 @@ task_t *get_task_by_name(const char *name) { return NULL; } -task_t *get_task_for_cpu(unsigned int cpu) { - task_t *task; - - list_for_each_entry (task, &tasks, list) { - if (task->cpu == cpu) - return task; - } - - return NULL; -} - -void schedule_task(task_t *task, unsigned int cpu) { +int schedule_task(task_t *task, cpu_t *cpu) { ASSERT(task); - if (cpu > get_nr_cpus() - 1) - panic("CPU[%u] does not exist.\n", cpu); + if (!cpu) { + printk("Unable to schedule task: %s. CPU does not exist.", task->name); + return -EEXIST; + } BUG_ON(get_task_state(task) != TASK_STATE_READY); - printk("CPU[%u]: Scheduling task %s[%u]\n", cpu, task->name, task->id); + printk("CPU[%u]: Scheduling task %s[%u]\n", cpu->id, task->name, task->id); + spin_lock(&cpu->lock); + list_add_tail(&task->list, &cpu->task_queue); task->cpu = cpu; + cpu->scheduled = true; set_task_state(task, TASK_STATE_SCHEDULED); + spin_unlock(&cpu->lock); + + return 0; } static void run_task(task_t *task) { @@ -201,21 +176,21 @@ static void run_task(task_t *task) { wait_for_task_state(task, TASK_STATE_SCHEDULED); - printk("CPU[%u]: Running task %s[%u]\n", task->cpu, task->name, task->id); + printk("CPU[%u]: Running task %s[%u]\n", task->cpu->id, task->name, task->id); set_task_state(task, TASK_STATE_RUNNING); task->result = task->func(task->arg); set_task_state(task, TASK_STATE_DONE); } -void wait_for_task_group(task_group_t group) { - task_t *task; +void wait_for_task_group(const cpu_t *cpu, task_group_t group) { + task_t *task, *safe; bool busy; do { busy = false; - list_for_each_entry (task, &tasks, list) { + list_for_each_entry_safe (task, safe, &cpu->task_queue, list) { /* When group is unspecified the functions waits for all tasks. */ if (group != TASK_GROUP_ALL && task->gid != group) continue; @@ -229,21 +204,40 @@ void wait_for_task_group(task_group_t group) { } while (busy); } -void run_tasks(unsigned int cpu) { - task_t *task; +void run_tasks(cpu_t *cpu) { + task_t *task, *safe; - while ((task = get_task_for_cpu(cpu))) { - switch (task->state) { - case TASK_STATE_DONE: - printk("Task '%s' finished with result %lu\n", task->name, task->result); - destroy_task(task); - break; - case TASK_STATE_SCHEDULED: - run_task(task); - break; - default: - break; - } - cpu_relax(); + spin_lock(&cpu->lock); + if (!cpu->scheduled) { + cpu->done = true; + spin_unlock(&cpu->lock); + return; } + spin_unlock(&cpu->lock); + + while (list_is_empty(&cpu->task_queue)) + cpu_relax(); + + do { + list_for_each_entry_safe (task, safe, &cpu->task_queue, list) { + switch (task->state) { + case TASK_STATE_DONE: + printk("%s task '%s' finished on CPU[%u] with result %ld\n", + task->type == TASK_TYPE_KERNEL ? "Kernel" : "User", task->name, + cpu->id, task->result); + destroy_task(task); + break; + case TASK_STATE_SCHEDULED: + run_task(task); + break; + default: + BUG(); + } + cpu_relax(); + } + } while (!list_is_empty(&cpu->task_queue)); + + spin_lock(&cpu->lock); + cpu->done = true; + spin_unlock(&cpu->lock); } diff --git a/drivers/acpi/acpica/osl.c b/drivers/acpi/acpica/osl.c index 4159277f..764e8b9c 100644 --- a/drivers/acpi/acpica/osl.c +++ b/drivers/acpi/acpica/osl.c @@ -374,12 +374,16 @@ ACPI_STATUS AcpiOsExecute(ACPI_EXECUTE_TYPE Type, ACPI_OSD_EXEC_CALLBACK Functio return AE_NO_MEMORY; set_task_group(task, TASK_GROUP_ACPI); - schedule_task(task, cpu->id); + schedule_task(task, cpu); return AE_OK; } -void AcpiOsWaitEventsComplete(void) { wait_for_task_group(TASK_GROUP_ACPI); } +void AcpiOsWaitEventsComplete(void) { + cpu_t *cpu = get_cpu(smp_processor_id()); + + wait_for_task_group(cpu, TASK_GROUP_ACPI); +} /* Synchronization and locking functions */ diff --git a/include/cpu.h b/include/cpu.h index 339b83e7..dcf47e8e 100644 --- a/include/cpu.h +++ b/include/cpu.h @@ -35,11 +35,13 @@ struct cpu { list_head_t list; unsigned int id; - unsigned int bsp : 1, enabled : 1, done : 1; + unsigned int bsp : 1, enabled : 1, scheduled : 1, done : 1; percpu_t *percpu; spinlock_t lock; + + list_head_t task_queue; }; typedef struct cpu cpu_t; diff --git a/include/sched.h b/include/sched.h index 005f1607..cb4a9671 100644 --- a/include/sched.h +++ b/include/sched.h @@ -25,6 +25,7 @@ #ifndef KTF_SCHED_H #define KTF_SCHED_H +#include #include #include #include @@ -66,7 +67,7 @@ struct task { task_group_t gid; task_state_t state; - unsigned int cpu; + cpu_t *cpu; const char *name; task_func_t func; @@ -79,19 +80,19 @@ typedef struct task task_t; /* External declarations */ extern void init_tasks(void); -extern task_t *get_task_by_id(tid_t id); -extern task_t *get_task_by_name(const char *name); -extern task_t *get_task_for_cpu(unsigned int cpu); -extern void schedule_task(task_t *task, unsigned int cpu); -extern void run_tasks(unsigned int cpu); -extern void wait_for_task_group(task_group_t group); +extern task_t *get_task_by_name(cpu_t *cpu, const char *name); extern task_t *new_task(const char *name, task_func_t func, void *arg, task_type_t type); +extern int schedule_task(task_t *task, cpu_t *cpu); +extern void run_tasks(cpu_t *cpu); +extern void wait_for_task_group(const cpu_t *cpu, task_group_t group); /* Static declarations */ static inline void set_task_group(task_t *task, task_group_t gid) { task->gid = gid; } -static inline void wait_for_all_tasks(void) { wait_for_task_group(TASK_GROUP_ALL); } +static inline void wait_for_cpu_tasks(cpu_t *cpu) { + wait_for_task_group(cpu, TASK_GROUP_ALL); +} static inline task_t *new_kernel_task(const char *name, task_func_t func, void *arg) { return new_task(name, func, arg, TASK_TYPE_KERNEL); diff --git a/include/smp/smp.h b/include/smp/smp.h index 402e6962..2ab912f9 100644 --- a/include/smp/smp.h +++ b/include/smp/smp.h @@ -29,8 +29,6 @@ #include #include -#define INVALID_CPU (~0U) - /* External declarations */ extern void init_smp(void); diff --git a/smp/smp.c b/smp/smp.c index ca6d83f4..311e630d 100644 --- a/smp/smp.c +++ b/smp/smp.c @@ -59,7 +59,7 @@ void __noreturn ap_startup(void) { ap_callin = true; smp_wmb(); - run_tasks(cpu->id); + run_tasks(cpu); while (true) halt(); From 43d4b283b22c75cbebd56e788183bd09e3f11ddc Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Tue, 19 Jul 2022 09:50:42 +0200 Subject: [PATCH 4/4] cpu,sched: implement SMP task start synchornization Instead of using regular bitfields and extensive locking, use atomic variable to implement the CPUs run state. Two states are added: UNBLOCKED (default CPU is blocked and waiting before starting the tasks) and FINISHED (CPU has done all its tasks). Signed-off-by: Pawel Wieczorkiewicz --- common/cpu.c | 20 +++++++++++++++----- common/kernel.c | 4 ++-- common/sched.c | 16 ++-------------- include/cpu.h | 34 +++++++++++++++++++++++++++++++++- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/common/cpu.c b/common/cpu.c index 798b2fba..1cdab4d6 100644 --- a/common/cpu.c +++ b/common/cpu.c @@ -42,8 +42,10 @@ static void init_cpu(cpu_t *cpu, unsigned int id, bool is_bsp, bool enabled) { cpu->id = id; cpu->bsp = is_bsp; cpu->enabled = enabled; - cpu->scheduled = false; - cpu->done = false; + + init_cpu_runstate(cpu); + if (is_bsp) + set_cpu_unblocked(cpu); cpu->percpu = get_percpu_page(id); BUG_ON(!cpu->percpu); @@ -100,15 +102,23 @@ void for_each_cpu(void (*func)(cpu_t *cpu)) { func(cpu); } +void unblock_all_cpus(void) { + cpu_t *cpu; + + list_for_each_entry (cpu, &cpus, list) + set_cpu_unblocked(cpu); +} + void wait_for_all_cpus(void) { cpu_t *cpu, *safe; do { list_for_each_entry_safe (cpu, safe, &cpus, list) { - spin_lock(&cpu->lock); - if (cpu->done) + if (is_cpu_finished(cpu)) { + spin_lock(&cpu->lock); list_unlink(&cpu->list); - spin_unlock(&cpu->lock); + spin_unlock(&cpu->lock); + } } } while (!list_is_empty(&cpus)); } diff --git a/common/kernel.c b/common/kernel.c index 7749ac04..500096b1 100644 --- a/common/kernel.c +++ b/common/kernel.c @@ -31,7 +31,6 @@ #include #include #include -#include #ifdef KTF_PMU #include #endif @@ -65,10 +64,11 @@ void kernel_main(void) { tests_task = new_kernel_task("tests", test_main, NULL); cpu = get_bsp_cpu(); + schedule_task(tests_task, cpu); run_tasks(cpu); - + unblock_all_cpus(); wait_for_all_cpus(); printk("All tasks done.\n"); diff --git a/common/sched.c b/common/sched.c index 22c019b3..3a2a9b18 100644 --- a/common/sched.c +++ b/common/sched.c @@ -163,7 +163,6 @@ int schedule_task(task_t *task, cpu_t *cpu) { spin_lock(&cpu->lock); list_add_tail(&task->list, &cpu->task_queue); task->cpu = cpu; - cpu->scheduled = true; set_task_state(task, TASK_STATE_SCHEDULED); spin_unlock(&cpu->lock); @@ -207,16 +206,7 @@ void wait_for_task_group(const cpu_t *cpu, task_group_t group) { void run_tasks(cpu_t *cpu) { task_t *task, *safe; - spin_lock(&cpu->lock); - if (!cpu->scheduled) { - cpu->done = true; - spin_unlock(&cpu->lock); - return; - } - spin_unlock(&cpu->lock); - - while (list_is_empty(&cpu->task_queue)) - cpu_relax(); + wait_cpu_unblocked(cpu); do { list_for_each_entry_safe (task, safe, &cpu->task_queue, list) { @@ -237,7 +227,5 @@ void run_tasks(cpu_t *cpu) { } } while (!list_is_empty(&cpu->task_queue)); - spin_lock(&cpu->lock); - cpu->done = true; - spin_unlock(&cpu->lock); + set_cpu_finished(cpu); } diff --git a/include/cpu.h b/include/cpu.h index dcf47e8e..3c3787d1 100644 --- a/include/cpu.h +++ b/include/cpu.h @@ -25,17 +25,23 @@ #ifndef KTF_CPU_H #define KTF_CPU_H +#include #include #include #include #include #include +#define CPU_UNBLOCKED (1 << 0) +#define CPU_FINISHED (1 << 1) + struct cpu { list_head_t list; unsigned int id; - unsigned int bsp : 1, enabled : 1, scheduled : 1, done : 1; + unsigned int bsp : 1, enabled : 1; + + atomic_t run_state; percpu_t *percpu; @@ -53,6 +59,32 @@ extern cpu_t *get_cpu(unsigned int id); extern cpu_t *get_bsp_cpu(void); extern unsigned int get_nr_cpus(void); extern void for_each_cpu(void (*func)(cpu_t *cpu)); +extern void unblock_all_cpus(void); extern void wait_for_all_cpus(void); +/* Static declarations */ + +static inline void init_cpu_runstate(cpu_t *cpu) { atomic_set(&cpu->run_state, 0); } + +static inline bool is_cpu_finished(cpu_t *cpu) { + return atomic_test_bit(CPU_FINISHED, &cpu->run_state); +} + +static inline void set_cpu_finished(cpu_t *cpu) { + atomic_test_and_set_bit(CPU_FINISHED, &cpu->run_state); +} + +static inline bool is_cpu_unblocked(cpu_t *cpu) { + return atomic_test_bit(CPU_UNBLOCKED, &cpu->run_state); +} + +static inline void set_cpu_unblocked(cpu_t *cpu) { + atomic_test_and_set_bit(CPU_UNBLOCKED, &cpu->run_state); +} + +static inline void wait_cpu_unblocked(cpu_t *cpu) { + while (!is_cpu_unblocked(cpu)) + cpu_relax(); +} + #endif /* KTF_CPU_H */