tests: smp_boot_delay: extend to test custom CPU start func

This extends the smp_boot_delay test to test the newly
introduced function k_smp_cpu_custom_start().

Signed-off-by: Daniel Leung <daniel.leung@intel.com>

tests/kernel/smp_boot_delay/boards/qemu_x86_64.conf

@@ -0,0 +1,5 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+CONFIG_MP_MAX_NUM_CPUS=4

tests/kernel/smp_boot_delay/boards/qemu_x86_64.overlay

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/ {
+	cpus {
+		cpu@2 {
+			device_type = "cpu";
+			compatible = "intel,x86";
+			d-cache-line-size = <64>;
+			reg = <2>;
+		};
+
+		cpu@3 {
+			device_type = "cpu";
+			compatible = "intel,x86";
+			d-cache-line-size = <64>;
+			reg = <3>;
+		};
+	};
+};

tests/kernel/smp_boot_delay/prj.conf

@@ -1,3 +1,4 @@
 CONFIG_ZTEST=y
 CONFIG_SMP=y
 CONFIG_SMP_BOOT_DELAY=y
+CONFIG_SCHED_CPU_MASK=y

tests/kernel/smp_boot_delay/src/main.c

@@ -23,7 +23,7 @@ char stack[STACKSZ];
 
 volatile bool mp_flag;
 
-struct k_thread cpu1_thr;
+struct k_thread cpu_thr;
 K_THREAD_STACK_DEFINE(thr_stack, STACKSZ);
 
 static void thread_fn(void *a, void *b, void *c)
@@ -37,7 +37,7 @@ ZTEST(smp_boot_delay, test_smp_boot_delay)
 	 * another CPU if it was available, but will not preempt us
 	 * unless we block (which we do not).
 	 */
-	k_thread_create(&cpu1_thr, thr_stack, K_THREAD_STACK_SIZEOF(thr_stack),
+	k_thread_create(&cpu_thr, thr_stack, K_THREAD_STACK_SIZEOF(thr_stack),
 			thread_fn, NULL, NULL, NULL,
 			1, 0, K_NO_WAIT);
 
@@ -52,8 +52,8 @@ ZTEST(smp_boot_delay, test_smp_boot_delay)
 	k_busy_wait(CPU_START_DELAY);
 	zassert_true(mp_flag, "CPU1 did not start");
 
-	k_thread_abort(&cpu1_thr);
-	k_thread_join(&cpu1_thr, K_FOREVER);
+	k_thread_abort(&cpu_thr);
+	k_thread_join(&cpu_thr, K_FOREVER);
 
 	/* Spawn the same thread to do the same thing, but this time
 	 * expect that the thread is going to run synchronously on the
@@ -61,12 +61,67 @@ ZTEST(smp_boot_delay, test_smp_boot_delay)
 	 * IPIs were correctly set up on the runtime-launched CPU.
 	 */
 	mp_flag = false;
-	k_thread_create(&cpu1_thr, thr_stack, K_THREAD_STACK_SIZEOF(thr_stack),
+	k_thread_create(&cpu_thr, thr_stack, K_THREAD_STACK_SIZEOF(thr_stack),
 			thread_fn, NULL, NULL, NULL,
 			1, 0, K_NO_WAIT);
 
 	k_busy_wait(CPU_IPI_DELAY);
+
+	k_thread_abort(&cpu_thr);
+	k_thread_join(&cpu_thr, K_FOREVER);
+
 	zassert_true(mp_flag, "CPU1 did not start thread via IPI");
 }
 
+volatile bool custom_init_flag;
+
+void custom_init_fn(void *arg)
+{
+	volatile bool *flag = (void *)arg;
+
+	*flag = true;
+}
+
+ZTEST(smp_boot_delay, test_smp_custom_start)
+{
+	k_tid_t thr;
+
+	if (CONFIG_MP_MAX_NUM_CPUS <= 2) {
+		/* CPU#1 has been started in test_smp_boot_delay
+		 * so we need another CPU for this test.
+		 */
+		ztest_test_skip();
+	}
+
+	mp_flag = false;
+	custom_init_flag = false;
+
+	/* Create a thread pinned on CPU#2 so that it will not
+	 * run on other CPUs.
+	 */
+	thr = k_thread_create(&cpu_thr, thr_stack, K_THREAD_STACK_SIZEOF(thr_stack),
+			      thread_fn, NULL, NULL, NULL,
+			      1, 0, K_FOREVER);
+	(void)k_thread_cpu_pin(thr, 2);
+	k_thread_start(thr);
+
+	/* Make sure that thread has not run (because the cpu is halted) */
+	k_busy_wait(CPU_START_DELAY);
+	zassert_false(mp_flag, "CPU2 must not be running yet");
+
+	/* Start the third CPU */
+	k_smp_cpu_start(2, custom_init_fn, (void *)&custom_init_flag);
+
+	/* Verify the thread ran */
+	k_busy_wait(CPU_START_DELAY);
+	zassert_true(mp_flag, "CPU2 did not start");
+
+	/* Verify that the custom init function has been called. */
+	zassert_true(custom_init_flag, "Custom init function has not been called.");
+
+	k_thread_abort(&cpu_thr);
+	k_thread_join(&cpu_thr, K_FOREVER);
+}
+
+
 ZTEST_SUITE(smp_boot_delay, NULL, NULL, NULL, NULL, NULL);