zephyr: Fix verify run all check

In scenarios where test_main is overridden ztest_run_all
may be invoked multiple times leading to the verify check to
fail inadvertently.

Signed-off-by: Al Semjonovs <asemjonovs@google.com>

subsys/testsuite/ztest/Kconfig

@@ -98,6 +98,12 @@ config ZTEST_RULE_1CPU
 
 endmenu
 
+config ZTEST_VERIFY_RUN_ALL
+	bool "Validates all defined tests have ran"
+	default y
+	help
+	  This rule will fail the project if not all tests have been run.
+
 config ZTEST_SHUFFLE
 	bool "Shuffle the order of tests and suites"
 	select TEST_RANDOM_GENERATOR if !ENTROPY_HAS_DRIVER

subsys/testsuite/ztest/src/ztest_defaults.c

@@ -30,7 +30,6 @@ const char *ztest_relative_filename(const char *file)
 void z_ztest_run_all(const void *state)
 {
 	ztest_run_test_suites(state);
-	ztest_verify_all_test_suites_ran();
 }
 
 /**

subsys/testsuite/ztest/src/ztest_new.c

@@ -648,30 +648,39 @@ void ztest_verify_all_test_suites_ran(void)
 	struct ztest_suite_node *suite;
 	struct ztest_unit_test *test;
 
-	for (suite = _ztest_suite_node_list_start; suite < _ztest_suite_node_list_end; ++suite) {
-		if (suite->stats->run_count < 1) {
-			PRINT("ERROR: Test suite '%s' did not run.\n", suite->name);
-			all_tests_run = false;
+	if (IS_ENABLED(CONFIG_ZTEST_VERIFY_RUN_ALL)) {
+		for (suite = _ztest_suite_node_list_start; suite < _ztest_suite_node_list_end;
+		     ++suite) {
+			if (suite->stats->run_count < 1) {
+				PRINT("ERROR: Test suite '%s' did not run.\n", suite->name);
+				all_tests_run = false;
+			}
 		}
-	}
 
-	for (test = _ztest_unit_test_list_start; test < _ztest_unit_test_list_end; ++test) {
-		suite = ztest_find_test_suite(test->test_suite_name);
-		if (suite == NULL) {
-			PRINT("ERROR: Test '%s' assigned to test suite '%s' which doesn't exist\n",
-			      test->name, test->test_suite_name);
-			all_tests_run = false;
+		for (test = _ztest_unit_test_list_start; test < _ztest_unit_test_list_end; ++test) {
+			suite = ztest_find_test_suite(test->test_suite_name);
+			if (suite == NULL) {
+				PRINT("ERROR: Test '%s' assigned to test suite '%s' which doesn't "
+				      "exist\n",
+				      test->name, test->test_suite_name);
+				all_tests_run = false;
+			}
 		}
-	}
 
-	if (!all_tests_run) {
-		test_status = 1;
+		if (!all_tests_run) {
+			test_status = 1;
+		}
 	}
 }
 
 void ztest_run_all(const void *state) { ztest_api.run_all(state); }
 
-void __weak test_main(void) { ztest_run_all(NULL); }
+void __weak test_main(void)
+{
+	ztest_run_all(NULL);
+
+	ztest_verify_all_test_suites_ran();
+}
 
 #ifndef KERNEL
 int main(void)

subsys/testsuite/ztest/src/ztest_posix.c

@@ -132,7 +132,6 @@ void z_ztest_run_all(const void *state)
 		z_ztest_list_tests();
 	} else {
 		ztest_run_test_suites(state);
-		ztest_verify_all_test_suites_ran();
 	}
 }