zara
/
zephyr
mirror of https://github.com/zephyrproject-rtos/zephyr
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Primary Git Repository for the Zephyr Project. Zephyr is a new generation, scalable, optimized, secure RTOS for multiple hardware architectures.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
118379 Commits
82 Branches
204 Tags
1.7 GiB
 
 
 
 
 
 
Tree: 061036f9e3
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '061036f9e3'
${ noResults }
zephyr/scripts/tests/twister/test_testplan.py

1793 lines
60 KiB
Raw Blame History

#!/usr/bin/env python3
# Copyright (c) 2020-2024 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
'''
This test file contains testsuites for testsuite.py module of twister
'''
import sys
import os
import mock
import pytest
from contextlib import nullcontext
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
from twisterlib.statuses import TwisterStatus
from twisterlib.testplan import TestPlan, change_skip_to_error_if_integration
from twisterlib.testinstance import TestInstance
from twisterlib.testsuite import TestSuite
from twisterlib.platform import Platform
from twisterlib.quarantine import Quarantine
from twisterlib.error import TwisterRuntimeError
def test_testplan_add_testsuites_short(class_testplan):
""" Testing add_testcase function of Testsuite class in twister """
# Test 1: Check the list of testsuites after calling add testsuites function is as expected
class_testplan.SAMPLE_FILENAME = 'test_sample_app.yaml'
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
class_testplan.add_testsuites()
tests_rel_dir = 'scripts/tests/twister/test_data/testsuites/tests/'
expected_testsuites = ['test_b.check_1',
'test_b.check_2',
'test_c.check_1',
'test_c.check_2',
'test_a.check_1',
'test_a.check_2',
'test_d.check_1',
'test_e.check_1',
'sample_test.app',
'test_config.main']
testsuite_list = []
for key in sorted(class_testplan.testsuites.keys()):
testsuite_list.append(os.path.basename(os.path.normpath(key)))
assert sorted(testsuite_list) == sorted(expected_testsuites)
# Test 2 : Assert Testcase name is expected & all testsuites values are testcase class objects
suite = class_testplan.testsuites.get(tests_rel_dir + 'test_a/test_a.check_1')
assert suite.name == tests_rel_dir + 'test_a/test_a.check_1'
assert all(isinstance(n, TestSuite) for n in class_testplan.testsuites.values())
@pytest.mark.parametrize("board_root_dir", [("board_config_file_not_exist"), ("board_config")])
def test_add_configurations_short(test_data, class_env, board_root_dir):
""" Testing add_configurations function of TestPlan class in Twister
Test : Asserting on default platforms list
"""
class_env.board_roots = [os.path.abspath(test_data + board_root_dir)]
plan = TestPlan(class_env)
plan.parse_configuration(config_file=class_env.test_config)
if board_root_dir == "board_config":
plan.add_configurations()
print(sorted(plan.default_platforms))
assert sorted(plan.default_platforms) == sorted(['demo_board_1/unit_testing', 'demo_board_3/unit_testing'])
elif board_root_dir == "board_config_file_not_exist":
plan.add_configurations()
assert sorted(plan.default_platforms) != sorted(['demo_board_1'])
def test_get_all_testsuites_short(class_testplan, all_testsuites_dict):
""" Testing get_all_testsuites function of TestPlan class in Twister """
plan = class_testplan
plan.testsuites = all_testsuites_dict
expected_tests = ['sample_test.app', 'test_a.check_1.1a',
'test_a.check_1.1c',
'test_a.check_1.2a', 'test_a.check_1.2b',
'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a',
'test_a.check_1.unit_1b', 'test_a.check_2.1a',
'test_a.check_2.1c', 'test_a.check_2.2a',
'test_a.check_2.2b', 'test_a.check_2.Unit_1c',
'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
'test_c.check_2', 'test_d.check_1.unit_1a',
'test_d.check_1.unit_1b',
'test_e.check_1.feature5.1a',
'test_e.check_1.feature5.1b',
'test_config.main']
assert sorted(plan.get_all_tests()) == sorted(expected_tests)
def test_get_platforms_short(class_testplan, platforms_list):
""" Testing get_platforms function of TestPlan class in Twister """
plan = class_testplan
plan.platforms = platforms_list
platform = plan.get_platform("demo_board_1")
assert isinstance(platform, Platform)
assert platform.name == "demo_board_1/unit_testing"
TESTDATA_PART1 = [
("toolchain_allow", ['gcc'], None, None, "Not in testsuite toolchain allow list"),
("platform_allow", ['demo_board_1/unit_testing'], None, None, "Not in testsuite platform allow list"),
("toolchain_exclude", ['zephyr'], None, None, "In test case toolchain exclude"),
("platform_exclude", ['demo_board_2'], None, None, "In test case platform exclude"),
("arch_exclude", ['x86'], None, None, "In test case arch exclude"),
("arch_allow", ['arm'], None, None, "Not in test case arch allow list"),
("skip", True, None, None, "Skip filter"),
("tags", set(['sensor', 'bluetooth']), "ignore_tags", ['bluetooth'], "Excluded tags per platform (exclude_tags)"),
("min_flash", "2024", "flash", "1024", "Not enough FLASH"),
("min_ram", "500", "ram", "256", "Not enough RAM"),
("None", "None", "env", ['BSIM_OUT_PATH', 'demo_env'], "Environment (BSIM_OUT_PATH, demo_env) not satisfied"),
("build_on_all", True, None, None, "Platform is excluded on command line."),
("build_on_all", True, "level", "foobar", "Unknown test level 'foobar'"),
(None, None, "supported_toolchains", ['gcc', 'xcc', 'xt-clang'], "Not supported by the toolchain"),
]
@pytest.mark.parametrize("tc_attribute, tc_value, plat_attribute, plat_value, expected_discards",
TESTDATA_PART1)
def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list,
tc_attribute, tc_value, plat_attribute, plat_value, expected_discards):
""" Testing apply_filters function of TestPlan class in Twister
Part 1: Response of apply_filters function have
appropriate values according to the filters
"""
plan = class_testplan
if tc_attribute is None and plat_attribute is None:
plan.apply_filters()
plan.platforms = platforms_list
plan.platform_names = [p.name for p in platforms_list]
plan.testsuites = all_testsuites_dict
for plat in plan.platforms:
if plat_attribute == "ignore_tags":
plat.ignore_tags = plat_value
if plat_attribute == "flash":
plat.flash = plat_value
if plat_attribute == "ram":
plat.ram = plat_value
if plat_attribute == "env":
plat.env = plat_value
plat.env_satisfied = False
if plat_attribute == "supported_toolchains":
plat.supported_toolchains = plat_value
for _, testcase in plan.testsuites.items():
if tc_attribute == "toolchain_allow":
testcase.toolchain_allow = tc_value
if tc_attribute == "platform_allow":
testcase.platform_allow = tc_value
if tc_attribute == "toolchain_exclude":
testcase.toolchain_exclude = tc_value
if tc_attribute == "platform_exclude":
testcase.platform_exclude = tc_value
if tc_attribute == "arch_exclude":
testcase.arch_exclude = tc_value
if tc_attribute == "arch_allow":
testcase.arch_allow = tc_value
if tc_attribute == "skip":
testcase.skip = tc_value
if tc_attribute == "tags":
testcase.tags = tc_value
if tc_attribute == "min_flash":
testcase.min_flash = tc_value
if tc_attribute == "min_ram":
testcase.min_ram = tc_value
if plat_attribute == "level":
plan.options.level = plat_value
if tc_attribute == "build_on_all":
for _, testcase in plan.testsuites.items():
testcase.build_on_all = tc_value
plan.apply_filters(exclude_platform=['demo_board_1'])
elif plat_attribute == "supported_toolchains":
plan.apply_filters(force_toolchain=False,
exclude_platform=['demo_board_1'],
platform=['demo_board_2/unit_testing'])
elif tc_attribute is None and plat_attribute is None:
plan.apply_filters()
else:
plan.apply_filters(exclude_platform=['demo_board_1'],
platform=['demo_board_2/unit_testing'])
filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, plan.instances.values()))
for d in filtered_instances:
assert d.reason == expected_discards
TESTDATA_PART2 = [
("runnable", "True", "Not runnable on device"),
("exclude_tag", ['test_a'], "Command line testsuite exclude filter"),
("run_individual_tests", ['scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1'], "TestSuite name filter"),
("arch", ['arm_test'], "Command line testsuite arch filter"),
("tag", ['test_d'], "Command line testsuite tag filter")
]
@pytest.mark.parametrize("extra_filter, extra_filter_value, expected_discards", TESTDATA_PART2)
def test_apply_filters_part2(class_testplan, all_testsuites_dict,
platforms_list, extra_filter, extra_filter_value, expected_discards):
""" Testing apply_filters function of TestPlan class in Twister
Part 2 : Response of apply_filters function (discard dictionary) have
appropriate values according to the filters
"""
class_testplan.platforms = platforms_list
class_testplan.platform_names = [p.name for p in platforms_list]
class_testplan.testsuites = all_testsuites_dict
kwargs = {
extra_filter : extra_filter_value,
"exclude_platform" : [
'demo_board_1'
],
"platform" : [
'demo_board_2'
]
}
class_testplan.apply_filters(**kwargs)
filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, class_testplan.instances.values()))
for d in filtered_instances:
assert d.reason == expected_discards
TESTDATA_PART3 = [
(20, 20, -1, 0),
(-2, -1, 10, 20),
(0, 0, 0, 0)
]
@pytest.mark.parametrize("tc_min_flash, plat_flash, tc_min_ram, plat_ram",
TESTDATA_PART3)
def test_apply_filters_part3(class_testplan, all_testsuites_dict, platforms_list,
tc_min_flash, plat_flash, tc_min_ram, plat_ram):
""" Testing apply_filters function of TestPlan class in Twister
Part 3 : Testing edge cases for ram and flash values of platforms & testsuites
"""
class_testplan.platforms = platforms_list
class_testplan.platform_names = [p.name for p in platforms_list]
class_testplan.testsuites = all_testsuites_dict
for plat in class_testplan.platforms:
plat.flash = plat_flash
plat.ram = plat_ram
for _, testcase in class_testplan.testsuites.items():
testcase.min_ram = tc_min_ram
testcase.min_flash = tc_min_flash
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
platform=['demo_board_2'])
filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, class_testplan.instances.values()))
assert not filtered_instances
def test_add_instances_short(tmp_path, class_env, all_testsuites_dict, platforms_list):
""" Testing add_instances() function of TestPlan class in Twister
Test 1: instances dictionary keys have expected values (Platform Name + Testcase Name)
Test 2: Values of 'instances' dictionary in Testsuite class are an
instance of 'TestInstance' class
Test 3: Values of 'instances' dictionary have expected values.
"""
class_env.outdir = tmp_path
plan = TestPlan(class_env)
plan.platforms = platforms_list
platform = plan.get_platform("demo_board_2")
instance_list = []
for _, testcase in all_testsuites_dict.items():
instance = TestInstance(testcase, platform, 'zephyr', class_env.outdir)
instance_list.append(instance)
plan.add_instances(instance_list)
assert list(plan.instances.keys()) == \
[platform.name + '/zephyr/' + s for s in list(all_testsuites_dict.keys())]
assert all(isinstance(n, TestInstance) for n in list(plan.instances.values()))
assert list(plan.instances.values()) == instance_list
QUARANTINE_BASIC = {
'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3'
}
QUARANTINE_WITH_REGEXP = {
'demo_board_2/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86',
'demo_board_1/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
'demo_board_3/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
'demo_board_2/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
'demo_board_2/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86'
}
QUARANTINE_PLATFORM = {
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_e/test_e.check_1' : 'all on board_3',
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_config/test_config.main' : 'all on board_3'
}
QUARANTINE_MULTIFILES = {
**QUARANTINE_BASIC,
**QUARANTINE_WITH_REGEXP
}
@pytest.mark.parametrize(
("quarantine_files, quarantine_verify, expected_val"),
[
(['basic.yaml'], False, QUARANTINE_BASIC),
(['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP),
(['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP),
(['platform.yaml'], False, QUARANTINE_PLATFORM),
(['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES),
(['empty.yaml'], False, {})
],
ids=[
'basic',
'with_regexp',
'quarantine_verify',
'platform',
'multifiles',
'empty'
])
def test_quarantine_short(class_testplan, platforms_list, test_data,
quarantine_files, quarantine_verify, expected_val):
""" Testing quarantine feature in Twister
"""
class_testplan.options.all = True
class_testplan.platforms = platforms_list
class_testplan.platform_names = [p.name for p in platforms_list]
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
class_testplan.add_testsuites()
quarantine_list = [
os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files
]
class_testplan.quarantine = Quarantine(quarantine_list)
class_testplan.options.quarantine_verify = quarantine_verify
class_testplan.apply_filters()
for testname, instance in class_testplan.instances.items():
if quarantine_verify:
if testname in expected_val:
assert instance.status == TwisterStatus.NONE
else:
assert instance.status == TwisterStatus.SKIP
assert instance.reason == "Not under quarantine"
else:
if testname in expected_val:
assert instance.status == TwisterStatus.SKIP
assert instance.reason == "Quarantine: " + expected_val[testname]
else:
assert instance.status == TwisterStatus.NONE
TESTDATA_PART4 = [
(os.path.join('test_d', 'test_d.check_1'), ['dummy'],
None, 'Snippet not supported'),
(os.path.join('test_c', 'test_c.check_1'), ['cdc-acm-console'],
0, None),
(os.path.join('test_d', 'test_d.check_1'), ['dummy', 'cdc-acm-console'],
2, 'Snippet not supported'),
]
@pytest.mark.parametrize(
'testpath, required_snippets, expected_filtered_len, expected_filtered_reason',
TESTDATA_PART4,
ids=['app', 'global', 'multiple']
)
def test_required_snippets_short(
class_testplan,
all_testsuites_dict,
platforms_list,
testpath,
required_snippets,
expected_filtered_len,
expected_filtered_reason
):
""" Testing required_snippets function of TestPlan class in Twister """
plan = class_testplan
testpath = os.path.join('scripts', 'tests', 'twister', 'test_data',
'testsuites', 'tests', testpath)
testsuite = class_testplan.testsuites.get(testpath)
plan.platforms = platforms_list
plan.platform_names = [p.name for p in platforms_list]
plan.testsuites = {testpath: testsuite}
for _, testcase in plan.testsuites.items():
testcase.exclude_platform = []
testcase.required_snippets = required_snippets
testcase.build_on_all = True
plan.apply_filters()
filtered_instances = list(
filter(lambda item: item.status == TwisterStatus.FILTER, plan.instances.values())
)
if expected_filtered_len is not None:
assert len(filtered_instances) == expected_filtered_len
if expected_filtered_reason is not None:
for d in filtered_instances:
assert d.reason == expected_filtered_reason
def test_testplan_get_level():
testplan = TestPlan(env=mock.Mock())
lvl1 = mock.Mock()
lvl1.name = 'a lvl'
lvl2 = mock.Mock()
lvl2.name = 'a lvl'
lvl3 = mock.Mock()
lvl3.name = 'other lvl'
testplan.levels.append(lvl1)
testplan.levels.append(lvl2)
testplan.levels.append(lvl3)
name = 'a lvl'
res = testplan.get_level(name)
assert res == lvl1
res = testplan.get_level(name)
assert res == lvl1
lvl_missed = mock.Mock()
lvl_missed.name = 'missed lvl'
res = testplan.get_level('missed_lvl')
assert res is None
testplan.levels.remove(lvl1)
testplan.levels.remove(lvl2)
res = testplan.get_level(name)
assert res is None
TESTDATA_1 = [
('', {}),
(
"""\
levels:
- name: lvl1
adds:
- sc1
- sc2
inherits: []
- name: lvl2
adds:
- sc1-1
- sc1-2
inherits: [lvl1]
""",
{
'lvl1': ['sc1', 'sc2'],
'lvl2': ['sc1-1', 'sc1-2', 'sc1', 'sc2']
}
),
]
@pytest.mark.parametrize(
'config_yaml, expected_scenarios',
TESTDATA_1,
ids=['no config', 'valid config']
)
def test_testplan_parse_configuration(tmp_path, config_yaml, expected_scenarios):
testplan = TestPlan(env=mock.Mock())
testplan.scenarios = ['sc1', 'sc1-1', 'sc1-2', 'sc2']
tmp_config_file = tmp_path / 'config_file.yaml'
if config_yaml:
tmp_config_file.write_text(config_yaml)
with pytest.raises(TwisterRuntimeError) if not config_yaml else nullcontext():
testplan.parse_configuration(tmp_config_file)
if not testplan.levels:
assert expected_scenarios == {}
for level in testplan.levels:
assert sorted(level.scenarios) == sorted(expected_scenarios[level.name])
TESTDATA_2 = [
([], [], False),
(['ts1.tc3'], [], True),
(['ts2.tc2'], ['- ts2'], False),
]
@pytest.mark.parametrize(
'sub_tests, expected_outs, expect_error',
TESTDATA_2,
ids=['no subtests', 'subtests not found', 'valid subtests']
)
def test_testplan_find_subtests(
capfd,
sub_tests,
expected_outs,
expect_error
):
testplan = TestPlan(env=mock.Mock())
testplan.options = mock.Mock(sub_test=sub_tests)
testplan.run_individual_testsuite = []
testplan.testsuites = {
'ts1': mock.Mock(
testcases=[
mock.Mock(),
mock.Mock(),
]
),
'ts2': mock.Mock(
testcases=[
mock.Mock(),
mock.Mock(),
mock.Mock(),
]
)
}
testplan.testsuites['ts1'].name = 'ts1'
testplan.testsuites['ts1'].testcases[0].name = 'ts1.tc1'
testplan.testsuites['ts1'].testcases[1].name = 'ts1.tc2'
testplan.testsuites['ts2'].name = 'ts2'
testplan.testsuites['ts2'].testcases[0].name = 'ts2.tc1'
testplan.testsuites['ts2'].testcases[1].name = 'ts2.tc2'
testplan.testsuites['ts2'].testcases[2].name = 'ts2.tc3'
with pytest.raises(TwisterRuntimeError) if expect_error else nullcontext():
testplan.find_subtests()
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stdout.write(err)
assert all([printout in out for printout in expected_outs])
TESTDATA_3 = [
(0, 0, [], False, [], TwisterRuntimeError, []),
(1, 1, [], False, [], TwisterRuntimeError, []),
(1, 0, [], True, [], TwisterRuntimeError, ['No quarantine list given to be verified']),
# (1, 0, ['qfile.yaml'], False, ['# empty'], None, ['Quarantine file qfile.yaml is empty']),
(1, 0, ['qfile.yaml'], False, ['- platforms:\n - demo_board_3\n comment: "board_3"'], None, []),
]
@pytest.mark.parametrize(
'added_testsuite_count, load_errors, ql, qv, ql_data, exception, expected_logs',
TESTDATA_3,
ids=['no tests', 'load errors', 'quarantine verify without quarantine list',
# 'empty quarantine file',
'valid quarantine file']
)
def test_testplan_discover(
tmp_path,
caplog,
added_testsuite_count,
load_errors,
ql,
qv,
ql_data,
exception,
expected_logs
):
for qf, data in zip(ql, ql_data):
tmp_qf = tmp_path / qf
tmp_qf.write_text(data)
testplan = TestPlan(env=mock.Mock())
testplan.options = mock.Mock(
test='ts1',
quarantine_list=[tmp_path / qf for qf in ql],
quarantine_verify=qv,
)
testplan.testsuites = {
'ts1': mock.Mock(id=1),
'ts2': mock.Mock(id=2),
}
testplan.run_individual_testsuite = 'ts0'
testplan.load_errors = load_errors
testplan.add_testsuites = mock.Mock(return_value=added_testsuite_count)
testplan.find_subtests = mock.Mock()
testplan.report_duplicates = mock.Mock()
testplan.parse_configuration = mock.Mock()
testplan.add_configurations = mock.Mock()
with pytest.raises(exception) if exception else nullcontext():
testplan.discover()
testplan.add_testsuites.assert_called_once_with(testsuite_filter='ts1')
assert all([log in caplog.text for log in expected_logs])
TESTDATA_4 = [
(None, None, None, None, '00',
TwisterRuntimeError, [], []),
(None, True, None, None, '6/4',
TwisterRuntimeError, set(['t-p3', 't-p4', 't-p1', 't-p2']), []),
(None, None, 'load_tests.json', None, '0/4',
TwisterRuntimeError, set(['lt-p1', 'lt-p3', 'lt-p4', 'lt-p2']), []),
('suffix', None, None, True, '2/4',
None, set(['ts-p4', 'ts-p2', 'ts-p1', 'ts-p3']), [2, 4]),
]
@pytest.mark.parametrize(
'report_suffix, only_failed, load_tests, test_only, subset,' \
' exception, expected_selected_platforms, expected_generate_subset_args',
TESTDATA_4,
ids=['apply_filters only', 'only failed', 'load tests', 'test only']
)
def test_testplan_load(
tmp_path,
report_suffix,
only_failed,
load_tests,
test_only,
subset,
exception,
expected_selected_platforms,
expected_generate_subset_args
):
twister_json = """\
{
"testsuites": [
{
"name": "ts1",
"platform": "t-p1",
"toolchain": "zephyr",
"testcases": []
},
{
"name": "ts1",
"platform": "t-p2",
"toolchain": "zephyr",
"testcases": []
},
{
"name": "ts2",
"platform": "t-p3",
"toolchain": "zephyr",
"testcases": []
},
{
"name": "ts2",
"platform": "t-p4",
"toolchain": "zephyr",
"testcases": []
}
]
}
"""
twister_file = tmp_path / 'twister.json'
twister_file.write_text(twister_json)
twister_suffix_json = """\
{
"testsuites": [
{
"name": "ts1",
"platform": "ts-p1",
"toolchain": "zephyr",
"testcases": []
},
{
"name": "ts1",
"platform": "ts-p2",
"toolchain": "zephyr",
"testcases": []
},
{
"name": "ts2",
"platform": "ts-p3",
"toolchain": "zephyr",
"testcases": []
},
{
"name": "ts2",
"platform": "ts-p4",
"toolchain": "zephyr",
"testcases": []
}
]
}
"""
twister_suffix_file = tmp_path / 'twister_suffix.json'
twister_suffix_file.write_text(twister_suffix_json)
load_tests_json = """\
{
"testsuites": [
{
"name": "ts1",
"platform": "lt-p1",
"toolchain": "zephyr",
"testcases": []
},
{
"name": "ts1",
"platform": "lt-p2",
"toolchain": "zephyr",
"testcases": []
},
{
"name": "ts2",
"platform": "lt-p3",
"toolchain": "zephyr",
\"testcases": []
},
{
"name": "ts2",
"platform": "lt-p4",
"toolchain": "zephyr",
"testcases": []
}
]
}
"""
load_tests_file = tmp_path / 'load_tests.json'
load_tests_file.write_text(load_tests_json)
testplan = TestPlan(env=mock.Mock(outdir=tmp_path))
testplan.testsuites = {
'ts1': mock.Mock(testcases=[], extra_configs=[]),
'ts2': mock.Mock(testcases=[], extra_configs=[]),
}
testplan.testsuites['ts1'].name = 'ts1'
testplan.testsuites['ts2'].name = 'ts2'
testplan.options = mock.Mock(
report_summary=None,
outdir=tmp_path,
report_suffix=report_suffix,
only_failed=only_failed,
load_tests=tmp_path / load_tests if load_tests else None,
test_only=test_only,
exclude_platform=['t-p0', 't-p1',
'ts-p0', 'ts-p1',
'lt-p0', 'lt-p1'],
platform=['t-p1', 't-p2', 't-p3', 't-p4',
'ts-p1', 'ts-p2', 'ts-p3', 'ts-p4',
'lt-p1', 'lt-p2', 'lt-p3', 'lt-p4'],
subset=subset
)
testplan.platforms=[mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(),
mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(),
mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock()]
testplan.platforms[0].name = 't-p1'
testplan.platforms[1].name = 't-p2'
testplan.platforms[2].name = 't-p3'
testplan.platforms[3].name = 't-p4'
testplan.platforms[4].name = 'ts-p1'
testplan.platforms[5].name = 'ts-p2'
testplan.platforms[6].name = 'ts-p3'
testplan.platforms[7].name = 'ts-p4'
testplan.platforms[8].name = 'lt-p1'
testplan.platforms[9].name = 'lt-p2'
testplan.platforms[10].name = 'lt-p3'
testplan.platforms[11].name = 'lt-p4'
testplan.platforms[0].aliases = ['t-p1']
testplan.platforms[1].aliases = ['t-p2']
testplan.platforms[2].aliases = ['t-p3']
testplan.platforms[3].aliases = ['t-p4']
testplan.platforms[4].aliases = ['ts-p1']
testplan.platforms[5].aliases = ['ts-p2']
testplan.platforms[6].aliases = ['ts-p3']
testplan.platforms[7].aliases = ['ts-p4']
testplan.platforms[8].aliases = ['lt-p1']
testplan.platforms[9].aliases = ['lt-p2']
testplan.platforms[10].aliases = ['lt-p3']
testplan.platforms[11].aliases = ['lt-p4']
testplan.platforms[0].normalized_name = 't-p1'
testplan.platforms[1].normalized_name = 't-p2'
testplan.platforms[2].normalized_name = 't-p3'
testplan.platforms[3].normalized_name = 't-p4'
testplan.platforms[4].normalized_name = 'ts-p1'
testplan.platforms[5].normalized_name = 'ts-p2'
testplan.platforms[6].normalized_name = 'ts-p3'
testplan.platforms[7].normalized_name = 'ts-p4'
testplan.platforms[8].normalized_name = 'lt-p1'
testplan.platforms[9].normalized_name = 'lt-p2'
testplan.platforms[10].normalized_name = 'lt-p3'
testplan.platforms[11].normalized_name = 'lt-p4'
testplan.generate_subset = mock.Mock()
testplan.apply_filters = mock.Mock()
with mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()), \
mock.patch('twisterlib.testinstance.TestInstance.check_runnable', return_value=True), \
pytest.raises(exception) if exception else nullcontext():
testplan.load()
assert testplan.selected_platforms == expected_selected_platforms
if expected_generate_subset_args:
testplan.generate_subset.assert_called_once_with(*expected_generate_subset_args)
else:
testplan.generate_subset.assert_not_called()
TESTDATA_5 = [
(False, False, None, 1, 2,
['plat1/testA', 'plat1/testB', 'plat1/testC',
'plat3/testA', 'plat3/testB', 'plat3/testC']),
(False, False, None, 1, 5,
['plat1/testA',
'plat3/testA', 'plat3/testB', 'plat3/testC']),
(False, False, None, 2, 2,
['plat2/testA', 'plat2/testB']),
(True, False, None, 1, 2,
['plat1/testA', 'plat2/testA', 'plat1/testB',
'plat3/testA', 'plat3/testB', 'plat3/testC']),
(True, False, None, 2, 2,
['plat2/testB', 'plat1/testC']),
(True, True, 123, 1, 2,
['plat2/testA', 'plat2/testB', 'plat1/testC',
'plat3/testB', 'plat3/testA', 'plat3/testC']),
(True, True, 123, 2, 2,
['plat1/testB', 'plat1/testA']),
]
@pytest.mark.parametrize(
'device_testing, shuffle, seed, subset, sets, expected_subset',
TESTDATA_5,
ids=['subset 1', 'subset 1 out of 5', 'subset 2',
'device testing, subset 1', 'device testing, subset 2',
'device testing, shuffle with seed, subset 1',
'device testing, shuffle with seed, subset 2']
)
def test_testplan_generate_subset(
device_testing,
shuffle,
seed,
subset,
sets,
expected_subset
):
testplan = TestPlan(env=mock.Mock())
testplan.options = mock.Mock(
device_testing=device_testing,
shuffle_tests=shuffle,
shuffle_tests_seed=seed
)
testplan.instances = {
'plat1/testA': mock.Mock(status=TwisterStatus.NONE),
'plat1/testB': mock.Mock(status=TwisterStatus.NONE),
'plat1/testC': mock.Mock(status=TwisterStatus.NONE),
'plat2/testA': mock.Mock(status=TwisterStatus.NONE),
'plat2/testB': mock.Mock(status=TwisterStatus.NONE),
'plat3/testA': mock.Mock(status=TwisterStatus.SKIP),
'plat3/testB': mock.Mock(status=TwisterStatus.SKIP),
'plat3/testC': mock.Mock(status=TwisterStatus.ERROR),
}
testplan.generate_subset(subset, sets)
assert [instance for instance in testplan.instances.keys()] == \
expected_subset
def test_testplan_handle_modules():
testplan = TestPlan(env=mock.Mock())
modules = [mock.Mock(meta={'name': 'name1'}),
mock.Mock(meta={'name': 'name2'})]
with mock.patch('twisterlib.testplan.parse_modules', return_value=modules):
testplan.handle_modules()
assert testplan.modules == ['name1', 'name2']
TESTDATA_6 = [
(True, False, False, 0, 'report_test_tree'),
(True, True, False, 0, 'report_test_tree'),
(True, False, True, 0, 'report_test_tree'),
(True, True, True, 0, 'report_test_tree'),
(False, True, False, 0, 'report_test_list'),
(False, True, True, 0, 'report_test_list'),
(False, False, True, 0, 'report_tag_list'),
(False, False, False, 1, None),
]
@pytest.mark.parametrize(
'test_tree, list_tests, list_tags, expected_res, expected_method',
TESTDATA_6,
ids=['test tree', 'test tree + test list', 'test tree + tag list',
'test tree + test list + tag list', 'test list',
'test list + tag list', 'tag list', 'no report']
)
def test_testplan_report(
test_tree,
list_tests,
list_tags,
expected_res,
expected_method
):
testplan = TestPlan(env=mock.Mock())
testplan.report_test_tree = mock.Mock()
testplan.report_test_list = mock.Mock()
testplan.report_tag_list = mock.Mock()
testplan.options = mock.Mock(
test_tree=test_tree,
list_tests=list_tests,
list_tags=list_tags,
)
res = testplan.report()
assert res == expected_res
methods = ['report_test_tree', 'report_test_list', 'report_tag_list']
if expected_method:
methods.remove(expected_method)
getattr(testplan, expected_method).assert_called_once()
for method in methods:
getattr(testplan, method).assert_not_called()
TESTDATA_7 = [
(
[
mock.Mock(
yamlfile='a.yaml',
scenarios=['scenario1', 'scenario2']
),
mock.Mock(
yamlfile='b.yaml',
scenarios=['scenario1']
)
],
TwisterRuntimeError,
'Duplicated test scenarios found:\n' \
'- scenario1 found in:\n' \
' - a.yaml\n' \
' - b.yaml\n',
[]
),
(
[
mock.Mock(
yamlfile='a.yaml',
scenarios=['scenario.a.1', 'scenario.a.2']
),
mock.Mock(
yamlfile='b.yaml',
scenarios=['scenario.b.1']
)
],
None,
None,
['No duplicates found.']
),
]
@pytest.mark.parametrize(
'testsuites, expected_error, error_msg, expected_logs',
TESTDATA_7,
ids=['a duplicate', 'no duplicates']
)
def test_testplan_report_duplicates(
capfd,
caplog,
testsuites,
expected_error,
error_msg,
expected_logs
):
def mock_get(name):
return list(filter(lambda x: name in x.scenarios, testsuites))
testplan = TestPlan(env=mock.Mock())
testplan.scenarios = [scenario for testsuite in testsuites \
for scenario in testsuite.scenarios]
testplan.get_testsuite = mock.Mock(side_effect=mock_get)
with pytest.raises(expected_error) if expected_error is not None else \
nullcontext() as err:
testplan.report_duplicates()
if expected_error:
assert str(err._excinfo[1]) == error_msg
assert all([log in caplog.text for log in expected_logs])
def test_testplan_report_tag_list(capfd):
testplan = TestPlan(env=mock.Mock())
testplan.testsuites = {
'testsuite0': mock.Mock(tags=set(['tag1', 'tag2'])),
'testsuite1': mock.Mock(tags=set(['tag1', 'tag2', 'tag3'])),
'testsuite2': mock.Mock(tags=set(['tag1', 'tag3'])),
'testsuite3': mock.Mock(tags=set(['tag']))
}
testplan.report_tag_list()
out,err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert '- tag' in out
assert '- tag1' in out
assert '- tag2' in out
assert '- tag3' in out
def test_testplan_report_test_tree(capfd):
testplan = TestPlan(env=mock.Mock())
testplan.get_tests_list = mock.Mock(
return_value=['1.dummy.case.1', '1.dummy.case.2',
'2.dummy.case.1', '2.dummy.case.2',
'3.dummy.case.1', '3.dummy.case.2',
'4.dummy.case.1', '4.dummy.case.2',
'5.dummy.case.1', '5.dummy.case.2',
'sample.group1.case1', 'sample.group1.case2',
'sample.group2.case', 'sample.group3.case1',
'sample.group3.case2', 'sample.group3.case3']
)
testplan.report_test_tree()
out,err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
expected = """
Testsuite
├── Samples
│ ├── group1
│ │ ├── sample.group1.case1
│ │ └── sample.group1.case2
│ ├── group2
│ │ └── sample.group2.case
│ └── group3
│ ├── sample.group3.case1
│ ├── sample.group3.case2
│ └── sample.group3.case3
└── Tests
├── 1
│ └── dummy
│ ├── 1.dummy.case.1
│ └── 1.dummy.case.2
├── 2
│ └── dummy
│ ├── 2.dummy.case.1
│ └── 2.dummy.case.2
├── 3
│ └── dummy
│ ├── 3.dummy.case.1
│ └── 3.dummy.case.2
├── 4
│ └── dummy
│ ├── 4.dummy.case.1
│ └── 4.dummy.case.2
└── 5
└── dummy
├── 5.dummy.case.1
└── 5.dummy.case.2
"""
expected = expected[1:]
assert expected in out
def test_testplan_report_test_list(capfd):
testplan = TestPlan(env=mock.Mock())
testplan.get_tests_list = mock.Mock(
return_value=['4.dummy.case.1', '4.dummy.case.2',
'3.dummy.case.2', '2.dummy.case.2',
'1.dummy.case.1', '1.dummy.case.2',
'3.dummy.case.1', '2.dummy.case.1',
'5.dummy.case.1', '5.dummy.case.2']
)
testplan.report_test_list()
out,err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert ' - 1.dummy.case.1\n' \
' - 1.dummy.case.2\n' \
' - 2.dummy.case.1\n' \
' - 2.dummy.case.2\n' \
' - 3.dummy.case.1\n' \
' - 3.dummy.case.2\n' \
' - 4.dummy.case.1\n' \
' - 4.dummy.case.2\n' \
' - 5.dummy.case.1\n' \
' - 5.dummy.case.2\n' \
'10 total.' in out
def test_testplan_info(capfd):
TestPlan.info('dummy text')
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert 'dummy text\n' in out
TESTDATA_8 = [
(False, ['p1e2/unit_testing', 'p2/unit_testing', 'p3/unit_testing'], ['p2/unit_testing', 'p3/unit_testing']),
(True, ['p1e2/unit_testing', 'p2/unit_testing', 'p3/unit_testing'], ['p3/unit_testing']),
]
@pytest.mark.parametrize(
'override_default_platforms, expected_platform_names, expected_defaults',
TESTDATA_8,
ids=['no override defaults', 'override defaults']
)
def test_testplan_add_configurations(
tmp_path,
override_default_platforms,
expected_platform_names,
expected_defaults
):
env = mock.Mock(board_roots=[tmp_path / 'boards'], soc_roots=[tmp_path], arch_roots=[tmp_path])
testplan = TestPlan(env=env)
testplan.test_config = {
'platforms': {
'override_default_platforms': override_default_platforms,
'default_platforms': ['p3', 'p1e1']
}
}
def mock_gen_plat(board_roots, soc_roots, arch_roots):
assert [tmp_path] == board_roots
assert [tmp_path] == soc_roots
assert [tmp_path] == arch_roots
platforms = [
mock.Mock(aliases=['p1e1/unit_testing', 'p1e1'], twister=False, default=False),
mock.Mock(aliases=['p1e2/unit_testing', 'p1e2'], twister=True, default=False),
mock.Mock(aliases=['p2/unit_testing', 'p2'], twister=True, default=True),
mock.Mock(aliases=['p3/unit_testing', 'p3'], twister=True, default=True),
]
for platform in platforms:
type(platform).name = mock.PropertyMock(return_value=platform.aliases[0])
yield platform
with mock.patch('twisterlib.testplan.generate_platforms', mock_gen_plat):
testplan.add_configurations()
if expected_defaults is not None:
print(expected_defaults)
print(testplan.default_platforms)
assert sorted(expected_defaults) == sorted(testplan.default_platforms)
if expected_platform_names is not None:
print(expected_platform_names)
print(testplan.platform_names)
platform_names = [p.name for p in testplan.platforms]
assert sorted(expected_platform_names) == sorted(platform_names)
def test_testplan_get_all_tests():
testplan = TestPlan(env=mock.Mock())
tc1 = mock.Mock()
tc1.name = 'tc1'
tc2 = mock.Mock()
tc2.name = 'tc2'
tc3 = mock.Mock()
tc3.name = 'tc3'
tc4 = mock.Mock()
tc4.name = 'tc4'
tc5 = mock.Mock()
tc5.name = 'tc5'
ts1 = mock.Mock(testcases=[tc1, tc2])
ts2 = mock.Mock(testcases=[tc3, tc4, tc5])
testplan.testsuites = {
'ts1': ts1,
'ts2': ts2
}
res = testplan.get_all_tests()
assert sorted(res) == ['tc1', 'tc2', 'tc3', 'tc4', 'tc5']
TESTDATA_9 = [
([], False, True, 11, 1),
([], False, False, 7, 2),
([], True, False, 9, 1),
([], True, True, 9, 1),
([], True, False, 9, 1),
(['good_test/dummy.common.1', 'good_test/dummy.common.2', 'good_test/dummy.common.3'], False, True, 3, 1),
(['good_test/dummy.common.1', 'good_test/dummy.common.2',
'duplicate_test/dummy.common.1', 'duplicate_test/dummy.common.2'], False, True, 4, 1),
(['dummy.common.1', 'dummy.common.2'], False, False, 2, 1),
(['good_test/dummy.common.1', 'good_test/dummy.common.2', 'good_test/dummy.common.3'], True, True, 0, 1),
]
@pytest.mark.parametrize(
'testsuite_filter, use_alt_root, detailed_id, expected_suite_count, expected_errors',
TESTDATA_9,
ids=[
'no testsuite filter, detailed id',
'no testsuite filter, short id',
'no testsuite filter, alt root, detailed id',
'no filter, alt root, detailed id',
'no filter, alt root, short id',
'testsuite filter',
'testsuite filter and valid duplicate',
'testsuite filter, short id and duplicate',
'testsuite filter, alt root',
]
)
def test_testplan_add_testsuites(tmp_path, testsuite_filter, use_alt_root, detailed_id,
expected_errors, expected_suite_count):
# tmp_path
# ├ tests <- test root
# │ ├ good_test
# │ │ └ testcase.yaml
# │ ├ wrong_test
# │ │ └ testcase.yaml
# │ ├ good_sample
# │ │ └ sample.yaml
# │ ├ duplicate_test
# │ │ └ testcase.yaml
# │ └ others
# │ └ other.txt
# └ other_tests <- alternate test root
# └ good_test
# └ testcase.yaml
tmp_test_root_dir = tmp_path / 'tests'
tmp_test_root_dir.mkdir()
tmp_good_test_dir = tmp_test_root_dir / 'good_test'
tmp_good_test_dir.mkdir()
testcase_yaml_1 = """\
tests:
dummy.common.1:
build_on_all: true
dummy.common.2:
build_on_all: true
dummy.common.3:
build_on_all: true
dummy.special:
build_on_all: false
"""
testfile_1 = tmp_good_test_dir / 'testcase.yaml'
testfile_1.write_text(testcase_yaml_1)
tmp_bad_test_dir = tmp_test_root_dir / 'wrong_test'
tmp_bad_test_dir.mkdir()
testcase_yaml_2 = """\
tests:
wrong:
yaml: {]}
"""
testfile_2 = tmp_bad_test_dir / 'testcase.yaml'
testfile_2.write_text(testcase_yaml_2)
tmp_good_sample_dir = tmp_test_root_dir / 'good_sample'
tmp_good_sample_dir.mkdir()
samplecase_yaml_1 = """\
tests:
sample.dummy.common.1:
tags:
- samples
sample.dummy.common.2:
tags:
- samples
sample.dummy.special.1:
tags:
- samples
"""
samplefile_1 = tmp_good_sample_dir / 'sample.yaml'
samplefile_1.write_text(samplecase_yaml_1)
tmp_duplicate_test_dir = tmp_test_root_dir / 'duplicate_test'
tmp_duplicate_test_dir.mkdir()
# The duplicate needs to have the same number of tests as these configurations
# can be read either with duplicate_test first, or good_test first, so number
# of selected tests needs to be the same in both situations.
testcase_yaml_4 = """\
tests:
dummy.common.1:
build_on_all: true
dummy.common.2:
build_on_all: true
dummy.common.3:
build_on_all: true
dummy.special:
build_on_all: false
"""
testfile_4 = tmp_duplicate_test_dir / 'testcase.yaml'
testfile_4.write_text(testcase_yaml_4)
tmp_other_dir = tmp_test_root_dir / 'others'
tmp_other_dir.mkdir()
_ = tmp_other_dir / 'other.txt'
tmp_alt_test_root_dir = tmp_path / 'other_tests'
tmp_alt_test_root_dir.mkdir()
tmp_alt_good_test_dir = tmp_alt_test_root_dir / 'good_test'
tmp_alt_good_test_dir.mkdir()
testcase_yaml_3 = """\
tests:
dummy.alt.1:
build_on_all: true
dummy.alt.2:
build_on_all: true
"""
testfile_3 = tmp_alt_good_test_dir / 'testcase.yaml'
testfile_3.write_text(testcase_yaml_3)
env = mock.Mock(
test_roots=[tmp_test_root_dir],
options=mock.Mock(detailed_test_id=detailed_id),
alt_config_root=[tmp_alt_test_root_dir] if use_alt_root else []
)
testplan = TestPlan(env=env)
res = testplan.add_testsuites(testsuite_filter)
assert res == expected_suite_count
assert testplan.load_errors == expected_errors
def test_testplan_str():
testplan = TestPlan(env=mock.Mock())
testplan.name = 'my name'
res = testplan.__str__()
assert res == 'my name'
TESTDATA_10 = [
('a platform', True),
('other platform', False),
]
@pytest.mark.parametrize(
'name, expect_found',
TESTDATA_10,
ids=['platform exists', 'no platform']
)
def test_testplan_get_platform(name, expect_found):
testplan = TestPlan(env=mock.Mock())
p1 = mock.Mock()
p1.name = 'some platform'
p1.aliases = [p1.name]
p2 = mock.Mock()
p2.name = 'a platform'
p2.aliases = [p2.name]
testplan.platforms = [p1, p2]
res = testplan.get_platform(name)
if expect_found:
assert res.name == name
else:
assert res is None
TESTDATA_11 = [
(True, 'runnable'),
(False, 'buildable'),
]
@pytest.mark.parametrize(
'device_testing, expected_tfilter',
TESTDATA_11,
ids=['device testing', 'no device testing']
)
def test_testplan_load_from_file(caplog, device_testing, expected_tfilter):
def get_platform(name):
p = mock.Mock()
p.name = name
p.normalized_name = name
return p
ts1tc1 = mock.Mock()
ts1tc1.name = 'TS1.tc1'
ts1 = mock.Mock(testcases=[ts1tc1])
ts1.name = 'TestSuite 1'
ts1.toolchain = 'zephyr'
ts2 = mock.Mock(testcases=[])
ts2.name = 'TestSuite 2'
ts2.toolchain = 'zephyr'
ts3tc1 = mock.Mock()
ts3tc1.name = 'TS3.tc1'
ts3tc2 = mock.Mock()
ts3tc2.name = 'TS3.tc2'
ts3 = mock.Mock(testcases=[ts3tc1, ts3tc2])
ts3.name = 'TestSuite 3'
ts3.toolchain = 'zephyr'
ts4tc1 = mock.Mock()
ts4tc1.name = 'TS4.tc1'
ts4 = mock.Mock(testcases=[ts4tc1])
ts4.name = 'TestSuite 4'
ts4.toolchain = 'zephyr'
ts5 = mock.Mock(testcases=[])
ts5.name = 'TestSuite 5'
ts5.toolchain = 'zephyr'
testplan = TestPlan(env=mock.Mock(outdir=os.path.join('out', 'dir')))
testplan.options = mock.Mock(device_testing=device_testing, test_only=True, report_summary=None)
testplan.testsuites = {
'TestSuite 1': ts1,
'TestSuite 2': ts2,
'TestSuite 3': ts3,
'TestSuite 4': ts4,
'TestSuite 5': ts5
}
testplan.get_platform = mock.Mock(side_effect=get_platform)
testplan_data = """\
{
"testsuites": [
{
"name": "TestSuite 1",
"platform": "Platform 1",
"run_id": 1,
"execution_time": 60.00,
"used_ram": 4096,
"available_ram": 12278,
"used_rom": 1024,
"available_rom": 1047552,
"status": "passed",
"toolchain": "zephyr",
"reason": "OK",
"testcases": [
{
"identifier": "TS1.tc1",
"status": "passed",
"reason": "passed",
"execution_time": 60.00,
"log": ""
}
]
},
{
"name": "TestSuite 2",
"platform": "Platform 1",
"toolchain": "zephyr"
},
{
"name": "TestSuite 3",
"platform": "Platform 1",
"run_id": 1,
"execution_time": 360.00,
"used_ram": 4096,
"available_ram": 12278,
"used_rom": 1024,
"available_rom": 1047552,
"status": "error",
"toolchain": "zephyr",
"reason": "File Not Found Error",
"testcases": [
{
"identifier": "TS3.tc1",
"status": "error",
"reason": "File Not Found Error.",
"execution_time": 360.00,
"log": "[ERROR]: File 'dummy.yaml' not found!\\nClosing..."
},
{
"identifier": "TS3.tc2"
}
]
},
{
"name": "TestSuite 4",
"platform": "Platform 1",
"execution_time": 360.00,
"used_ram": 4096,
"available_ram": 12278,
"used_rom": 1024,
"available_rom": 1047552,
"status": "skipped",
"toolchain": "zephyr",
"reason": "Not in requested test list.",
"testcases": [
{
"identifier": "TS4.tc1",
"status": "skipped",
"reason": "Not in requested test list.",
"execution_time": 360.00,
"log": "[INFO] Parsing..."
},
{
"identifier": "TS3.tc2"
}
]
},
{
"name": "TestSuite 5",
"platform": "Platform 2",
"toolchain": "zephyr"
}
]
}
"""
filter_platform = ['Platform 1']
check_runnable_mock = mock.Mock(return_value=True)
with mock.patch('builtins.open', mock.mock_open(read_data=testplan_data)), \
mock.patch('twisterlib.testinstance.TestInstance.check_runnable', check_runnable_mock), \
mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()):
testplan.load_from_file('dummy.yaml', filter_platform)
expected_instances = {
'Platform 1/zephyr/TestSuite 1': {
'metrics': {
'handler_time': 60.0,
'used_ram': 4096,
'used_rom': 1024,
'available_ram': 12278,
'available_rom': 1047552
},
'retries': 0,
'toolchain': 'zephyr',
'testcases': {
'TS1.tc1': {
'status': TwisterStatus.PASS,
'reason': 'passed',
'duration': 60.0,
'output': ''
}
}
},
'Platform 1/zephyr/TestSuite 2': {
'metrics': {
'handler_time': 0,
'used_ram': 0,
'used_rom': 0,
'available_ram': 0,
'available_rom': 0
},
'retries': 0,
'toolchain': 'zephyr',
'testcases': []
},
'Platform 1/zephyr/TestSuite 3': {
'metrics': {
'handler_time': 360.0,
'used_ram': 4096,
'used_rom': 1024,
'available_ram': 12278,
'available_rom': 1047552
},
'retries': 1,
'toolchain': 'zephyr',
'testcases': {
'TS3.tc1': {
'status': TwisterStatus.ERROR,
'reason': None,
'duration': 360.0,
'output': '[ERROR]: File \'dummy.yaml\' not found!\nClosing...'
},
'TS3.tc2': {
'status': TwisterStatus.NONE,
'reason': None,
'duration': 0,
'output': ''
}
}
},
'Platform 1/zephyr/TestSuite 4': {
'metrics': {
'handler_time': 360.0,
'used_ram': 4096,
'used_rom': 1024,
'available_ram': 12278,
'available_rom': 1047552
},
'retries': 0,
'toolchain': 'zephyr',
'testcases': {
'TS4.tc1': {
'status': TwisterStatus.SKIP,
'reason': 'Not in requested test list.',
'duration': 360.0,
'output': '[INFO] Parsing...'
}
}
},
}
for n, i in testplan.instances.items():
assert expected_instances[n]['metrics'] == i.metrics
assert expected_instances[n]['retries'] == i.retries
for t in i.testcases:
assert expected_instances[n]['testcases'][str(t)]['status'] == t.status
assert expected_instances[n]['testcases'][str(t)]['reason'] == t.reason
assert expected_instances[n]['testcases'][str(t)]['duration'] == t.duration
assert expected_instances[n]['testcases'][str(t)]['output'] == t.output
check_runnable_mock.assert_called_with(mock.ANY, mock.ANY)
expected_logs = [
'loading TestSuite 1...',
'loading TestSuite 2...',
'loading TestSuite 3...',
'loading TestSuite 4...',
]
assert all([log in caplog.text for log in expected_logs])
def test_testplan_add_instances():
testplan = TestPlan(env=mock.Mock())
instance1 = mock.Mock()
instance1.name = 'instance 1'
instance2 = mock.Mock()
instance2.name = 'instance 2'
instance_list = [instance1, instance2]
testplan.add_instances(instance_list)
assert testplan.instances == {
'instance 1': instance1,
'instance 2': instance2,
}
def test_testplan_get_testcase():
testplan = TestPlan(env=mock.Mock())
testplan.testsuites = {
'test1.suite0': mock.Mock(testcases=[mock.Mock(), mock.Mock()]),
'test1.suite1': mock.Mock(testcases=[mock.Mock(), mock.Mock()]),
'test1.suite2': mock.Mock(testcases=[mock.Mock(), mock.Mock()]),
'test1.suite3': mock.Mock(testcases=[])
}
testplan.testsuites['test1.suite0'].testcases[0].name = 'test1.suite0.case0'
testplan.testsuites['test1.suite0'].testcases[1].name = 'test1.suite0.case1'
#
testplan.testsuites['test1.suite1'].testcases[0].name = 'test1.suite1.case0'
testplan.testsuites['test1.suite1'].testcases[1].name = 'test1.suite1.case0' # in suite duplicate
#
testplan.testsuites['test1.suite2'].testcases[0].name = 'test1.suite2.case0'
testplan.testsuites['test1.suite2'].testcases[1].name = 'test1.suite1.case0' # out suite duplicate
id = 'test1.suite1.case0'
res = testplan.get_testcase(id)
assert len(res) == 3
assert testplan.testsuites['test1.suite1'] in res
assert testplan.testsuites['test1.suite2'] in res
def test_testplan_verify_platforms_existence(caplog):
testplan = TestPlan(env=mock.Mock())
testplan.platform_names = ['a platform', 'other platform']
platform_names = ['other platform', 'some platform']
log_info = 'PLATFORM ERROR'
with pytest.raises(SystemExit) as se:
testplan.verify_platforms_existence(platform_names, log_info)
assert str(se.value) == '2'
assert 'PLATFORM ERROR - unrecognized platform - some platform'
TESTDATA_12 = [
(True),
(False)
]
@pytest.mark.parametrize(
'exists',
TESTDATA_12,
ids=['links dir exists', 'links dir does not exist']
)
def test_testplan_create_build_dir_links(exists):
outdir = os.path.join('out', 'dir')
instances_linked = []
def mock_link(links_dir_path, instance):
assert links_dir_path == os.path.join(outdir, 'twister_links')
instances_linked.append(instance)
instances = {
'inst0': mock.Mock(status=TwisterStatus.PASS),
'inst1': mock.Mock(status=TwisterStatus.SKIP),
'inst2': mock.Mock(status=TwisterStatus.ERROR),
}
expected_instances = [instances['inst0'], instances['inst2']]
testplan = TestPlan(env=mock.Mock(outdir=outdir))
testplan._create_build_dir_link = mock.Mock(side_effect=mock_link)
testplan.instances = instances
with mock.patch('os.path.exists', return_value=exists), \
mock.patch('os.mkdir', mock.Mock()) as mkdir_mock:
testplan.create_build_dir_links()
if not exists:
mkdir_mock.assert_called_once()
assert expected_instances == instances_linked
TESTDATA_13 = [
('nt'),
('Linux')
]
@pytest.mark.parametrize(
'os_name',
TESTDATA_13,
)
def test_testplan_create_build_dir_link(os_name):
def mock_makedirs(path, exist_ok=False):
assert exist_ok
assert path == instance_build_dir
def mock_symlink(source, target):
assert source == instance_build_dir
assert target == os.path.join('links', 'path', 'test_0')
def mock_call(cmd, shell=False):
assert shell
assert cmd == ['mklink', '/J', os.path.join('links', 'path', 'test_0'),
instance_build_dir]
def mock_join(*paths):
slash = "\\" if os.name == 'nt' else "/"
return slash.join(paths)
with mock.patch('os.name', os_name), \
mock.patch('os.symlink', side_effect=mock_symlink), \
mock.patch('os.makedirs', side_effect=mock_makedirs), \
mock.patch('subprocess.call', side_effect=mock_call), \
mock.patch('os.path.join', side_effect=mock_join):
testplan = TestPlan(env=mock.Mock())
links_dir_path = os.path.join('links', 'path')
instance_build_dir = os.path.join('some', 'far', 'off', 'build', 'dir')
instance = mock.Mock(build_dir=instance_build_dir)
testplan._create_build_dir_link(links_dir_path, instance)
assert instance.build_dir == os.path.join('links', 'path', 'test_0')
assert testplan.link_dir_counter == 1
TESTDATA_14 = [
('bad platform', 'dummy reason', [],
'dummy status', 'dummy reason'),
('good platform', 'quarantined', [],
'dummy status', 'quarantined'),
('good platform', 'dummy reason', [{'type': 'command line filter'}],
'dummy status', 'dummy reason'),
('good platform', 'dummy reason', [{'type': 'Skip filter'}],
'dummy status', 'dummy reason'),
('good platform', 'dummy reason', [{'type': 'platform key filter'}],
'dummy status', 'dummy reason'),
('good platform', 'dummy reason', [{'type': 'Toolchain filter'}],
'dummy status', 'dummy reason'),
('good platform', 'dummy reason', [{'type': 'Module filter'}],
'dummy status', 'dummy reason'),
('good platform', 'dummy reason', [{'type': 'testsuite filter'}],
TwisterStatus.ERROR, 'dummy reason but is one of the integration platforms'),
]
@pytest.mark.parametrize(
'platform_name, reason, filters,' \
' expected_status, expected_reason',
TESTDATA_14,
ids=['wrong platform', 'quarantined', 'command line filtered',
'skip filtered', 'platform key filtered', 'toolchain filtered',
'module filtered', 'skip to error change']
)
def test_change_skip_to_error_if_integration(
platform_name,
reason,
filters,
expected_status,
expected_reason
):
options = mock.Mock()
platform = mock.Mock()
platform.name = platform_name
testsuite = mock.Mock(integration_platforms=['good platform', 'a platform'])
instance = mock.Mock(
testsuite=testsuite,
platform=platform,
filters=filters,
status='dummy status',
reason=reason
)
change_skip_to_error_if_integration(options, instance)
assert instance.status == expected_status
assert instance.reason == expected_reason