diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst index d9ac4646f7b..42f09ef020e 100644 --- a/doc/develop/test/twister.rst +++ b/doc/develop/test/twister.rst @@ -993,13 +993,30 @@ framework and the pytest harness of twister. The following options apply to the shell harness: -shell_params_file: (default empty) +shell_commands: (default empty) + Specify a list of shell commands to be executed and their expected output. + For example: + + .. code-block:: yaml + + harness_config: + shell_commands: + - command: "kernel cycles" + expected: "cycles: .* hw cycles" + - command: "kernel version" + expected: "Zephyr version .*" + - command: "kernel sleep 100" + + + If expected output is not provided, the command will be executed and the output + will be logged. + +shell_commands_file: (default empty) Specify a file containing test parameters to be used in the test. The file should contain a list of commands and their expected output. For example: - .. code-block:: none + .. code-block:: yaml - test_shell_harness: - command: "mpu mtest 1" expected: "The value is: 0x.*" - command: "mpu mtest 2" @@ -1007,7 +1024,8 @@ shell_params_file: (default empty) If no file is specified, the shell harness will use the default file - ``test_shell.yml`` in the test directory. + ``test_shell.yml`` in the test directory. + ``shell_commands`` will take precedence over ``shell_commands_file``. Selecting platform scope ************************ diff --git a/samples/arch/mpu/mpu_test/test_shell.yml b/samples/arch/mpu/mpu_test/test_shell.yml index 36815e9b684..fcce9e59ce9 100644 --- a/samples/arch/mpu/mpu_test/test_shell.yml +++ b/samples/arch/mpu/mpu_test/test_shell.yml @@ -1,4 +1,3 @@ -test_shell_harness: - command: "mpu mtest 1" expected: "The value is: 0x.*" - command: "mpu mtest 2" diff --git a/samples/drivers/flash_shell/test_shell.yml b/samples/drivers/flash_shell/test_shell.yml index f04555d8935..2e4dbee98a0 100644 --- a/samples/drivers/flash_shell/test_shell.yml +++ b/samples/drivers/flash_shell/test_shell.yml @@ -1,3 +1,2 @@ -test_shell_harness: - command: "flash page_info 0" expected: "Page for address 0x0" diff --git a/samples/subsys/shell/shell_module/test_shell.yml b/samples/subsys/shell/shell_module/test_shell.yml index 1f78f105227..172651f7847 100644 --- a/samples/subsys/shell/shell_module/test_shell.yml +++ b/samples/subsys/shell/shell_module/test_shell.yml @@ -1,4 +1,3 @@ -test_shell_harness: - command: "kernel cycles" expected: "cycles: .* hw cycles" - command: "kernel version" diff --git a/samples/subsys/testsuite/pytest/shell/testcase.yaml b/samples/subsys/testsuite/pytest/shell/testcase.yaml index f256cd9df8f..adbbc85088b 100644 --- a/samples/subsys/testsuite/pytest/shell/testcase.yaml +++ b/samples/subsys/testsuite/pytest/shell/testcase.yaml @@ -1,28 +1,34 @@ +common: + tags: + - test_framework + - pytest + - shell + filter: CONFIG_SERIAL and not CONFIG_SMP and dt_chosen_enabled("zephyr,shell-uart") + extra_configs: + - arch:posix:CONFIG_NATIVE_UART_0_ON_STDINOUT=y + min_ram: 40 + integration_platforms: + - native_sim + - qemu_cortex_m3 tests: sample.pytest.shell: - filter: CONFIG_SERIAL and dt_chosen_enabled("zephyr,shell-uart") - min_ram: 40 harness: pytest - extra_configs: - - arch:posix:CONFIG_NATIVE_UART_0_ON_STDINOUT=y - integration_platforms: - - native_sim - - qemu_cortex_m3 - tags: - - test_framework - - pytest - - shell sample.pytest.shell.vt100_colors_off: - filter: CONFIG_SERIAL and dt_chosen_enabled("zephyr,shell-uart") - min_ram: 40 harness: pytest extra_configs: - - arch:posix:CONFIG_NATIVE_UART_0_ON_STDINOUT=y - CONFIG_SHELL_VT100_COLORS=n - integration_platforms: - - native_sim - - qemu_cortex_m3 - tags: - - test_framework - - pytest - - shell + sample.harness.shell: + harness: shell + harness_config: + shell_commands: &kernel_commands + - command: "kernel cycles" + expected: "cycles: .* hw cycles" + - command: "kernel version" + expected: "Zephyr version .*" + - command: "kernel sleep 100" + sample.harness.shell.vt100_colors_off: + harness: shell + extra_configs: + - CONFIG_SHELL_VT100_COLORS=n + harness_config: + shell_commands: *kernel_commands diff --git a/scripts/pylib/shell-twister-harness/test_shell.py b/scripts/pylib/shell-twister-harness/test_shell.py index 0a3317329cf..62c22453730 100644 --- a/scripts/pylib/shell-twister-harness/test_shell.py +++ b/scripts/pylib/shell-twister-harness/test_shell.py @@ -19,14 +19,19 @@ def testdata_path(request): def get_next_commands(testdata_path): with open(testdata_path) as yaml_file: data = yaml.safe_load(yaml_file) - for entry in data['test_shell_harness']: - yield entry['command'], entry['expected'] + for entry in data: + yield entry['command'], entry.get('expected', None) def test_shell_harness(shell: Shell, testdata_path): + if not testdata_path: + pytest.skip('testdata not provided') for command, expected in get_next_commands(testdata_path): logger.info('send command: %s', command) lines = shell.exec_command(command) + if not expected: + logger.debug('no expected response') + continue match = False for line in lines: if re.match(expected, line): diff --git a/scripts/pylib/twister/twisterlib/harness.py b/scripts/pylib/twister/twisterlib/harness.py index dae3531a03a..83b895d73b9 100644 --- a/scripts/pylib/twister/twisterlib/harness.py +++ b/scripts/pylib/twister/twisterlib/harness.py @@ -17,6 +17,7 @@ from collections import OrderedDict from enum import Enum import junitparser.junitparser as junit +import yaml from pytest import ExitCode from twisterlib.constants import SUPPORTED_SIMS_IN_PYTEST from twisterlib.environment import PYTEST_PLUGIN_INSTALLED, ZEPHYR_BASE @@ -628,6 +629,7 @@ class Pytest(Harness): self.status = TwisterStatus.SKIP self.instance.reason = 'No tests collected' + class Shell(Pytest): def generate_command(self): config = self.instance.testsuite.harness_config @@ -635,13 +637,28 @@ class Shell(Pytest): config['pytest_root'] = pytest_root command = super().generate_command() - if config.get('shell_params_file'): - p_file = os.path.join(self.source_dir, config.get('shell_params_file')) - command.append(f'--testdata={p_file}') + if test_shell_file := self._get_shell_commands_file(config): + command.append(f'--testdata={test_shell_file}') else: - command.append(f'--testdata={os.path.join(self.source_dir, "test_shell.yml")}') + logger.warning('No shell commands provided') return command + def _get_shell_commands_file(self, harness_config): + if shell_commands := harness_config.get('shell_commands'): + test_shell_file = os.path.join(self.running_dir, 'test_shell.yml') + with open(test_shell_file, 'w') as f: + yaml.dump(shell_commands, f) + return test_shell_file + + test_shell_file = harness_config.get('shell_commands_file', 'test_shell.yml') + test_shell_file = os.path.join( + self.source_dir, os.path.expanduser(os.path.expandvars(test_shell_file)) + ) + if os.path.exists(test_shell_file): + return test_shell_file + return None + + class Gtest(Harness): ANSI_ESCAPE = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') _NAME_PATTERN = "[a-zA-Z_][a-zA-Z0-9_]*" diff --git a/scripts/schemas/twister/testsuite-schema.yaml b/scripts/schemas/twister/testsuite-schema.yaml index 3c2007da4d4..55bfbcb7b9c 100644 --- a/scripts/schemas/twister/testsuite-schema.yaml +++ b/scripts/schemas/twister/testsuite-schema.yaml @@ -107,9 +107,20 @@ schema;scenario-schema: type: map required: false mapping: - "shell_params_file": + "shell_commands_file": type: str required: false + "shell_commands": + type: seq + required: false + sequence: + - type: map + mapping: + "command": + type: str + required: true + "expected": + type: str "type": type: str required: false