twister: shell harness with commands alongside the harness_config

Allow user to add shell commands in testcase/sample yaml file
alongside the harness_config like in the console harness.

Signed-off-by: Grzegorz Chwierut <grzegorz.chwierut@nordicsemi.no>

doc/develop/test/twister.rst

@@ -993,13 +993,30 @@ framework and the pytest harness of twister.
 
 The following options apply to the shell harness:
 
-shell_params_file: <string> (default empty)
+shell_commands: <list of pairs of commands and their expected output> (default empty)
+    Specify a list of shell commands to be executed and their expected output.
+    For example:
+
+    .. code-block:: yaml
+
+        harness_config:
+          shell_commands:
+          - command: "kernel cycles"
+            expected: "cycles: .* hw cycles"
+          - command: "kernel version"
+            expected: "Zephyr version .*"
+          - command: "kernel sleep 100"
+
+
+    If expected output is not provided, the command will be executed and the output
+    will be logged.
+
+shell_commands_file: <string> (default empty)
     Specify a file containing test parameters to be used in the test.
     The file should contain a list of commands and their expected output. For example:
 
-    .. code-block:: none
+    .. code-block:: yaml
 
-      test_shell_harness:
       - command: "mpu mtest 1"
         expected: "The value is: 0x.*"
       - command: "mpu mtest 2"
@@ -1008,6 +1025,7 @@ shell_params_file: <string> (default empty)
 
     If no file is specified, the shell harness will use the default file
     ``test_shell.yml`` in the test directory.
+    ``shell_commands`` will take precedence over ``shell_commands_file``.
 
 Selecting platform scope
 ************************

samples/arch/mpu/mpu_test/test_shell.yml

@@ -1,4 +1,3 @@
-test_shell_harness:
 - command: "mpu mtest 1"
   expected: "The value is: 0x.*"
 - command: "mpu mtest 2"

samples/drivers/flash_shell/test_shell.yml

@@ -1,3 +1,2 @@
-test_shell_harness:
 - command: "flash page_info 0"
   expected: "Page for address 0x0"

samples/subsys/shell/shell_module/test_shell.yml

@@ -1,4 +1,3 @@
-test_shell_harness:
 - command: "kernel cycles"
   expected: "cycles: .* hw cycles"
 - command: "kernel version"

samples/subsys/testsuite/pytest/shell/testcase.yaml

@@ -1,28 +1,34 @@
-tests:
+common:
-  sample.pytest.shell:
-    filter: CONFIG_SERIAL and dt_chosen_enabled("zephyr,shell-uart")
-    min_ram: 40
-    harness: pytest
-    extra_configs:
-      - arch:posix:CONFIG_NATIVE_UART_0_ON_STDINOUT=y
-    integration_platforms:
-      - native_sim
-      - qemu_cortex_m3
   tags:
     - test_framework
     - pytest
     - shell
-  sample.pytest.shell.vt100_colors_off:
+  filter: CONFIG_SERIAL and not CONFIG_SMP and dt_chosen_enabled("zephyr,shell-uart")
-    filter: CONFIG_SERIAL and dt_chosen_enabled("zephyr,shell-uart")
-    min_ram: 40
-    harness: pytest
   extra_configs:
     - arch:posix:CONFIG_NATIVE_UART_0_ON_STDINOUT=y
-      - CONFIG_SHELL_VT100_COLORS=n
+  min_ram: 40
   integration_platforms:
     - native_sim
     - qemu_cortex_m3
-    tags:
+tests:
-      - test_framework
+  sample.pytest.shell:
-      - pytest
+    harness: pytest
-      - shell
+  sample.pytest.shell.vt100_colors_off:
+    harness: pytest
+    extra_configs:
+      - CONFIG_SHELL_VT100_COLORS=n
+  sample.harness.shell:
+    harness: shell
+    harness_config:
+      shell_commands: &kernel_commands
+        - command: "kernel cycles"
+          expected: "cycles: .* hw cycles"
+        - command: "kernel version"
+          expected: "Zephyr version .*"
+        - command: "kernel sleep 100"
+  sample.harness.shell.vt100_colors_off:
+    harness: shell
+    extra_configs:
+      - CONFIG_SHELL_VT100_COLORS=n
+    harness_config:
+      shell_commands: *kernel_commands

scripts/pylib/shell-twister-harness/test_shell.py

@@ -19,14 +19,19 @@ def testdata_path(request):
 def get_next_commands(testdata_path):
     with open(testdata_path) as yaml_file:
         data = yaml.safe_load(yaml_file)
-    for entry in data['test_shell_harness']:
+    for entry in data:
-        yield entry['command'], entry['expected']
+        yield entry['command'], entry.get('expected', None)
 
 
 def test_shell_harness(shell: Shell, testdata_path):
+    if not testdata_path:
+        pytest.skip('testdata not provided')
     for command, expected in get_next_commands(testdata_path):
         logger.info('send command: %s', command)
         lines = shell.exec_command(command)
+        if not expected:
+            logger.debug('no expected response')
+            continue
         match = False
         for line in lines:
             if re.match(expected, line):

scripts/pylib/twister/twisterlib/harness.py

@@ -17,6 +17,7 @@ from collections import OrderedDict
 from enum import Enum
 
 import junitparser.junitparser as junit
+import yaml
 from pytest import ExitCode
 from twisterlib.constants import SUPPORTED_SIMS_IN_PYTEST
 from twisterlib.environment import PYTEST_PLUGIN_INSTALLED, ZEPHYR_BASE
@@ -628,6 +629,7 @@ class Pytest(Harness):
             self.status = TwisterStatus.SKIP
             self.instance.reason = 'No tests collected'
 
+
 class Shell(Pytest):
     def generate_command(self):
         config = self.instance.testsuite.harness_config
@@ -635,13 +637,28 @@ class Shell(Pytest):
         config['pytest_root'] = pytest_root
 
         command = super().generate_command()
-        if config.get('shell_params_file'):
+        if test_shell_file := self._get_shell_commands_file(config):
-            p_file = os.path.join(self.source_dir, config.get('shell_params_file'))
+            command.append(f'--testdata={test_shell_file}')
-            command.append(f'--testdata={p_file}')
         else:
-            command.append(f'--testdata={os.path.join(self.source_dir, "test_shell.yml")}')
+            logger.warning('No shell commands provided')
         return command
 
+    def _get_shell_commands_file(self, harness_config):
+        if shell_commands := harness_config.get('shell_commands'):
+            test_shell_file = os.path.join(self.running_dir, 'test_shell.yml')
+            with open(test_shell_file, 'w') as f:
+                yaml.dump(shell_commands, f)
+            return test_shell_file
+
+        test_shell_file = harness_config.get('shell_commands_file', 'test_shell.yml')
+        test_shell_file = os.path.join(
+            self.source_dir, os.path.expanduser(os.path.expandvars(test_shell_file))
+        )
+        if os.path.exists(test_shell_file):
+            return test_shell_file
+        return None
+
+
 class Gtest(Harness):
     ANSI_ESCAPE = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
     _NAME_PATTERN = "[a-zA-Z_][a-zA-Z0-9_]*"

scripts/schemas/twister/testsuite-schema.yaml

@@ -107,9 +107,20 @@ schema;scenario-schema:
       type: map
       required: false
       mapping:
-        "shell_params_file":
+        "shell_commands_file":
           type: str
           required: false
+        "shell_commands":
+          type: seq
+          required: false
+          sequence:
+            - type: map
+              mapping:
+                "command":
+                  type: str
+                  required: true
+                "expected":
+                  type: str
         "type":
           type: str
           required: false