diff --git a/cmake/modules/dts.cmake b/cmake/modules/dts.cmake index ec0999d366b..19b168c7485 100644 --- a/cmake/modules/dts.cmake +++ b/cmake/modules/dts.cmake @@ -94,6 +94,8 @@ find_package(Dtc 1.4.6) # The directory containing devicetree related scripts. set(DT_SCRIPTS ${ZEPHYR_BASE}/scripts/dts) +# This parses and collects the DT information +set(GEN_EDT_SCRIPT ${DT_SCRIPTS}/gen_edt.py) # This generates DT information needed by the C macro APIs, # along with a few other things. set(GEN_DEFINES_SCRIPT ${DT_SCRIPTS}/gen_defines.py) @@ -216,7 +218,7 @@ foreach(dts_root ${DTS_ROOT}) set(vendor_prefixes ${dts_root}/${VENDOR_PREFIXES}) if(EXISTS ${vendor_prefixes}) - list(APPEND EXTRA_GEN_DEFINES_ARGS --vendor-prefixes ${vendor_prefixes}) + list(APPEND EXTRA_GEN_EDT_ARGS --vendor-prefixes ${vendor_prefixes}) endif() endforeach() @@ -266,23 +268,44 @@ toolchain_parse_make_rule(${DTS_DEPS} set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${DTS_INCLUDE_FILES} + ${GEN_EDT_SCRIPT} ${GEN_DEFINES_SCRIPT} ${GEN_DRIVER_KCONFIG_SCRIPT} ${GEN_DTS_CMAKE_SCRIPT} ) # -# Run GEN_DEFINES_SCRIPT. +# Run GEN_EDT_SCRIPT. # string(REPLACE ";" " " EXTRA_DTC_FLAGS_RAW "${EXTRA_DTC_FLAGS}") -set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT} +set(CMD_GEN_EDT ${PYTHON_EXECUTABLE} ${GEN_EDT_SCRIPT} --dts ${DTS_POST_CPP} --dtc-flags '${EXTRA_DTC_FLAGS_RAW}' --bindings-dirs ${DTS_ROOT_BINDINGS} ---header-out ${DEVICETREE_GENERATED_H}.new --dts-out ${ZEPHYR_DTS}.new # for debugging and dtc ---edt-pickle-out ${EDT_PICKLE} +--edt-pickle-out ${EDT_PICKLE}.new +${EXTRA_GEN_EDT_ARGS} +) + +execute_process( + COMMAND ${CMD_GEN_EDT} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + COMMAND_ERROR_IS_FATAL ANY + ) +zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT) +zephyr_file_copy(${EDT_PICKLE}.new ${EDT_PICKLE} ONLY_IF_DIFFERENT) +file(REMOVE ${ZEPHYR_DTS}.new ${EDT_PICKLE}.new) +message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}") +message(STATUS "Generated pickled edt: ${EDT_PICKLE}") + +# +# Run GEN_DEFINES_SCRIPT. +# + +set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT} +--header-out ${DEVICETREE_GENERATED_H}.new +--edt-pickle ${EDT_PICKLE} ${EXTRA_GEN_DEFINES_ARGS} ) @@ -291,7 +314,6 @@ execute_process( WORKING_DIRECTORY ${PROJECT_BINARY_DIR} COMMAND_ERROR_IS_FATAL ANY ) -zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT) zephyr_file_copy(${DEVICETREE_GENERATED_H}.new ${DEVICETREE_GENERATED_H} ONLY_IF_DIFFERENT) file(REMOVE ${ZEPHYR_DTS}.new ${DEVICETREE_GENERATED_H}.new) message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}") diff --git a/doc/releases/release-notes-4.0.rst b/doc/releases/release-notes-4.0.rst index cfb37072851..fadc76e2930 100644 --- a/doc/releases/release-notes-4.0.rst +++ b/doc/releases/release-notes-4.0.rst @@ -133,6 +133,17 @@ Build system and Infrastructure * Added support for .elf files to the west flash command for jlink, pyocd and linkserver runners. +* Extracted pickled EDT generation from gen_defines.py into gen_edt.py. This moved the following + parameters from the cmake variable ``EXTRA_GEN_DEFINES_ARGS`` to ``EXTRA_GEN_EDT_ARGS``: + + * ``--dts`` + * ``--dtc-flags`` + * ``--bindings-dirs`` + * ``--dts-out`` + * ``--edt-pickle-out`` + * ``--vendor-prefixes`` + * ``--edtlib-Werror`` + Documentation ************* diff --git a/scripts/dts/edtlib_logger.py b/scripts/dts/edtlib_logger.py new file mode 100644 index 00000000000..de3461f617c --- /dev/null +++ b/scripts/dts/edtlib_logger.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA +# Copyright (c) 2019 Linaro Limited +# Copyright (c) 2024 SILA Embedded Solutions GmbH + +import logging +import sys + + +class LogFormatter(logging.Formatter): + '''A log formatter that prints the level name in lower case, + for compatibility with earlier versions of edtlib.''' + + def __init__(self): + super().__init__(fmt='%(levelnamelower)s: %(message)s') + + def format(self, record): + record.levelnamelower = record.levelname.lower() + return super().format(record) + + +def setup_edtlib_logging() -> None: + # The edtlib module emits logs using the standard 'logging' module. + # Configure it so that warnings and above are printed to stderr, + # using the LogFormatter class defined above to format each message. + + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(LogFormatter()) + + logger = logging.getLogger('edtlib') + logger.setLevel(logging.WARNING) + logger.addHandler(handler) diff --git a/scripts/dts/gen_defines.py b/scripts/dts/gen_defines.py index f049f7bc2e4..fda61526dc8 100755 --- a/scripts/dts/gen_defines.py +++ b/scripts/dts/gen_defines.py @@ -2,16 +2,11 @@ # Copyright (c) 2019 - 2020 Nordic Semiconductor ASA # Copyright (c) 2019 Linaro Limited +# Copyright (c) 2024 SILA Embedded Solutions GmbH # SPDX-License-Identifier: BSD-3-Clause -# This script uses edtlib to generate a header file from a devicetree -# (.dts) file. Information from binding files in YAML format is used -# as well. -# -# Bindings are files that describe devicetree nodes. Devicetree nodes are -# usually mapped to bindings via their 'compatible = "..."' property. -# -# See Zephyr's Devicetree user guide for details. +# This script uses edtlib to generate a header file from a pickled +# edt file. # # Note: Do not access private (_-prefixed) identifiers from edtlib here (and # also note that edtlib is not meant to expose the dtlib API directly). @@ -20,7 +15,6 @@ import argparse from collections import defaultdict -import logging import os import pathlib import pickle @@ -31,18 +25,9 @@ from typing import Iterable, NoReturn, Optional sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'python-devicetree', 'src')) +import edtlib_logger from devicetree import edtlib -class LogFormatter(logging.Formatter): - '''A log formatter that prints the level name in lower case, - for compatibility with earlier versions of edtlib.''' - - def __init__(self): - super().__init__(fmt='%(levelnamelower)s: %(message)s') - - def format(self, record): - record.levelnamelower = record.levelname.lower() - return super().format(record) def main(): global header_file @@ -50,30 +35,13 @@ def main(): args = parse_args() - setup_edtlib_logging() + edtlib_logger.setup_edtlib_logging() - vendor_prefixes = {} - for prefixes_file in args.vendor_prefixes: - vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file)) - - try: - edt = edtlib.EDT(args.dts, args.bindings_dirs, - # Suppress this warning if it's suppressed in dtc - warn_reg_unit_address_mismatch= - "-Wno-simple_bus_reg" not in args.dtc_flags, - default_prop_types=True, - infer_binding_for_paths=["/zephyr,user"], - werror=args.edtlib_Werror, - vendor_prefixes=vendor_prefixes) - except edtlib.EDTError as e: - sys.exit(f"devicetree error: {e}") + with open(args.edt_pickle, 'rb') as f: + edt = pickle.load(f) flash_area_num = 0 - # Save merged DTS source, as a debugging aid - with open(args.dts_out, "w", encoding="utf-8") as f: - print(edt.dts_source, file=f) - # Create the generated header. with open(args.header_out, "w", encoding="utf-8") as header_file: write_top_comment(edt) @@ -133,22 +101,6 @@ def main(): write_chosen(edt) write_global_macros(edt) - if args.edt_pickle_out: - write_pickled_edt(edt, args.edt_pickle_out) - - -def setup_edtlib_logging() -> None: - # The edtlib module emits logs using the standard 'logging' module. - # Configure it so that warnings and above are printed to stderr, - # using the LogFormatter class defined above to format each message. - - handler = logging.StreamHandler(sys.stderr) - handler.setFormatter(LogFormatter()) - - logger = logging.getLogger('edtlib') - logger.setLevel(logging.WARNING) - logger.addHandler(handler) - def node_z_path_id(node: edtlib.Node) -> str: # Return the node specific bit of the node's path identifier: @@ -173,27 +125,10 @@ def parse_args() -> argparse.Namespace: # Returns parsed command-line arguments parser = argparse.ArgumentParser(allow_abbrev=False) - parser.add_argument("--dts", required=True, help="DTS file") - parser.add_argument("--dtc-flags", - help="'dtc' devicetree compiler flags, some of which " - "might be respected here") - parser.add_argument("--bindings-dirs", nargs='+', required=True, - help="directory with bindings in YAML format, " - "we allow multiple") parser.add_argument("--header-out", required=True, help="path to write header to") - parser.add_argument("--dts-out", required=True, - help="path to write merged DTS source code to (e.g. " - "as a debugging aid)") - parser.add_argument("--edt-pickle-out", - help="path to write pickled edtlib.EDT object to") - parser.add_argument("--vendor-prefixes", action='append', default=[], - help="vendor-prefixes.txt path; used for validation; " - "may be given multiple times") - parser.add_argument("--edtlib-Werror", action="store_true", - help="if set, edtlib-specific warnings become errors. " - "(this does not apply to warnings shared " - "with dtc.)") + parser.add_argument("--edt-pickle", + help="path to read pickled edtlib.EDT object from") return parser.parse_args() @@ -1099,21 +1034,6 @@ def quote_str(s: str) -> str: return f'"{escape(s)}"' -def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None: - # Writes the edt object in pickle format to out_file. - - with open(out_file, 'wb') as f: - # Pickle protocol version 4 is the default as of Python 3.8 - # and was introduced in 3.4, so it is both available and - # recommended on all versions of Python that Zephyr supports - # (at time of writing, Python 3.6 was Zephyr's minimum - # version, and 3.8 the most recent CPython release). - # - # Using a common protocol version here will hopefully avoid - # reproducibility issues in different Python installations. - pickle.dump(edt, f, protocol=4) - - def err(s: str) -> NoReturn: raise Exception(s) diff --git a/scripts/dts/gen_edt.py b/scripts/dts/gen_edt.py new file mode 100755 index 00000000000..093524515dd --- /dev/null +++ b/scripts/dts/gen_edt.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA +# Copyright (c) 2019 Linaro Limited +# Copyright (c) 2024 SILA Embedded Solutions GmbH +# SPDX-License-Identifier: Apache-2.0 + +# This script uses edtlib to generate a pickled edt from a devicetree +# (.dts) file. Information from binding files in YAML format is used +# as well. +# +# Bindings are files that describe devicetree nodes. Devicetree nodes are +# usually mapped to bindings via their 'compatible = "..."' property. +# +# See Zephyr's Devicetree user guide for details. +# +# Note: Do not access private (_-prefixed) identifiers from edtlib here (and +# also note that edtlib is not meant to expose the dtlib API directly). +# Instead, think of what API you need, and add it as a public documented API in +# edtlib. This will keep this script simple. + +import argparse +import os +import pickle +import sys +from typing import NoReturn + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'python-devicetree', + 'src')) + +import edtlib_logger +from devicetree import edtlib + + +def main(): + args = parse_args() + + edtlib_logger.setup_edtlib_logging() + + vendor_prefixes = {} + for prefixes_file in args.vendor_prefixes: + vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file)) + + try: + edt = edtlib.EDT(args.dts, args.bindings_dirs, + # Suppress this warning if it's suppressed in dtc + warn_reg_unit_address_mismatch= + "-Wno-simple_bus_reg" not in args.dtc_flags, + default_prop_types=True, + infer_binding_for_paths=["/zephyr,user"], + werror=args.edtlib_Werror, + vendor_prefixes=vendor_prefixes) + except edtlib.EDTError as e: + sys.exit(f"devicetree error: {e}") + + # Save merged DTS source, as a debugging aid + with open(args.dts_out, "w", encoding="utf-8") as f: + print(edt.dts_source, file=f) + + write_pickled_edt(edt, args.edt_pickle_out) + + +def parse_args() -> argparse.Namespace: + # Returns parsed command-line arguments + + parser = argparse.ArgumentParser(allow_abbrev=False) + parser.add_argument("--dts", required=True, help="DTS file") + parser.add_argument("--dtc-flags", + help="'dtc' devicetree compiler flags, some of which " + "might be respected here") + parser.add_argument("--bindings-dirs", nargs='+', required=True, + help="directory with bindings in YAML format, " + "we allow multiple") + parser.add_argument("--dts-out", required=True, + help="path to write merged DTS source code to (e.g. " + "as a debugging aid)") + parser.add_argument("--edt-pickle-out", + help="path to write pickled edtlib.EDT object to", required=True) + parser.add_argument("--vendor-prefixes", action='append', default=[], + help="vendor-prefixes.txt path; used for validation; " + "may be given multiple times") + parser.add_argument("--edtlib-Werror", action="store_true", + help="if set, edtlib-specific warnings become errors. " + "(this does not apply to warnings shared " + "with dtc.)") + + return parser.parse_args() + + +def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None: + # Writes the edt object in pickle format to out_file. + + with open(out_file, 'wb') as f: + # Pickle protocol version 4 is the default as of Python 3.8 + # and was introduced in 3.4, so it is both available and + # recommended on all versions of Python that Zephyr supports + # (at time of writing, Python 3.6 was Zephyr's minimum + # version, and 3.10 the most recent CPython release). + # + # Using a common protocol version here will hopefully avoid + # reproducibility issues in different Python installations. + pickle.dump(edt, f, protocol=4) + + +def err(s: str) -> NoReturn: + raise Exception(s) + + +if __name__ == "__main__": + main() diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 430e43846ea..b0a6777edbd 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -330,10 +330,10 @@ class CMake: if not self.options.disable_warnings_as_errors: warnings_as_errors = 'y' - gen_defines_args = "--edtlib-Werror" + gen_edt_args = "--edtlib-Werror" else: warnings_as_errors = 'n' - gen_defines_args = "" + gen_edt_args = "" warning_command = 'CONFIG_COMPILER_WARNINGS_AS_ERRORS' if self.instance.sysbuild: @@ -345,7 +345,7 @@ class CMake: f'-DTC_RUNID={self.instance.run_id}', f'-DTC_NAME={self.instance.testsuite.name}', f'-D{warning_command}={warnings_as_errors}', - f'-DEXTRA_GEN_DEFINES_ARGS={gen_defines_args}', + f'-DEXTRA_GEN_EDT_ARGS={gen_edt_args}', f'-G{self.env.generator}' ] diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index 8d7973f5e05..9fc4a8fa269 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -371,7 +371,7 @@ TESTDATA_2_2 = [ [os.path.join('dummy', 'cmake'), '-B' + os.path.join('build', 'dir'), '-DTC_RUNID=1', '-DTC_NAME=testcase', '-DSB_CONFIG_COMPILER_WARNINGS_AS_ERRORS=y', - '-DEXTRA_GEN_DEFINES_ARGS=--edtlib-Werror', '-Gdummy_generator', + '-DEXTRA_GEN_EDT_ARGS=--edtlib-Werror', '-Gdummy_generator', '-S' + os.path.join('source', 'dir'), 'arg1', 'arg2', '-DBOARD=', @@ -385,7 +385,7 @@ TESTDATA_2_2 = [ [os.path.join('dummy', 'cmake'), '-B' + os.path.join('build', 'dir'), '-DTC_RUNID=1', '-DTC_NAME=testcase', '-DSB_CONFIG_COMPILER_WARNINGS_AS_ERRORS=n', - '-DEXTRA_GEN_DEFINES_ARGS=', '-Gdummy_generator', + '-DEXTRA_GEN_EDT_ARGS=', '-Gdummy_generator', '-Szephyr_base/share/sysbuild', '-DAPP_DIR=' + os.path.join('source', 'dir'), 'arg1', 'arg2',