cmake: yaml: switch to yaml for intermediate file

The intermediate file used to expand the generator expressions is now in
YAML format. This allows for a more robust handling of the data, as the
single quoted strings are a lot easier to escape: inside them every
character is a literal except for two single quotes (which intuitively
map to a literal single quote).

When saving a simple YAML file without genexes, the single quotes in
strings need to be escaped. However, doing so when saving the
intermediate file would make it harder to properly escape them later,
since it could be possible that the expansion of the generator
expressions would introduce new single quotes. For this reason, when
genexes are enabled, the escaping is now done in a single pass inside
the yaml-filter.cmake script.

Signed-off-by: Luca Burelli <l.burelli@arduino.cc>

cmake/modules/yaml.cmake

@@ -491,7 +491,11 @@ function(yaml_save)
 
   zephyr_get_scoped(genex ${ARG_YAML_NAME} GENEX)
   zephyr_get_scoped(json_content ${ARG_YAML_NAME} JSON)
-  to_yaml("${json_content}" 0 yaml_out ${genex})
+  if(genex)
+    to_yaml("${json_content}" 0 yaml_out DIRECT_GENEX)
+   else()
+    to_yaml("${json_content}" 0 yaml_out DIRECT)
+   endif()
 
   if(EXISTS ${yaml_file})
     FILE(RENAME ${yaml_file} ${yaml_file}.bak)
@@ -529,20 +533,42 @@ function(yaml_save)
 
     cmake_path(SET yaml_path "${yaml_file}")
     cmake_path(GET yaml_path STEM yaml_file_no_ext)
-    set(expanded_file ${yaml_file_no_ext}_${genex_save_count}.json)
+    set(expanded_file ${yaml_file_no_ext}_${genex_save_count}.yaml)
     set_property(TARGET ${save_target} PROPERTY expanded_file ${expanded_file})
 
     # comment this to keep the temporary files
     set_property(TARGET ${save_target} APPEND PROPERTY temp_files ${expanded_file})
 
-    FILE(GENERATE OUTPUT ${expanded_file}
-      CONTENT "${json_content}"
-    )
+    to_yaml("${json_content}" 0 yaml_out TEMP_GENEX)
+    FILE(GENERATE OUTPUT ${expanded_file} CONTENT "${yaml_out}")
   endif()
 endfunction()
 
-function(to_yaml in_json level yaml genex)
+function(to_yaml in_json level yaml mode)
   zephyr_string(ESCAPE json "${in_json}")
+
+  if(mode STREQUAL "DIRECT")
+    # Direct output mode, no genexes: write a standard YAML
+    set(expand_lists TRUE)
+    set(escape_quotes TRUE)
+    set(comment_genexes FALSE)
+  elseif(mode STREQUAL "DIRECT_GENEX" OR mode STREQUAL "FINAL_GENEX")
+    # Direct output mode with genexes enabled, or final write of post-processed
+    # file: write a standard YAML, comment entries with genexes if they are
+    # (still) present in the file
+    set(expand_lists TRUE)
+    set(escape_quotes TRUE)
+    set(comment_genexes TRUE)
+  elseif(mode STREQUAL "TEMP_GENEX")
+    # Temporary output mode for genex expansion: save single quotes with no
+    # special processing, since they will be fixed up by yaml-filter.cmake
+    set(expand_lists FALSE)
+    set(escape_quotes FALSE)
+    set(comment_genexes FALSE)
+  else()
+    message(FATAL_ERROR "to_yaml(... ${mode} ) is malformed.")
+  endif()
+
   if(level GREATER 0)
     math(EXPR level_dec "${level} - 1")
     set(indent_${level} "${indent_${level_dec}}  ")
@@ -564,7 +590,7 @@ function(to_yaml in_json level yaml genex)
       # JSON object -> YAML dictionary
       set(${yaml} "${${yaml}}${indent_${level}}${member}:\n")
       math(EXPR sublevel "${level} + 1")
-      to_yaml("${subjson}" ${sublevel} ${yaml} ${genex})
+      to_yaml("${subjson}" ${sublevel} ${yaml} ${mode})
     elseif(type STREQUAL ARRAY)
       # JSON array -> YAML list
       set(${yaml} "${${yaml}}${indent_${level}}${member}:")
@@ -581,13 +607,15 @@ function(to_yaml in_json level yaml genex)
           string(JSON length ERROR_VARIABLE ignore LENGTH "${item}")
           if(length)
             set(non_indent_yaml)
-            to_yaml("${item}" 0 non_indent_yaml FALSE)
+            to_yaml("${item}" 0 non_indent_yaml ${mode})
             string(REGEX REPLACE "\n$" "" non_indent_yaml "${non_indent_yaml}")
             string(REPLACE "\n" "\n${indent_${level}}   " indent_yaml "${non_indent_yaml}")
             set(${yaml} "${${yaml}}${indent_${level}} - ${indent_yaml}\n")
           else()
-            # Assume a string, escape single quotes.
+            # Assume a string, escape single quotes when required (see comment below).
+            if(escape_quotes)
               string(REPLACE "'" "''" item "${item}")
+            endif()
             set(${yaml} "${${yaml}}${indent_${level}} - '${item}'\n")
           endif()
         endforeach()
@@ -597,13 +625,17 @@ function(to_yaml in_json level yaml genex)
       # - with unexpanded generator expressions: save as YAML comment
       # - if it matches the special prefix: convert to YAML list
       # - otherwise: save as YAML scalar
-      # Single quotes must be escaped in the value.
+      # Single quotes must be escaped in the value _unless_ this will be used
+      # to expand generator expressions, because then the escaping will be
+      # addressed once in the yaml-filter.cmake script.
+      if(escape_quotes)
         string(REPLACE "'" "''" subjson "${subjson}")
-      if(subjson MATCHES "\\$<.*>" AND ${genex})
+      endif()
+      if(subjson MATCHES "\\$<.*>" AND comment_genexes)
         # Yet unexpanded generator expression: save as comment
         string(SUBSTRING ${indent_${level}} 1 -1 short_indent)
         set(${yaml} "${${yaml}}#${short_indent}${member}: '${subjson}'\n")
-      elseif(subjson MATCHES "^@YAML-LIST@")
+      elseif(subjson MATCHES "^@YAML-LIST@" AND expand_lists)
         # List-as-string: convert to list
         set(${yaml} "${${yaml}}${indent_${level}}${member}:")
         list(POP_FRONT subjson)

cmake/yaml-filter.cmake

@@ -1,21 +1,28 @@
 # Copyright (c) 2024 Arduino SA
 # SPDX-License-Identifier: Apache-2.0
 
-# Simple second stage filter for YAML generation, used when generator
-# expressions have been used for some of the data and the conversion to
-# YAML needs to happen after cmake has completed processing.
+# Second stage filter for YAML generation, called when generator expressions
+# have been used in some of the data and cleanup needs to happen after CMake
+# has completed processing.
 #
-# This scripts expects as input:
-# - EXPANDED_FILE: the name of the input file, in JSON format, that contains
-#              the expanded generator expressions.
+# Two issues are addressed here:
+#
+#  - the intermediate YAML may have non-escaped single quotes in its strings.
+#    These may have been introduced directly via yaml_set() in the main CMake
+#    script or by some generator expressions; at this stage they are however
+#    finalized and can be escaped in one single pass.
+#
+#  - in the input YAML, lists have been stored as a CMake-format string to
+#    allow generator expressions to seamlessly expand into multiple items.
+#    These now need to be converted back into a proper YAML list.
+#
+# This scripts expects as input the following variables:
+#
+# - EXPANDED_FILE: the name of the file that contains the expanded generator
+#                  expressions.
 # - OUTPUT_FILE: the name of the final output YAML file.
 # - TEMP_FILES: a list of temporary files that need to be removed after
 #               the conversion is done.
-#
-# This script loads the Zephyr yaml module and reuses its `to_yaml()`
-# function to convert the fully expanded JSON content to YAML, taking
-# into account the special format that was used to store lists.
-# Temporary files are then removed.
 
 cmake_minimum_required(VERSION 3.20.0)
 
@@ -23,13 +30,40 @@ set(ZEPHYR_BASE ${CMAKE_CURRENT_LIST_DIR}/../)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
 include(yaml)
 
-file(READ ${EXPANDED_FILE} json_content)
-to_yaml("${json_content}" 0 yaml_out TRUE)
+# Fix all quotes in the input YAML file. Strings in it will be in one of the
+# following formats:
+#
+#   name: 'string with 'single quotes' in it'
+#       - 'string with 'single quotes' in it'
+#
+# To address this, every single quote is duplicated, then the first and last
+# occurrences of two single quotes are removed (they are the string beginning
+# and end markers). The result is written to a temporary file.
+file(STRINGS ${EXPANDED_FILE} yaml_content)
+foreach(line ${yaml_content})
+  string(REPLACE "'" "''" line "${line}") # escape every single quote in the string
+  string(REGEX REPLACE "^([^']* )''" "\\1'" line "${line}") # fix opening quote
+  string(REGEX REPLACE "''$" "'" line "${line}") # fix closing quote
+  # semicolons in the string are not to be confused with string separators
+  string(REPLACE ";" "\\;" line "${line}")
+  list(APPEND tmp_content "${line}")
+endforeach()
+list(JOIN tmp_content "\n" tmp_content)
+file(WRITE ${EXPANDED_FILE}.tmp "${tmp_content}")
+
+# Load the now-fixed YAML file and convert the CMake-format lists back into
+# proper YAML format using the to_yaml() function. The result is written to
+# the final destination file.
+yaml_load(FILE ${EXPANDED_FILE}.tmp NAME yaml_saved)
+zephyr_get_scoped(json_content yaml_saved JSON)
+to_yaml("${json_content}" 0 yaml_out FINAL_GENEX)
 file(WRITE ${OUTPUT_FILE} "${yaml_out}")
 
 # Remove unused temporary files. EXPANDED_FILE needs to be kept, or the
-# build system will complain there is no rule to rebuild it
+# build system will complain there is no rule to rebuild it, but the
+# .tmp file that was just created can be removed.
 list(REMOVE_ITEM TEMP_FILES ${EXPANDED_FILE})
+list(APPEND TEMP_FILES ${EXPANDED_FILE}.tmp)
 foreach(file ${TEMP_FILES})
   file(REMOVE ${file})
 endforeach()