diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e01bd482b..a424058ea 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -182,7 +182,7 @@ jobs:
     # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here.
     - name: Configure C++17
       run: >
-        cmake -S . -B build2
+        cmake -S . -B build2 -Werror=dev
         -DPYBIND11_WERROR=ON
         -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF
         -DPYBIND11_PYTEST_ARGS=-v
diff --git a/.gitignore b/.gitignore
index 43d5094c9..4daf6119c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,4 @@ pybind11Targets.cmake
 /docs/_build/*
 .ipynb_checkpoints/
 tests/main.cpp
+CMakeUserPresents.json
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c4e62e09b..8fbcea524 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -129,16 +129,17 @@ set(PYBIND11_HEADERS
     include/pybind11/detail/cpp_conduit.h
     include/pybind11/detail/descr.h
     include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
+    include/pybind11/detail/exception_translation.h
     include/pybind11/detail/function_record_pyobject.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
     include/pybind11/detail/native_enum_data.h
+    include/pybind11/detail/pybind11_namespace_macros.h
     include/pybind11/detail/struct_smart_holder.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/typeid.h
     include/pybind11/detail/using_smart_holder.h
     include/pybind11/detail/value_and_holder.h
-    include/pybind11/detail/exception_translation.h
     include/pybind11/attr.h
     include/pybind11/buffer_info.h
     include/pybind11/cast.h
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 830c0374a..9ee8131b1 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -412,22 +412,26 @@ function(pybind11_enable_warnings target_name)
   endif()
 
   if(PYBIND11_WERROR)
-    if(MSVC)
-      target_compile_options(${target_name} PRIVATE /WX)
-    elseif(PYBIND11_CUDA_TESTS)
-      target_compile_options(${target_name} PRIVATE "SHELL:-Werror all-warnings")
-    elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Clang|IntelLLVM)")
-      target_compile_options(${target_name} PRIVATE -Werror)
-    elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
       if(CMAKE_CXX_STANDARD EQUAL 17) # See PR #3570
         target_compile_options(${target_name} PRIVATE -Wno-conversion)
       endif()
-      target_compile_options(
-        ${target_name}
-        PRIVATE
-          -Werror-all
-          # "Inlining inhibited by limit max-size", "Inlining inhibited by limit max-total-size"
-          -diag-disable 11074,11076)
+      # "Inlining inhibited by limit max-size", "Inlining inhibited by limit max-total-size"
+      target_compile_options(${target_name} PRIVATE -diag-disable 11074,11076)
+    endif()
+
+    if(CMAKE_VERSION VERSION_LESS "3.24")
+      if(MSVC)
+        target_compile_options(${target_name} PRIVATE /WX)
+      elseif(PYBIND11_CUDA_TESTS)
+        target_compile_options(${target_name} PRIVATE "SHELL:-Werror all-warnings")
+      elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Clang|IntelLLVM)")
+        target_compile_options(${target_name} PRIVATE -Werror)
+      elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+        target_compile_options(${target_name} PRIVATE -Werror-all)
+      endif()
+    else()
+      set_target_properties(${target_name} PROPERTIES COMPILE_WARNING_AS_ERROR ON)
     endif()
   endif()
 endfunction()