console: add hook for debug server to handle outgoing characters

A debug server might want to redirect outgoing characters on the console
to another output if it piggybacks on the serial line normally reserved
for the console.

Change-Id: I534f5b35456306940a3926f52fe5cce2d5c94da4
Signed-off-by: Benjamin Walsh <benjamin.walsh@windriver.com>

drivers/console/Kconfig

@@ -83,6 +83,18 @@ config UART_CONSOLE_INIT_PRIORITY
 	  Console has to be initialized after the UART driver
 	  it uses.
 
+config UART_CONSOLE_DEBUG_SERVER_HOOKS
+	bool
+	prompt "Debug server hooks in debug console"
+	default n
+	depends on UART_CONSOLE
+	help
+	This option allows a debug server agent such as GDB to take over the
+	handling of traffic that goes through the console logic. The debug
+	server looks at characters received and decides to handle them itself if
+	they are some sort of control characters, or let the regular console
+	code handle them if they are of no special significance to it.
+
 config RAM_CONSOLE
 	bool
 	prompt "Use RAM console"

drivers/console/uart_console.c

@@ -44,6 +44,24 @@
 
 static struct device *uart_console_dev;
 
+#ifdef CONFIG_UART_CONSOLE_DEBUG_SERVER_HOOKS
+static UART_CONSOLE_OUT_DEBUG_HOOK_SIG(debug_hook_out_nop)
+{
+	ARG_UNUSED(c);
+	return !UART_CONSOLE_DEBUG_HOOK_HANDLED;
+}
+
+static uart_console_out_debug_hook_t *debug_hook_out = debug_hook_out_nop;
+void uart_console_out_debug_hook_install(uart_console_out_debug_hook_t *hook)
+{
+	debug_hook_out = hook;
+}
+#define HANDLE_DEBUG_HOOK_OUT(c) \
+	(debug_hook_out(c) == UART_CONSOLE_DEBUG_HOOK_HANDLED)
+#else
+#define HANDLE_DEBUG_HOOK_OUT(c) 0
+#endif /* CONFIG_UART_CONSOLE_DEBUG_SERVER_HOOKS */
+
 #if 0 /* NOTUSED */
 /**
  *
@@ -77,6 +95,12 @@ static int console_in(void)
 
 static int console_out(int c)
 {
+	int handled_by_debug_server = HANDLE_DEBUG_HOOK_OUT(c);
+
+	if (handled_by_debug_server) {
+		return c;
+	}
+
 	uart_poll_out(uart_console_dev, (unsigned char)c);
 	if ('\n' == c) {
 		uart_poll_out(uart_console_dev, (unsigned char)'\r');

include/drivers/console/uart_console.h

@@ -46,6 +46,19 @@ struct uart_console_input {
  */
 void uart_register_input(struct nano_fifo *avail, struct nano_fifo *lines);
 
+/*
+ * Allows having debug hooks in the console driver for handling incoming
+ * control characters, and letting other ones through.
+ */
+#ifdef CONFIG_UART_CONSOLE_DEBUG_SERVER_HOOKS
+#define UART_CONSOLE_DEBUG_HOOK_HANDLED 1
+#define UART_CONSOLE_OUT_DEBUG_HOOK_SIG(x) int(x)(char c)
+typedef UART_CONSOLE_OUT_DEBUG_HOOK_SIG(uart_console_out_debug_hook_t);
+
+extern void uart_console_out_debug_hook_install(
+				uart_console_out_debug_hook_t *hook);
+#endif
+
 #ifdef __cplusplus
 }
 #endif