diff --git a/drivers/video/CMakeLists.txt b/drivers/video/CMakeLists.txt index 5da14958d12..e678071bdec 100644 --- a/drivers/video/CMakeLists.txt +++ b/drivers/video/CMakeLists.txt @@ -15,3 +15,4 @@ zephyr_library_sources_ifdef(CONFIG_VIDEO_STM32_DCMI video_stm32_dcmi.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_OV5640 ov5640.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_OV7670 ov7670.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_ESP32 video_esp32_dvp.c) +zephyr_library_sources_ifdef(CONFIG_VIDEO_MCUX_SDMA video_mcux_smartdma.c) diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index c7693ec0e0a..6ff70bbcfca 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -72,4 +72,6 @@ source "drivers/video/Kconfig.ov7670" source "drivers/video/Kconfig.gc2145" +source "drivers/video/Kconfig.mcux_sdma" + endif # VIDEO diff --git a/drivers/video/Kconfig.mcux_sdma b/drivers/video/Kconfig.mcux_sdma new file mode 100644 index 00000000000..240e2ee3a48 --- /dev/null +++ b/drivers/video/Kconfig.mcux_sdma @@ -0,0 +1,10 @@ +# NXP MCUX SDMA driver configuration options + +# Copyright 2024 NXP +# SPDX-License-Identifier: Apache-2.0 + +config VIDEO_MCUX_SDMA + bool "NXP MCUX Video SMARTDMA driver" + default y + select DMA + depends on DT_HAS_NXP_VIDEO_SMARTDMA_ENABLED diff --git a/drivers/video/video_mcux_smartdma.c b/drivers/video/video_mcux_smartdma.c new file mode 100644 index 00000000000..860ce1476b7 --- /dev/null +++ b/drivers/video/video_mcux_smartdma.c @@ -0,0 +1,391 @@ +/* + * Copyright 2024 NXP + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#define DT_DRV_COMPAT nxp_video_smartdma + +#include +#include + +#include +#include +#include +#include + +#define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL +#include +LOG_MODULE_REGISTER(nxp_video_sdma); + +struct nxp_video_sdma_config { + const struct device *dma_dev; + const struct device *sensor_dev; + const struct pinctrl_dev_config *pincfg; + uint8_t vsync_pin; + uint8_t hsync_pin; + uint8_t pclk_pin; +}; + +/* Firmware reads 30 lines of data per video buffer */ +#define SDMA_LINE_COUNT 30 +/* Firmware only supports 320x240 */ +#define SDMA_VBUF_HEIGHT 240 +#define SDMA_VBUF_WIDTH 320 + +struct nxp_video_sdma_data { + /* Must be aligned on 4 byte boundary, as lower 2 bits of ARM2SDMA register + * are used to enable interrupts + */ + smartdma_camera_param_t params __aligned(4); + uint32_t smartdma_stack[64] __aligned(32); + struct k_fifo fifo_in; + struct k_fifo fifo_out; + struct k_sem stream_empty; /* Signals stream has run out of buffers */ + bool stream_starved; + bool buf_reload_flag; + struct video_buffer *active_buf; + struct video_buffer *queued_buf; + const struct nxp_video_sdma_config *config; + uint32_t frame_idx; +}; + +/* Executed in interrupt context */ +static void nxp_video_sdma_callback(const struct device *dev, void *user_data, + uint32_t channel, int status) +{ + struct nxp_video_sdma_data *data = user_data; + + if (status < 0) { + LOG_ERR("Transfer failed: %d, stopping DMA", status); + dma_stop(data->config->dma_dev, 0); + return; + } + /* + * SmartDMA engine streams 15 lines of RGB565 data, then interrupts the + * system. The engine will reload the framebuffer pointer after sending + * the first interrupt, and before sending the second interrupt. + * + * Based on this, we alternate between reloading the framebuffer + * pointer and queueing a completed frame every other interrupt + */ + if (data->buf_reload_flag) { + /* Save old framebuffer, we will dequeue it next interrupt */ + data->active_buf = data->queued_buf; + /* Load new framebuffer */ + data->queued_buf = k_fifo_get(&data->fifo_in, K_NO_WAIT); + if (data->queued_buf == NULL) { + data->stream_starved = true; + } else { + data->params.p_buffer_ping_pong = (uint32_t *)data->queued_buf->buffer; + } + } else { + if (data->stream_starved) { + /* Signal any waiting threads */ + k_sem_give(&data->stream_empty); + } + data->active_buf->line_offset = (data->frame_idx / 2) * SDMA_LINE_COUNT; + data->active_buf->timestamp = k_uptime_get_32(); + k_fifo_put(&data->fifo_out, data->active_buf); + + } + /* Toggle buffer reload flag*/ + data->buf_reload_flag = !data->buf_reload_flag; +} + +static int nxp_video_sdma_stream_start(const struct device *dev) +{ + const struct nxp_video_sdma_config *config = dev->config; + struct nxp_video_sdma_data *data = dev->data; + struct dma_config sdma_config = {0}; + int ret; + + /* Setup dma configuration for SmartDMA */ + sdma_config.dma_slot = kSMARTDMA_CameraDiv16FrameQVGA; + sdma_config.dma_callback = nxp_video_sdma_callback; + sdma_config.user_data = data; + /* Setting bit 1 here enables the SmartDMA to interrupt ARM core + * when writing to SMARTDMA2ARM register + */ + sdma_config.head_block = (struct dma_block_config *)(((uint32_t)&data->params) | 0x2); + + /* Setup parameters for SmartDMA engine */ + data->params.smartdma_stack = data->smartdma_stack; + /* SmartDMA continuously streams data once started. If user + * has not provided a framebuffer, we can't start DMA. + */ + data->queued_buf = k_fifo_get(&data->fifo_in, K_NO_WAIT); + if (data->queued_buf == NULL) { + return -EIO; + } + data->params.p_buffer_ping_pong = (uint32_t *)data->queued_buf->buffer; + /* The firmware writes the index of the frame slice + * (from 0-15) into this buffer + */ + data->params.p_stripe_index = &data->frame_idx; + + /* Start DMA engine */ + ret = dma_config(config->dma_dev, 0, &sdma_config); + if (ret < 0) { + return ret; + } + /* Reset stream state variables */ + k_sem_reset(&data->stream_empty); + data->buf_reload_flag = true; + data->stream_starved = false; + + ret = dma_start(config->dma_dev, 0); + if (ret < 0) { + return ret; + } + + return 0; +} + +static int nxp_video_sdma_stream_stop(const struct device *dev) +{ + const struct nxp_video_sdma_config *config = dev->config; + + /* Stop DMA engine */ + return dma_stop(config->dma_dev, 0); +} + +static int nxp_video_sdma_enqueue(const struct device *dev, + enum video_endpoint_id ep, + struct video_buffer *vbuf) +{ + struct nxp_video_sdma_data *data = dev->data; + + if (ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + /* SmartDMA will read 30 lines of RGB565 video data into framebuffer */ + vbuf->bytesused = SDMA_VBUF_WIDTH * SDMA_LINE_COUNT * sizeof(uint16_t); + if (vbuf->size < vbuf->bytesused) { + return -EINVAL; + } + + /* Put buffer into FIFO */ + k_fifo_put(&data->fifo_in, vbuf); + if (data->stream_starved) { + /* Kick SmartDMA off */ + nxp_video_sdma_stream_start(dev); + } + return 0; +} + +static int nxp_video_sdma_dequeue(const struct device *dev, + enum video_endpoint_id ep, + struct video_buffer **vbuf, + k_timeout_t timeout) +{ + struct nxp_video_sdma_data *data = dev->data; + + if (ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + *vbuf = k_fifo_get(&data->fifo_out, timeout); + if (*vbuf == NULL) { + return -EAGAIN; + } + + return 0; +} + +static int nxp_video_sdma_flush(const struct device *dev, + enum video_endpoint_id ep, + bool cancel) +{ + const struct nxp_video_sdma_config *config = dev->config; + struct nxp_video_sdma_data *data = dev->data; + struct video_buf *vbuf; + + if (!cancel) { + /* Wait for DMA to signal it is empty */ + k_sem_take(&data->stream_empty, K_FOREVER); + } else { + /* Stop DMA engine */ + dma_stop(config->dma_dev, 0); + /* Forward all buffers in fifo_in to fifo_out */ + while ((vbuf = k_fifo_get(&data->fifo_in, K_NO_WAIT))) { + k_fifo_put(&data->fifo_out, vbuf); + } + } + return 0; +} + +/* SDMA only supports 320x240 RGB565 */ +static const struct video_format_cap fmts[] = { + { + .pixelformat = VIDEO_PIX_FMT_RGB565, + .width_min = SDMA_VBUF_WIDTH, + .width_max = SDMA_VBUF_WIDTH, + .height_min = SDMA_VBUF_HEIGHT, + .height_max = SDMA_VBUF_HEIGHT, + .width_step = 0, + .height_step = 0, + }, + { 0 }, +}; + +static int nxp_video_sdma_set_format(const struct device *dev, + enum video_endpoint_id ep, + struct video_format *fmt) +{ + const struct nxp_video_sdma_config *config = dev->config; + + if (fmt == NULL || ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + if (!device_is_ready(config->sensor_dev)) { + LOG_ERR("Sensor device not ready"); + return -ENODEV; + } + + if ((fmt->pixelformat != fmts[0].pixelformat) || + (fmt->width != fmts[0].width_min) || + (fmt->height != fmts[0].height_min) || + (fmt->pitch != fmts[0].width_min * 2)) { + LOG_ERR("Unsupported format"); + return -ENOTSUP; + } + + /* Forward format to sensor device */ + return video_set_format(config->sensor_dev, ep, fmt); +} + +static int nxp_video_sdma_get_format(const struct device *dev, + enum video_endpoint_id ep, + struct video_format *fmt) +{ + const struct nxp_video_sdma_config *config = dev->config; + int ret; + + if (fmt == NULL || ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + if (!device_is_ready(config->sensor_dev)) { + LOG_ERR("Sensor device not ready"); + return -ENODEV; + } + + /* + * Check sensor format. If it is not RGB565 320x240 + * reconfigure the sensor, + * as this is the only format supported. + */ + ret = video_get_format(config->sensor_dev, VIDEO_EP_OUT, fmt); + if (ret < 0) { + return ret; + } + + /* Verify that format is RGB565 */ + if ((fmt->pixelformat != fmts[0].pixelformat) || + (fmt->width != fmts[0].width_min) || + (fmt->height != fmts[0].height_min) || + (fmt->pitch != fmts[0].width_min * 2)) { + /* Update format of sensor */ + fmt->pixelformat = fmts[0].pixelformat; + fmt->width = fmts[0].width_min; + fmt->height = fmts[0].height_min; + fmt->pitch = fmts[0].width_min * 2; + ret = video_set_format(config->sensor_dev, VIDEO_EP_OUT, fmt); + if (ret < 0) { + LOG_ERR("Sensor device does not support RGB565"); + return ret; + } + } + + return 0; +} + +static int nxp_video_sdma_get_caps(const struct device *dev, + enum video_endpoint_id ep, + struct video_caps *caps) +{ + if (ep != VIDEO_EP_OUT) { + return -EINVAL; + } + + /* SmartDMA needs at least two buffers allocated before starting */ + caps->min_vbuf_count = 2; + /* Firmware reads 30 lines per queued vbuf */ + caps->min_line_count = caps->max_line_count = SDMA_LINE_COUNT; + caps->format_caps = fmts; + return 0; +} + +static int nxp_video_sdma_init(const struct device *dev) +{ + const struct nxp_video_sdma_config *config = dev->config; + struct nxp_video_sdma_data *data = dev->data; + int ret; + + if (!device_is_ready(config->dma_dev)) { + LOG_ERR("SmartDMA not ready"); + return -ENODEV; + } + + INPUTMUX_Init(INPUTMUX0); + /* Attach Camera VSYNC, HSYNC, and PCLK as inputs 0, 1, and 2 of the SmartDMA */ + INPUTMUX_AttachSignal(INPUTMUX0, 0, + config->vsync_pin + (SMARTDMAARCHB_INMUX0 << PMUX_SHIFT)); + INPUTMUX_AttachSignal(INPUTMUX0, 1, + config->hsync_pin + (SMARTDMAARCHB_INMUX0 << PMUX_SHIFT)); + INPUTMUX_AttachSignal(INPUTMUX0, 2, + config->pclk_pin + (SMARTDMAARCHB_INMUX0 << PMUX_SHIFT)); + /* Turnoff clock to inputmux to save power. Clock is only needed to make changes */ + INPUTMUX_Deinit(INPUTMUX0); + + k_fifo_init(&data->fifo_in); + k_fifo_init(&data->fifo_out); + /* Given to when the DMA engine runs out of buffers */ + k_sem_init(&data->stream_empty, 0, 1); + + /* Install camera firmware used by SmartDMA */ + dma_smartdma_install_fw(config->dma_dev, (uint8_t *)s_smartdmaCameraFirmware, + s_smartdmaCameraFirmwareSize); + ret = pinctrl_apply_state(config->pincfg, PINCTRL_STATE_DEFAULT); + if (ret < 0) { + return ret; + } + + return 0; +} + +static const struct video_driver_api nxp_video_sdma_api = { + .get_format = nxp_video_sdma_get_format, + .set_format = nxp_video_sdma_set_format, + .get_caps = nxp_video_sdma_get_caps, + .stream_start = nxp_video_sdma_stream_start, + .stream_stop = nxp_video_sdma_stream_stop, + .enqueue = nxp_video_sdma_enqueue, + .dequeue = nxp_video_sdma_dequeue, + .flush = nxp_video_sdma_flush +}; + +#define NXP_VIDEO_SDMA_INIT(inst) \ + PINCTRL_DT_INST_DEFINE(inst); \ + const struct nxp_video_sdma_config sdma_config_##inst = { \ + .dma_dev = DEVICE_DT_GET(DT_INST_PARENT(inst)), \ + .sensor_dev = DEVICE_DT_GET(DT_INST_PHANDLE(inst, sensor)), \ + .pincfg = PINCTRL_DT_INST_DEV_CONFIG_GET(inst), \ + .vsync_pin = DT_INST_PROP(inst, vsync_pin), \ + .hsync_pin = DT_INST_PROP(inst, hsync_pin), \ + .pclk_pin = DT_INST_PROP(inst, pclk_pin), \ + }; \ + struct nxp_video_sdma_data sdma_data_##inst = { \ + .config = &sdma_config_##inst, \ + }; \ + \ + DEVICE_DT_INST_DEFINE(inst, nxp_video_sdma_init, NULL, \ + &sdma_data_##inst, &sdma_config_##inst, \ + POST_KERNEL, \ + CONFIG_KERNEL_INIT_PRIORITY_DEVICE, \ + &nxp_video_sdma_api); + +DT_INST_FOREACH_STATUS_OKAY(NXP_VIDEO_SDMA_INIT) diff --git a/dts/bindings/video/nxp,video-smartdma.yaml b/dts/bindings/video/nxp,video-smartdma.yaml new file mode 100644 index 00000000000..5312f2b3bc3 --- /dev/null +++ b/dts/bindings/video/nxp,video-smartdma.yaml @@ -0,0 +1,29 @@ +# Copyright 2024 NXP +# SPDX-License-Identifier: Apache-2.0 + +description: NXP SmartDMA Video Driver + +compatible: "nxp,video-smartdma" + +include: [base.yaml, pinctrl-device.yaml] + +properties: + sensor: + required: true + type: phandle + description: phandle of connected sensor device + vsync-pin: + required: true + type: int + description: | + GPIO0 pin index to use for VSYNC input. Only pins 0-15 may be used. + hsync-pin: + required: true + type: int + description: | + GPIO0 pin index to use for HSYNC input. Only pins 0-15 may be used. + pclk-pin: + required: true + type: int + description: | + GPIO0 pin index to use for PCLK input. Only pins 0-15 may be used.