From ba547ddcc63fd8e7fdf5b0bf71b2a26483eb406d Mon Sep 17 00:00:00 2001 From: Ville Juven Date: Thu, 18 Apr 2024 14:30:07 +0300 Subject: [PATCH 1/2] arm64/imx9: Add DMA memory allocator Add a simple allocator for DMA safe memory. It will provide contiguous blocks of memory with D-Cache line size alignment. NOTE: The optimal granule size is the D-Cache line size (64), but due to restrictions in the granule allocator this would result in a maximum block size of 2K only, thus use 256B granules instead givin 8K max block size. Once the granule allocator is fixed this limitation can be removed. --- arch/arm64/src/imx9/Kconfig | 18 +++ arch/arm64/src/imx9/Make.defs | 4 + arch/arm64/src/imx9/imx9_dma_alloc.c | 145 ++++++++++++++++++ arch/arm64/src/imx9/imx9_dma_alloc.h | 88 +++++++++++ .../imx9/imx93-evk/configs/nsh/defconfig | 3 + .../arm64/imx9/imx93-evk/src/imx9_bringup.c | 12 ++ 6 files changed, 270 insertions(+) create mode 100644 arch/arm64/src/imx9/imx9_dma_alloc.c create mode 100644 arch/arm64/src/imx9/imx9_dma_alloc.h diff --git a/arch/arm64/src/imx9/Kconfig b/arch/arm64/src/imx9/Kconfig index ba72f37268aec..d5e50b8573ec0 100644 --- a/arch/arm64/src/imx9/Kconfig +++ b/arch/arm64/src/imx9/Kconfig @@ -24,6 +24,24 @@ endchoice # i.MX9 Chip Selection endmenu # "i.MX9 Chip Selection" +config IMX9_DMA_ALLOC + bool "Enable DMA capable memory allocator" + depends on GRAN + default y if CONFIG_FAT_DMAMEMORY + +menu "DMA Allocator Configuration" + depends on IMX9_DMA_ALLOC + +config IMX9_DMA_ALLOC_POOL_SIZE + int "DMA allocator memory pool size in bytes" + default 4096 + +config IMX9_DMA_ALLOC_SECT + string "Section for DMA allocator memory pool, default is .bss" + default ".bss" + +endmenu # DMA Allocator Configuration + config IMX9_FLEXIO_PWM bool select PWM_MULTICHAN diff --git a/arch/arm64/src/imx9/Make.defs b/arch/arm64/src/imx9/Make.defs index dcca949e0f42e..9bce853d4626a 100644 --- a/arch/arm64/src/imx9/Make.defs +++ b/arch/arm64/src/imx9/Make.defs @@ -55,3 +55,7 @@ endif ifeq ($(CONFIG_IMX9_EDMA), y) CHIP_CSRCS += imx9_edma.c endif + +ifeq ($(CONFIG_IMX9_DMA_ALLOC),y) + CHIP_CSRCS += imx9_dma_alloc.c +endif diff --git a/arch/arm64/src/imx9/imx9_dma_alloc.c b/arch/arm64/src/imx9/imx9_dma_alloc.c new file mode 100644 index 0000000000000..b397f9112745d --- /dev/null +++ b/arch/arm64/src/imx9/imx9_dma_alloc.c @@ -0,0 +1,145 @@ +/**************************************************************************** + * arch/arm64/src/imx9/imx9_dma_alloc.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include + +#include + +#include + +#if defined(CONFIG_IMX9_DMA_ALLOC) + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/* DMA buffers must be aligned with the D-Cache line boundaries to facilitate + * cache operations on the DMA buffers when the D-Cache is enabled. + */ + +#define DMA_ALIGN ARMV8A_DCACHE_LINESIZE +#define DMA_ALIGN_MASK (DMA_ALIGN - 1) +#define DMA_ALIGN_UP(n) (((n) + DMA_ALIGN_MASK) & ~DMA_ALIGN_MASK) + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +static GRAN_HANDLE dma_allocator; + +/* The DMA heap size constrains the total number of things that can be + * ready to do DMA at a time. + * + * For example, FAT DMA depends on one sector-sized buffer per filesystem + * plus one sector-sized buffer per file. + * + * We use a fundamental alignment / granule size of 64B; it fulfills the + * requirement for any DMA engine. + */ + +static uint8_t g_dma_heap[CONFIG_IMX9_DMA_ALLOC_POOL_SIZE] +aligned_data(DMA_ALIGN) locate_data(CONFIG_IMX9_DMA_ALLOC_SECT); + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: imx9_dma_alloc_init + * + * Description: + * Initialize the DMA memory allocator. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int imx9_dma_alloc_init(void) +{ + /* Allocate 64B granules with 64B alignment */ + + /* REVISIT: Use 256B granule size to get 8K maximum allocation. This is a + * limitation in the granule allocator itself. + */ + + dma_allocator = gran_initialize(g_dma_heap, sizeof(g_dma_heap), 8, 6); + + if (dma_allocator == NULL) + { + return -ENOMEM; + } + + return OK; +} + +/**************************************************************************** + * Name: imx9_dma_alloc + * + * Description: + * Allocate a contiguous block of physical memory for DMA. + * + * Input Parameters: + * size - Size of the requested block in bytes. + * + * Returned Value: + * Physical address of the first page on success; NULL on failure. + * + ****************************************************************************/ + +void *imx9_dma_alloc(size_t size) +{ + return gran_alloc(dma_allocator, size); +} + +/**************************************************************************** + * Name: imx9_dma_free + * + * Description: + * Free a previously allocated DMA memory block. + * + * Input Parameters: + * memory - Physical address of the first page of DMA memory. + * size - Size of the allocated block in bytes. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void imx9_dma_free(void *memory, size_t size) +{ + gran_free(dma_allocator, memory, size); +} + +#endif /* CONFIG_IMX9_DMA_ALLOC */ diff --git a/arch/arm64/src/imx9/imx9_dma_alloc.h b/arch/arm64/src/imx9/imx9_dma_alloc.h new file mode 100644 index 0000000000000..dd98bdf3695b9 --- /dev/null +++ b/arch/arm64/src/imx9/imx9_dma_alloc.h @@ -0,0 +1,88 @@ +/**************************************************************************** + * arch/arm64/src/imx9/imx9_dma_alloc.h + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __ARCH_ARM64_SRC_IMX9_IMX9_DMA_ALLOC_H +#define __ARCH_ARM64_SRC_IMX9_IMX9_DMA_ALLOC_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include + +/**************************************************************************** + * Public Function Prototypes + ****************************************************************************/ + +/**************************************************************************** + * Name: imx9_dma_alloc_init + * + * Description: + * Initialize the DMA memory allocator. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int imx9_dma_alloc_init(void); + +/**************************************************************************** + * Name: imx9_dma_alloc + * + * Description: + * Allocate a contiguous block of physical memory for DMA. + * + * Input Parameters: + * size - Size of the requested block in bytes. + * + * Returned Value: + * Physical address of the first page on success; NULL on failure. + * + ****************************************************************************/ + +void *imx9_dma_alloc(size_t size); + +/**************************************************************************** + * Name: imx9_dma_free + * + * Description: + * Free a previously allocated DMA memory block. + * + * Input Parameters: + * memory - Physical address of the first page of DMA memory. + * size - Size of the allocated block in bytes. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void imx9_dma_free(void *memory, size_t size); + +#ifdef CONFIG_FAT_DMAMEMORY +# define fat_dma_alloc(s) imx9_dma_alloc(s) +# define fat_dma_free(m,s) imx9_dma_free(m,s) +#endif + +#endif /* __ARCH_ARM64_SRC_IMX9_IMX9_DMA_ALLOC_H */ diff --git a/boards/arm64/imx9/imx93-evk/configs/nsh/defconfig b/boards/arm64/imx9/imx93-evk/configs/nsh/defconfig index 8c685db7a90db..9cd5129d39349 100644 --- a/boards/arm64/imx9/imx93-evk/configs/nsh/defconfig +++ b/boards/arm64/imx9/imx93-evk/configs/nsh/defconfig @@ -26,11 +26,14 @@ CONFIG_EXAMPLES_HELLO=y CONFIG_EXPERIMENTAL=y CONFIG_FS_PROCFS=y CONFIG_FS_ROMFS=y +CONFIG_GRAN=y CONFIG_HAVE_CXX=y CONFIG_HAVE_CXXINITIALIZE=y CONFIG_I2C=y CONFIG_I2C_RESET=y CONFIG_IDLETHREAD_STACKSIZE=8192 +CONFIG_IMX9_DMA_ALLOC=y +CONFIG_IMX9_DMA_ALLOC_POOL_SIZE=81920 CONFIG_IMX9_EDMA=y CONFIG_IMX9_FLEXIO1_PWM=y CONFIG_IMX9_GPIO_IRQ=y diff --git a/boards/arm64/imx9/imx93-evk/src/imx9_bringup.c b/boards/arm64/imx9/imx93-evk/src/imx9_bringup.c index acd7b309a0afe..fc27bc3c9567b 100644 --- a/boards/arm64/imx9/imx93-evk/src/imx9_bringup.c +++ b/boards/arm64/imx9/imx93-evk/src/imx9_bringup.c @@ -29,6 +29,8 @@ #include +#include "imx9_dma_alloc.h" + #include "imx93-evk.h" /**************************************************************************** @@ -57,6 +59,16 @@ int imx9_bringup(void) } #endif +#ifdef CONFIG_IMX9_DMA_ALLOC + /* Initialize the DMA memory allocator */ + + ret = imx9_dma_alloc_init(); + if (ret < 0) + { + syslog(LOG_ERR, "ERROR: Failed initialize DMA allocator: %d\n", ret); + } +#endif + #ifdef CONFIG_PWM /* Configure PWM outputs */ From 182511ff3b9a45007a3ed819972fb7a29b85b63e Mon Sep 17 00:00:00 2001 From: Ville Juven Date: Thu, 18 Apr 2024 15:46:31 +0300 Subject: [PATCH 2/2] arch/imx9_lpspi: Use DMA safe buffers to do the DMA transfers Using user allocated buffers for DMA transfers is not safe for two reasons: - User space memory is virtual memory, DMA needs physical memory - User memory buffer alignment cannot be guaranteed -> cache line ops are not safe --- arch/arm64/src/imx9/Kconfig | 7 +++++ arch/arm64/src/imx9/imx9_lpspi.c | 54 ++++++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/arch/arm64/src/imx9/Kconfig b/arch/arm64/src/imx9/Kconfig index d5e50b8573ec0..01e027bf45f56 100644 --- a/arch/arm64/src/imx9/Kconfig +++ b/arch/arm64/src/imx9/Kconfig @@ -718,6 +718,13 @@ config IMX9_LPSPI_DMATHRESHOLD by polling logic. But we need a threshold value to determine what is small. +config IMX9_LPSPI_DMA_BUFFER_SIZE + int "LPSPI DMA buffer size" + default 4096 + depends on IMX9_LPSPI_DMA + ---help--- + Set the LPSPI driver DMA buffer size. + config IMX9_LPSPI1_DMA bool "LPSPI1 DMA" default n diff --git a/arch/arm64/src/imx9/imx9_lpspi.c b/arch/arm64/src/imx9/imx9_lpspi.c index 48d8ccf582a79..b373b04c8f213 100644 --- a/arch/arm64/src/imx9/imx9_lpspi.c +++ b/arch/arm64/src/imx9/imx9_lpspi.c @@ -67,6 +67,7 @@ #include "arm64_internal.h" #include "imx9_ccm.h" #include "imx9_clockconfig.h" +#include "imx9_dma_alloc.h" #include "imx9_gpio.h" #include "imx9_iomuxc.h" #include "imx9_lpspi.h" @@ -131,6 +132,8 @@ struct imx9_lpspidev_s DMACH_HANDLE txdma; /* DMA channel handle for TX transfers */ sem_t rxsem; /* Wait for RX DMA to complete */ sem_t txsem; /* Wait for TX DMA to complete */ + void *txbuf; /* Driver DMA safe buffer for TX */ + void *rxbuf; /* Driver DMA safe buffer for RX */ #endif }; @@ -1305,13 +1308,13 @@ static void imx9_lpspi_exchange(struct spi_dev_s *dev, const void *txbuffer, void *rxbuffer, size_t nwords) { + struct imx9_lpspidev_s *priv = (struct imx9_lpspidev_s *)dev; int ret; size_t adjust; ssize_t nbytes; static uint8_t rxdummy[4] aligned_data(4); static const uint16_t txdummy = 0xffff; uint32_t regval; - struct imx9_lpspidev_s *priv = (struct imx9_lpspidev_s *)dev; DEBUGASSERT(priv != NULL); DEBUGASSERT(priv && priv->spibase); @@ -1338,6 +1341,17 @@ static void imx9_lpspi_exchange(struct spi_dev_s *dev, return; } + /* Check if the transfer is too long */ + + if (nbytes > CONFIG_IMX9_LPSPI_DMA_BUFFER_SIZE) + { + /* Transfer is too long, revert to slow non-DMA method */ + + spiwarn("frame %lu too long, fall back to no DMA transfer\n", nbytes); + imx9_lpspi_exchange_nodma(dev, txbuffer, rxbuffer, nwords); + return; + } + /* Disable SPI when we are configuring it */ imx9_lpspi_modifyreg32(priv, IMX9_LPSPI_CR_OFFSET, LPSPI_CR_MEN, 0); @@ -1362,13 +1376,22 @@ static void imx9_lpspi_exchange(struct spi_dev_s *dev, if (txbuffer) { - up_clean_dcache((uintptr_t)txbuffer, (uintptr_t)txbuffer + nbytes); + /* Move the user data to device internal buffer */ + + memcpy(priv->txbuf, txbuffer, nbytes); + + /* And flush it to RAM */ + + up_clean_dcache((uintptr_t)priv->txbuf, + (uintptr_t)priv->txbuf + nbytes); } if (rxbuffer) { - up_invalidate_dcache((uintptr_t)rxbuffer, - (uintptr_t)rxbuffer + nbytes); + /* Prepare the RX buffer for DMA */ + + up_invalidate_dcache((uintptr_t)priv->rxbuf, + (uintptr_t)priv->rxbuf + nbytes); } /* Set up the DMA */ @@ -1378,7 +1401,7 @@ static void imx9_lpspi_exchange(struct spi_dev_s *dev, struct imx9_edma_xfrconfig_s config; config.saddr = priv->spibase + IMX9_LPSPI_RDR_OFFSET; - config.daddr = (uintptr_t) (rxbuffer ? rxbuffer : rxdummy); + config.daddr = (uintptr_t) (rxbuffer ? priv->rxbuf : rxdummy); config.soff = 0; config.doff = rxbuffer ? adjust : 0; config.iter = nbytes; @@ -1391,7 +1414,7 @@ static void imx9_lpspi_exchange(struct spi_dev_s *dev, #endif imx9_dmach_xfrsetup(priv->rxdma, &config); - config.saddr = (uintptr_t) (txbuffer ? txbuffer : &txdummy); + config.saddr = (uintptr_t) (txbuffer ? priv->txbuf : &txdummy); config.daddr = priv->spibase + IMX9_LPSPI_TDR_OFFSET; config.soff = txbuffer ? adjust : 0; config.doff = 0; @@ -1436,10 +1459,16 @@ static void imx9_lpspi_exchange(struct spi_dev_s *dev, imx9_lpspi_putreg32(priv, IMX9_LPSPI_DER_OFFSET, 0); - if (rxbuffer) + if (rxbuffer && ret >= 0) { - up_invalidate_dcache((uintptr_t)rxbuffer, - (uintptr_t)rxbuffer + nbytes); + /* Flush the RX data to ram */ + + up_invalidate_dcache((uintptr_t)priv->rxbuf, + (uintptr_t)priv->rxbuf + nbytes); + + /* Copy data to user buffer */ + + memcpy(rxbuffer, priv->rxbuf, nbytes); } } @@ -2044,6 +2073,13 @@ struct spi_dev_s *imx9_lpspibus_initialize(int bus) priv->rxdma = imx9_dmach_alloc(priv->rxch, 0); DEBUGASSERT(priv->rxdma && priv->txdma); } + + if (priv->txbuf == NULL && priv->rxbuf == NULL) + { + priv->txbuf = imx9_dma_alloc(CONFIG_IMX9_LPSPI_DMA_BUFFER_SIZE); + priv->rxbuf = imx9_dma_alloc(CONFIG_IMX9_LPSPI_DMA_BUFFER_SIZE); + DEBUGASSERT(priv->txbuf && priv->rxbuf); + } } else {