Wear-leveling EEPROM drivers: embedded_flash, spi_flash, legacy

builddefs/build_keyboard.mk

@@ -13,6 +13,14 @@ endif
 include paths.mk
 include $(BUILDDEFS_PATH)/message.mk
 
+# Helper to add defines with a 'QMK_' prefix
+define add_qmk_prefix_defs
+    ifdef $1
+        # Need to cater for 'STM32L4xx+'
+        OPT_DEFS += -DQMK_$(2)="$($1)" -DQMK_$(2)_$(shell echo $($1) | sed -e 's@+@Plus@g' -e 's@[^a-zA-Z0-9]@_@g' | tr '[:lower:]' '[:upper:]')
+    endif
+endef
+
 # Set the qmk cli to use
 QMK_BIN ?= qmk
 
@@ -438,6 +446,14 @@ else
     include $(TMK_PATH)/protocol/$(PLATFORM_KEY).mk
 endif
 
+# Setup definitions based on the selected MCU
+$(eval $(call add_qmk_prefix_defs,MCU_ORIG,MCU))
+$(eval $(call add_qmk_prefix_defs,MCU_ARCH,MCU_ARCH))
+$(eval $(call add_qmk_prefix_defs,MCU_PORT_NAME,MCU_PORT_NAME))
+$(eval $(call add_qmk_prefix_defs,MCU_FAMILY,MCU_FAMILY))
+$(eval $(call add_qmk_prefix_defs,MCU_SERIES,MCU_SERIES))
+$(eval $(call add_qmk_prefix_defs,BOARD,BOARD))
+
 # TODO: remove this bodge?
 PROJECT_DEFS := $(OPT_DEFS)
 PROJECT_INC := $(VPATH) $(EXTRAINCDIRS) $(KEYBOARD_PATHS)

builddefs/common_features.mk

@@ -173,7 +173,7 @@ ifeq ($(strip $(QUANTUM_PAINTER_ENABLE)), yes)
     include $(QUANTUM_DIR)/painter/rules.mk
 endif
 
-VALID_EEPROM_DRIVER_TYPES := vendor custom transient i2c spi
+VALID_EEPROM_DRIVER_TYPES := vendor custom transient i2c spi wear_leveling
 EEPROM_DRIVER ?= vendor
 ifeq ($(filter $(EEPROM_DRIVER),$(VALID_EEPROM_DRIVER_TYPES)),)
   $(call CATASTROPHIC_ERROR,Invalid EEPROM_DRIVER,EEPROM_DRIVER="$(EEPROM_DRIVER)" is not a valid EEPROM driver)
@@ -186,6 +186,10 @@ else
     # Custom EEPROM implementation -- only needs to implement init/erase/read_block/write_block
     OPT_DEFS += -DEEPROM_DRIVER -DEEPROM_CUSTOM
     SRC += eeprom_driver.c
+  else ifeq ($(strip $(EEPROM_DRIVER)), wear_leveling)
+    # Wear-leveling EEPROM implementation
+    OPT_DEFS += -DEEPROM_DRIVER -DEEPROM_WEAR_LEVELING
+    SRC += eeprom_driver.c eeprom_wear_leveling.c
   else ifeq ($(strip $(EEPROM_DRIVER)), i2c)
     # External I2C EEPROM implementation
     OPT_DEFS += -DEEPROM_DRIVER -DEEPROM_I2C
@@ -237,17 +241,47 @@ else
   endif
 endif
 
+VALID_WEAR_LEVELING_DRIVER_TYPES := custom embedded_flash spi_flash legacy
+WEAR_LEVELING_DRIVER ?= none
+ifneq ($(strip $(WEAR_LEVELING_DRIVER)),none)
+  ifeq ($(filter $(WEAR_LEVELING_DRIVER),$(VALID_WEAR_LEVELING_DRIVER_TYPES)),)
+    $(call CATASTROPHIC_ERROR,Invalid WEAR_LEVELING_DRIVER,WEAR_LEVELING_DRIVER="$(WEAR_LEVELING_DRIVER)" is not a valid wear leveling driver)
+  else
+    FNV_ENABLE := yes
+    OPT_DEFS += -DWEAR_LEVELING_ENABLE
+    OPT_DEFS += -DWEAR_LEVELING_$(strip $(shell echo $(WEAR_LEVELING_DRIVER) | tr '[:lower:]' '[:upper:]'))
+    COMMON_VPATH += $(PLATFORM_PATH)/$(PLATFORM_KEY)/$(DRIVER_DIR)/wear_leveling
+    COMMON_VPATH += $(DRIVER_PATH)/wear_leveling
+    COMMON_VPATH += $(QUANTUM_DIR)/wear_leveling
+    SRC += wear_leveling.c
+    ifeq ($(strip $(WEAR_LEVELING_DRIVER)), embedded_flash)
+      OPT_DEFS += -DHAL_USE_EFL
+      SRC += wear_leveling_efl.c
+      POST_CONFIG_H += $(PLATFORM_PATH)/$(PLATFORM_KEY)/$(DRIVER_DIR)/wear_leveling/wear_leveling_efl_config.h
+    else ifeq ($(strip $(WEAR_LEVELING_DRIVER)), spi_flash)
+      FLASH_DRIVER := spi
+      SRC += wear_leveling_flash_spi.c
+      POST_CONFIG_H += $(DRIVER_PATH)/wear_leveling/wear_leveling_flash_spi_config.h
+    else ifeq ($(strip $(WEAR_LEVELING_DRIVER)), legacy)
+      COMMON_VPATH += $(PLATFORM_PATH)/$(PLATFORM_KEY)/$(DRIVER_DIR)/flash
+      SRC += flash_stm32.c wear_leveling_legacy.c
+      POST_CONFIG_H += $(PLATFORM_PATH)/$(PLATFORM_KEY)/$(DRIVER_DIR)/wear_leveling/wear_leveling_legacy_config.h
+    endif
+  endif
+endif
+
 VALID_FLASH_DRIVER_TYPES := spi
-FLASH_DRIVER ?= no
-ifneq ($(strip $(FLASH_DRIVER)), no)
+FLASH_DRIVER ?= none
+ifneq ($(strip $(FLASH_DRIVER)), none)
     ifeq ($(filter $(FLASH_DRIVER),$(VALID_FLASH_DRIVER_TYPES)),)
-        $(error FLASH_DRIVER="$(FLASH_DRIVER)" is not a valid FLASH driver)
+        $(call CATASTROPHIC_ERROR,Invalid FLASH_DRIVER,FLASH_DRIVER="$(FLASH_DRIVER)" is not a valid flash driver)
     else
         OPT_DEFS += -DFLASH_ENABLE
-        ifeq ($(strip $(FLASH_DRIVER)), spi)
+        ifeq ($(strip $(FLASH_DRIVER)),spi)
             OPT_DEFS += -DFLASH_DRIVER -DFLASH_SPI
             COMMON_VPATH += $(DRIVER_PATH)/flash
             SRC += flash_spi.c
+            QUANTUM_LIB_SRC += spi_master.c
         endif
     endif
 endif

docs/_summary.md

@@ -146,6 +146,7 @@
       * [SPI Driver](spi_driver.md)
       * [WS2812 Driver](ws2812_driver.md)
       * [EEPROM Driver](eeprom_driver.md)
+      * [Flash Driver](flash_driver.md)
       * ['serial' Driver](serial_driver.md)
       * [UART Driver](uart_driver.md)
     * [GPIO Controls](gpio_control.md)

docs/eeprom_driver.md

@@ -2,12 +2,15 @@
 
 The EEPROM driver can be swapped out depending on the needs of the keyboard, or whether extra hardware is present.
 
+Selecting the EEPROM driver is done in your keyboard's `rules.mk`:
+
 Driver                             | Description
 -----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 `EEPROM_DRIVER = vendor` (default) | Uses the on-chip driver provided by the chip manufacturer. For AVR, this is provided by avr-libc. This is supported on ARM for a subset of chips -- STM32F3xx, STM32F1xx, and STM32F072xB will be emulated by writing to flash. STM32L0xx and STM32L1xx will use the onboard dedicated true EEPROM. Other chips will generally act as "transient" below.
 `EEPROM_DRIVER = i2c`              | Supports writing to I2C-based 24xx EEPROM chips. See the driver section below.
 `EEPROM_DRIVER = spi`              | Supports writing to SPI-based 25xx EEPROM chips. See the driver section below.
 `EEPROM_DRIVER = transient`        | Fake EEPROM driver -- supports reading/writing to RAM, and will be discarded when power is lost.
+`EEPROM_DRIVER = wear_leveling`    | Frontend driver for the wear_leveling system, allowing for EEPROM emulation on top of flash -- both in-MCU and external SPI NOR flash.
 
 ## Vendor Driver Configuration :id=vendor-eeprom-driver-configuration
 
@@ -55,13 +58,13 @@ MB85RC256V FRAM  | `#define EEPROM_I2C_MB85RC256V` | <https://www.adafruit.com/p
 
 Currently QMK supports 25xx-series chips over SPI. As such, requires a working spi_master driver configuration. You can override the driver configuration via your config.h:
 
-`config.h` override                            | Description                                                                          | Default Value
------------------------------------------------|--------------------------------------------------------------------------------------|--------------
-`#define EXTERNAL_EEPROM_SPI_SLAVE_SELECT_PIN` | SPI Slave select pin in order to inform that the EEPROM is currently being addressed | _none_
-`#define EXTERNAL_EEPROM_SPI_CLOCK_DIVISOR`    | Clock divisor used to divide the peripheral clock to derive the SPI frequency        | `64`
-`#define EXTERNAL_EEPROM_BYTE_COUNT`           | Total size of the EEPROM in bytes                                                    | 8192
-`#define EXTERNAL_EEPROM_PAGE_SIZE`            | Page size of the EEPROM in bytes, as specified in the datasheet                      | 32
-`#define EXTERNAL_EEPROM_ADDRESS_SIZE`         | The number of bytes to transmit for the memory location within the EEPROM            | 2
+`config.h` override                            | Default Value | Description
+-----------------------------------------------|---------------|-------------------------------------------------------------------------------------
+`#define EXTERNAL_EEPROM_SPI_SLAVE_SELECT_PIN` | _none_        | SPI Slave select pin in order to inform that the EEPROM is currently being addressed
+`#define EXTERNAL_EEPROM_SPI_CLOCK_DIVISOR`    | `64`          | Clock divisor used to divide the peripheral clock to derive the SPI frequency
+`#define EXTERNAL_EEPROM_BYTE_COUNT`           | `8192`        | Total size of the EEPROM in bytes
+`#define EXTERNAL_EEPROM_PAGE_SIZE`            | `32`          | Page size of the EEPROM in bytes, as specified in the datasheet
+`#define EXTERNAL_EEPROM_ADDRESS_SIZE`         | `2`           | The number of bytes to transmit for the memory location within the EEPROM
 
 !> There's no way to determine if there is an SPI EEPROM actually responding. Generally, this will result in reads of nothing but zero.
 
@@ -74,3 +77,69 @@ The only configurable item for the transient EEPROM driver is its size:
 `#define TRANSIENT_EEPROM_SIZE` | Total size of the EEPROM storage in bytes | 64
 
 Default values and extended descriptions can be found in `drivers/eeprom/eeprom_transient.h`.
+
+## Wear-leveling Driver Configuration :id=wear_leveling-eeprom-driver-configuration
+
+The wear-leveling driver uses an algorithm to minimise the number of erase cycles on the underlying MCU flash memory.
+
+There is no specific configuration for this driver, but the wear-leveling system used by this driver may need configuration. See the [wear-leveling configuration](#wear_leveling-configuration) section for more information.
+
+# Wear-leveling Configuration :id=wear_leveling-configuration
+
+The wear-leveling driver has a few possible _backing stores_ that may be used by adding to your keyboard's `rules.mk` file:
+
+Driver                                  | Description
+----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+`WEAR_LEVELING_DRIVER = embedded_flash` | This driver is used for emulating EEPROM by writing to embedded flash on the MCU.
+`WEAR_LEVELING_DRIVER = spi_flash`      | This driver is used to address external SPI NOR Flash peripherals.
+`WEAR_LEVELING_DRIVER = legacy`         | This driver is the "legacy" emulated EEPROM provided in historical revisions of QMK. Currently used for STM32F0xx and STM32F4x1, but slated for deprecation and removal once `embedded_flash` support for those MCU families is complete.
+
+!> All wear-leveling drivers require an amount of RAM equivalent to the selected logical EEPROM size. Increasing the size to 32kB of EEPROM requires 32kB of RAM, which a significant number of MCUs simply do not have.
+
+## Wear-leveling Embedded Flash Driver Configuration :id=wear_leveling-efl-driver-configuration
+
+This driver performs writes to the embedded flash storage embedded in the MCU. In most circumstances, the last few of sectors of flash are used in order to minimise the likelihood of collision with program code.
+
+Configurable options in your keyboard's `config.h`:
+
+`config.h` override                      | Default     | Description
+-----------------------------------------|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+`#define WEAR_LEVELING_EFL_FIRST_SECTOR` | _unset_     | The first sector on the MCU to use. By default this is not defined and calculated at runtime based on the MCU. However, different flash sizes on MCUs may require custom configuration.
+`#define WEAR_LEVELING_EFL_FLASH_SIZE`   | _unset_     | Allows overriding the flash size available for use for wear-leveling. Under normal circumstances this is automatically calculated and should not need to be overridden. Specifying a size larger than the amount actually available in flash will usually prevent the MCU from booting.
+`#define WEAR_LEVELING_LOGICAL_SIZE`     | `1024`      | Number of bytes "exposed" to the rest of QMK and denotes the size of the usable EEPROM.
+`#define WEAR_LEVELING_BACKING_SIZE`     | `2048`      | Number of bytes used by the wear-leveling algorithm for its underlying storage, and needs to be a multiple of the logical size.
+`#define BACKING_STORE_WRITE_SIZE`       | _automatic_ | The byte width of the underlying write used on the MCU, and is usually automatically determined from the selected MCU family. If an error occurs in the auto-detection, you'll need to consult the MCU's datasheet and determine this value, specifying it directly.
+
+!> If your MCU does not boot after swapping to the EFL wear-leveling driver, it's likely that the flash size is incorrectly detected, usually as an MCU with larger flash and may require overriding.
+
+## Wear-leveling SPI Flash Driver Configuration :id=wear_leveling-flash_spi-driver-configuration
+
+This driver performs writes to an external SPI NOR Flash peripheral. It also requires a working configuration for the SPI NOR Flash peripheral -- see the [flash driver](flash_driver.md) documentation for more information.
+
+Configurable options in your keyboard's `config.h`:
+
+`config.h` override                                 | Default                        | Description
+----------------------------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------
+`#define WEAR_LEVELING_EXTERNAL_FLASH_BLOCK_COUNT`  | `1`                            | Number of blocks in the external flash used by the wear-leveling algorithm.
+`#define WEAR_LEVELING_EXTERNAL_FLASH_BLOCK_OFFSET` | `0`                            | The index first block in the external flash used by the wear-leveling algorithm.
+`#define WEAR_LEVELING_LOGICAL_SIZE`                | `((block_count*block_size)/2)` | Number of bytes "exposed" to the rest of QMK and denotes the size of the usable EEPROM. Result must be <= 64kB.
+`#define WEAR_LEVELING_BACKING_SIZE`                | `(block_count*block_size)`     | Number of bytes used by the wear-leveling algorithm for its underlying storage, and needs to be a multiple of the logical size.
+`#define BACKING_STORE_WRITE_SIZE`                  | `8`                            | The write width used whenever a write is performed on the external flash peripheral.
+
+!> There is currently a limit of 64kB for the EEPROM subsystem within QMK, so using a larger flash is not going to be beneficial as the logical size cannot be increased beyond 65536. The backing size may be increased to a larger value, but erase timing may suffer as a result.
+
+## Wear-leveling Legacy EEPROM Emulation Driver Configuration :id=wear_leveling-legacy-driver-configuration
+
+This driver performs writes to the embedded flash storage embedded in the MCU much like the normal Embedded Flash Driver, and is only for use with STM32F0xx and STM32F4x1 devices. This flash implementation is still currently provided as the EFL driver is currently non-functional for the previously mentioned families.
+
+By default, `1024` bytes of emulated EEPROM is provided:
+
+MCU       | EEPROM Provided | Flash Used
+----------|-----------------|--------------
+STM32F042 | `1024` bytes    | `2048` bytes
+STM32F070 | `1024` bytes    | `2048` bytes
+STM32F072 | `1024` bytes    | `2048` bytes
+STM32F401 | `1024` bytes    | `16384` bytes
+STM32F411 | `1024` bytes    | `16384` bytes
+
+Under normal circumstances configuration of this driver requires intimate knowledge of the MCU's flash structure -- reconfiguration is at your own risk and will require referring to the code.

drivers/eeprom/eeprom_wear_leveling.c

@@ -0,0 +1,23 @@
+// Copyright 2022 Nick Brassel (@tzarc)
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <stdint.h>
+#include <string.h>
+
+#include "eeprom_driver.h"
+#include "wear_leveling.h"
+
+void eeprom_driver_init(void) {
+    wear_leveling_init();
+}
+
+void eeprom_driver_erase(void) {
+    wear_leveling_erase();
+}
+
+void eeprom_read_block(void *buf, const void *addr, size_t len) {
+    wear_leveling_read((uint32_t)addr, buf, len);
+}
+
+void eeprom_write_block(const void *buf, void *addr, size_t len) {
+    wear_leveling_write((uint32_t)addr, buf, len);
+}

drivers/wear_leveling/wear_leveling_flash_spi.c

@@ -0,0 +1,101 @@
+// Copyright 2022 Nick Brassel (@tzarc)
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <stdbool.h>
+#include <hal.h>
+#include "util.h"
+#include "timer.h"
+#include "wear_leveling.h"
+#include "wear_leveling_internal.h"
+
+#ifndef WEAR_LEVELING_EXTERNAL_FLASH_BULK_COUNT
+#    define WEAR_LEVELING_EXTERNAL_FLASH_BULK_COUNT 32
+#endif // WEAR_LEVELING_EXTERNAL_FLASH_BULK_COUNT
+
+bool backing_store_init(void) {
+    bs_dprintf("Init\n");
+    flash_init();
+    return true;
+}
+
+bool backing_store_unlock(void) {
+    bs_dprintf("Unlock\n");
+    // No-op -- handled by the flash driver as it is.
+    return true;
+}
+
+bool backing_store_erase(void) {
+#ifdef WEAR_LEVELING_DEBUG_OUTPUT
+    uint32_t start = timer_read32();
+#endif
+
+    bool ret = true;
+    for (int i = 0; i < (WEAR_LEVELING_EXTERNAL_FLASH_BLOCK_COUNT); ++i) {
+        flash_status_t status = flash_erase_block(((WEAR_LEVELING_EXTERNAL_FLASH_BLOCK_OFFSET) + i) * (EXTERNAL_FLASH_BLOCK_SIZE));
+        if (status != FLASH_STATUS_SUCCESS) {
+            ret = false;
+            break;
+        }
+    }
+
+    bs_dprintf("Backing store erase took %ldms to complete\n", ((long)(timer_read32() - start)));
+    return ret;
+}
+
+bool backing_store_write(uint32_t address, backing_store_int_t value) {
+    return backing_store_write_bulk(address, &value, 1);
+}
+
+bool backing_store_lock(void) {
+    bs_dprintf("Lock  \n");
+    // No-op -- handled by the flash driver as it is.
+    return true;
+}
+
+bool backing_store_read(uint32_t address, backing_store_int_t *value) {
+    return backing_store_read_bulk(address, value, 1);
+}
+
+bool backing_store_read_bulk(uint32_t address, backing_store_int_t *values, size_t item_count) {
+    bs_dprintf("Read  ");
+    uint32_t       offset = (WEAR_LEVELING_EXTERNAL_FLASH_BLOCK_OFFSET) * (EXTERNAL_FLASH_BLOCK_SIZE) + address;
+    flash_status_t status = flash_read_block(offset, values, sizeof(backing_store_int_t) * item_count);
+    if (status == FLASH_STATUS_SUCCESS) {
+        for (size_t i = 0; i < item_count; ++i) {
+            values[i] = ~values[i];
+        }
+        wl_dump(offset, values, sizeof(backing_store_int_t) * item_count);
+    }
+    return status == FLASH_STATUS_SUCCESS;
+}
+
+bool backing_store_write_bulk(uint32_t address, backing_store_int_t *values, size_t item_count) {
+    uint32_t            offset = (WEAR_LEVELING_EXTERNAL_FLASH_BLOCK_OFFSET) * (EXTERNAL_FLASH_BLOCK_SIZE) + address;
+    size_t              index  = 0;
+    backing_store_int_t temp[WEAR_LEVELING_EXTERNAL_FLASH_BULK_COUNT];
+    do {
+        // Copy out the block of data we want to transmit first
+        size_t this_loop = MIN(item_count, WEAR_LEVELING_EXTERNAL_FLASH_BULK_COUNT);
+        for (size_t i = 0; i < this_loop; ++i) {
+            temp[i] = values[index + i];
+        }
+
+        bs_dprintf("Write ");
+        wl_dump(offset, temp, sizeof(backing_store_int_t) * this_loop);
+
+        // Take the complement instead
+        for (size_t i = 0; i < this_loop; ++i) {
+            temp[i] = ~temp[i];
+        }
+
+        // Write out the block
+        if (flash_write_block(offset, temp, sizeof(backing_store_int_t) * this_loop) != FLASH_STATUS_SUCCESS) {
+            return false;
+        }
+
+        offset += this_loop * sizeof(backing_store_int_t);
+        index += this_loop;
+        item_count -= this_loop;
+    } while (item_count > 0);
+
+    return true;
+}