Unittest fixes (ARM-software#143)

- Remove import error when generating tests without tflite_micro
installed
- Update LSTM unittests variable names to match refactored names
- Update refactored int8 LSTM test generator to use the tflite_micro
interpreter
- Update schema.fbs default path
- Regenerate LSTM test data

Tests/UnitTest/RefactoredTestGen/Lib/op_conv.py

@@ -22,12 +22,6 @@
 import keras
 import numpy as np
 
-try:
-    import tflite_micro
-except ModuleNotFoundError:
-    pass
-
-
 def generate_data(tflite_fname, params):
     tensors = {}
     effective_scales = {}
@@ -41,6 +35,8 @@ def generate_data(tflite_fname, params):
     aliases["output"] = "output_ref"
 
     if params["tflite_generator"] == "json":
+        import tflite_micro # Only tflite_micro interpreter supports int4 convolution
+
         interpreter = tflite_micro.runtime.Interpreter.from_file(
             model_path=str(tflite_fname), arena_size=params["arena_size"],
             intrepreter_config=tflite_micro.runtime.InterpreterConfig.kPreserveAllTensors)

Tests/UnitTest/RefactoredTestGen/Lib/test.py

@@ -52,10 +52,10 @@ def generate(params, args, fpaths):
 
     # Check if test is valid, skip otherwise
     if (params["interpreter"] == "tflite_runtime") and (not tflite_runtime_imported):
-        print("Skipping...")
+        print("Skipping due to tflite_runtime not being installed...")
         return
     if (params["interpreter"] == "tflite_micro") and (not tflite_micro_imported):
-        print("Skipping...")
+        print("Skipping due to tflite_micro not being installed...")
         return
 
     op_type = get_op_type(params["op_type"])

Tests/UnitTest/RefactoredTestGen/generate_test_data.py

@@ -53,7 +53,7 @@ def main():
 
     parser.add_argument("--schema",
                         type=pathlib.Path,
-                        default="../../../tflite_micro/tensorflow/lite/schema/schema.fbs",
+                        default="../../../tflite_micro/tensorflow/compiler/mlir/lite/schema/schema.fbs",
                         help="Path to the schema-file needed for generating tflite-files with flatc")
     parser.add_argument("--list", action="store_true", help="Only list tests in test plan")
 

Tests/UnitTest/RefactoredTestGen/test_plan.json

@@ -551,7 +551,7 @@
     "input_data_type": "int8_t",
     "weights_data_type": "int8_t",
     "bias_data_type": "int32_t",
-    "interpreter": "tensorflow",
+    "interpreter": "tflite_micro",
     "tflite_generator": "keras",
     "tests" : [
         {"name" : "lstm_1",

Tests/UnitTest/TestCases/TestData/lstm_1/cell_gate_bias.h

@@ -0,0 +1,6 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int32_t lstm_1_cell_gate_bias[11] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

Tests/UnitTest/TestCases/TestData/lstm_1/cell_gate_hidden_weights.h

@@ -0,0 +1,12 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int8_t lstm_1_cell_gate_hidden_weights[121] = {
+    112, -105, -82, 46,   66,  -73, 74,  110, 81,  100, -52,  58,   78,   -51,  -90, 33,   86,   -34,  49,  52,  113,
+    -68, 41,   49,  -103, -79, -59, -87, -26, 61,  -32, 109,  35,   7,    88,   -38, 48,   116,  -119, 90,  87,  -30,
+    -56, 100,  80,  -22,  -62, 56,  69,  118, 81,  98,  3,    -100, -127, -83,  17,  57,   118,  119,  -41, -78, 57,
+    38,  -93,  -35, -106, 89,  -10, 126, 27,  88,  -94, -32,  39,   -59,  -125, -74, -107, -119, -80,  14,  -58, 122,
+    68,  -45,  73,  -7,   -31, -82, 122, 63,  113, -62, -44,  -117, -40,  56,   84,  -38,  22,   -84,  33,  -87, -112,
+    -13, 63,   -45, 118,  -96, 98,  -82, -96, 98,  -88, -122, -110, -17,  4,    -81, -64};

Tests/UnitTest/TestCases/TestData/lstm_1/cell_gate_input_weights.h

@@ -0,0 +1,19 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int8_t lstm_1_cell_gate_input_weights[242] = {
+    -106, -106, 87,   27,   2,    -44, -70,  63,   -115, -118, 9,    48,   49,  51,   93,  -68,  -121, -42, -77,
+    -83,  0,    35,   -1,   16,   -25, -110, 108,  -18,  -2,   106,  -114, -43, 17,   67,  80,   -125, -70, -100,
+    -123, -107, -119, -4,   -50,  49,  -38,  -29,  -18,  -22,  30,   -4,   -91, -126, 119, -107, -1,   -72, 1,
+    90,   -112, 89,   117,  -60,  91,  54,   -122, -1,   72,   64,   100,  -20, -90,  -59, 84,   117,  -62, -10,
+    -115, -4,   68,   -127, -121, -51, -88,  -35,  13,   40,   29,   -103, 60,  -90,  -50, 83,   97,   -29, 47,
+    -21,  -8,   -18,  -4,   -101, 42,  -79,  -71,  -34,  18,   116,  -46,  -46, 102,  45,  -43,  98,   38,  -98,
+    101,  -29,  91,   -42,  46,   -62, 124,  -9,   -43,  -30,  -111, -68,  67,  45,   -31, -13,  117,  -77, -105,
+    -35,  -57,  39,   -64,  95,   91,  -81,  28,   -39,  -90,  109,  126,  127, -41,  64,  36,   -99,  127, -94,
+    76,   76,   122,  19,   118,  -1,  3,    -109, -79,  60,   84,   -29,  108, 66,   91,  -45,  17,   58,  -112,
+    10,   -29,  107,  73,   -112, 1,   1,    114,  67,   124,  -50,  -32,  43,  -65,  88,  98,   -54,  -14, -98,
+    -120, 3,    112,  53,   4,    122, -59,  64,   -105, -65,  -70,  -66,  95,  39,   85,  42,   43,   -58, -105,
+    -115, 95,   -78,  45,   51,   22,  -21,  84,   -100, 49,   79,   28,   -70, 77,   -3,  -25,  -65,  34,  -126,
+    87,   101,  99,   -116, 71,   -5,  -116, 126,  -89,  -34,  -120, -13,  117, -120};

Tests/UnitTest/TestCases/TestData/lstm_1/config_data.h

@@ -1,38 +1,34 @@
-// Generated by test_settings.py using tensorflow version 2.15.0 (Keras version 2.15.0).
-// Interpreter from tflite_micro version 0.dev20231214171620-g47f77ab and revision None.
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
 #pragma once
-#define LSTM_1_BUFFER_SIZE 11
-#define LSTM_1_INPUT_BATCHES 1
-#define LSTM_1_DST_SIZE 110
+#define LSTM_1_TIME_MAJOR true
+#define LSTM_1_BATCH_SIZE 1
 #define LSTM_1_TIME_STEPS 10
-#define LSTM_1_NUMBER_UNITS 11
-#define LSTM_1_NUMBER_INPUTS 22
-#define LSTM_1_TIME_MAJOR 1
-#define LSTM_1_IN_ACTIVATION_MIN -32768
-#define LSTM_1_IN_ACTIVATION_MAX 32767
-#define LSTM_1_IN_TO_INPUT_MULTIPLIER 1075906048
-#define LSTM_1_IN_TO_INPUT_SHIFT -2
-#define LSTM_1_IN_TO_FORGET_MULTIPLIER 1085883136
-#define LSTM_1_IN_TO_FORGET_SHIFT -2
-#define LSTM_1_IN_TO_CELL_MULTIPLIER 1084231552
-#define LSTM_1_IN_TO_CELL_SHIFT -2
-#define LSTM_1_IN_TO_OUTPUT_MULTIPLIER 1085274240
-#define LSTM_1_IN_TO_OUTPUT_SHIFT -2
-#define LSTM_1_RECURRENT_TO_INPUT_MULTIPLIER 1523696256
-#define LSTM_1_RECURRENT_TO_INPUT_SHIFT -2
-#define LSTM_1_RECURRENT_TO_FORGET_MULTIPLIER 1511291392
-#define LSTM_1_RECURRENT_TO_FORGET_SHIFT -2
-#define LSTM_1_RECURRENT_TO_CELL_MULTIPLIER 1523716992
-#define LSTM_1_RECURRENT_TO_CELL_SHIFT -2
-#define LSTM_1_RECURRENT_TO_OUTPUT_MULTIPLIER 1525092864
-#define LSTM_1_RECURRENT_TO_OUTPUT_SHIFT -2
-#define LSTM_1_FORGET_MULTIPLIER 1073741824
-#define LSTM_1_FORGET_SHIFT -14
-#define LSTM_1_INPUT_MULTIPLIER 1073741824
-#define LSTM_1_INPUT_SHIFT -17
-#define LSTM_1_HIDDEN_MULTIPLIER 1522160019
-#define LSTM_1_HIDDEN_SHIFT -22
-#define LSTM_1_HIDDEN_OFFSET -23
-#define LSTM_1_DATA_OFFSET 128
-#define LSTM_1_OUTPUT_STATE_OFFSET -23
-#define LSTM_1_CELL_STATE_SHIFT -12
+#define LSTM_1_INPUT_SIZE 22
+#define LSTM_1_HIDDEN_SIZE 11
+#define LSTM_1_CELL_SCALE_POWER -14
+#define LSTM_1_INPUT_ZERO_POINT 128
+#define LSTM_1_OUTPUT_ZERO_POINT 1
+#define LSTM_1_CELL_CLIP 32767
+#define LSTM_1_FORGET_TO_CELL_MULTIPLIER 1073741824
+#define LSTM_1_FORGET_TO_CELL_SHIFT -14
+#define LSTM_1_INPUT_TO_CELL_MULTIPLIER 1073741824
+#define LSTM_1_INPUT_TO_CELL_SHIFT -15
+#define LSTM_1_OUTPUT_MULTIPLIER 1441324494
+#define LSTM_1_OUTPUT_SHIFT -22
+#define LSTM_1_OUTPUT_GATE_HIDDEN_MULTIPLIER 1611693952
+#define LSTM_1_OUTPUT_GATE_HIDDEN_SHIFT -2
+#define LSTM_1_CELL_GATE_HIDDEN_MULTIPLIER 1599350272
+#define LSTM_1_CELL_GATE_HIDDEN_SHIFT -2
+#define LSTM_1_FORGET_GATE_HIDDEN_MULTIPLIER 1591598720
+#define LSTM_1_FORGET_GATE_HIDDEN_SHIFT -2
+#define LSTM_1_INPUT_GATE_HIDDEN_MULTIPLIER 1584295680
+#define LSTM_1_INPUT_GATE_HIDDEN_SHIFT -2
+#define LSTM_1_OUTPUT_GATE_INPUT_MULTIPLIER 1080487424
+#define LSTM_1_OUTPUT_GATE_INPUT_SHIFT -2
+#define LSTM_1_CELL_GATE_INPUT_MULTIPLIER 1082850560
+#define LSTM_1_CELL_GATE_INPUT_SHIFT -2
+#define LSTM_1_FORGET_GATE_INPUT_MULTIPLIER 1084475904
+#define LSTM_1_FORGET_GATE_INPUT_SHIFT -2
+#define LSTM_1_INPUT_GATE_INPUT_MULTIPLIER 1080990464
+#define LSTM_1_INPUT_GATE_INPUT_SHIFT -2

Tests/UnitTest/TestCases/TestData/lstm_1/forget_gate_bias.h

@@ -0,0 +1,6 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int32_t lstm_1_forget_gate_bias[11] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

Tests/UnitTest/TestCases/TestData/lstm_1/forget_gate_hidden_weights.h

@@ -0,0 +1,12 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int8_t lstm_1_forget_gate_hidden_weights[121] = {
+    126,  19,   121,  74,   -99, 46,  49,  98,  97,  -53,  -51, -60,  -77, 84,  127,  -97, 50,  43,  -74, 73,   11,
+    85,   36,   54,   123,  -5,  -17, 120, 11,  120, -97,  -44, -111, 63,  12,  48,   56,  32,  -10, 115, 25,   -127,
+    -43,  -122, 42,   100,  3,   -36, 70,  -17, 71,  -109, -89, 38,   -53, 91,  -34,  -34, -36, 41,  30,  0,    -127,
+    45,   -112, 32,   42,   1,   48,  42,  92,  -66, 57,   70,  -54,  26,  96,  -104, 104, 50,  2,   65,  -15,  94,
+    -109, 62,   -116, -107, -76, -50, 12,  55,  12,  59,   107, -107, 90,  -15, -40,  38,  -45, 32,  -85, -120, -42,
+    -80,  -80,  -88,  -111, 107, -57, -88, 67,  76,  -71,  23,  -30,  -54, 20,  123,  -50};

Tests/UnitTest/TestCases/TestData/lstm_1/forget_gate_input_weights.h

@@ -0,0 +1,19 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int8_t lstm_1_forget_gate_input_weights[242] = {
+    -81,  -33,  -29,  -20,  87,   -58,  -99,  83,   -79,  -56,  58,   89,  -76,  94,  80,  -32, 34,   40,   -74,
+    -84,  -61,  8,    15,   64,   -38,  -23,  33,   -51,  8,    96,   -80, -50,  -53, 74,  50,  123,  -120, 31,
+    94,   78,   54,   -3,   -24,  36,   -55,  71,   -119, -66,  -106, -35, -15,  -77, 32,  70,  9,    66,   75,
+    73,   62,   32,   -111, -11,  41,   110,  12,   -101, -103, 60,   -10, -2,   -59, 55,  -22, -126, -11,  -29,
+    -112, 39,   0,    -46,  42,   -70,  -97,  -21,  52,   -50,  54,   -39, -111, 24,  83,  6,   67,   -66,  125,
+    127,  -104, 31,   5,    -117, -76,  100,  89,   81,   122,  22,   33,  -64,  90,  58,  9,   46,   -59,  104,
+    -56,  16,   41,   0,    111,  -102, -39,  115,  -95,  -5,   -11,  -30, -40,  118, -54, 81,  -7,   -116, 93,
+    -111, 82,   -54,  72,   -2,   86,   27,   -104, 2,    -91,  11,   -55, -77,  2,   -36, -36, -96,  -55,  121,
+    -25,  35,   -20,  -16,  -101, -82,  -22,  26,   -122, 106,  98,   9,   -37,  95,  -85, 76,  1,    49,   -96,
+    29,   66,   -64,  37,   122,  -35,  -115, 65,   1,    -25,  106,  53,  -43,  -22, 72,  -83, -43,  -13,  -84,
+    -46,  -125, -101, 89,   116,  102,  -10,  48,   42,   -45,  -32,  4,   49,   43,  93,  52,  30,   -95,  -99,
+    119,  114,  63,   99,   -60,  -70,  -15,  -34,  75,   -64,  86,   -6,  2,    -1,  -90, -22, 71,   57,   3,
+    -44,  73,   -79,  4,    -22,  -89,  -89,  -117, 34,   -43,  18,   -4,  7,    -55};

Tests/UnitTest/TestCases/TestData/lstm_1/input_gate_bias.h

@@ -0,0 +1,6 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int32_t lstm_1_input_gate_bias[11] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

Tests/UnitTest/TestCases/TestData/lstm_1/input_gate_hidden_weights.h

@@ -0,0 +1,12 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int8_t lstm_1_input_gate_hidden_weights[121] = {
+    -109, -8,  -126, 9,    -104, 117,  -37,  -81,  55,   105, -113, 61,  -50,  113,  99,  -79,  112, 55,  56,  94,  -36,
+    35,   35,  -88,  -11,  89,   -115, 22,   -76,  -19,  24,  114,  103, -110, 28,   -31, -24,  109, 105, -13, 56,  30,
+    56,   -6,  92,   -40,  47,   -115, -93,  82,   64,   43,  81,   29,  37,   86,   4,   -113, 64,  63,  -91, -26, -24,
+    17,   -1,  -62,  -90,  -71,  17,   56,   -112, -9,   12,  -55,  107, 96,   -111, 126, 67,   -51, -90, -77, 116, 17,
+    117,  -63, 116,  -123, 115,  -2,   -126, 13,   -123, 2,   14,   116, -55,  -127, 48,  -97,  -84, -30, -31, 51,  28,
+    -109, -28, 42,   -32,  19,   -36,  94,   100,  -12,  -83, -40,  23,  -127, -91,  31,  -20};

Tests/UnitTest/TestCases/TestData/lstm_1/input_gate_input_weights.h

@@ -0,0 +1,19 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int8_t lstm_1_input_gate_input_weights[242] = {
+    -76,  -123, 48,  68,   29,  32,   -88,  -120, -72,  77,   16,   -123, -55,  0,    42,   -1,   -34,  -105, -109,
+    -6,   -112, -20, 9,    40,  86,   -119, 80,   118,  -71,  72,   -46,  -121, 41,   60,   -2,   73,   53,   -70,
+    -98,  -8,   32,  -113, -37, 99,   122,  6,    -106, 32,   -106, -2,   -20,  107,  100,  -120, -107, -28,  104,
+    -46,  -104, -47, -57,  46,  -111, -107, -101, -95,  -110, -66,  -73,  -76,  -62,  -19,  95,   96,   113,  -56,
+    79,   -50,  -65, -15,  95,  14,   101,  40,   -86,  25,   106,  4,    -47,  -108, -90,  -85,  3,    -75,  -67,
+    -112, 35,   -55, -99,  65,  45,   -102, 85,   -3,   -68,  -83,  -80,  25,   -103, 24,   54,   -19,  3,    2,
+    78,   42,   -26, -22,  64,  -66,  -5,   -92,  -79,  -47,  -45,  102,  -99,  -18,  -114, -115, 54,   -82,  -118,
+    -52,  -42,  89,  21,   -20, 89,   -95,  39,   59,   103,  -12,  -75,  50,   92,   85,   -51,  -97,  30,   100,
+    -1,   -127, 75,  -98,  46,  114,  115,  -80,  5,    -63,  -7,   48,   33,   -88,  -4,   -98,  -65,  -57,  33,
+    58,   12,   -86, -96,  51,  -100, -65,  34,   -80,  15,   -26,  -39,  -87,  93,   54,   -31,  -24,  75,   -72,
+    127,  119,  13,  46,   43,  33,   -115, 105,  61,   -73,  -108, -42,  98,   48,   105,  14,   113,  -61,  -103,
+    -40,  -30,  47,  13,   -50, 23,   -1,   4,    109,  122,  -40,  124,  77,   -109, -87,  -15,  69,   -56,  114,
+    -99,  -72,  -93, 29,   108, -92,  49,   6,    -11,  -62,  -60,  -39,  -76,  -49};

Tests/UnitTest/TestCases/TestData/lstm_1/input_tensor.h

@@ -0,0 +1,17 @@
+// Generated by generate_test_data.py using tensorflow version 2.17.0 (Keras version 2.17.0).
+// Interpreter from tflite_micro runtime version 0.dev20240814215812-g07e7f33.
+#pragma once
+#include <stdint.h>
+
+const int8_t lstm_1_input_tensor[220] = {
+    111,  -58, -104, 106,  -50,  10,   103, 111, -78,  8,    -51, -13,  -68,  -60,  -30,  85,   94,  58,  -54,  -46,
+    26,   -96, 78,   33,   8,    111,  114, -46, -93,  115,  89,  -19,  0,    -119, -78,  41,   -72, 1,   -16,  -14,
+    -105, -41, 26,   -114, 88,   -91,  31,  25,  0,    -57,  -98, 126,  64,   98,   -33,  -119, 70,  -48, -72,  77,
+    -102, 21,  43,   -6,   -115, 117,  34,  -25, 109,  -5,   43,  -63,  43,   75,   -100, -48,  26,  107, -126, 71,
+    -59,  45,  -104, -52,  41,   -57,  122, 48,  73,   -89,  -91, 23,   0,    104,  -23,  -44,  -46, -17, 11,   -85,
+    27,   24,  -75,  118,  35,   -82,  22,  3,   -116, -6,   80,  -6,   -61,  -111, 121,  -33,  11,  -10, 36,   -16,
+    41,   -70, -17,  26,   9,    -120, -27, 24,  34,   -126, -96, 103,  -85,  67,   -33,  87,   -71, 2,   64,   -28,
+    -18,  61,  19,   40,   67,   -33,  108, 100, -59,  -40,  -48, -105, 95,   93,   -117, -66,  96,  -78, 32,   -28,
+    49,   -10, -91,  -51,  -19,  35,   -56, 30,  -70,  -52,  -80, 75,   -2,   -119, 105,  -101, -16, 17,  79,   74,
+    2,    67,  -82,  19,   -15,  -83,  43,  2,   -100, -19,  -32, 43,   35,   -14,  -110, -72,  -34, -25, 79,   -33,
+    -64,  106, 71,   58,   -36,  21,   -37, -65, -41,  6,    74,  -80,  -116, 81,   92,   -24,  -19, 2,   -119, -83};