diff --git a/tensorflow/lite/experimental/microfrontend/lib/BUILD b/tensorflow/lite/experimental/microfrontend/lib/BUILD index 8ba5f9a2f23..4ad9d68301a 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/BUILD +++ b/tensorflow/lite/experimental/microfrontend/lib/BUILD @@ -20,6 +20,8 @@ cc_library( "fft_util.h", ], deps = [ + ":fprintf_shim", + ":memory_util", "@kissfft//:kiss_fftr_16", ], ) @@ -37,9 +39,19 @@ cc_library( deps = [ ":bits", ":fft", + ":fprintf_shim", + ":memory_util", ], ) +cc_library( + name = "fprintf_shim", + hdrs = [ + "fprintf_shim.h", + ], + defines = ["MICROFRONTEND_USE_FPRINTF"], +) + cc_library( name = "frontend", srcs = [ @@ -54,6 +66,7 @@ cc_library( ":bits", ":fft", ":filterbank", + ":fprintf_shim", ":log_scale", ":noise_reduction", ":pcan_gain_control", @@ -78,6 +91,16 @@ cc_library( ], ) +cc_library( + name = "memory_util", + srcs = [ + "memory_util_stdlib.c", + ], + hdrs = [ + "memory_util.h", + ], +) + cc_library( name = "noise_reduction", srcs = [ @@ -88,6 +111,10 @@ cc_library( "noise_reduction.h", "noise_reduction_util.h", ], + deps = [ + ":fprintf_shim", + ":memory_util", + ], ) cc_library( @@ -102,6 +129,8 @@ cc_library( ], deps = [ ":bits", + ":fprintf_shim", + ":memory_util", ], ) @@ -115,6 +144,10 @@ cc_library( "window.h", "window_util.h", ], + deps = [ + "fprintf_shim", + ":memory_util", + ], ) cc_test( @@ -162,6 +195,18 @@ cc_test( ], ) +cc_test( + name = "memory_util_test", + srcs = ["memory_util_test.cc"], + # Setting copts for experimental code to [], but this code should be fixed + # to build with the default copts (micro_copts()) + copts = [], + deps = [ + ":memory_util", + "//tensorflow/lite/micro/testing:micro_test", + ], +) + cc_test( name = "noise_reduction_test", srcs = ["noise_reduction_test.cc"], diff --git a/tensorflow/lite/experimental/microfrontend/lib/fft_io.c b/tensorflow/lite/experimental/microfrontend/lib/fft_io.c index 820221c120f..0d2fb187ab2 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/fft_io.c +++ b/tensorflow/lite/experimental/microfrontend/lib/fft_io.c @@ -15,19 +15,23 @@ limitations under the License. #include "tensorflow/lite/experimental/microfrontend/lib/fft_io.h" void FftWriteMemmapPreamble(FILE* fp, const struct FftState* state) { - fprintf(fp, "static int16_t fft_input[%zu];\n", state->fft_size); - fprintf(fp, "static struct complex_int16_t fft_output[%zu];\n", - state->fft_size / 2 + 1); - fprintf(fp, "static char fft_scratch[%zu];\n", state->scratch_size); - fprintf(fp, "\n"); + MICROFRONTEND_FPRINTF(fp, "static int16_t fft_input[%zu];\n", + state->fft_size); + MICROFRONTEND_FPRINTF(fp, "static struct complex_int16_t fft_output[%zu];\n", + state->fft_size / 2 + 1); + MICROFRONTEND_FPRINTF(fp, "static char fft_scratch[%zu];\n", + state->scratch_size); + MICROFRONTEND_FPRINTF(fp, "\n"); } void FftWriteMemmap(FILE* fp, const struct FftState* state, const char* variable) { - fprintf(fp, "%s->input = fft_input;\n", variable); - fprintf(fp, "%s->output = fft_output;\n", variable); - fprintf(fp, "%s->fft_size = %zu;\n", variable, state->fft_size); - fprintf(fp, "%s->input_size = %zu;\n", variable, state->input_size); - fprintf(fp, "%s->scratch = fft_scratch;\n", variable); - fprintf(fp, "%s->scratch_size = %zu;\n", variable, state->scratch_size); + MICROFRONTEND_FPRINTF(fp, "%s->input = fft_input;\n", variable); + MICROFRONTEND_FPRINTF(fp, "%s->output = fft_output;\n", variable); + MICROFRONTEND_FPRINTF(fp, "%s->fft_size = %zu;\n", variable, state->fft_size); + MICROFRONTEND_FPRINTF(fp, "%s->input_size = %zu;\n", variable, + state->input_size); + MICROFRONTEND_FPRINTF(fp, "%s->scratch = fft_scratch;\n", variable); + MICROFRONTEND_FPRINTF(fp, "%s->scratch_size = %zu;\n", variable, + state->scratch_size); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/fft_util.cc b/tensorflow/lite/experimental/microfrontend/lib/fft_util.cc index d516b464585..190935f42ac 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/fft_util.cc +++ b/tensorflow/lite/experimental/microfrontend/lib/fft_util.cc @@ -14,11 +14,11 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/experimental/microfrontend/lib/fft_util.h" -#include - #define FIXED_POINT 16 #include "kiss_fft.h" #include "tools/kiss_fftr.h" +#include "tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h" +#include "tensorflow/lite/experimental/microfrontend/lib/memory_util.h" int FftPopulateState(struct FftState* state, size_t input_size) { state->input_size = input_size; @@ -28,16 +28,16 @@ int FftPopulateState(struct FftState* state, size_t input_size) { } state->input = reinterpret_cast( - malloc(state->fft_size * sizeof(*state->input))); + microfrontend_alloc(state->fft_size * sizeof(*state->input))); if (state->input == nullptr) { - fprintf(stderr, "Failed to alloc fft input buffer\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to alloc fft input buffer\n"); return 0; } - state->output = reinterpret_cast( - malloc((state->fft_size / 2 + 1) * sizeof(*state->output) * 2)); + state->output = reinterpret_cast(microfrontend_alloc( + (state->fft_size / 2 + 1) * sizeof(*state->output) * 2)); if (state->output == nullptr) { - fprintf(stderr, "Failed to alloc fft output buffer\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to alloc fft output buffer\n"); return 0; } @@ -46,12 +46,12 @@ int FftPopulateState(struct FftState* state, size_t input_size) { kiss_fftr_cfg kfft_cfg = kiss_fftr_alloc( state->fft_size, 0, nullptr, &scratch_size); if (kfft_cfg != nullptr) { - fprintf(stderr, "Kiss memory sizing failed.\n"); + MICROFRONTEND_FPRINTF(stderr, "Kiss memory sizing failed.\n"); return 0; } - state->scratch = malloc(scratch_size); + state->scratch = microfrontend_alloc(scratch_size); if (state->scratch == nullptr) { - fprintf(stderr, "Failed to alloc fft scratch buffer\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to alloc fft scratch buffer\n"); return 0; } state->scratch_size = scratch_size; @@ -59,14 +59,15 @@ int FftPopulateState(struct FftState* state, size_t input_size) { kfft_cfg = kiss_fftr_alloc(state->fft_size, 0, state->scratch, &scratch_size); if (kfft_cfg != state->scratch) { - fprintf(stderr, "Kiss memory preallocation strategy failed.\n"); + MICROFRONTEND_FPRINTF(stderr, + "Kiss memory preallocation strategy failed.\n"); return 0; } return 1; } void FftFreeStateContents(struct FftState* state) { - free(state->input); - free(state->output); - free(state->scratch); + microfrontend_free(state->input); + microfrontend_free(state->output); + microfrontend_free(state->scratch); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/filterbank_io.c b/tensorflow/lite/experimental/microfrontend/lib/filterbank_io.c index 6ce4c7c7964..c6873adf615 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/filterbank_io.c +++ b/tensorflow/lite/experimental/microfrontend/lib/filterbank_io.c @@ -16,15 +16,15 @@ limitations under the License. static void PrintArray(FILE* fp, const char* name, const int16_t* values, size_t size) { - fprintf(fp, "static int16_t filterbank_%s[] = {", name); + MICROFRONTEND_FPRINTF(fp, "static int16_t filterbank_%s[] = {", name); int i; for (i = 0; i < size; ++i) { - fprintf(fp, "%d", values[i]); + MICROFRONTEND_FPRINTF(fp, "%d", values[i]); if (i < size - 1) { - fprintf(fp, ", "); + MICROFRONTEND_FPRINTF(fp, ", "); } } - fprintf(fp, "};\n"); + MICROFRONTEND_FPRINTF(fp, "};\n"); } void FilterbankWriteMemmapPreamble(FILE* fp, @@ -44,24 +44,31 @@ void FilterbankWriteMemmapPreamble(FILE* fp, PrintArray(fp, "weights", state->weights, num_weights); PrintArray(fp, "unweights", state->unweights, num_weights); - fprintf(fp, "static uint64_t filterbank_work[%d];\n", num_channels_plus_1); - fprintf(fp, "\n"); + MICROFRONTEND_FPRINTF(fp, "static uint64_t filterbank_work[%d];\n", + num_channels_plus_1); + MICROFRONTEND_FPRINTF(fp, "\n"); } void FilterbankWriteMemmap(FILE* fp, const struct FilterbankState* state, const char* variable) { - fprintf(fp, "%s->num_channels = %d;\n", variable, state->num_channels); - fprintf(fp, "%s->start_index = %d;\n", variable, state->start_index); - fprintf(fp, "%s->end_index = %d;\n", variable, state->end_index); + MICROFRONTEND_FPRINTF(fp, "%s->num_channels = %d;\n", variable, + state->num_channels); + MICROFRONTEND_FPRINTF(fp, "%s->start_index = %d;\n", variable, + state->start_index); + MICROFRONTEND_FPRINTF(fp, "%s->end_index = %d;\n", variable, + state->end_index); - fprintf( + MICROFRONTEND_FPRINTF( fp, "%s->channel_frequency_starts = filterbank_channel_frequency_starts;\n", variable); - fprintf(fp, "%s->channel_weight_starts = filterbank_channel_weight_starts;\n", - variable); - fprintf(fp, "%s->channel_widths = filterbank_channel_widths;\n", variable); - fprintf(fp, "%s->weights = filterbank_weights;\n", variable); - fprintf(fp, "%s->unweights = filterbank_unweights;\n", variable); - fprintf(fp, "%s->work = filterbank_work;\n", variable); + MICROFRONTEND_FPRINTF( + fp, "%s->channel_weight_starts = filterbank_channel_weight_starts;\n", + variable); + MICROFRONTEND_FPRINTF(fp, "%s->channel_widths = filterbank_channel_widths;\n", + variable); + MICROFRONTEND_FPRINTF(fp, "%s->weights = filterbank_weights;\n", variable); + MICROFRONTEND_FPRINTF(fp, "%s->unweights = filterbank_unweights;\n", + variable); + MICROFRONTEND_FPRINTF(fp, "%s->work = filterbank_work;\n", variable); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/filterbank_util.c b/tensorflow/lite/experimental/microfrontend/lib/filterbank_util.c index f18ebf54750..84f2a74a9e3 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/filterbank_util.c +++ b/tensorflow/lite/experimental/microfrontend/lib/filterbank_util.c @@ -16,7 +16,10 @@ limitations under the License. #include #include -#include +#include + +#include "tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h" +#include "tensorflow/lite/experimental/microfrontend/lib/memory_util.h" #define kFilterbankIndexAlignment 4 #define kFilterbankChannelBlockSize 4 @@ -65,29 +68,29 @@ int FilterbankPopulateState(const struct FilterbankConfig* config, ? 1 : kFilterbankIndexAlignment / sizeof(int16_t)); - state->channel_frequency_starts = - malloc(num_channels_plus_1 * sizeof(*state->channel_frequency_starts)); - state->channel_weight_starts = - malloc(num_channels_plus_1 * sizeof(*state->channel_weight_starts)); + state->channel_frequency_starts = microfrontend_alloc( + num_channels_plus_1 * sizeof(*state->channel_frequency_starts)); + state->channel_weight_starts = microfrontend_alloc( + num_channels_plus_1 * sizeof(*state->channel_weight_starts)); state->channel_widths = - malloc(num_channels_plus_1 * sizeof(*state->channel_widths)); - state->work = malloc(num_channels_plus_1 * sizeof(*state->work)); + microfrontend_alloc(num_channels_plus_1 * sizeof(*state->channel_widths)); + state->work = microfrontend_alloc(num_channels_plus_1 * sizeof(*state->work)); float* center_mel_freqs = - malloc(num_channels_plus_1 * sizeof(*center_mel_freqs)); + microfrontend_alloc(num_channels_plus_1 * sizeof(*center_mel_freqs)); int16_t* actual_channel_starts = - malloc(num_channels_plus_1 * sizeof(*actual_channel_starts)); + microfrontend_alloc(num_channels_plus_1 * sizeof(*actual_channel_starts)); int16_t* actual_channel_widths = - malloc(num_channels_plus_1 * sizeof(*actual_channel_widths)); + microfrontend_alloc(num_channels_plus_1 * sizeof(*actual_channel_widths)); if (state->channel_frequency_starts == NULL || state->channel_weight_starts == NULL || state->channel_widths == NULL || center_mel_freqs == NULL || actual_channel_starts == NULL || actual_channel_widths == NULL) { - free(center_mel_freqs); - free(actual_channel_starts); - free(actual_channel_widths); - fprintf(stderr, "Failed to allocate channel buffers\n"); + microfrontend_free(center_mel_freqs); + microfrontend_free(actual_channel_starts); + microfrontend_free(actual_channel_widths); + MICROFRONTEND_FPRINTF(stderr, "Failed to allocate channel buffers\n"); return 0; } @@ -160,15 +163,19 @@ int FilterbankPopulateState(const struct FilterbankConfig* config, // Allocate the two arrays to store the weights - weight_index_start contains // the index of what would be the next set of weights that we would need to // add, so that's how many weights we need to allocate. - state->weights = calloc(weight_index_start, sizeof(*state->weights)); - state->unweights = calloc(weight_index_start, sizeof(*state->unweights)); + state->weights = + microfrontend_alloc(weight_index_start * sizeof(*state->weights)); + memset(state->weights, 0, (weight_index_start * sizeof(*state->weights))); + state->unweights = + microfrontend_alloc(weight_index_start * sizeof(*state->unweights)); + memset(state->unweights, 0, (weight_index_start * sizeof(*state->unweights))); // If the alloc failed, we also need to nuke the arrays. if (state->weights == NULL || state->unweights == NULL) { - free(center_mel_freqs); - free(actual_channel_starts); - free(actual_channel_widths); - fprintf(stderr, "Failed to allocate weights or unweights\n"); + microfrontend_free(center_mel_freqs); + microfrontend_free(actual_channel_starts); + microfrontend_free(actual_channel_widths); + MICROFRONTEND_FPRINTF(stderr, "Failed to allocate weights or unweights\n"); return 0; } @@ -200,21 +207,22 @@ int FilterbankPopulateState(const struct FilterbankConfig* config, } } - free(center_mel_freqs); - free(actual_channel_starts); - free(actual_channel_widths); + microfrontend_free(center_mel_freqs); + microfrontend_free(actual_channel_starts); + microfrontend_free(actual_channel_widths); if (state->end_index >= spectrum_size) { - fprintf(stderr, "Filterbank end_index is above spectrum size.\n"); + MICROFRONTEND_FPRINTF(stderr, + "Filterbank end_index is above spectrum size.\n"); return 0; } return 1; } void FilterbankFreeStateContents(struct FilterbankState* state) { - free(state->channel_frequency_starts); - free(state->channel_weight_starts); - free(state->channel_widths); - free(state->weights); - free(state->unweights); - free(state->work); + microfrontend_free(state->channel_frequency_starts); + microfrontend_free(state->channel_weight_starts); + microfrontend_free(state->channel_widths); + microfrontend_free(state->weights); + microfrontend_free(state->unweights); + microfrontend_free(state->work); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h b/tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h new file mode 100644 index 00000000000..e293db7b428 --- /dev/null +++ b/tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h @@ -0,0 +1,37 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FPRINTF_SHIM_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FPRINTF_SHIM_H_ + +// This code is shared between the TensorFlow training environment and +// the embedded Micro codebase. On Micro, many platforms don't support +// stdio, so we stub out the fprintf call so it does nothing. In the +// Bazel build files for the training ops, we enable this macro so that +// useful debug logging will still be output. +#ifdef MICROFRONTEND_USE_FPRINTF + +#include + +// Redirect to the real fprintf. +#define MICROFRONTEND_FPRINTF fprintf + +#else // MICROFRONTEND_USE_FPRINTF + +// Stub out calls to fprintf so they do nothing. +#define MICROFRONTEND_FPRINTF(stream, format, ...) + +#endif // MICROFRONTEND_USE_FPRINTF + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_MEMORY_UTIL_H_ diff --git a/tensorflow/lite/experimental/microfrontend/lib/frontend_io.c b/tensorflow/lite/experimental/microfrontend/lib/frontend_io.c index b422d078a6f..333f4433727 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/frontend_io.c +++ b/tensorflow/lite/experimental/microfrontend/lib/frontend_io.c @@ -27,43 +27,47 @@ int WriteFrontendStateMemmap(const char* header, const char* source, // Write a header that just has our init function. FILE* fp = fopen(header, "w"); if (!fp) { - fprintf(stderr, "Failed to open header '%s' for write\n", header); + MICROFRONTEND_FPRINTF(stderr, "Failed to open header '%s' for write\n", + header); return 0; } - fprintf(fp, "#ifndef FRONTEND_STATE_MEMMAP_H_\n"); - fprintf(fp, "#define FRONTEND_STATE_MEMMAP_H_\n"); - fprintf(fp, "\n"); - fprintf(fp, "#include \"frontend.h\"\n"); - fprintf(fp, "\n"); - fprintf(fp, "struct FrontendState* GetFrontendStateMemmap();\n"); - fprintf(fp, "\n"); - fprintf(fp, "#endif // FRONTEND_STATE_MEMMAP_H_\n"); + MICROFRONTEND_FPRINTF(fp, "#ifndef FRONTEND_STATE_MEMMAP_H_\n"); + MICROFRONTEND_FPRINTF(fp, "#define FRONTEND_STATE_MEMMAP_H_\n"); + MICROFRONTEND_FPRINTF(fp, "\n"); + MICROFRONTEND_FPRINTF(fp, "#include \"frontend.h\"\n"); + MICROFRONTEND_FPRINTF(fp, "\n"); + MICROFRONTEND_FPRINTF(fp, + "struct FrontendState* GetFrontendStateMemmap();\n"); + MICROFRONTEND_FPRINTF(fp, "\n"); + MICROFRONTEND_FPRINTF(fp, "#endif // FRONTEND_STATE_MEMMAP_H_\n"); fclose(fp); // Write out the source file that actually has everything in it. fp = fopen(source, "w"); if (!fp) { - fprintf(stderr, "Failed to open source '%s' for write\n", source); + MICROFRONTEND_FPRINTF(stderr, "Failed to open source '%s' for write\n", + source); return 0; } - fprintf(fp, "#include \"%s\"\n", header); - fprintf(fp, "\n"); + MICROFRONTEND_FPRINTF(fp, "#include \"%s\"\n", header); + MICROFRONTEND_FPRINTF(fp, "\n"); WindowWriteMemmapPreamble(fp, &state->window); FftWriteMemmapPreamble(fp, &state->fft); FilterbankWriteMemmapPreamble(fp, &state->filterbank); NoiseReductionWriteMemmapPreamble(fp, &state->noise_reduction); - fprintf(fp, "static struct FrontendState state;\n"); - fprintf(fp, "struct FrontendState* GetFrontendStateMemmap() {\n"); + MICROFRONTEND_FPRINTF(fp, "static struct FrontendState state;\n"); + MICROFRONTEND_FPRINTF(fp, + "struct FrontendState* GetFrontendStateMemmap() {\n"); WindowWriteMemmap(fp, &state->window, " (&state.window)"); FftWriteMemmap(fp, &state->fft, " (&state.fft)"); FilterbankWriteMemmap(fp, &state->filterbank, " (&state.filterbank)"); NoiseReductionWriteMemmap(fp, &state->noise_reduction, " (&state.noise_reduction)"); LogScaleWriteMemmap(fp, &state->log_scale, " (&state.log_scale)"); - fprintf(fp, " FftInit(&state.fft);\n"); - fprintf(fp, " FrontendReset(&state);\n"); - fprintf(fp, " return &state;\n"); - fprintf(fp, "}\n"); + MICROFRONTEND_FPRINTF(fp, " FftInit(&state.fft);\n"); + MICROFRONTEND_FPRINTF(fp, " FrontendReset(&state);\n"); + MICROFRONTEND_FPRINTF(fp, " return &state;\n"); + MICROFRONTEND_FPRINTF(fp, "}\n"); fclose(fp); return 1; } diff --git a/tensorflow/lite/experimental/microfrontend/lib/frontend_main.c b/tensorflow/lite/experimental/microfrontend/lib/frontend_main.c index 861778c77a1..b377263ac90 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/frontend_main.c +++ b/tensorflow/lite/experimental/microfrontend/lib/frontend_main.c @@ -26,14 +26,14 @@ int main(int argc, char** argv) { struct FrontendState frontend_state; if (!FrontendPopulateState(&frontend_config, &frontend_state, sample_rate)) { - fprintf(stderr, "Failed to populate frontend state\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to populate frontend state\n"); FrontendFreeStateContents(&frontend_state); return 1; } FILE* fp = fopen(filename, "r"); if (fp == NULL) { - fprintf(stderr, "Failed to open %s for read\n", filename); + MICROFRONTEND_FPRINTF(stderr, "Failed to open %s for read\n", filename); return 1; } fseek(fp, 0L, SEEK_END); @@ -43,7 +43,7 @@ int main(int argc, char** argv) { int16_t* original_audio_data = audio_data; if (audio_file_size != fread(audio_data, sizeof(int16_t), audio_file_size, fp)) { - fprintf(stderr, "Failed to read in all audio data\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to read in all audio data\n"); fclose(fp); return 1; } diff --git a/tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_generator.c b/tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_generator.c index d62433a541c..262a67b1623 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_generator.c +++ b/tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_generator.c @@ -20,10 +20,11 @@ limitations under the License. int main(int argc, char** argv) { if (argc != 3) { - fprintf(stderr, - "%s requires exactly two parameters - the names of the header and " - "source files to save\n", - argv[0]); + MICROFRONTEND_FPRINTF( + stderr, + "%s requires exactly two parameters - the names of the header and " + "source files to save\n", + argv[0]); return 1; } struct FrontendConfig frontend_config; @@ -32,13 +33,13 @@ int main(int argc, char** argv) { int sample_rate = 16000; struct FrontendState frontend_state; if (!FrontendPopulateState(&frontend_config, &frontend_state, sample_rate)) { - fprintf(stderr, "Failed to populate frontend state\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to populate frontend state\n"); FrontendFreeStateContents(&frontend_state); return 1; } if (!WriteFrontendStateMemmap(argv[1], argv[2], &frontend_state)) { - fprintf(stderr, "Failed to write memmap\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to write memmap\n"); FrontendFreeStateContents(&frontend_state); return 1; } diff --git a/tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_main.c b/tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_main.c index e9c89b569e7..d70ade31d9a 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_main.c +++ b/tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_main.c @@ -23,7 +23,7 @@ int main(int argc, char** argv) { char* filename = argv[1]; FILE* fp = fopen(filename, "r"); if (fp == NULL) { - fprintf(stderr, "Failed to open %s for read\n", filename); + MICROFRONTEND_FPRINTF(stderr, "Failed to open %s for read\n", filename); return 1; } fseek(fp, 0L, SEEK_END); @@ -33,7 +33,7 @@ int main(int argc, char** argv) { int16_t* original_audio_data = audio_data; if (audio_file_size != fread(audio_data, sizeof(int16_t), audio_file_size, fp)) { - fprintf(stderr, "Failed to read in all audio data\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to read in all audio data\n"); fclose(fp); return 1; } diff --git a/tensorflow/lite/experimental/microfrontend/lib/frontend_util.c b/tensorflow/lite/experimental/microfrontend/lib/frontend_util.c index 27224f6d29a..d7db9fb8620 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/frontend_util.c +++ b/tensorflow/lite/experimental/microfrontend/lib/frontend_util.c @@ -14,10 +14,10 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h" -#include #include #include "tensorflow/lite/experimental/microfrontend/lib/bits.h" +#include "tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h" void FrontendFillConfigWithDefaults(struct FrontendConfig* config) { WindowFillConfigWithDefaults(&config->window); @@ -32,26 +32,26 @@ int FrontendPopulateState(const struct FrontendConfig* config, memset(state, 0, sizeof(*state)); if (!WindowPopulateState(&config->window, &state->window, sample_rate)) { - fprintf(stderr, "Failed to populate window state\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to populate window state\n"); return 0; } if (!FftPopulateState(&state->fft, state->window.size)) { - fprintf(stderr, "Failed to populate fft state\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to populate fft state\n"); return 0; } FftInit(&state->fft); if (!FilterbankPopulateState(&config->filterbank, &state->filterbank, sample_rate, state->fft.fft_size / 2 + 1)) { - fprintf(stderr, "Failed to populate filterbank state\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to populate filterbank state\n"); return 0; } if (!NoiseReductionPopulateState(&config->noise_reduction, &state->noise_reduction, state->filterbank.num_channels)) { - fprintf(stderr, "Failed to populate noise reduction state\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to populate noise reduction state\n"); return 0; } @@ -61,12 +61,13 @@ int FrontendPopulateState(const struct FrontendConfig* config, &config->pcan_gain_control, &state->pcan_gain_control, state->noise_reduction.estimate, state->filterbank.num_channels, state->noise_reduction.smoothing_bits, input_correction_bits)) { - fprintf(stderr, "Failed to populate pcan gain control state\n"); + MICROFRONTEND_FPRINTF(stderr, + "Failed to populate pcan gain control state\n"); return 0; } if (!LogScalePopulateState(&config->log_scale, &state->log_scale)) { - fprintf(stderr, "Failed to populate log scale state\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to populate log scale state\n"); return 0; } diff --git a/tensorflow/lite/experimental/microfrontend/lib/log_scale_io.c b/tensorflow/lite/experimental/microfrontend/lib/log_scale_io.c index a04760de58e..91807227997 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/log_scale_io.c +++ b/tensorflow/lite/experimental/microfrontend/lib/log_scale_io.c @@ -16,6 +16,8 @@ limitations under the License. void LogScaleWriteMemmap(FILE* fp, const struct LogScaleState* state, const char* variable) { - fprintf(fp, "%s->enable_log = %d;\n", variable, state->enable_log); - fprintf(fp, "%s->scale_shift = %d;\n", variable, state->scale_shift); + MICROFRONTEND_FPRINTF(fp, "%s->enable_log = %d;\n", variable, + state->enable_log); + MICROFRONTEND_FPRINTF(fp, "%s->scale_shift = %d;\n", variable, + state->scale_shift); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/memory_util.h b/tensorflow/lite/experimental/microfrontend/lib/memory_util.h new file mode 100644 index 00000000000..b1239639cec --- /dev/null +++ b/tensorflow/lite/experimental/microfrontend/lib/memory_util.h @@ -0,0 +1,34 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_MEMORY_UTIL_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_MEMORY_UTIL_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// On platforms that support dynamic memory allocation, these just call into +// malloc and free, but for embedded systems with no heap we have an alternate +// implementation that allocates linearly from a fixed-size arena. +void* microfrontend_alloc(size_t size); +void microfrontend_free(void* ptr); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_MEMORY_UTIL_H_ diff --git a/tensorflow/lite/experimental/microfrontend/lib/memory_util_fixed_pool.c b/tensorflow/lite/experimental/microfrontend/lib/memory_util_fixed_pool.c new file mode 100644 index 00000000000..b0a666d0f01 --- /dev/null +++ b/tensorflow/lite/experimental/microfrontend/lib/memory_util_fixed_pool.c @@ -0,0 +1,44 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This version of the allocation utilities uses a fixed pool of memory so that +// a standard library heap isn't required. Include this in the list of files to +// compile instead of the memory_util_stdlib.c version if your platform doesn't +// have a heap available. + +#include "tensorflow/lite/experimental/microfrontend/lib/memory_util.h" + +// This size has been determined by experimentation, based on the largest +// allocations used by the micro speech example and tests. +#define FIXED_POOL_SIZE (30 * 1024) + +void* microfrontend_alloc(size_t size) { + static unsigned char fixed_pool[FIXED_POOL_SIZE]; + static int fixed_pool_used = 0; + + int next_used = fixed_pool_used + size; + if (next_used > FIXED_POOL_SIZE) { + return 0; + } + + void* result = &fixed_pool[fixed_pool_used]; + fixed_pool_used += size; + + return result; +} + +void microfrontend_free(void* ptr) { + // Do nothing. +} diff --git a/tensorflow/lite/experimental/microfrontend/lib/memory_util_stdlib.c b/tensorflow/lite/experimental/microfrontend/lib/memory_util_stdlib.c new file mode 100644 index 00000000000..af5a56f388f --- /dev/null +++ b/tensorflow/lite/experimental/microfrontend/lib/memory_util_stdlib.c @@ -0,0 +1,25 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This version of the allocation utilities uses standard malloc/free +// implementations for the memory required by the frontend. + +#include + +#include "tensorflow/lite/experimental/microfrontend/lib/memory_util.h" + +void* microfrontend_alloc(size_t size) { return malloc(size); } + +void microfrontend_free(void* ptr) { free(ptr); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/memory_util_test.cc b/tensorflow/lite/experimental/microfrontend/lib/memory_util_test.cc new file mode 100644 index 00000000000..4118d483a46 --- /dev/null +++ b/tensorflow/lite/experimental/microfrontend/lib/memory_util_test.cc @@ -0,0 +1,31 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/experimental/microfrontend/lib/memory_util.h" + +#include "tensorflow/lite/micro/testing/micro_test.h" + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(MemoryUtil_CheckAlloc) { + TF_LITE_MICRO_EXPECT_NE(nullptr, microfrontend_alloc(256)); +} + +TF_LITE_MICRO_TEST(MemoryUtil_CheckFree) { + void* ptr = microfrontend_alloc(128); + TF_LITE_MICRO_EXPECT_NE(nullptr, ptr); + microfrontend_free(ptr); +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_io.c b/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_io.c index 19c32b32759..dc3b7b24f24 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_io.c +++ b/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_io.c @@ -16,19 +16,23 @@ limitations under the License. void NoiseReductionWriteMemmapPreamble( FILE* fp, const struct NoiseReductionState* state) { - fprintf(fp, "static uint32_t noise_reduction_estimate[%zu];\n", - state->num_channels); - fprintf(fp, "\n"); + MICROFRONTEND_FPRINTF(fp, "static uint32_t noise_reduction_estimate[%zu];\n", + state->num_channels); + MICROFRONTEND_FPRINTF(fp, "\n"); } void NoiseReductionWriteMemmap(FILE* fp, const struct NoiseReductionState* state, const char* variable) { - fprintf(fp, "%s->even_smoothing = %d;\n", variable, state->even_smoothing); - fprintf(fp, "%s->odd_smoothing = %d;\n", variable, state->odd_smoothing); - fprintf(fp, "%s->min_signal_remaining = %d;\n", variable, - state->min_signal_remaining); - fprintf(fp, "%s->num_channels = %d;\n", variable, state->num_channels); + MICROFRONTEND_FPRINTF(fp, "%s->even_smoothing = %d;\n", variable, + state->even_smoothing); + MICROFRONTEND_FPRINTF(fp, "%s->odd_smoothing = %d;\n", variable, + state->odd_smoothing); + MICROFRONTEND_FPRINTF(fp, "%s->min_signal_remaining = %d;\n", variable, + state->min_signal_remaining); + MICROFRONTEND_FPRINTF(fp, "%s->num_channels = %d;\n", variable, + state->num_channels); - fprintf(fp, "%s->estimate = noise_reduction_estimate;\n", variable); + MICROFRONTEND_FPRINTF(fp, "%s->estimate = noise_reduction_estimate;\n", + variable); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.c b/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.c index a6c9234eb88..760f6752e02 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.c +++ b/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.c @@ -14,7 +14,10 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.h" -#include +#include + +#include "tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h" +#include "tensorflow/lite/experimental/microfrontend/lib/memory_util.h" void NoiseReductionFillConfigWithDefaults(struct NoiseReductionConfig* config) { config->smoothing_bits = 10; @@ -32,14 +35,16 @@ int NoiseReductionPopulateState(const struct NoiseReductionConfig* config, state->min_signal_remaining = config->min_signal_remaining * (1 << kNoiseReductionBits); state->num_channels = num_channels; - state->estimate = calloc(state->num_channels, sizeof(*state->estimate)); + state->estimate = + microfrontend_alloc(state->num_channels * sizeof(*state->estimate)); + memset(state->estimate, 0, (state->num_channels * sizeof(*state->estimate))); if (state->estimate == NULL) { - fprintf(stderr, "Failed to alloc estimate buffer\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to alloc estimate buffer\n"); return 0; } return 1; } void NoiseReductionFreeStateContents(struct NoiseReductionState* state) { - free(state->estimate); + microfrontend_free(state->estimate); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.c b/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.c index 22d58767433..6b11de6931f 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.c +++ b/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.c @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h" #include "tensorflow/lite/experimental/microfrontend/lib/bits.h" +#include "tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h" int16_t WideDynamicFunction(const uint32_t x, const int16_t* lut) { if (x <= 2) { diff --git a/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.c b/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.c index e850d439e8c..f4bbfffec2c 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.c +++ b/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.c @@ -15,7 +15,9 @@ limitations under the License. #include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.h" #include -#include + +#include "tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h" +#include "tensorflow/lite/experimental/microfrontend/lib/memory_util.h" #define kint16max 0x00007FFF @@ -52,9 +54,10 @@ int PcanGainControlPopulateState(const struct PcanGainControlConfig* config, } state->noise_estimate = noise_estimate; state->num_channels = num_channels; - state->gain_lut = malloc(kWideDynamicFunctionLUTSize * sizeof(int16_t)); + state->gain_lut = + microfrontend_alloc(kWideDynamicFunctionLUTSize * sizeof(int16_t)); if (state->gain_lut == NULL) { - fprintf(stderr, "Failed to allocate gain LUT\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to allocate gain LUT\n"); return 0; } state->snr_shift = config->gain_bits - input_correction_bits - kPcanSnrBits; @@ -88,5 +91,5 @@ int PcanGainControlPopulateState(const struct PcanGainControlConfig* config, } void PcanGainControlFreeStateContents(struct PcanGainControlState* state) { - free(state->gain_lut); + microfrontend_free(state->gain_lut); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/window_io.c b/tensorflow/lite/experimental/microfrontend/lib/window_io.c index d12cac2c853..0f2ee7b99e1 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/window_io.c +++ b/tensorflow/lite/experimental/microfrontend/lib/window_io.c @@ -15,29 +15,32 @@ limitations under the License. #include "tensorflow/lite/experimental/microfrontend/lib/window_io.h" void WindowWriteMemmapPreamble(FILE* fp, const struct WindowState* state) { - fprintf(fp, "static int16_t window_coefficients[] = {\n"); + MICROFRONTEND_FPRINTF(fp, "static int16_t window_coefficients[] = {\n"); int i; for (i = 0; i < state->size; ++i) { - fprintf(fp, "%d", state->coefficients[i]); + MICROFRONTEND_FPRINTF(fp, "%d", state->coefficients[i]); if (i < state->size - 1) { - fprintf(fp, ", "); + MICROFRONTEND_FPRINTF(fp, ", "); } } - fprintf(fp, "};\n"); - fprintf(fp, "static int16_t window_input[%zu];\n", state->size); - fprintf(fp, "static int16_t window_output[%zu];\n", state->size); - fprintf(fp, "\n"); + MICROFRONTEND_FPRINTF(fp, "};\n"); + MICROFRONTEND_FPRINTF(fp, "static int16_t window_input[%zu];\n", state->size); + MICROFRONTEND_FPRINTF(fp, "static int16_t window_output[%zu];\n", + state->size); + MICROFRONTEND_FPRINTF(fp, "\n"); } void WindowWriteMemmap(FILE* fp, const struct WindowState* state, const char* variable) { - fprintf(fp, "%s->size = %zu;\n", variable, state->size); - fprintf(fp, "%s->coefficients = window_coefficients;\n", variable); - fprintf(fp, "%s->step = %zu;\n", variable, state->step); + MICROFRONTEND_FPRINTF(fp, "%s->size = %zu;\n", variable, state->size); + MICROFRONTEND_FPRINTF(fp, "%s->coefficients = window_coefficients;\n", + variable); + MICROFRONTEND_FPRINTF(fp, "%s->step = %zu;\n", variable, state->step); - fprintf(fp, "%s->input = window_input;\n", variable); - fprintf(fp, "%s->input_used = %zu;\n", variable, state->input_used); - fprintf(fp, "%s->output = window_output;\n", variable); - fprintf(fp, "%s->max_abs_output_value = %d;\n", variable, - state->max_abs_output_value); + MICROFRONTEND_FPRINTF(fp, "%s->input = window_input;\n", variable); + MICROFRONTEND_FPRINTF(fp, "%s->input_used = %zu;\n", variable, + state->input_used); + MICROFRONTEND_FPRINTF(fp, "%s->output = window_output;\n", variable); + MICROFRONTEND_FPRINTF(fp, "%s->max_abs_output_value = %d;\n", variable, + state->max_abs_output_value); } diff --git a/tensorflow/lite/experimental/microfrontend/lib/window_util.c b/tensorflow/lite/experimental/microfrontend/lib/window_util.c index eee6e7b56ef..f3ace6a99e1 100644 --- a/tensorflow/lite/experimental/microfrontend/lib/window_util.c +++ b/tensorflow/lite/experimental/microfrontend/lib/window_util.c @@ -15,10 +15,12 @@ limitations under the License. #include "tensorflow/lite/experimental/microfrontend/lib/window_util.h" #include -#include #include #include +#include "tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h" +#include "tensorflow/lite/experimental/microfrontend/lib/memory_util.h" + // Some platforms don't have M_PI #ifndef M_PI #define M_PI 3.14159265358979323846 @@ -34,9 +36,10 @@ int WindowPopulateState(const struct WindowConfig* config, state->size = config->size_ms * sample_rate / 1000; state->step = config->step_size_ms * sample_rate / 1000; - state->coefficients = malloc(state->size * sizeof(*state->coefficients)); + state->coefficients = + microfrontend_alloc(state->size * sizeof(*state->coefficients)); if (state->coefficients == NULL) { - fprintf(stderr, "Failed to allocate window coefficients\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to allocate window coefficients\n"); return 0; } @@ -51,15 +54,15 @@ int WindowPopulateState(const struct WindowConfig* config, } state->input_used = 0; - state->input = malloc(state->size * sizeof(*state->input)); + state->input = microfrontend_alloc(state->size * sizeof(*state->input)); if (state->input == NULL) { - fprintf(stderr, "Failed to allocate window input\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to allocate window input\n"); return 0; } - state->output = malloc(state->size * sizeof(*state->output)); + state->output = microfrontend_alloc(state->size * sizeof(*state->output)); if (state->output == NULL) { - fprintf(stderr, "Failed to allocate window output\n"); + MICROFRONTEND_FPRINTF(stderr, "Failed to allocate window output\n"); return 0; } @@ -67,7 +70,7 @@ int WindowPopulateState(const struct WindowConfig* config, } void WindowFreeStateContents(struct WindowState* state) { - free(state->coefficients); - free(state->input); - free(state->output); + microfrontend_free(state->coefficients); + microfrontend_free(state->input); + microfrontend_free(state->output); } diff --git a/tensorflow/lite/micro/benchmarks/Makefile.inc b/tensorflow/lite/micro/benchmarks/Makefile.inc index 2106ae3bfed..cf16affa58e 100644 --- a/tensorflow/lite/micro/benchmarks/Makefile.inc +++ b/tensorflow/lite/micro/benchmarks/Makefile.inc @@ -9,9 +9,17 @@ tensorflow/lite/micro/benchmarks/micro_benchmark.h PERSON_DETECTION_BENCHMARK_SRCS := \ tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc \ $(MAKEFILE_DIR)/downloads/person_model_int8/no_person_image_data.cc \ -$(MAKEFILE_DIR)/downloads/person_model_int8/person_detect_model_data.cc \ $(MAKEFILE_DIR)/downloads/person_model_int8/person_image_data.cc +ifeq ($(CO_PROCESSOR),ethos_u) + # Ethos-U use a Vela optimized version of the original model. + PERSON_DETECTION_BENCHMARK_SRCS += \ + $(MAKEFILE_DIR)/downloads/person_model_int8/person_detect_model_data_vela.cc +else + PERSON_DETECTION_BENCHMARK_SRCS += \ + $(MAKEFILE_DIR)/downloads/person_model_int8/person_detect_model_data.cc +endif + PERSON_DETECTION_BENCHMARK_HDRS := \ tensorflow/lite/micro/examples/person_detection/person_detect_model_data.h \ tensorflow/lite/micro/examples/person_detection/no_person_image_data.h \ diff --git a/tensorflow/lite/micro/benchmarks/README.md b/tensorflow/lite/micro/benchmarks/README.md index 74de7599c97..7ee5978013c 100644 --- a/tensorflow/lite/micro/benchmarks/README.md +++ b/tensorflow/lite/micro/benchmarks/README.md @@ -11,6 +11,7 @@ platform. - [Run on x86](#run-on-x86) - [Run on Xtensa XPG Simulator](#run-on-xtensa-xpg-simulator) - [Run on Sparkfun Edge](#run-on-sparkfun-edge) +- [Run on FVP based on Arm Corstone-300 software](#run-on-fvp-based-on-arm-corstone-300-software) ## Keyword benchmark @@ -64,3 +65,34 @@ make -f tensorflow/lite/micro/tools/make/Makefile TARGET=sparkfun_edge person_de Refer to flashing instructions in the [Person Detection Example](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/person_detection/README.md#running-on-sparkfun-edge). +## Run on FVP based on Arm Corstone-300 software + +For more info about the Corstone-300 software see: +[tensorflow/lite/micro/cortex_m_corstone_300/README.md](../cortex_m_corstone_300/README.md). + +Disclaimer: Executing the benchmark test on the Corstone-300 software will +provide a general metric of instructions executed. The estimates are not cycle +accurate, however it aligns to instruction per cycle, and is a consistent +environment. This means it can detect if code changes changed performance. + +The person detection benchmark can also run with Ethos-U enabled, as the +downloaded model will be optimized for Ethos-U. For more info see: +[tensorflow/lite/micro/kernels/ethos_u/README.md](../kernels/ethos_u/README.md). + +To run the keyword benchmark on FVP: + +``` +make -j -f tensorflow/lite/micro/tools/make/Makefile TARGET=cortex_m_corstone_300 TARGET_ARCH=cortex-m55 run_keyword_benchmark +``` + +To run the person detection benchmark on FVP: + +``` +make -j -f tensorflow/lite/micro/tools/make/Makefile TARGET=cortex_m_corstone_300 TARGET_ARCH=cortex-m55 run_person_detection_benchmark +``` + +To run the person detection benchmark on FVP with Ethos-U: + +``` +make -j -f tensorflow/lite/micro/tools/make/Makefile CO_PROCESSOR=ethos_u TARGET=cortex_m_corstone_300 TARGET_ARCH=cortex-m55 run_person_detection_benchmark +``` diff --git a/tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc b/tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc index 1e98bbd53a9..388178343a6 100644 --- a/tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc +++ b/tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc @@ -63,13 +63,13 @@ void PersonDetectionNIerations(const int8_t* input, int iterations, PersonDetectionBenchmarkRunner& benchmark_runner, MicroProfiler& profiler) { benchmark_runner.SetInput(input); - int32_t ticks = 0; + uint32_t ticks = 0; for (int i = 0; i < iterations; ++i) { profiler.ClearEvents(); benchmark_runner.RunSingleIteration(); ticks += profiler.GetTotalTicks(); } - MicroPrintf("%s took %d ticks (%d ms)", tag, ticks, TicksToMs(ticks)); + MicroPrintf("%s took %u ticks (%u ms)", tag, ticks, TicksToMs(ticks)); } } // namespace tflite diff --git a/tensorflow/lite/micro/cortex_m_corstone_300/micro_time.cc b/tensorflow/lite/micro/cortex_m_corstone_300/micro_time.cc new file mode 100644 index 00000000000..a7db6e482ac --- /dev/null +++ b/tensorflow/lite/micro/cortex_m_corstone_300/micro_time.cc @@ -0,0 +1,21 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// This file is empty to ensure that a specialized implementation of +// micro_time.h is used (instead of the default implementation from +// tensorflow/lite/micro/micro_time.cc). +// +// The actual target-specific implementation of micro_time.h is in +// system_setup.cc since that allows us to consolidate all the target-specific +// specializations into one source file. diff --git a/tensorflow/lite/micro/cortex_m_corstone_300/system_setup.cc b/tensorflow/lite/micro/cortex_m_corstone_300/system_setup.cc index 4b716e90ef7..dc2178ee340 100644 --- a/tensorflow/lite/micro/cortex_m_corstone_300/system_setup.cc +++ b/tensorflow/lite/micro/cortex_m_corstone_300/system_setup.cc @@ -21,10 +21,50 @@ limitations under the License. #include CMSIS_DEVICE_ARM_CORTEX_M_XX_HEADER_FILE #endif #include "tensorflow/lite/micro/micro_error_reporter.h" +#include "tensorflow/lite/micro/micro_time.h" #include "tensorflow/lite/micro/system_setup.h" +// DWT (Data Watchpoint and Trace) registers, only exists on ARM Cortex with a +// DWT unit. +#define KIN1_DWT_CONTROL (*((volatile uint32_t*)0xE0001000)) + +// DWT Control register. +#define KIN1_DWT_CYCCNTENA_BIT (1UL << 0) + +// CYCCNTENA bit in DWT_CONTROL register. +#define KIN1_DWT_CYCCNT (*((volatile uint32_t*)0xE0001004)) + +// DWT Cycle Counter register. +#define KIN1_DEMCR (*((volatile uint32_t*)0xE000EDFC)) + +// DEMCR: Debug Exception and Monitor Control Register. +#define KIN1_TRCENA_BIT (1UL << 24) + +// Trace enable bit in DEMCR register. +#define KIN1_LAR (*((volatile uint32_t*)0xE0001FB0)) + +// Unlock access to DWT (ITM, etc.)registers. +#define KIN1_UnlockAccessToDWT() KIN1_LAR = 0xC5ACCE55; + +// TRCENA: Enable trace and debug block DEMCR (Debug Exception and Monitor +// Control Register. +#define KIN1_InitCycleCounter() KIN1_DEMCR |= KIN1_TRCENA_BIT + +#define KIN1_ResetCycleCounter() KIN1_DWT_CYCCNT = 0 +#define KIN1_EnableCycleCounter() KIN1_DWT_CONTROL |= KIN1_DWT_CYCCNTENA_BIT +#define KIN1_DisableCycleCounter() KIN1_DWT_CONTROL &= ~KIN1_DWT_CYCCNTENA_BIT +#define KIN1_GetCycleCounter() KIN1_DWT_CYCCNT + namespace tflite { +namespace { +constexpr int kClocksPerSecond = 25e6; +} // namespace + +int32_t ticks_per_second() { return kClocksPerSecond; } + +int32_t GetCurrentTimeTicks() { return KIN1_GetCycleCounter(); } + #ifdef ETHOS_U void ethosuIrqHandler0() { ethosu_irq_handler(); } #endif @@ -36,6 +76,11 @@ void uart_init(void); void InitializeTarget() { uart_init(); + KIN1_UnlockAccessToDWT(); + KIN1_InitCycleCounter(); + KIN1_ResetCycleCounter(); + KIN1_EnableCycleCounter(); + #ifdef ETHOS_U constexpr int ethosu_base_address = 0x48102000; constexpr int ethosu_irq = 56; diff --git a/tensorflow/lite/micro/examples/micro_speech/Makefile.inc b/tensorflow/lite/micro/examples/micro_speech/Makefile.inc index a4ec24f48e1..3291c39684b 100644 --- a/tensorflow/lite/micro/examples/micro_speech/Makefile.inc +++ b/tensorflow/lite/micro/examples/micro_speech/Makefile.inc @@ -63,6 +63,7 @@ tensorflow/lite/experimental/microfrontend/lib/frontend_util.c \ tensorflow/lite/experimental/microfrontend/lib/log_lut.c \ tensorflow/lite/experimental/microfrontend/lib/log_scale.c \ tensorflow/lite/experimental/microfrontend/lib/log_scale_util.c \ +tensorflow/lite/experimental/microfrontend/lib/memory_util_fixed_pool.c \ tensorflow/lite/experimental/microfrontend/lib/noise_reduction.c \ tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.c \ tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.c \ @@ -77,11 +78,13 @@ tensorflow/lite/experimental/microfrontend/lib/fft.h \ tensorflow/lite/experimental/microfrontend/lib/fft_util.h \ tensorflow/lite/experimental/microfrontend/lib/filterbank.h \ tensorflow/lite/experimental/microfrontend/lib/filterbank_util.h \ +tensorflow/lite/experimental/microfrontend/lib/fprintf_shim.h \ tensorflow/lite/experimental/microfrontend/lib/frontend.h \ tensorflow/lite/experimental/microfrontend/lib/frontend_util.h \ tensorflow/lite/experimental/microfrontend/lib/log_lut.h \ tensorflow/lite/experimental/microfrontend/lib/log_scale.h \ tensorflow/lite/experimental/microfrontend/lib/log_scale_util.h \ +tensorflow/lite/experimental/microfrontend/lib/memory_util.h \ tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h \ tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.h \ tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h \ @@ -90,6 +93,13 @@ tensorflow/lite/experimental/microfrontend/lib/window.h \ tensorflow/lite/experimental/microfrontend/lib/window_util.h \ $(KISSFFT_LIB_HDRS) +MICRO_FEATURES_LIB_MEMORY_UTIL_TEST_SRCS := \ +tensorflow/lite/experimental/microfrontend/lib/memory_util_test.c \ +$(MICRO_FEATURES_LIB_SRCS) + +MICRO_FEATURES_LIB_MEMORY_UTIL_TEST_HDRS := \ +$(MICRO_FEATURES_LIB_HDRS) + MICRO_FEATURES_GENERATOR_SRCS := \ tensorflow/lite/micro/examples/micro_speech/micro_features/micro_features_generator.cc \ tensorflow/lite/micro/examples/micro_speech/micro_features/micro_model_settings.cc \ @@ -241,6 +251,10 @@ include $(wildcard tensorflow/lite/micro/examples/micro_speech/*/Makefile.inc) # way to build third_party code with a reduced list of CFLAGS. CCFLAGS := $(filter-out $(CC_WARNINGS),$(CCFLAGS)) +# Test the code for feature generation. +$(eval $(call microlite_test,micro_features_lib_memory_util_test,\ +$(MICRO_FEATURES_LIB_MEMORY_UTIL_TEST_SRCS), $(MICRO_FEATURES_LIB_MEMORY_UTIL_TEST_HDRS))) + # Test the code for feature generation. $(eval $(call microlite_test,micro_features_generator_test,\ $(MICRO_FEATURES_GENERATOR_TEST_SRCS), $(MICRO_FEATURES_GENERATOR_TEST_HDRS))) diff --git a/tensorflow/lite/micro/examples/micro_speech/simple_features/bluepill/simple_features_generator_test.cc b/tensorflow/lite/micro/examples/micro_speech/simple_features/bluepill/simple_features_generator_test.cc new file mode 100644 index 00000000000..28a101fb4eb --- /dev/null +++ b/tensorflow/lite/micro/examples/micro_speech/simple_features/bluepill/simple_features_generator_test.cc @@ -0,0 +1,35 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/examples/micro_speech/simple_features/simple_features_generator.h" + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/micro/examples/micro_speech/no_30ms_sample_data.h" +#include "tensorflow/lite/micro/examples/micro_speech/simple_features/no_power_spectrum_data.h" +#include "tensorflow/lite/micro/examples/micro_speech/simple_features/yes_power_spectrum_data.h" +#include "tensorflow/lite/micro/examples/micro_speech/yes_30ms_sample_data.h" +#include "tensorflow/lite/micro/micro_error_reporter.h" +#include "tensorflow/lite/micro/testing/micro_test.h" + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(TestSimpleFeaturesGenerator) { + tflite::MicroErrorReporter micro_error_reporter; + // This test times out on a Bluepill because of the very slow FFT + // calculations, so replace it with a version that does nothing for this + // platform. +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/kernels/ethos_u/README.md b/tensorflow/lite/micro/kernels/ethos_u/README.md index 11c69df4a9d..b29a8c0dcc5 100644 --- a/tensorflow/lite/micro/kernels/ethos_u/README.md +++ b/tensorflow/lite/micro/kernels/ethos_u/README.md @@ -52,6 +52,8 @@ In order to run a test with Ethos-U55 enabled, a platform with corresponding har On top of that the .tflite model needs to be modified according subchapter "Ethos-U custom operator" above. +The log level of the Ethos-U driver can be set in the build command. For example: ETHOSU_LOG_SEVERITY=ETHOSU_LOG_INFO. + ## Example using network tester See tensorflow/lite/micro/examples/network_tester/README.md for more info. diff --git a/tensorflow/lite/micro/testing/test_with_arm_corstone_300.sh b/tensorflow/lite/micro/testing/test_with_arm_corstone_300.sh index b51f49caeb4..9b39ee4adf5 100755 --- a/tensorflow/lite/micro/testing/test_with_arm_corstone_300.sh +++ b/tensorflow/lite/micro/testing/test_with_arm_corstone_300.sh @@ -39,11 +39,14 @@ FVP+='-C mps3_board.uart0.unbuffered_output=1 ' FVP+='-C mps3_board.uart0.shutdown_on_eot=1' ${FVP} ${BINARY_TO_TEST} | tee ${MICRO_LOG_FILENAME} -if grep -q "$PASS_STRING" ${MICRO_LOG_FILENAME} +if [[ ${2} != "non_test_binary" ]] then - echo "$BINARY_TO_TEST: PASS" - exit 0 -else - echo "$BINARY_TO_TEST: FAIL - '$PASS_STRING' not found in logs." - exit 1 + if grep -q "$PASS_STRING" ${MICRO_LOG_FILENAME} + then + echo "$BINARY_TO_TEST: PASS" + exit 0 + else + echo "$BINARY_TO_TEST: FAIL - '$PASS_STRING' not found in logs." + exit 1 + fi fi diff --git a/tensorflow/lite/micro/tools/make/ethos_u_core_platform_download.sh b/tensorflow/lite/micro/tools/make/ethos_u_core_platform_download.sh index 5c02a39be20..e5e4ed62db1 100755 --- a/tensorflow/lite/micro/tools/make/ethos_u_core_platform_download.sh +++ b/tensorflow/lite/micro/tools/make/ethos_u_core_platform_download.sh @@ -80,6 +80,9 @@ else sed -i '/rodata/d' ${LINKER_PATH}/platform_parsed.ld sed -i 's/network_model_sec/\.rodata\*/' ${LINKER_PATH}/platform_parsed.ld + # Allow tensor_arena in namespace. This will put tensor arena in SRAM intended by linker file. + sed -i 's/tensor_arena/\*tensor_arena\*/' ${LINKER_PATH}/platform_parsed.ld + # Patch retarget.c so that g++ can find _exit symbol. cat <> ${DOWNLOADED_ETHOS_U_CORE_PLATFORM_PATH}/targets/corstone-300/retarget.c diff --git a/tensorflow/lite/micro/tools/make/ext_libs/ethos_u.inc b/tensorflow/lite/micro/tools/make/ext_libs/ethos_u.inc index c9b7548d207..e676b6cc2ae 100644 --- a/tensorflow/lite/micro/tools/make/ext_libs/ethos_u.inc +++ b/tensorflow/lite/micro/tools/make/ext_libs/ethos_u.inc @@ -42,10 +42,15 @@ INCLUDES += -I$(ETHOSU_DRIVER_PATH)/include \ -I$(CMSIS_PATH)/CMSIS/Core/Include GENERATED_PROJECT_INCLUDES += -I./$(ETHOSU_DRIVER_PATH)/include -ETHOSU_LOG_SEVERITY := ETHOSU_LOG_INFO +ETHOSU_LOG_SEVERITY := ETHOSU_LOG_WARN CCFLAGS += -DETHOSU_LOG_SEVERITY=$(ETHOSU_LOG_SEVERITY) # TODO(#47718): resolve warnings. CCFLAGS += \ -Wno-return-type \ -Wno-format + +ifeq ($(TOOLCHAIN), gcc) + CCFLAGS += \ + -Wno-unused-but-set-variable +endif \ No newline at end of file diff --git a/tensorflow/lite/micro/tools/make/helper_functions.inc b/tensorflow/lite/micro/tools/make/helper_functions.inc index c102ee2866a..b9d32b1db00 100644 --- a/tensorflow/lite/micro/tools/make/helper_functions.inc +++ b/tensorflow/lite/micro/tools/make/helper_functions.inc @@ -495,7 +495,7 @@ $(1)_LOCAL_SRCS := $$(call specialize,$$($(1)_LOCAL_SRCS)) ALL_SRCS += $$($(1)_LOCAL_SRCS) $(1)_LOCAL_HDRS := $(3) $(1)_LOCAL_OBJS := $$(addprefix $$(OBJDIR), \ -$$(patsubst %.cc,%.o,$$(patsubst %.c,%.o,$$($(1)_LOCAL_SRCS)))) +$$(patsubst %.S,%.o,$$(patsubst %.cc,%.o,$$(patsubst %.c,%.o,$$($(1)_LOCAL_SRCS))))) $(1)_BINARY := $$(BINDIR)$(1) $$($(1)_BINARY): $$($(1)_LOCAL_OBJS) $$(MICROLITE_LIB_PATH) @mkdir -p $$(dir $$@) diff --git a/tensorflow/lite/micro/tools/make/targets/bluepill_makefile.inc b/tensorflow/lite/micro/tools/make/targets/bluepill_makefile.inc index 572b3898bb4..09ed6735598 100644 --- a/tensorflow/lite/micro/tools/make/targets/bluepill_makefile.inc +++ b/tensorflow/lite/micro/tools/make/targets/bluepill_makefile.inc @@ -69,11 +69,10 @@ EXCLUDED_TESTS := \ tensorflow/lite/micro/memory_arena_threshold_test.cc MICROLITE_TEST_SRCS := $(filter-out $(EXCLUDED_TESTS), $(MICROLITE_TEST_SRCS)) -EXCLUDED_EXAMPLE_TESTS := \ +EXCLUDED_EXAMPLE_TEST_MAKEFILES := \ tensorflow/lite/micro/examples/magic_wand/Makefile.inc \ - tensorflow/lite/micro/examples/micro_speech/Makefile.inc \ tensorflow/lite/micro/examples/image_recognition_experimental/Makefile.inc -MICRO_LITE_EXAMPLE_TESTS := $(filter-out $(EXCLUDED_EXAMPLE_TESTS), $(MICRO_LITE_EXAMPLE_TESTS)) +MICRO_LITE_EXAMPLE_TESTS := $(filter-out $(EXCLUDED_EXAMPLE_TESTS), $(MICRO_LITE_EXAMPLE_TEST_MAKEFILES)) TEST_SCRIPT := tensorflow/lite/micro/testing/test_with_renode.sh diff --git a/tensorflow/lite/micro/tools/make/targets/cortex_m_corstone_300_makefile.inc b/tensorflow/lite/micro/tools/make/targets/cortex_m_corstone_300_makefile.inc index 9e593b8150e..33f324d611b 100644 --- a/tensorflow/lite/micro/tools/make/targets/cortex_m_corstone_300_makefile.inc +++ b/tensorflow/lite/micro/tools/make/targets/cortex_m_corstone_300_makefile.inc @@ -169,9 +169,6 @@ INCLUDES += \ -I$(CMSIS_PATH)/Device/ARM/$(ARM_CPU)/Include \ -I$(CMSIS_PATH)/CMSIS/Core/Include -# TODO(#47071): Examine why Micro benchmarks fails. -MICRO_LITE_BENCHMARKS := $(filter-out tensorflow/lite/micro/benchmarks/Makefile.inc, $(MICRO_LITE_BENCHMARKS)) - # TODO(#47070): Examine why some tests fail here. EXCLUDED_TESTS := \ tensorflow/lite/micro/micro_interpreter_test.cc \ diff --git a/tensorflow/lite/micro/tools/make/third_party_downloads.inc b/tensorflow/lite/micro/tools/make/third_party_downloads.inc index 8dafd904f0e..ec26e305e40 100644 --- a/tensorflow/lite/micro/tools/make/third_party_downloads.inc +++ b/tensorflow/lite/micro/tools/make/third_party_downloads.inc @@ -39,9 +39,6 @@ KISSFFT_MD5="438ba1fef5783cc5f5f201395cc477ca" RUY_URL="https://github.com/google/ruy/archive/d37128311b445e758136b8602d1bbd2a755e115d.zip" RUY_MD5="abf7a91eb90d195f016ebe0be885bb6e" -IMAGE_RECOGNITION_MODEL_URL := "https://storage.googleapis.com/download.tensorflow.org/models/tflite/cifar_image_recognition_model_2020_05_27.zip" -IMAGE_RECOGNITION_MODEL_MD5 := "1f4607b05ac45b8a6146fb883dbc2d7b" - PERSON_MODEL_URL := "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_grayscale_2020_05_27.zip" PERSON_MODEL_MD5 := "55b85f76e2995153e660391d4a209ef1"