Skip to content

Commit

Permalink
Updated sync and use of shared TFL/TFLM code.
Browse files Browse the repository at this point in the history 
 * The TFLM Makefile globs all the shared TFL/TFLM code.
 * This allows us to move the explicit list of sources and headers to the
   sync script since that is where we determine what code needs to be
   sync'd from upstream TF.

With this change, we are ready to have the tflite_micro repository be
the source of truth for all TFLM code and sync only the shared TFL/TFLM
code from the tensorflow repo.

Bug: http://b/182914089
  • Loading branch information
@advaitjain
advaitjain committed May 27, 2021
1 parent 300d510 commit 6adb838
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 138 deletions.
148 changes: 99 additions & 49 deletions ci/sync_from_upstream_tf.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,64 +21,114 @@
# of the TFLM code via this script.
#

set -x
set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR=${SCRIPT_DIR}/..
cd "${ROOT_DIR}"

rm -rf /tmp/tensorflow
rm -rf /tmp/tflm-tree

git clone https://github.com/tensorflow/tensorflow.git --depth=1 /tmp/tensorflow

# Manually apply some patches to avoid having to upstream changes to TF that
# are very specific to the new TFLM repo.
cp ci/temp_patches/tf_update_visibility.patch /tmp/tensorflow
cd /tmp/tensorflow
git apply tf_update_visibility.patch

# TODO(b/184886633): the downloads should happen as part of the create_tflm_tree
# script.
make -f tensorflow/lite/micro/tools/make/Makefile third_party_downloads
python3 tensorflow/lite/micro/tools/project_generation/create_tflm_tree.py /tmp/tflm-tree
cd -

rsync -r --delete /tmp/tflm-tree/tensorflow/lite tensorflow/

# The entire micro directory will be copied from upstream TF (instead of being
# copied from the output of the project generation).
git checkout tensorflow/lite/micro/

# TfLite BUILD files are manually maintained in the TFLM repo.
git checkout \
tensorflow/BUILD \
tensorflow/lite/BUILD \
tensorflow/lite/build_def.bzl \
tensorflow/lite/c/BUILD \
tensorflow/lite/core/api/BUILD \
tensorflow/lite/kernels/BUILD \
tensorflow/lite/kernels/internal/BUILD \
tensorflow/lite/schema/BUILD

git checkout \
tensorflow/lite/kernels/internal/optimized/neon_check.h \
tensorflow/lite/experimental/microfrontend

rsync -r --delete /tmp/tensorflow/tensorflow/lite/micro tensorflow/lite/


SHARED_TFL_CODE="\
tensorflow/lite/c/common.c \
tensorflow/lite/core/api/error_reporter.cc \
tensorflow/lite/core/api/flatbuffer_conversions.cc \
tensorflow/lite/core/api/op_resolver.cc \
tensorflow/lite/core/api/tensor_utils.cc \
tensorflow/lite/kernels/internal/quantization_util.cc \
tensorflow/lite/kernels/kernel_util.cc \
tensorflow/lite/schema/schema_utils.cc
tensorflow/lite/c/builtin_op_data.h \
tensorflow/lite/c/c_api_types.h \
tensorflow/lite/c/common.h \
tensorflow/lite/core/api/error_reporter.h \
tensorflow/lite/core/api/flatbuffer_conversions.h \
tensorflow/lite/core/api/op_resolver.h \
tensorflow/lite/core/api/tensor_utils.h \
tensorflow/lite/kernels/internal/common.h \
tensorflow/lite/kernels/internal/compatibility.h \
tensorflow/lite/kernels/internal/quantization_util.h \
tensorflow/lite/kernels/internal/reference/add.h \
tensorflow/lite/kernels/internal/reference/add_n.h \
tensorflow/lite/kernels/internal/reference/arg_min_max.h \
tensorflow/lite/kernels/internal/reference/batch_to_space_nd.h \
tensorflow/lite/kernels/internal/reference/binary_function.h \
tensorflow/lite/kernels/internal/reference/ceil.h \
tensorflow/lite/kernels/internal/reference/comparisons.h \
tensorflow/lite/kernels/internal/reference/concatenation.h \
tensorflow/lite/kernels/internal/reference/conv.h \
tensorflow/lite/kernels/internal/reference/cumsum.h \
tensorflow/lite/kernels/internal/reference/depth_to_space.h \
tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h \
tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h \
tensorflow/lite/kernels/internal/reference/dequantize.h \
tensorflow/lite/kernels/internal/reference/elu.h \
tensorflow/lite/kernels/internal/reference/exp.h \
tensorflow/lite/kernels/internal/reference/fill.h \
tensorflow/lite/kernels/internal/reference/floor.h \
tensorflow/lite/kernels/internal/reference/floor_div.h \
tensorflow/lite/kernels/internal/reference/floor_mod.h \
tensorflow/lite/kernels/internal/reference/fully_connected.h \
tensorflow/lite/kernels/internal/reference/hard_swish.h \
tensorflow/lite/kernels/internal/reference/integer_ops/add.h \
tensorflow/lite/kernels/internal/reference/integer_ops/conv.h \
tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h \
tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h \
tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h \
tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h \
tensorflow/lite/kernels/internal/reference/integer_ops/mean.h \
tensorflow/lite/kernels/internal/reference/integer_ops/mul.h \
tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h \
tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h \
tensorflow/lite/kernels/internal/reference/integer_ops/transpose_conv.h \
tensorflow/lite/kernels/internal/reference/l2normalization.h \
tensorflow/lite/kernels/internal/reference/leaky_relu.h \
tensorflow/lite/kernels/internal/reference/log_softmax.h \
tensorflow/lite/kernels/internal/reference/maximum_minimum.h \
tensorflow/lite/kernels/internal/reference/mul.h \
tensorflow/lite/kernels/internal/reference/neg.h \
tensorflow/lite/kernels/internal/reference/pad.h \
tensorflow/lite/kernels/internal/reference/pooling.h \
tensorflow/lite/kernels/internal/reference/prelu.h \
tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h \
tensorflow/lite/kernels/internal/reference/quantize.h \
tensorflow/lite/kernels/internal/reference/reduce.h \
tensorflow/lite/kernels/internal/reference/requantize.h \
tensorflow/lite/kernels/internal/reference/resize_bilinear.h \
tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h \
tensorflow/lite/kernels/internal/reference/round.h \
tensorflow/lite/kernels/internal/reference/softmax.h \
tensorflow/lite/kernels/internal/reference/space_to_batch_nd.h \
tensorflow/lite/kernels/internal/reference/sub.h \
tensorflow/lite/kernels/internal/reference/logistic.h \
tensorflow/lite/kernels/internal/reference/strided_slice.h \
tensorflow/lite/kernels/internal/reference/tanh.h \
tensorflow/lite/kernels/internal/reference/transpose.h \
tensorflow/lite/kernels/internal/reference/transpose_conv.h \
tensorflow/lite/kernels/internal/cppmath.h \
tensorflow/lite/kernels/internal/max.h \
tensorflow/lite/kernels/internal/min.h \
tensorflow/lite/kernels/internal/portable_tensor.h \
tensorflow/lite/kernels/internal/strided_slice_logic.h \
tensorflow/lite/kernels/internal/tensor_ctypes.h \
tensorflow/lite/kernels/internal/types.h \
tensorflow/lite/kernels/kernel_util.h \
tensorflow/lite/kernels/op_macros.h \
tensorflow/lite/kernels/padding.h \
tensorflow/lite/portable_type_to_tflitetype.h \
tensorflow/lite/schema/schema_generated.h \
tensorflow/lite/schema/schema_utils.h \
"

for filepath in ${SHARED_TFL_CODE}
do
mkdir -p $(dirname ${filepath})
/bin/cp /tmp/tensorflow/${filepath} ${filepath}
done

# The microfrontend is sync'd from upstream but not as part of the explicitly
# specified SHARED_TFL_CODE since this is only needed for the examples.
rm -rf tensorflow/lite/experimental/microfrontend/lib
cp -r /tmp/tensorflow/tensorflow/lite/experimental/microfrontend/lib tensorflow/lite/experimental/microfrontend/lib

rm -rf tensorflow/lite/micro/tools/ci_build/tflm_bazel

# Any TFLM-repo specific files that are not in upstream TF will be deleted with
# the rsync command and any files whose source of truth is the new TFLM repo
# should be manually restored.
git checkout \
tensorflow/lite/micro/tools/ci_build/helper_functions.sh \
tensorflow/lite/micro/tools/ci_build/test_bazel.sh \
tensorflow/lite/micro/tools/ci_build/test_makefile.sh

99 changes: 10 additions & 89 deletions tensorflow/lite/micro/tools/make/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -399,105 +399,26 @@ tensorflow/lite/micro/kernels/zeros_like.cc
MICROLITE_TEST_HDRS := \
$(wildcard tensorflow/lite/micro/testing/*.h)

# The explicitly specified list of sources and headers that are shared between
# TfLite and TFLM are in the ci/sync_from_upstream_tf.sh script.
TFL_CC_SRCS := \
$(shell find tensorflow/lite/ -type d \( -path tensorflow/lite/experimental -o -path tensorflow/lite/micro \) -prune -false -o -name "*.cc" -o -name "*.c")

TFL_CC_HDRS := \
$(shell find tensorflow/lite/ -type d \( -path tensorflow/lite/experimental -o -path tensorflow/lite/micro \) -prune -false -o -name "*.h")

MICROLITE_CC_BASE_SRCS := \
$(wildcard tensorflow/lite/micro/*.cc) \
$(wildcard tensorflow/lite/micro/memory_planner/*.cc) \
tensorflow/lite/c/common.c \
tensorflow/lite/core/api/error_reporter.cc \
tensorflow/lite/core/api/flatbuffer_conversions.cc \
tensorflow/lite/core/api/op_resolver.cc \
tensorflow/lite/core/api/tensor_utils.cc \
tensorflow/lite/kernels/internal/quantization_util.cc \
tensorflow/lite/kernels/kernel_util.cc \
tensorflow/lite/schema/schema_utils.cc
$(TFL_CC_SRCS)

MICROLITE_CC_HDRS := \
$(wildcard tensorflow/lite/micro/*.h) \
$(wildcard tensorflow/lite/micro/benchmarks/*model_data.h) \
$(wildcard tensorflow/lite/micro/kernels/*.h) \
$(wildcard tensorflow/lite/micro/memory_planner/*.h) \
LICENSE \
tensorflow/lite/c/builtin_op_data.h \
tensorflow/lite/c/c_api_types.h \
tensorflow/lite/c/common.h \
tensorflow/lite/core/api/error_reporter.h \
tensorflow/lite/core/api/flatbuffer_conversions.h \
tensorflow/lite/core/api/op_resolver.h \
tensorflow/lite/core/api/tensor_utils.h \
tensorflow/lite/kernels/internal/common.h \
tensorflow/lite/kernels/internal/compatibility.h \
tensorflow/lite/kernels/internal/optimized/neon_check.h \
tensorflow/lite/kernels/internal/quantization_util.h \
tensorflow/lite/kernels/internal/reference/add.h \
tensorflow/lite/kernels/internal/reference/add_n.h \
tensorflow/lite/kernels/internal/reference/arg_min_max.h \
tensorflow/lite/kernels/internal/reference/batch_to_space_nd.h \
tensorflow/lite/kernels/internal/reference/binary_function.h \
tensorflow/lite/kernels/internal/reference/ceil.h \
tensorflow/lite/kernels/internal/reference/comparisons.h \
tensorflow/lite/kernels/internal/reference/concatenation.h \
tensorflow/lite/kernels/internal/reference/conv.h \
tensorflow/lite/kernels/internal/reference/cumsum.h \
tensorflow/lite/kernels/internal/reference/depth_to_space.h \
tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h \
tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h \
tensorflow/lite/kernels/internal/reference/dequantize.h \
tensorflow/lite/kernels/internal/reference/elu.h \
tensorflow/lite/kernels/internal/reference/exp.h \
tensorflow/lite/kernels/internal/reference/fill.h \
tensorflow/lite/kernels/internal/reference/floor.h \
tensorflow/lite/kernels/internal/reference/floor_div.h \
tensorflow/lite/kernels/internal/reference/floor_mod.h \
tensorflow/lite/kernels/internal/reference/fully_connected.h \
tensorflow/lite/kernels/internal/reference/hard_swish.h \
tensorflow/lite/kernels/internal/reference/integer_ops/add.h \
tensorflow/lite/kernels/internal/reference/integer_ops/conv.h \
tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h \
tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h \
tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h \
tensorflow/lite/kernels/internal/reference/integer_ops/l2normalization.h \
tensorflow/lite/kernels/internal/reference/integer_ops/mean.h \
tensorflow/lite/kernels/internal/reference/integer_ops/mul.h \
tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h \
tensorflow/lite/kernels/internal/reference/integer_ops/tanh.h \
tensorflow/lite/kernels/internal/reference/integer_ops/transpose_conv.h \
tensorflow/lite/kernels/internal/reference/l2normalization.h \
tensorflow/lite/kernels/internal/reference/leaky_relu.h \
tensorflow/lite/kernels/internal/reference/log_softmax.h \
tensorflow/lite/kernels/internal/reference/maximum_minimum.h \
tensorflow/lite/kernels/internal/reference/mul.h \
tensorflow/lite/kernels/internal/reference/neg.h \
tensorflow/lite/kernels/internal/reference/pad.h \
tensorflow/lite/kernels/internal/reference/pooling.h \
tensorflow/lite/kernels/internal/reference/prelu.h \
tensorflow/lite/kernels/internal/reference/process_broadcast_shapes.h \
tensorflow/lite/kernels/internal/reference/quantize.h \
tensorflow/lite/kernels/internal/reference/reduce.h \
tensorflow/lite/kernels/internal/reference/requantize.h \
tensorflow/lite/kernels/internal/reference/resize_bilinear.h \
tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h \
tensorflow/lite/kernels/internal/reference/round.h \
tensorflow/lite/kernels/internal/reference/softmax.h \
tensorflow/lite/kernels/internal/reference/space_to_batch_nd.h \
tensorflow/lite/kernels/internal/reference/sub.h \
tensorflow/lite/kernels/internal/reference/logistic.h \
tensorflow/lite/kernels/internal/reference/strided_slice.h \
tensorflow/lite/kernels/internal/reference/tanh.h \
tensorflow/lite/kernels/internal/reference/transpose.h \
tensorflow/lite/kernels/internal/reference/transpose_conv.h \
tensorflow/lite/kernels/internal/cppmath.h \
tensorflow/lite/kernels/internal/max.h \
tensorflow/lite/kernels/internal/min.h \
tensorflow/lite/kernels/internal/portable_tensor.h \
tensorflow/lite/kernels/internal/strided_slice_logic.h \
tensorflow/lite/kernels/internal/tensor_ctypes.h \
tensorflow/lite/kernels/internal/types.h \
tensorflow/lite/kernels/kernel_util.h \
tensorflow/lite/kernels/op_macros.h \
tensorflow/lite/kernels/padding.h \
tensorflow/lite/portable_type_to_tflitetype.h \
tensorflow/lite/schema/schema_generated.h \
tensorflow/lite/schema/schema_utils.h
$(TFL_CC_HDRS)

# Load custom kernels.
include $(MAKEFILE_DIR)/additional_kernels.inc
Expand Down

0 comments on commit 6adb838

Please sign in to comment.