Make API Codegen Work on Linux (#255)

* Use a 'gsed' alias in /api/codegen.sh script
* Delete duplicated /api/add_license_headers.sh script
* Move existing helper scripts from /hack to /tools
* Remove .travis.yml and git_push.sh from API code generation

Resolves #254

Signed-off-by: Christian Kadner <[email protected]>

Makefile

@@ -19,5 +19,5 @@ check_doc_links: ## Check markdown files for invalid links
 .PHONY: check_license
 check_license: ## Make sure source files have license header
 	@git grep -L "SPDX-License-Identifier: Apache-2.0" -- *.py *.yml *.yaml *.sh *.html *.js *.css *.ts *.tsx ':!*.bundle.js' | \
-		grep . && echo "Missing license headers in files above. Run './hack/add_license_headers.sh'" && exit 1 || \
+		grep . && echo "Missing license headers in files above. Run './tools/bash/add_license_headers.sh'" && exit 1 || \
 		echo "$@: OK"

api/client/.swagger-codegen-ignore

@@ -22,4 +22,6 @@
 # Then explicitly reverse the ignore rule for a single file:
 #!docs/README.md
 
+.travis.yml
+git_push.sh
 requirements.txt

api/codegen.sh

@@ -41,12 +41,28 @@ swagger-codegen generate -i swagger/swagger.yaml -l python       -o client 2>&1
 echo "Generating Python server:"
 swagger-codegen generate -i swagger/swagger.yaml -l python-flask -o server 2>&1 | grep -v -E "writing file|/test/" | sed "s|${SCRIPT_DIR}|.|g"
 
+# set interactive mode to enable defining a gsed alias
+shopt -s expand_aliases
+
+# we use sed to make in-file text replacements, but sed works differently on macOS and Linux
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then  # Linux
+    alias gsed="sed -i"
+elif [[ "$OSTYPE" == "darwin"* ]]; then  # macOS
+    alias gsed="sed -i ''"
+elif [[ "$OSTYPE" == "cygwin" ]]; then  # POSIX compatible emulation for Windows
+    alias gsed="sed -i"
+else
+    echo "FAILED. OS not compatible with script '${BASH_SOURCE[0]}'"
+    exit 1
+fi
+export gsed
+
 # we need to modify the generated controller methods to 'do some magic!' ...
 # replace:
 #     return 'do some magic!'
 # with:
 #     return util.invoke_controller_impl(__name__, locals())
-sed -i '' "s/'do some magic\!'/util.invoke_controller_impl()/g" "${SCRIPT_DIR}"/server/swagger_server/controllers/*.py
+gsed "s/'do some magic\!'/util.invoke_controller_impl()/g" "${SCRIPT_DIR}"/server/swagger_server/controllers/*.py
 
 # and add the 'magic' utility method to forward the controller invocations ... unless we already did
 grep "invoke_controller_impl" "${UTIL_FILE}" -q || cat <<'EOF' >> "${UTIL_FILE}"

api/generate_code.sh

@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # define global variables
-REPO_URL="git@github.com:machine-learning-exchange/mlx.git"
+REPO_URL="https://github.com/machine-learning-exchange/mlx.git"
 SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd)"
 PROJECT_DIR="${TRAVIS_BUILD_DIR:-$(cd "${SCRIPT_DIR%/api}"; pwd)}"
 TEMP_DIR="${PROJECT_DIR}/temp"
@@ -30,14 +30,15 @@ clone_repo_to_tempdir() {
         --filter=blob:none #-q
     cd "${CLONE_DIR}"
     git config core.sparseCheckout true
-    echo "api/*"> .git/info/sparse-checkout
+    echo "api/*" > .git/info/sparse-checkout
+    echo "tools/*" >> .git/info/sparse-checkout
     git checkout main #-- api
     cd - &> /dev/null
   else
     echo "Get latest commit from main ..."
     cd "${CLONE_DIR}"
     git fetch origin # -q
-    git -c advice.detachedHead=false checkout "${VERSION}" -f -- api #-q
+    git -c advice.detachedHead=false checkout "${VERSION}" -f -- api tools #-q
     cd - &> /dev/null
   fi
 }
@@ -49,7 +50,7 @@ generate_reverse_patch_for_undesired_codegen_results() {
   echo "========================================================================"
   cd "${CLONE_DIR}"
   ./api/codegen.sh || exit 1
-  cd api && ./add_license_headers.sh && cd -
+  ./tools/bash/add_license_headers.sh || exit 1
   #git diff --output="${TEMP_DIR}/undesired_codegen_results_$(date +"%Y-%m-%d").patch"
   git diff -R --output="${TEMP_DIR}/revert_undesired_codegen_results_$(date +"%Y-%m-%d").patch"
   #git checkout .  # undo local changes
@@ -63,7 +64,7 @@ generate_code() {
   echo "========================================================================"
   cd "${PROJECT_DIR}"
   ./api/codegen.sh || exit 1
-  cd api && ./add_license_headers.sh && cd -
+  ./tools/bash/add_license_headers.sh || exit 1
 }
 
 revert_undesired_codegen_results() {

api/server/.swagger-codegen-ignore

@@ -28,8 +28,10 @@ swagger_server/models/protobuf_any.py
 swagger_server/util.py
 swagger_server/__main__.py
 
-Dockerfile
 .dockerignore
+.travis.yml
+Dockerfile
 README.md
+git_push.sh
 requirements.txt
 tox.ini

hack/add_license_headers.sh → tools/bash/add_license_headers.sh

@@ -14,15 +14,15 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then  # macOS
 elif [[ "$OSTYPE" == "cygwin" ]]; then  # POSIX compatible emulation for Windows
     alias gsed="sed -i"
 else
-    echo "FAILED. OS not compatible with script '/hack/add_license_headers.sh'"
+    echo "FAILED. OS not compatible with script '${BASH_SOURCE[0]}'"
     exit 1
 fi
 export gsed
 
 hash_comment () {
-  echo "$1"
   if ! grep -q "SPDX-License-Identifier" "$1"
   then
+    echo "$1"
     gsed '1i\
     # Copyright 2021 The MLX Contributors\
     #\
@@ -32,9 +32,9 @@ hash_comment () {
 }
 
 slash_comment () {
-  echo "$1"
   if ! grep -q "SPDX-License-Identifier" "$1"
   then
+    echo "$1"
     gsed '1i\
     // Copyright 2021 The MLX Contributors\
     //\
@@ -44,9 +44,9 @@ slash_comment () {
 }
 
 css_comment () {
-  echo "$1"
   if ! grep -q "SPDX-License-Identifier" "$1"
   then
+    echo "$1"
     gsed '1i\
     /*\
     * Copyright 2021 The MLX Contributors\
@@ -58,14 +58,14 @@ css_comment () {
 }
 
 html_comment () {
-  echo "$1"
   if ! grep -q "SPDX-License-Identifier" "$1"
   then
+    echo "$1"
     gsed '1i\
     <!--\
-     Copyright 2021 The MLX Contributors\
-      \
-      SPDX-License-Identifier: Apache-2.0\
+    Copyright 2021 The MLX Contributors\
+    \
+    SPDX-License-Identifier: Apache-2.0\
     -->\
     ' "$1"
   fi