From 442654be91d843d084c928f3cf038cc52d95d1aa Mon Sep 17 00:00:00 2001 From: Christian Kadner Date: Mon, 8 Nov 2021 12:02:47 -0800 Subject: [PATCH] Make API Codegen Work on Linux (#255) * Use a 'gsed' alias in /api/codegen.sh script * Delete duplicated /api/add_license_headers.sh script * Move existing helper scripts from /hack to /tools * Remove .travis.yml and git_push.sh from API code generation Resolves #254 Signed-off-by: Christian Kadner --- Makefile | 2 +- api/add_license_headers.sh | 75 ------------------- api/client/.swagger-codegen-ignore | 2 + api/client/.travis.yml | 18 ----- api/client/git_push.sh | 57 -------------- api/codegen.sh | 18 ++++- api/generate_code.sh | 11 +-- api/server/.swagger-codegen-ignore | 4 +- api/server/.travis.yml | 16 ---- api/server/git_push.sh | 57 -------------- {hack => tools/bash}/add_license_headers.sh | 16 ++-- .../python}/regenerate_catalog_upload_json.py | 0 12 files changed, 37 insertions(+), 239 deletions(-) delete mode 100755 api/add_license_headers.sh delete mode 100644 api/client/.travis.yml delete mode 100644 api/client/git_push.sh delete mode 100644 api/server/.travis.yml delete mode 100644 api/server/git_push.sh rename {hack => tools/bash}/add_license_headers.sh (89%) rename {hack => tools/python}/regenerate_catalog_upload_json.py (100%) diff --git a/Makefile b/Makefile index 254d1af5..8efb70de 100644 --- a/Makefile +++ b/Makefile @@ -19,5 +19,5 @@ check_doc_links: ## Check markdown files for invalid links .PHONY: check_license check_license: ## Make sure source files have license header @git grep -L "SPDX-License-Identifier: Apache-2.0" -- *.py *.yml *.yaml *.sh *.html *.js *.css *.ts *.tsx ':!*.bundle.js' | \ - grep . && echo "Missing license headers in files above. Run './hack/add_license_headers.sh'" && exit 1 || \ + grep . && echo "Missing license headers in files above. Run './tools/bash/add_license_headers.sh'" && exit 1 || \ echo "$@: OK" diff --git a/api/add_license_headers.sh b/api/add_license_headers.sh deleted file mode 100755 index b3f2a4fa..00000000 --- a/api/add_license_headers.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 The MLX Contributors -# -# SPDX-License-Identifier: Apache-2.0 - -hash_comment () { - echo "$1" - if ! grep -q "SPDX-License-Identifier" "$1" - then - sed -i '' '1i\ - # Copyright 2021 The MLX Contributors\ - #\ - # SPDX-License-Identifier: Apache-2.0\ - ' "$1" - fi -} - -slash_comment () { - echo "$1" - if ! grep -q "SPDX-License-Identifier" "$1" - then - sed -i '' '1i\ - // Copyright 2021 The MLX Contributors\ - //\ - // SPDX-License-Identifier: Apache-2.0\ - ' "$1" - fi -} - -css_comment () { - echo "$1" - if ! grep -q "SPDX-License-Identifier" "$1" - then - sed -i '' '1i\ - /*\ - * Copyright 2021 The MLX Contributors\ - *\ - * SPDX-License-Identifier: Apache-2.0\ - */\ - ' "$1" - fi -} - -html_comment () { - echo "$1" - if ! grep -q "SPDX-License-Identifier" "$1" - then - sed -i '' '1i\ - \ - ' "$1" - fi -} - -export -f hash_comment slash_comment css_comment html_comment - -echo "Adding license headers to ..." - -# Python, YAML, Bash -echo " - Python, YAML, Shell scripts" -find . -type f -not -path '*/temp/*' -a -not -path '*/\.*' -a \( -name '*.py' -o -name '*.yaml' -o -name '*.yml' -o -name '*.sh' \) -exec bash -c 'hash_comment "$0"' {} \; -find . -type f -not -path '*/temp/*' -a -name '.travis.yml' -exec bash -c 'hash_comment "$0"' {} \; - -# Javascript -find . -type f -not -path '*/temp/*' -a -not -path '*/\.*' -a \( -name '*.js' -o -name '*.ts' \) -exec bash -c 'slash_comment "$0"' {} \; - -# CSS -find . -type f -not -path '*/temp/*' -a -not -path '*/\.*' -a \( -name '*.css' -o -name '*.tsx' \) -exec bash -c 'css_comment "$0"' {} \; - -# HTML -find . -type f -not -path '*/temp/*' -a -not -path '*/\.*' -a -name '*.html' -exec bash -c 'html_comment "$0"' {} \; diff --git a/api/client/.swagger-codegen-ignore b/api/client/.swagger-codegen-ignore index c0dca84f..0338cd6b 100644 --- a/api/client/.swagger-codegen-ignore +++ b/api/client/.swagger-codegen-ignore @@ -22,4 +22,6 @@ # Then explicitly reverse the ignore rule for a single file: #!docs/README.md +.travis.yml +git_push.sh requirements.txt diff --git a/api/client/.travis.yml b/api/client/.travis.yml deleted file mode 100644 index ee5023f9..00000000 --- a/api/client/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2021 The MLX Contributors -# -# SPDX-License-Identifier: Apache-2.0 -# -# ref: https://docs.travis-ci.com/user/languages/python -language: python -python: - - "2.7" - - "3.2" - - "3.3" - - "3.4" - - "3.5" - #- "3.5-dev" # 3.5 development branch - #- "nightly" # points to the latest development branch e.g. 3.6-dev -# command to install dependencies -install: "pip install -r requirements.txt" -# command to run tests -script: nosetests diff --git a/api/client/git_push.sh b/api/client/git_push.sh deleted file mode 100644 index 089144db..00000000 --- a/api/client/git_push.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh - -# Copyright 2021 The MLX Contributors -# -# SPDX-License-Identifier: Apache-2.0 - -# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/ -# -# Usage example: /bin/sh ./git_push.sh wing328 swagger-petstore-perl "minor update" - -git_user_id=$1 -git_repo_id=$2 -release_note=$3 - -if [ "$git_user_id" = "" ]; then - git_user_id="GIT_USER_ID" - echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id" -fi - -if [ "$git_repo_id" = "" ]; then - git_repo_id="GIT_REPO_ID" - echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id" -fi - -if [ "$release_note" = "" ]; then - release_note="Minor update" - echo "[INFO] No command line input provided. Set \$release_note to $release_note" -fi - -# Initialize the local directory as a Git repository -git init - -# Adds the files in the local repository and stages them for commit. -git add . - -# Commits the tracked changes and prepares them to be pushed to a remote repository. -git commit -m "$release_note" - -# Sets the new remote -git_remote=`git remote` -if [ "$git_remote" = "" ]; then # git remote not defined - - if [ "$GIT_TOKEN" = "" ]; then - echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment." - git remote add origin https://github.com/${git_user_id}/${git_repo_id}.git - else - git remote add origin https://${git_user_id}:${GIT_TOKEN}@github.com/${git_user_id}/${git_repo_id}.git - fi - -fi - -git pull origin master - -# Pushes (Forces) the changes in the local repository up to the remote repository -echo "Git pushing to https://github.com/${git_user_id}/${git_repo_id}.git" -git push origin master 2>&1 | grep -v 'To https' - diff --git a/api/codegen.sh b/api/codegen.sh index 60218937..25bc73dd 100755 --- a/api/codegen.sh +++ b/api/codegen.sh @@ -41,12 +41,28 @@ swagger-codegen generate -i swagger/swagger.yaml -l python -o client 2>&1 echo "Generating Python server:" swagger-codegen generate -i swagger/swagger.yaml -l python-flask -o server 2>&1 | grep -v -E "writing file|/test/" | sed "s|${SCRIPT_DIR}|.|g" +# set interactive mode to enable defining a gsed alias +shopt -s expand_aliases + +# we use sed to make in-file text replacements, but sed works differently on macOS and Linux +if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux + alias gsed="sed -i" +elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS + alias gsed="sed -i ''" +elif [[ "$OSTYPE" == "cygwin" ]]; then # POSIX compatible emulation for Windows + alias gsed="sed -i" +else + echo "FAILED. OS not compatible with script '${BASH_SOURCE[0]}'" + exit 1 +fi +export gsed + # we need to modify the generated controller methods to 'do some magic!' ... # replace: # return 'do some magic!' # with: # return util.invoke_controller_impl(__name__, locals()) -sed -i '' "s/'do some magic\!'/util.invoke_controller_impl()/g" "${SCRIPT_DIR}"/server/swagger_server/controllers/*.py +gsed "s/'do some magic\!'/util.invoke_controller_impl()/g" "${SCRIPT_DIR}"/server/swagger_server/controllers/*.py # and add the 'magic' utility method to forward the controller invocations ... unless we already did grep "invoke_controller_impl" "${UTIL_FILE}" -q || cat <<'EOF' >> "${UTIL_FILE}" diff --git a/api/generate_code.sh b/api/generate_code.sh index 3387b489..de60459d 100755 --- a/api/generate_code.sh +++ b/api/generate_code.sh @@ -5,7 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 # define global variables -REPO_URL="git@github.com:machine-learning-exchange/mlx.git" +REPO_URL="https://github.com/machine-learning-exchange/mlx.git" SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd)" PROJECT_DIR="${TRAVIS_BUILD_DIR:-$(cd "${SCRIPT_DIR%/api}"; pwd)}" TEMP_DIR="${PROJECT_DIR}/temp" @@ -30,14 +30,15 @@ clone_repo_to_tempdir() { --filter=blob:none #-q cd "${CLONE_DIR}" git config core.sparseCheckout true - echo "api/*"> .git/info/sparse-checkout + echo "api/*" > .git/info/sparse-checkout + echo "tools/*" >> .git/info/sparse-checkout git checkout main #-- api cd - &> /dev/null else echo "Get latest commit from main ..." cd "${CLONE_DIR}" git fetch origin # -q - git -c advice.detachedHead=false checkout "${VERSION}" -f -- api #-q + git -c advice.detachedHead=false checkout "${VERSION}" -f -- api tools #-q cd - &> /dev/null fi } @@ -49,7 +50,7 @@ generate_reverse_patch_for_undesired_codegen_results() { echo "========================================================================" cd "${CLONE_DIR}" ./api/codegen.sh || exit 1 - cd api && ./add_license_headers.sh && cd - + ./tools/bash/add_license_headers.sh || exit 1 #git diff --output="${TEMP_DIR}/undesired_codegen_results_$(date +"%Y-%m-%d").patch" git diff -R --output="${TEMP_DIR}/revert_undesired_codegen_results_$(date +"%Y-%m-%d").patch" #git checkout . # undo local changes @@ -63,7 +64,7 @@ generate_code() { echo "========================================================================" cd "${PROJECT_DIR}" ./api/codegen.sh || exit 1 - cd api && ./add_license_headers.sh && cd - + ./tools/bash/add_license_headers.sh || exit 1 } revert_undesired_codegen_results() { diff --git a/api/server/.swagger-codegen-ignore b/api/server/.swagger-codegen-ignore index c60a2f8b..433f6725 100644 --- a/api/server/.swagger-codegen-ignore +++ b/api/server/.swagger-codegen-ignore @@ -28,8 +28,10 @@ swagger_server/models/protobuf_any.py swagger_server/util.py swagger_server/__main__.py -Dockerfile .dockerignore +.travis.yml +Dockerfile README.md +git_push.sh requirements.txt tox.ini diff --git a/api/server/.travis.yml b/api/server/.travis.yml deleted file mode 100644 index aee3676e..00000000 --- a/api/server/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2021 The MLX Contributors -# -# SPDX-License-Identifier: Apache-2.0 -# ref: https://docs.travis-ci.com/user/languages/python -language: python -python: - - "3.2" - - "3.3" - - "3.4" - - "3.5" - #- "3.5-dev" # 3.5 development branch - #- "nightly" # points to the latest development branch e.g. 3.6-dev -# command to install dependencies -install: "pip install -r requirements.txt" -# command to run tests -script: nosetests diff --git a/api/server/git_push.sh b/api/server/git_push.sh deleted file mode 100644 index 2f0b600c..00000000 --- a/api/server/git_push.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh - -# Copyright 2021 The MLX Contributors -# -# SPDX-License-Identifier: Apache-2.0 - -# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/ -# -# Usage example: /bin/sh ./git_push.sh wing328 swagger-petstore-perl "minor update" - -git_user_id=$1 -git_repo_id=$2 -release_note=$3 - -if [ "$git_user_id" = "" ]; then - git_user_id="GIT_USER_ID" - echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id" -fi - -if [ "$git_repo_id" = "" ]; then - git_repo_id="GIT_REPO_ID" - echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id" -fi - -if [ "$release_note" = "" ]; then - release_note="Minor update" - echo "[INFO] No command line input provided. Set \$release_note to $release_note" -fi - -# Initialize the local directory as a Git repository -git init - -# Adds the files in the local repository and stages them for commit. -git add . - -# Commits the tracked changes and prepares them to be pushed to a remote repository. -git commit -m "$release_note" - -# Sets the new remote -git_remote=`git remote` -if [ "$git_remote" = "" ]; then # git remote not defined - - if [ "$GIT_TOKEN" = "" ]; then - echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment." - git remote add origin https://github.com/${git_user_id}/${git_repo_id}.git - else - git remote add origin https://${git_user_id}:${GIT_TOKEN}@github.com/${git_user_id}/${git_repo_id}.git - fi - -fi - -git pull origin master - -# Pushes (Forces) the changes in the local repository up to the remote repository -echo "Git pushing to https://github.com/${git_user_id}/${git_repo_id}.git" -git push origin master 2>&1 | grep -v 'To https' - diff --git a/hack/add_license_headers.sh b/tools/bash/add_license_headers.sh similarity index 89% rename from hack/add_license_headers.sh rename to tools/bash/add_license_headers.sh index 35a49a28..02405b83 100755 --- a/hack/add_license_headers.sh +++ b/tools/bash/add_license_headers.sh @@ -14,15 +14,15 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS elif [[ "$OSTYPE" == "cygwin" ]]; then # POSIX compatible emulation for Windows alias gsed="sed -i" else - echo "FAILED. OS not compatible with script '/hack/add_license_headers.sh'" + echo "FAILED. OS not compatible with script '${BASH_SOURCE[0]}'" exit 1 fi export gsed hash_comment () { - echo "$1" if ! grep -q "SPDX-License-Identifier" "$1" then + echo "$1" gsed '1i\ # Copyright 2021 The MLX Contributors\ #\ @@ -32,9 +32,9 @@ hash_comment () { } slash_comment () { - echo "$1" if ! grep -q "SPDX-License-Identifier" "$1" then + echo "$1" gsed '1i\ // Copyright 2021 The MLX Contributors\ //\ @@ -44,9 +44,9 @@ slash_comment () { } css_comment () { - echo "$1" if ! grep -q "SPDX-License-Identifier" "$1" then + echo "$1" gsed '1i\ /*\ * Copyright 2021 The MLX Contributors\ @@ -58,14 +58,14 @@ css_comment () { } html_comment () { - echo "$1" if ! grep -q "SPDX-License-Identifier" "$1" then + echo "$1" gsed '1i\ \ ' "$1" fi diff --git a/hack/regenerate_catalog_upload_json.py b/tools/python/regenerate_catalog_upload_json.py similarity index 100% rename from hack/regenerate_catalog_upload_json.py rename to tools/python/regenerate_catalog_upload_json.py