Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix bugs in ipynb2py and app test #385

Closed
wants to merge 38 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
501ca42
add timer in run_app_tests
edieYoung May 18, 2018
7038008
add timer to run_app_tests
edieYoung May 18, 2018
e44f1fd
Merge branch 'master' of https://github.com/intel-analytics/analytics…
edieYoung May 21, 2018
9d55e9b
Merge branch 'master' of https://github.com/intel-analytics/analytics…
edieYoung May 21, 2018
3a7f565
Merge branch 'master' of https://github.com/intel-analytics/analytics…
edieYoung May 22, 2018
0cb58d1
Merge branch 'master' of https://github.com/intel-analytics/analytics…
edieYoung May 22, 2018
c765bc4
add_timer_to_app_tests
edieYoung May 22, 2018
b94c669
Merge branch 'master' of https://github.com/intel-analytics/analytics…
edieYoung May 22, 2018
27fd9b6
Merge branch 'master' of https://github.com/intel-analytics/analytics…
edieYoung Jun 11, 2018
14f91df
add exception exit in apps and example tests
edieYoung Jun 11, 2018
914b34d
switch test order
edieYoung Jun 11, 2018
103e62c
switch test order to verify
edieYoung Jun 11, 2018
6a1682e
test_update
edieYoung Jun 12, 2018
a765596
test_update
edieYoung Jun 12, 2018
1452dc7
test_update
edieYoung Jun 13, 2018
271a90b
Merge branch 'master' of https://github.com/intel-analytics/analytics…
edieYoung Jun 13, 2018
c8156f5
test_update
edieYoung Jun 13, 2018
e7b49a4
test_update
edieYoung Jun 13, 2018
f4f2358
test_update
edieYoung Jun 13, 2018
8869b47
test_update
edieYoung Jun 13, 2018
bacade9
test_update
edieYoung Jun 13, 2018
abb1b01
test_update
edieYoung Jun 13, 2018
f0d348a
test_update
edieYoung Jun 13, 2018
1a8e55b
test_update
edieYoung Jun 13, 2018
7339905
test_update
edieYoung Jun 13, 2018
884ef11
example_update
edieYoung Jun 13, 2018
1b32b96
example_update
edieYoung Jun 13, 2018
0de1cf4
test_update
edieYoung Jun 13, 2018
f467ebf
test_update
edieYoung Jun 13, 2018
ced59e6
test_update
edieYoung Jun 13, 2018
2bac41f
test_update
edieYoung Jun 13, 2018
d7f8be2
test_update
edieYoung Jun 14, 2018
140c915
fix_apptest
edieYoung Jun 14, 2018
b6242cd
Merge branch 'master' of https://github.com/intel-analytics/analytics…
edieYoung Jun 14, 2018
23f4c32
test_update
edieYoung Jun 14, 2018
99b1afb
test_update
edieYoung Jun 14, 2018
b499f4e
test_update
edieYoung Jun 14, 2018
12f3d82
test_update
edieYoung Jun 14, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test_update
edieYoung committed Jun 13, 2018

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 1452dc79bb74e2d4083ad6ec07024bc96997b3b1
2 changes: 1 addition & 1 deletion apps/ipynb2py.sh
100644 → 100755
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ else
sed -i 's/%%/#/' $1.tmp.ipynb
sed -i 's/%pylab/#/' $1.tmp.ipynb

jupyter nbconvert --to script $1.tmp.ipynb
jupyter nbconvert $1.tmp.ipynb --to python

mv $1.tmp.py $1.py
sed -i '1i# -*- coding: utf-8 -*-' $1.py
66 changes: 33 additions & 33 deletions apps/run-app-tests.sh
Original file line number Diff line number Diff line change
@@ -14,6 +14,32 @@ chmod +x ./apps/ipynb2py.sh

set -e

echo "#1 start app test for anomaly-detection-nyc-taxi"
#timer
start=$(date "+%s")
./apps/ipynb2py.sh ./apps/anomaly-detection/anomaly-detection-nyc-taxi

chmod +x $ANALYTICS_ZOO_HOME/scripts/data/NAB/nyc_taxi/get_nyc_taxi.sh

$ANALYTICS_ZOO_HOME/scripts/data/NAB/nyc_taxi/get_nyc_taxi.sh

${SPARK_HOME}/bin/spark-submit \
--master ${MASTER} \
--driver-cores 2 \
--driver-memory 12g \
--total-executor-cores 2 \
--executor-cores 2 \
--executor-memory 12g \
--conf spark.akka.frameSize=64 \
--py-files ${ANALYTICS_ZOO_PYZIP},${ANALYTICS_ZOO_HOME}/apps/anomaly-detection/anomaly-detection-nyc-taxi.py \
--properties-file ${ANALYTICS_ZOO_CONF} \
--jars ${ANALYTICS_ZOO_JAR} \
--conf spark.driver.extraClassPath=${ANALYTICS_ZOO_JAR} \
--conf spark.executor.extraClassPath=${ANALYTICS_ZOO_JAR} \
${ANALYTICS_ZOO_HOME}/apps/anomaly-detection/anomaly-detection-nyc-taxi.py
now=$(date "+%s")
time1=$((now-start))

echo "#2 start app test for object-detection"
#timer
start=$(date "+%s")
@@ -22,26 +48,26 @@ start=$(date "+%s")
FILENAME="$ANALYTICS_ZOO_HOME/apps/object-detection/analytics-zoo_ssd-mobilenet-300x300_PASCAL_0.1.0.model"
if [ -f "$FILENAME" ]
then
echo "$FILENAME already exists"
echo "$FILENAME already exists"
else
wget $FTP_URI/analytics-zoo-models-new/object-detection/analytics-zoo_ssd-mobilenet-300x300_PASCAL_0.1.0.model -P $ANALYTICS_ZOO_HOME/apps/object-detection/
fi
fi
if [ -f "$FILENAME" ]
then
echo "$FILENAME already exists"
echo "$FILENAME already exists"
else
wget https://s3-ap-southeast-1.amazonaws.com/analytics-zoo-models/object-detection/analytics-zoo_ssd-mobilenet-300x300_PASCAL_0.1.0.model -P $ANALYTICS_ZOO_HOME/apps/object-detection/
fi
fi
FILENAME="$ANALYTICS_ZOO_HOME/apps/object-detection/train_dog.mp4"
if [ -f "$FILENAME" ]
then
echo "$FILENAME already exists"
echo "$FILENAME already exists"
else
wget $FTP_URI/analytics-zoo-data/apps/object-detection/train_dog.mp4 -P $ANALYTICS_ZOO_HOME/apps/object-detection/
fi
fi
if [ -f "$FILENAME" ]
then
echo "$FILENAME already exists"
echo "$FILENAME already exists"
else
wget https://s3.amazonaws.com/analytics-zoo-data/train_dog.mp4 -P $ANALYTICS_ZOO_HOME/apps/object-detection/
fi
@@ -64,32 +90,6 @@ ${SPARK_HOME}/bin/spark-submit \
now=$(date "+%s")
time2=$((now-start))

echo "#1 start app test for anomaly-detection-nyc-taxi"
#timer
start=$(date "+%s")
./apps/ipynb2py.sh ./apps/anomaly-detection/anomaly-detection-nyc-taxi

chmod +x $ANALYTICS_ZOO_HOME/scripts/data/NAB/nyc_taxi/get_nyc_taxi.sh

$ANALYTICS_ZOO_HOME/scripts/data/NAB/nyc_taxi/get_nyc_taxi.sh

${SPARK_HOME}/bin/spark-submit \
--master ${MASTER} \
--driver-cores 2 \
--driver-memory 12g \
--total-executor-cores 2 \
--executor-cores 2 \
--executor-memory 12g \
--conf spark.akka.frameSize=64 \
--py-files ${ANALYTICS_ZOO_PYZIP},${ANALYTICS_ZOO_HOME}/apps/anomaly-detection/anomaly-detection-nyc-taxi.py \
--properties-file ${ANALYTICS_ZOO_CONF} \
--jars ${ANALYTICS_ZOO_JAR} \
--conf spark.driver.extraClassPath=${ANALYTICS_ZOO_JAR} \
--conf spark.executor.extraClassPath=${ANALYTICS_ZOO_JAR} \
${ANALYTICS_ZOO_HOME}/apps/anomaly-detection/anomaly-detection-nyc-taxi.py
now=$(date "+%s")
time1=$((now-start))

echo "#3 start app test for ncf-explicit-feedback"
#timer
start=$(date "+%s")