From 3a890366997772c36c51253530e45a8526fb8921 Mon Sep 17 00:00:00 2001 From: Kartikeya Sharma Date: Thu, 4 Mar 2021 15:11:48 -0500 Subject: [PATCH 1/7] added argoverse-download ability --- data/argoverse.yaml | 27 +++++++++++++++++ data/scripts/get_argoverse.sh | 57 +++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 data/argoverse.yaml create mode 100644 data/scripts/get_argoverse.sh diff --git a/data/argoverse.yaml b/data/argoverse.yaml new file mode 100644 index 000000000000..8328c401da2f --- /dev/null +++ b/data/argoverse.yaml @@ -0,0 +1,27 @@ +# Argoverse dataset (ring-front-center camera) https://www.argoverse.org/data.html +# Train command: python train.py --data argoverse.yaml +# Default dataset location is next to /yolov5: +# /parent_folder +# /argoverse +# /yolov5 + + +# download command/URL (optional) +download: bash data/scripts/get_argoverse.sh + +# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] +train: ../argoverse/Argoverse-1.1/tracking/train/ +val: ../argoverse/Argoverse-1.1/tracking/val/ +test: ../argoverse/Argoverse-1.1/tracking/test/ + +# number of classes +nc: 8 + +# class names +names: [ 'person', 'bicycle', 'car', 'motorcycle', 'bus', 'truck', 'traffic_light', 'stop_sign' ] + +# Print classes +# with open('data/coco.yaml') as f: +# d = yaml.load(f, Loader=yaml.FullLoader) # dict +# for i, x in enumerate(d['names']): +# print(i, x) diff --git a/data/scripts/get_argoverse.sh b/data/scripts/get_argoverse.sh new file mode 100644 index 000000000000..77a4fbe6bb5e --- /dev/null +++ b/data/scripts/get_argoverse.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Argoverse dataset (ring-front-center camera) https://www.argoverse.org/data.html +# Download command: bash data/scripts/get_argoverse.sh +# Train command: python train.py --data argoverse.yaml +# Default dataset location is next to /yolov5: +# /parent_folder +# /argoverse +# /yolov5 + +# Download/unzip images +p='../argoverse/' +d='../argoverse/Argoverse-1.1/' # unzip directory +url=https://s3.amazonaws.com/argoai-argoverse/ +f1='tracking_train1_v1.1.tar.gz' # 43G +f2='tracking_train2_v1.1.tar.gz' # 52G +f3='tracking_train3_v1.1.tar.gz' # 46G +f4='tracking_train4_v1.1.tar.gz' # 9.8G +f2='tracking_val_v1.1.tar.gz' # 57G +f3='tracking_test_v1.1.tar.gz' # 49G (optional) +mkdir $p +mkdir $d +for f in $f1 $f2 $f3 $f4 $f5 $f6; do + echo 'Downloading' $url$f '...' + wget $url$f -O $f && tar -xvzf $f -C $d && rm $f & # download, unzip, remove in background +done +wait # finish background tasks + +old_dir='../argoverse/Argoverse-1.1/argoverse-tracking/' +new_dir='../argoverse/Argoverse-1.1/images/' +mv $old_dir $new_dir + +# move all train files to single folder +mkdir '../argoverse/Argoverse-1.1/images/train' +mv ../argoverse/Argoverse-1.1/images/train1/* '../argoverse/Argoverse-1.1/images/train/' && rm -r ../argoverse/Argoverse-1.1/images/train1/ +mv ../argoverse/Argoverse-1.1/images/train2/* '../argoverse/Argoverse-1.1/images/train/' && rm -r ../argoverse/Argoverse-1.1/images/train2/ +mv ../argoverse/Argoverse-1.1/images/train3/* '../argoverse/Argoverse-1.1/images/train/' && rm -r ../argoverse/Argoverse-1.1/images/train3/ +mv ../argoverse/Argoverse-1.1/images/train4/* '../argoverse/Argoverse-1.1/images/train/' && rm -r ../argoverse/Argoverse-1.1/images/train4/ + +# get all images from ring_front_center camera and delete the rest +for split in "train" "test" "val"; do + for d in ../argoverse/Argoverse-1.1/images/$split/*; do + if [ -d "$d" ]; then + for f in $d/ring_front_center/*; do + mv $f ../argoverse/Argoverse-1.1/images/$split/ && rm -rf $d + done + fi + done +done + +# Download labels +labels_url = 'https://github.com/karthiksharma98/sap-starterkit/releases/download/yolov5-labels/' +labels_file = 'labels.zip' +mkdir '../argoverse/Argoverse-1.1/labels' +wget $labels_url -O $labels_file && unzip $labels_file -d ../argoverse/Argoverse-1.1/labels/ && rm $labels_file + + + From 6e4ade10d9ecc6502192b2d47b556f386498e420 Mon Sep 17 00:00:00 2001 From: Kartikeya Sharma Date: Thu, 4 Mar 2021 15:26:27 -0500 Subject: [PATCH 2/7] bugfix --- data/scripts/get_argoverse.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/scripts/get_argoverse.sh b/data/scripts/get_argoverse.sh index 77a4fbe6bb5e..acc8ba8895c9 100644 --- a/data/scripts/get_argoverse.sh +++ b/data/scripts/get_argoverse.sh @@ -15,8 +15,8 @@ f1='tracking_train1_v1.1.tar.gz' # 43G f2='tracking_train2_v1.1.tar.gz' # 52G f3='tracking_train3_v1.1.tar.gz' # 46G f4='tracking_train4_v1.1.tar.gz' # 9.8G -f2='tracking_val_v1.1.tar.gz' # 57G -f3='tracking_test_v1.1.tar.gz' # 49G (optional) +f5='tracking_val_v1.1.tar.gz' # 57G +f6='tracking_test_v1.1.tar.gz' # 49G (optional) mkdir $p mkdir $d for f in $f1 $f2 $f3 $f4 $f5 $f6; do From 3e722d1559b20c344114b9c846908f1cb6159483 Mon Sep 17 00:00:00 2001 From: Kartikeya Sharma Date: Mon, 8 Mar 2021 19:54:35 -0500 Subject: [PATCH 3/7] add support for Argoverse dataset --- data/argoverse.yaml | 6 +-- data/scripts/get_argoverse.sh | 90 +++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/data/argoverse.yaml b/data/argoverse.yaml index 8328c401da2f..f80197a7d381 100644 --- a/data/argoverse.yaml +++ b/data/argoverse.yaml @@ -10,9 +10,9 @@ download: bash data/scripts/get_argoverse.sh # train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] -train: ../argoverse/Argoverse-1.1/tracking/train/ -val: ../argoverse/Argoverse-1.1/tracking/val/ -test: ../argoverse/Argoverse-1.1/tracking/test/ +train: ../argoverse/Argoverse-1.1/images/train/ +val: ../argoverse/Argoverse-1.1/images/val/ +test: ../argoverse/Argoverse-1.1/images/test/ # number of classes nc: 8 diff --git a/data/scripts/get_argoverse.sh b/data/scripts/get_argoverse.sh index acc8ba8895c9..39ef55aecadc 100644 --- a/data/scripts/get_argoverse.sh +++ b/data/scripts/get_argoverse.sh @@ -8,50 +8,58 @@ # /yolov5 # Download/unzip images -p='../argoverse/' -d='../argoverse/Argoverse-1.1/' # unzip directory -url=https://s3.amazonaws.com/argoai-argoverse/ -f1='tracking_train1_v1.1.tar.gz' # 43G -f2='tracking_train2_v1.1.tar.gz' # 52G -f3='tracking_train3_v1.1.tar.gz' # 46G -f4='tracking_train4_v1.1.tar.gz' # 9.8G -f5='tracking_val_v1.1.tar.gz' # 57G -f6='tracking_test_v1.1.tar.gz' # 49G (optional) -mkdir $p +d='../argoverse/' # unzip directory mkdir $d -for f in $f1 $f2 $f3 $f4 $f5 $f6; do - echo 'Downloading' $url$f '...' - wget $url$f -O $f && tar -xvzf $f -C $d && rm $f & # download, unzip, remove in background -done +url=https://argoverse-hd.s3.us-east-2.amazonaws.com/ +f=Argoverse-HD-Full.zip +wget $url$f -O $f && unzip $f -d $d && rm $f & # download, unzip, remove in background wait # finish background tasks -old_dir='../argoverse/Argoverse-1.1/argoverse-tracking/' -new_dir='../argoverse/Argoverse-1.1/images/' -mv $old_dir $new_dir - -# move all train files to single folder -mkdir '../argoverse/Argoverse-1.1/images/train' -mv ../argoverse/Argoverse-1.1/images/train1/* '../argoverse/Argoverse-1.1/images/train/' && rm -r ../argoverse/Argoverse-1.1/images/train1/ -mv ../argoverse/Argoverse-1.1/images/train2/* '../argoverse/Argoverse-1.1/images/train/' && rm -r ../argoverse/Argoverse-1.1/images/train2/ -mv ../argoverse/Argoverse-1.1/images/train3/* '../argoverse/Argoverse-1.1/images/train/' && rm -r ../argoverse/Argoverse-1.1/images/train3/ -mv ../argoverse/Argoverse-1.1/images/train4/* '../argoverse/Argoverse-1.1/images/train/' && rm -r ../argoverse/Argoverse-1.1/images/train4/ - -# get all images from ring_front_center camera and delete the rest -for split in "train" "test" "val"; do - for d in ../argoverse/Argoverse-1.1/images/$split/*; do - if [ -d "$d" ]; then - for f in $d/ring_front_center/*; do - mv $f ../argoverse/Argoverse-1.1/images/$split/ && rm -rf $d - done - fi - done -done - -# Download labels -labels_url = 'https://github.com/karthiksharma98/sap-starterkit/releases/download/yolov5-labels/' -labels_file = 'labels.zip' -mkdir '../argoverse/Argoverse-1.1/labels' -wget $labels_url -O $labels_file && unzip $labels_file -d ../argoverse/Argoverse-1.1/labels/ && rm $labels_file +cd ../argoverse/Argoverse-1.1/ +ln -s tracking images + +cd ../Argoverse-HD/annotations/ + +python3 - "$@" < Date: Mon, 8 Mar 2021 20:24:49 -0500 Subject: [PATCH 4/7] Refactored code --- data/{argoverse.yaml => argoverse_hd.yaml} | 8 ++++---- data/scripts/{get_argoverse.sh => get_argoverse_hd.sh} | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename data/{argoverse.yaml => argoverse_hd.yaml} (67%) rename data/scripts/{get_argoverse.sh => get_argoverse_hd.sh} (100%) diff --git a/data/argoverse.yaml b/data/argoverse_hd.yaml similarity index 67% rename from data/argoverse.yaml rename to data/argoverse_hd.yaml index f80197a7d381..5b266f83cfcc 100644 --- a/data/argoverse.yaml +++ b/data/argoverse_hd.yaml @@ -1,5 +1,5 @@ -# Argoverse dataset (ring-front-center camera) https://www.argoverse.org/data.html -# Train command: python train.py --data argoverse.yaml +# Argoverse-HD dataset (ring-front-center camera) http://www.cs.cmu.edu/~mengtial/proj/streaming/ +# Train command: python train.py --data argoverse_hd.yaml # Default dataset location is next to /yolov5: # /parent_folder # /argoverse @@ -7,12 +7,12 @@ # download command/URL (optional) -download: bash data/scripts/get_argoverse.sh +download: bash data/scripts/get_argoverse_hd.sh # train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] train: ../argoverse/Argoverse-1.1/images/train/ val: ../argoverse/Argoverse-1.1/images/val/ -test: ../argoverse/Argoverse-1.1/images/test/ +test: ../argoverse/Argoverse-1.1/images/test/ #Submit to: https://eval.ai/web/challenges/challenge-page/800/overview # number of classes nc: 8 diff --git a/data/scripts/get_argoverse.sh b/data/scripts/get_argoverse_hd.sh similarity index 100% rename from data/scripts/get_argoverse.sh rename to data/scripts/get_argoverse_hd.sh From e0a1c92da37447c1d58d0fd3efed3914a350a74b Mon Sep 17 00:00:00 2001 From: Kartikeya Sharma Date: Mon, 8 Mar 2021 20:28:00 -0500 Subject: [PATCH 5/7] renamed to argoverse-HD --- data/argoverse_hd.yaml | 6 ------ data/scripts/get_argoverse_hd.sh | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/data/argoverse_hd.yaml b/data/argoverse_hd.yaml index 5b266f83cfcc..87568bc0a561 100644 --- a/data/argoverse_hd.yaml +++ b/data/argoverse_hd.yaml @@ -19,9 +19,3 @@ nc: 8 # class names names: [ 'person', 'bicycle', 'car', 'motorcycle', 'bus', 'truck', 'traffic_light', 'stop_sign' ] - -# Print classes -# with open('data/coco.yaml') as f: -# d = yaml.load(f, Loader=yaml.FullLoader) # dict -# for i, x in enumerate(d['names']): -# print(i, x) diff --git a/data/scripts/get_argoverse_hd.sh b/data/scripts/get_argoverse_hd.sh index 39ef55aecadc..454f14c5a5ed 100644 --- a/data/scripts/get_argoverse_hd.sh +++ b/data/scripts/get_argoverse_hd.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Argoverse dataset (ring-front-center camera) https://www.argoverse.org/data.html -# Download command: bash data/scripts/get_argoverse.sh -# Train command: python train.py --data argoverse.yaml +# Argoverse-HD dataset (ring-front-center camera) http://www.cs.cmu.edu/~mengtial/proj/streaming/ +# Download command: bash data/scripts/get_argoverse_hd.sh +# Train command: python train.py --data argoverse_hd.yaml # Default dataset location is next to /yolov5: # /parent_folder # /argoverse From 00a1a8c2807665ca61d6811a1a040cf9709fbf51 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 9 Mar 2021 18:43:33 -0800 Subject: [PATCH 6/7] unzip -q and YOLOv5 small cleanup items --- data/scripts/get_argoverse_hd.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/scripts/get_argoverse_hd.sh b/data/scripts/get_argoverse_hd.sh index 454f14c5a5ed..884862db03f5 100644 --- a/data/scripts/get_argoverse_hd.sh +++ b/data/scripts/get_argoverse_hd.sh @@ -12,7 +12,7 @@ d='../argoverse/' # unzip directory mkdir $d url=https://argoverse-hd.s3.us-east-2.amazonaws.com/ f=Argoverse-HD-Full.zip -wget $url$f -O $f && unzip $f -d $d && rm $f & # download, unzip, remove in background +wget $url$f -O $f && unzip -q $f -d $d && rm $f & # download, unzip, remove in background wait # finish background tasks cd ../argoverse/Argoverse-1.1/ @@ -24,7 +24,7 @@ python3 - "$@" < Date: Tue, 9 Mar 2021 18:48:03 -0800 Subject: [PATCH 7/7] add image counts --- data/argoverse_hd.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/argoverse_hd.yaml b/data/argoverse_hd.yaml index 87568bc0a561..df7a9361e769 100644 --- a/data/argoverse_hd.yaml +++ b/data/argoverse_hd.yaml @@ -10,9 +10,9 @@ download: bash data/scripts/get_argoverse_hd.sh # train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] -train: ../argoverse/Argoverse-1.1/images/train/ -val: ../argoverse/Argoverse-1.1/images/val/ -test: ../argoverse/Argoverse-1.1/images/test/ #Submit to: https://eval.ai/web/challenges/challenge-page/800/overview +train: ../argoverse/Argoverse-1.1/images/train/ # 39384 images +val: ../argoverse/Argoverse-1.1/images/val/ # 15062 iamges +test: ../argoverse/Argoverse-1.1/images/test/ # Submit to: https://eval.ai/web/challenges/challenge-page/800/overview # number of classes nc: 8