add cifar10 example for NAS

examples/trials/nas_cifar10/README.md

@@ -2,7 +2,14 @@
  ===	
 
 Now we have an NAS example [NNI-NAS-Example](https://github.com/Crysple/NNI-NAS-Example) run in NNI using NAS interface from our contributors.	
+
+We have included its trial code in this folder, and provided example config files to show how to use PPO tuner to tune the trial code.
+
+> Download data
+
+- `cd data && . download.sh`
+- `tar xzf cifar-10-python.tar.gz && mv cifar-batches cifar10`
 
 Thanks our lovely contributors. 	
 
-And welcome more and more people to join us!
+And welcome more and more people to join us!

examples/trials/nas_cifar10/config_pai_ppo.yml

@@ -0,0 +1,31 @@
+authorName: Unknown
+experimentName: enas_macro
+trialConcurrency: 20
+maxExecDuration: 2400h
+maxTrialNum: 20000
+#choice: local, remote
+trainingServicePlatform: pai
+#choice: true, false
+useAnnotation: true
+multiPhase: false
+versionCheck: false
+nniManagerIp: 0.0.0.0
+tuner:
+  builtinTunerName: PPOTuner
+  classArgs:
+    optimize_mode: maximize
+    trials_per_update: 60
+    epochs_per_update: 20
+    minibatch_size: 6
+trial:
+  command: sh ./macro_cifar10_pai.sh
+  codeDir: ./
+  gpuNum: 1
+  cpuNum: 1
+  memoryMB: 8196
+  image: msranni/nni:latest
+  virtualCluster: nni
+paiConfig:
+  userName: your_account
+  passWord: your_pwd
+  host: 0.0.0.0

examples/trials/nas_cifar10/config_ppo.yml

@@ -0,0 +1,21 @@
+authorName: Unknown
+experimentName: enas_macro
+trialConcurrency: 4
+maxExecDuration: 2400h
+maxTrialNum: 20000
+#choice: local, remote
+trainingServicePlatform: local
+#choice: true, false
+useAnnotation: true
+multiPhase: false
+tuner:
+  builtinTunerName: PPOTuner
+  classArgs:
+    optimize_mode: maximize
+    trials_per_update: 60
+    epochs_per_update: 12
+    minibatch_size: 10
+trial:
+  command: sh ./macro_cifar10.sh
+  codeDir: ./
+  gpuNum: 1

examples/trials/nas_cifar10/data/download.sh

@@ -0,0 +1 @@
+wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz

examples/trials/nas_cifar10/macro_cifar10.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+export PYTHONPATH="$(pwd)"
+
+python3 src/cifar10/nni_child_cifar10.py \
+  --data_format="NCHW" \
+  --search_for="macro" \
+  --reset_output_dir \
+  --data_path="data/cifar10" \
+  --output_dir="outputs" \
+  --train_data_size=45000 \
+  --batch_size=100 \
+  --num_epochs=8 \
+  --log_every=50 \
+  --eval_every_epochs=1 \
+  --child_use_aux_heads \
+  --child_num_layers=12 \
+  --child_out_filters=36 \
+  --child_l2_reg=0.0002 \
+  --child_num_branches=6 \
+  --child_num_cell_layers=5 \
+  --child_keep_prob=0.50 \
+  --child_drop_path_keep_prob=0.60 \
+  --child_lr_cosine \
+  --child_lr_max=0.05 \
+  --child_lr_min=0.001 \
+  --child_lr_T_0=10 \
+  --child_lr_T_mul=2 \
+  --controller_search_whole_channels \
+  --controller_train_every=1 \
+  --controller_num_aggregate=20 \
+  --controller_train_steps=50 \
+  --child_mode="subgraph" \
+  "$@"
+

examples/trials/nas_cifar10/macro_cifar10_pai.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+export PYTHONPATH="$(pwd)"
+
+python3 src/cifar10/nni_child_cifar10.py \
+  --data_format="NCHW" \
+  --search_for="macro" \
+  --reset_output_dir \
+  --data_path="data/cifar10" \
+  --output_dir="outputs" \
+  --train_data_size=45000 \
+  --batch_size=100 \
+  --num_epochs=30 \
+  --log_every=50 \
+  --eval_every_epochs=1 \
+  --child_use_aux_heads \
+  --child_num_layers=12 \
+  --child_out_filters=36 \
+  --child_l2_reg=0.0002 \
+  --child_num_branches=6 \
+  --child_num_cell_layers=5 \
+  --child_keep_prob=0.50 \
+  --child_drop_path_keep_prob=0.60 \
+  --child_lr_cosine \
+  --child_lr_max=0.05 \
+  --child_lr_min=0.001 \
+  --child_lr_T_0=10 \
+  --child_lr_T_mul=2 \
+  --controller_search_whole_channels \
+  --controller_train_every=1 \
+  --controller_num_aggregate=20 \
+  --controller_train_steps=50 \
+  --child_mode="subgraph" \
+  "$@"
+

examples/trials/nas_cifar10/src/cifar10/data_utils.py

@@ -0,0 +1,74 @@
+import os
+import sys
+import pickle
+import numpy as np
+import tensorflow as tf
+
+
+def _read_data(data_path, train_files):
+    """Reads CIFAR-10 format data. Always returns NHWC format.
+
+    Returns:
+        images: np tensor of size [N, H, W, C]
+        labels: np tensor of size [N]
+    """
+    images, labels = [], []
+    for file_name in train_files:
+        print(file_name)
+        full_name = os.path.join(data_path, file_name)
+        with open(full_name, "rb") as finp:
+            data = pickle.load(finp, encoding='latin1')
+            batch_images = data["data"].astype(np.float32) / 255.0
+            batch_labels = np.array(data["labels"], dtype=np.int32)
+            images.append(batch_images)
+            labels.append(batch_labels)
+    images = np.concatenate(images, axis=0)
+    labels = np.concatenate(labels, axis=0)
+    images = np.reshape(images, [-1, 3, 32, 32])
+    images = np.transpose(images, [0, 2, 3, 1])
+
+    return images, labels
+
+
+def read_data(data_path, num_valids=5000):
+    print("-" * 80)
+    print("Reading data")
+
+    images, labels = {}, {}
+
+    train_files = [
+        "data_batch_1",
+        "data_batch_2",
+        "data_batch_3",
+        "data_batch_4",
+        "data_batch_5",
+    ]
+    test_file = [
+        "test_batch",
+    ]
+    images["train"], labels["train"] = _read_data(data_path, train_files)
+
+    if num_valids:
+        images["valid"] = images["train"][-num_valids:]
+        labels["valid"] = labels["train"][-num_valids:]
+
+        images["train"] = images["train"][:-num_valids]
+        labels["train"] = labels["train"][:-num_valids]
+    else:
+        images["valid"], labels["valid"] = None, None
+
+    images["test"], labels["test"] = _read_data(data_path, test_file)
+
+    print("Prepropcess: [subtract mean], [divide std]")
+    mean = np.mean(images["train"], axis=(0, 1, 2), keepdims=True)
+    std = np.std(images["train"], axis=(0, 1, 2), keepdims=True)
+
+    print("mean: {}".format(np.reshape(mean * 255.0, [-1])))
+    print("std: {}".format(np.reshape(std * 255.0, [-1])))
+
+    images["train"] = (images["train"] - mean) / std
+    if num_valids:
+        images["valid"] = (images["valid"] - mean) / std
+    images["test"] = (images["test"] - mean) / std
+
+    return images, labels