update scripts

README.md

@@ -16,7 +16,20 @@ Searching CNNs
 ```
 ```
 
-Train the Searched RNN
+Train the searched CNN on CIFAR
+```
+bash ./scripts-cnn/train-imagenet.sh 0 GDAS_F1 52 14
+bash ./scripts-cnn/train-imagenet.sh 0 GDAS_V1 50 14
+```
+
+Train the searched CNN on ImageNet
+```
+bash ./scripts-cnn/train-imagenet.sh 0 GDAS_F1 52 14
+bash ./scripts-cnn/train-imagenet.sh 0 GDAS_V1 50 14
+```
+
+
+Train the searched RNN
 ```
 bash ./scripts-rnn/train-PTB.sh 0 DARTS_V1
 bash ./scripts-rnn/train-PTB.sh 0 DARTS_V2

exps-cnn/DARTS-Search.py

@@ -1,3 +1,4 @@
+# DARTS First Order, Refer to https://github.com/quark0/darts
 import os, sys, time, glob, random, argparse
 import numpy as np
 from copy import deepcopy

exps-cnn/train_base.py

@@ -13,25 +13,11 @@ if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
 from utils import AverageMeter, time_string, convert_secs2time
 from utils import print_log, obtain_accuracy
 from utils import Cutout, count_parameters_in_MB
-from nas import DARTS_V1, DARTS_V2, NASNet, PNASNet, AmoebaNet, ENASNet
+from nas import model_types as models
-from nas import DMS_V1, DMS_F1, GDAS_CC
-from meta_nas import META_V1, META_V2
 from train_utils import main_procedure
 from train_utils_imagenet import main_procedure_imagenet
 from scheduler import load_config
 
-models = {'DARTS_V1': DARTS_V1,
-          'DARTS_V2': DARTS_V2,
-          'NASNet'  : NASNet,
-          'PNASNet' : PNASNet,
-          'ENASNet' : ENASNet,
-          'DMS_V1'  : DMS_V1,
-          'DMS_F1'  : DMS_F1,
-          'GDAS_CC' : GDAS_CC,
-          'META_V1' : META_V1,
-          'META_V2' : META_V2,
-          'AmoebaNet' : AmoebaNet}
-
 
 parser = argparse.ArgumentParser("cifar")
 parser.add_argument('--data_path',         type=str,   help='Path to dataset')

exps-cnn/train_utils.py

@@ -10,6 +10,7 @@ from utils import time_string, convert_secs2time
 from utils import count_parameters_in_MB
 from utils import Cutout
 from nas import NetworkCIFAR as Network
+from datasets import get_datasets
 
 def obtain_best(accuracies):
   if len(accuracies) == 0: return (0, 0)
@@ -17,38 +18,10 @@ def obtain_best(accuracies):
   s2b = sorted( tops )
   return s2b[-1]
 
+
 def main_procedure(config, dataset, data_path, args, genotype, init_channels, layers, log):
   
-  # Mean + Std
+  train_data, test_data, class_num = get_datasets(dataset, data_path, args.cutout)
-  if dataset == 'cifar10':
-    mean = [x / 255 for x in [125.3, 123.0, 113.9]]
-    std = [x / 255 for x in [63.0, 62.1, 66.7]]
-  elif dataset == 'cifar100':
-    mean = [x / 255 for x in [129.3, 124.1, 112.4]]
-    std = [x / 255 for x in [68.2, 65.4, 70.4]]
-  else:
-    raise TypeError("Unknow dataset : {:}".format(dataset))
-  # Dataset Transformation
-  if dataset == 'cifar10' or dataset == 'cifar100':
-    lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(),
-             transforms.Normalize(mean, std)]
-    if config.cutout > 0 : lists += [Cutout(config.cutout)]
-    train_transform = transforms.Compose(lists)
-    test_transform  = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
-  else:
-    raise TypeError("Unknow dataset : {:}".format(dataset))
-  # Dataset Defination
-  if dataset == 'cifar10':
-    train_data = dset.CIFAR10(data_path, train= True, transform=train_transform, download=True)
-    test_data  = dset.CIFAR10(data_path, train=False, transform=test_transform , download=True)
-    class_num  = 10
-  elif dataset == 'cifar100':
-    train_data = dset.CIFAR100(data_path, train= True, transform=train_transform, download=True)
-    test_data  = dset.CIFAR100(data_path, train=False, transform=test_transform , download=True)
-    class_num  = 100
-  else:
-    raise TypeError("Unknow dataset : {:}".format(dataset))
-
 
   print_log('-------------------------------------- main-procedure', log)
   print_log('config        : {:}'.format(config), log)

exps-cnn/train_utils_imagenet.py

@@ -12,6 +12,7 @@ from utils import count_parameters_in_MB
 from utils import print_FLOPs
 from utils import Cutout
 from nas import NetworkImageNet as Network
+from datasets import get_datasets
 
 
 def obtain_best(accuracies):
@@ -40,30 +41,7 @@ class CrossEntropyLabelSmooth(nn.Module):
 def main_procedure_imagenet(config, data_path, args, genotype, init_channels, layers, log):
   
   # training data and testing data
-  traindir = os.path.join(data_path, 'train')
+  train_data, valid_data, class_num = get_datasets('imagenet-1k', data_path, -1)
-  validdir = os.path.join(data_path, 'val')
-  normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
-  train_data = dset.ImageFolder(
-    traindir,
-    transforms.Compose([
-      transforms.RandomResizedCrop(224),
-      transforms.RandomHorizontalFlip(),
-      transforms.ColorJitter(
-        brightness=0.4,
-        contrast=0.4,
-        saturation=0.4,
-        hue=0.2),
-      transforms.ToTensor(),
-      normalize,
-    ]))
-  valid_data = dset.ImageFolder(
-    validdir,
-    transforms.Compose([
-      transforms.Resize(256),
-      transforms.CenterCrop(224),
-      transforms.ToTensor(),
-      normalize,
-    ]))
 
   train_queue = torch.utils.data.DataLoader(
     train_data, batch_size=config.batch_size, shuffle= True, pin_memory=True, num_workers=args.workers)
@@ -73,7 +51,6 @@ def main_procedure_imagenet(config, data_path, args, genotype, init_channels, la
 
   class_num   = 1000
 
-
   print_log('-------------------------------------- main-procedure', log)
   print_log('config        : {:}'.format(config), log)
   print_log('genotype      : {:}'.format(genotype), log)
@@ -98,8 +75,7 @@ def main_procedure_imagenet(config, data_path, args, genotype, init_channels, la
   criterion_smooth = CrossEntropyLabelSmooth(class_num, config.label_smooth).cuda()
 
 
-  optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay)
+  optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay, nestero=True)
-  #optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay, nestero=True)
   if config.type == 'cosine':
     scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(config.epochs))
   elif config.type == 'steplr':

lib/datasets/__init__.py

@@ -1,3 +1,4 @@
 from .MetaBatchSampler import MetaBatchSampler
 from .TieredImageNet import TieredImageNet
 from .LanguageDataset import Corpus
+from .get_dataset_with_transform import get_datasets

lib/datasets/get_dataset_with_transform.py

@@ -0,0 +1,74 @@
+import os, sys, torch
+import os.path as osp
+import torchvision.datasets as dset
+import torch.backends.cudnn as cudnn
+import torchvision.transforms as transforms
+
+from utils import Cutout
+from .TieredImageNet import TieredImageNet
+
+Dataset2Class = {'cifar10' : 10,
+                 'cifar100': 100,
+                 'tiered'  : -1,
+                 'imagnet-1k'  : 1000,
+                 'imagenet-100': 100}
+
+
+def get_datasets(name, root, cutout):
+
+  # Mean + Std
+  if name == 'cifar10':
+    mean = [x / 255 for x in [125.3, 123.0, 113.9]]
+    std = [x / 255 for x in [63.0, 62.1, 66.7]]
+  elif name == 'cifar100':
+    mean = [x / 255 for x in [129.3, 124.1, 112.4]]
+    std = [x / 255 for x in [68.2, 65.4, 70.4]]
+  elif name == 'tiered':
+    mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
+  elif name == 'imagnet-1k' or name == 'imagenet-100':
+    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
+  else: raise TypeError("Unknow dataset : {:}".format(name))
+
+
+  # Data Argumentation
+  if name == 'cifar10' or name == 'cifar100':
+    lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(),
+             transforms.Normalize(mean, std)]
+    if cutout > 0 : lists += [Cutout(cutout)]
+    train_transform = transforms.Compose(lists)
+    test_transform  = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
+  elif name == 'tiered':
+    lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(80, padding=4), transforms.ToTensor(), transforms.Normalize(mean, std)]
+    if cutout > 0 : lists += [Cutout(cutout)]
+    train_transform = transforms.Compose(lists)
+    test_transform  = transforms.Compose([transforms.CenterCrop(80), transforms.ToTensor(), transforms.Normalize(mean, std)])
+  elif name == 'imagnet-1k' or name == 'imagenet-100':
+    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+    train_transform = transforms.Compose([
+      transforms.RandomResizedCrop(224),
+      transforms.RandomHorizontalFlip(),
+      transforms.ColorJitter(
+        brightness=0.4,
+        contrast=0.4,
+        saturation=0.4,
+        hue=0.2),
+      transforms.ToTensor(),
+      normalize,
+    ])
+    test_transform = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize])
+  else: raise TypeError("Unknow dataset : {:}".format(name))
+    train_data = TieredImageNet(root, 'train-val', train_transform)
+    test_data = None
+  if name == 'cifar10':
+    train_data = dset.CIFAR10(root, train=True, transform=train_transform, download=True)
+    test_data  = dset.CIFAR10(root, train=True, transform=test_transform , download=True)
+  elif name == 'cifar100':
+    train_data = dset.CIFAR100(root, train=True, transform=train_transform, download=True)
+    test_data  = dset.CIFAR100(root, train=True, transform=test_transform , download=True)
+  elif name == 'imagnet-1k' or name == 'imagenet-100':
+    train_data = dset.ImageFolder(osp.join(root, 'train'), train_transform)
+    test_data  = dset.ImageFolder(osp.join(root, 'val'), train_transform)
+  else: raise TypeError("Unknow dataset : {:}".format(name))
+  
+  class_num = Dataset2Class[name]
+  return train_data, test_data, class_num

lib/move.sh

@@ -1,4 +0,0 @@
-rm -rf pytorch
-git clone https://github.com/pytorch/pytorch.git
-cp -r ./pytorch/torch/nn xnn
-rm -rf pytorch

lib/nas/__init__.py

@@ -11,8 +11,6 @@ from .CifarNet import NetworkCIFAR
 from .ImageNet import NetworkImageNet
 
 # genotypes
-from .genotypes import DARTS_V1, DARTS_V2
+from .genotypes import model_types
-from .genotypes import NASNet, PNASNet, AmoebaNet, ENASNet
-from .genotypes import DMS_V1, DMS_F1, GDAS_CC
 
 from .construct_utils import return_alphas_str

lib/nas/genotypes.py

@@ -179,7 +179,7 @@ ENASNet = Genotype(
 DARTS = DARTS_V2
 
 # Search by normal and reduce
-DMS_V1 = Genotype(
+GDAS_V1 = Genotype(
   normal=[('skip_connect', 0, 0.13017432391643524), ('skip_connect', 1, 0.12947972118854523), ('skip_connect', 0, 0.13062666356563568), ('sep_conv_5x5', 2, 0.12980839610099792), ('sep_conv_3x3', 3, 0.12923765182495117), ('skip_connect', 0, 0.12901571393013), ('sep_conv_5x5', 4, 0.12938997149467468), ('sep_conv_3x3', 3, 0.1289220005273819)],
   normal_concat=range(2, 6),
   reduce=[('sep_conv_5x5', 0, 0.12862831354141235), ('sep_conv_3x3', 1, 0.12783904373645782), ('sep_conv_5x5', 2, 0.12725995481014252), ('sep_conv_5x5', 1, 0.12705285847187042), ('dil_conv_5x5', 2, 0.12797553837299347), ('sep_conv_3x3', 1, 0.12737272679805756), ('sep_conv_5x5', 0, 0.12833961844444275), ('sep_conv_5x5', 1, 0.12758426368236542)],
@@ -187,7 +187,7 @@ DMS_V1 = Genotype(
 )
 
 # Search by normal and fixing reduction
-DMS_F1 = Genotype(
+GDAS_F1 = Genotype(
   normal=[('skip_connect', 0, 0.16), ('skip_connect', 1, 0.13), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.16), ('sep_conv_3x3', 2, 0.15)],
   normal_concat=[2, 3, 4, 5],
   reduce=None,
@@ -201,3 +201,13 @@ GDAS_CC = Genotype(
   reduce=None,
   reduce_concat=range(2, 6)
 )
+
+model_types = {'DARTS_V1': DARTS_V1,
+               'DARTS_V2': DARTS_V2,
+               'NASNet'  : NASNet,
+               'PNASNet' : PNASNet, 
+               'AmoebaNet': AmoebaNet,
+               'ENASNet' : ENASNet,
+               'GDAS_V1' : GDAS_V1,
+               'GDAS_F1' : GDAS_F1,
+               'GDAS_CC' : GDAS_CC}

scripts-cnn/train-cifar.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env sh
-if [ "$#" -ne 2 ] ;then
+# bash scripts-cnn/train-cifar.sh 0 GDAS cifar10
+if [ "$#" -ne 3 ] ;then
   echo "Input illegal number of parameters " $#
-  echo "Need 2 parameters for the GPUs, the architecture"
+  echo "Need 3 parameters for the GPUs, the architecture, and the dataset-name"
   exit 1               
 fi 
 if [ "$TORCH_HOME" = "" ]; then
@@ -13,7 +14,7 @@ fi
 
 gpus=$1
 arch=$2
-dataset=cifar100
+dataset=$3
 SAVED=./snapshots/NAS/${arch}-${dataset}-E600
 
 CUDA_VISIBLE_DEVICES=${gpus} python ./exps-nas/train_base.py \

scripts-cnn/train-imagenet.sh

@@ -18,7 +18,7 @@ channels=$3
 layers=$4
 SAVED=./snapshots/NAS/${arch}-${dataset}-C${channels}-L${layers}-E250
 
-CUDA_VISIBLE_DEVICES=${gpus} python ./exps-nas/train_base.py \
+CUDA_VISIBLE_DEVICES=${gpus} python ./exps-cnn/train_base.py \
 	--data_path $TORCH_HOME/ILSVRC2012 \
 	--dataset ${dataset} --arch ${arch} \
 	--save_path ${SAVED} \

scripts-cnn/train-model-simple.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env sh
-if [ "$#" -ne 2 ] ;then
-  echo "Input illegal number of parameters " $#
-  echo "Need 2 parameters for the GPUs and the architecture"
-  exit 1               
-fi 
-if [ "$TORCH_HOME" = "" ]; then
-  echo "Must set TORCH_HOME envoriment variable for data dir saving"
-  exit 1
-else
-  echo "TORCH_HOME : $TORCH_HOME"
-fi
-
-gpus=$1
-arch=$2
-dataset=cifar10
-SAVED=./snapshots/NAS/${arch}-${dataset}-E100
-
-CUDA_VISIBLE_DEVICES=${gpus} python ./exps-nas/train_base.py \
-	--data_path $TORCH_HOME/cifar.python \
-	--dataset ${dataset} --arch ${arch} \
-	--save_path ${SAVED} \
-	--grad_clip 5 \
-	--model_config ./configs/nas-cifar-cos-simple.config \
-	--print_freq 100 --workers 8

scripts-cnn/train-model.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env sh
-if [ "$#" -ne 2 ] ;then
-  echo "Input illegal number of parameters " $#
-  echo "Need 2 parameters for the GPUs, the architecture"
-  exit 1               
-fi 
-if [ "$TORCH_HOME" = "" ]; then
-  echo "Must set TORCH_HOME envoriment variable for data dir saving"
-  exit 1
-else
-  echo "TORCH_HOME : $TORCH_HOME"
-fi
-
-gpus=$1
-arch=$2
-dataset=cifar10
-SAVED=./snapshots/NAS/${arch}-${dataset}-E600
-
-CUDA_VISIBLE_DEVICES=${gpus} python ./exps-nas/train_base.py \
-	--data_path $TORCH_HOME/cifar.python \
-	--dataset ${dataset} --arch ${arch} \
-	--save_path ${SAVED} \
-	--grad_clip 5 \
-	--init_channels 36 --layers 20 \
-	--model_config ./configs/nas-cifar-cos.config \
-	--print_freq 100 --workers 8