Update plots for NATS-Bench

exps/NATS-Bench/draw-fig6.py

@@ -33,7 +33,7 @@ def fetch_data(root_dir='./output/search', search_space='tss', dataset=None):
   alg2name['REA'] = 'R-EA-SS3'
   alg2name['REINFORCE'] = 'REINFORCE-0.01'
   alg2name['RANDOM'] = 'RANDOM'
-  alg2name['BOHB'] = 'BOHB'
+  # alg2name['BOHB'] = 'BOHB'
   for alg, name in alg2name.items():
     alg2path[alg] = os.path.join(ss_dir, dataset, name, 'results.pth')
     assert os.path.isfile(alg2path[alg]), 'invalid path : {:}'.format(alg2path[alg])
@@ -76,11 +76,18 @@ y_max_s = {('cifar10', 'tss'): 94.5,
            ('ImageNet16-120', 'tss'): 44,
            ('ImageNet16-120', 'sss'): 46}
 
+x_axis_s = {('cifar10', 'tss'): 200,
+            ('cifar10', 'sss'): 200,
+            ('cifar100', 'tss'): 400,
+            ('cifar100', 'sss'): 400,
+            ('ImageNet16-120', 'tss'): 1200,
+            ('ImageNet16-120', 'sss'): 600}
+
 name2label = {'cifar10': 'CIFAR-10',
               'cifar100': 'CIFAR-100',
               'ImageNet16-120': 'ImageNet-16-120'}
 
-def visualize_curve(api, vis_save_dir, search_space, max_time):
+def visualize_curve(api, vis_save_dir, search_space):
   vis_save_dir = vis_save_dir.resolve()
   vis_save_dir.mkdir(parents=True, exist_ok=True)
 
@@ -89,28 +96,36 @@ def visualize_curve(api, vis_save_dir, search_space, max_time):
   LabelSize, LegendFontsize = 16, 16
 
   def sub_plot_fn(ax, dataset):
+    xdataset, max_time = dataset.split('-T')
     alg2data = fetch_data(search_space=search_space, dataset=dataset)
     alg2accuracies = OrderedDict()
     total_tickets = 150
-    time_tickets = [float(i) / total_tickets * max_time for i in range(total_tickets)]
+    time_tickets = [float(i) / total_tickets * int(max_time) for i in range(total_tickets)]
     colors = ['b', 'g', 'c', 'm', 'y']
-    ax.set_xlim(0, 200)
+    ax.set_xlim(0, x_axis_s[(xdataset, search_space)])
-    ax.set_ylim(y_min_s[(dataset, search_space)], y_max_s[(dataset, search_space)])
+    ax.set_ylim(y_min_s[(xdataset, search_space)],
+                y_max_s[(xdataset, search_space)])
     for idx, (alg, data) in enumerate(alg2data.items()):
-      print('plot alg : {:}'.format(alg))
+      print('{:} plot alg : {:}'.format(time_string(), alg))
       accuracies = []
       for ticket in time_tickets:
-        accuracy = query_performance(api, data, dataset, ticket)
+        accuracy = query_performance(api, data, xdataset, ticket)
         accuracies.append(accuracy)
       alg2accuracies[alg] = accuracies
       ax.plot([x/100 for x in time_tickets], accuracies, c=colors[idx], label='{:}'.format(alg))
       ax.set_xlabel('Estimated wall-clock time (1e2 seconds)', fontsize=LabelSize)
-      ax.set_ylabel('Test accuracy on {:}'.format(name2label[dataset]), fontsize=LabelSize)
+      ax.set_ylabel('Test accuracy on {:}'.format(name2label[xdataset]), fontsize=LabelSize)
-      ax.set_title('Searching results on {:}'.format(name2label[dataset]), fontsize=LabelSize+4)
+      ax.set_title('Searching results on {:}'.format(name2label[xdataset]), fontsize=LabelSize+4)
     ax.legend(loc=4, fontsize=LegendFontsize)
 
   fig, axs = plt.subplots(1, 3, figsize=figsize)
-  datasets = ['cifar10', 'cifar100', 'ImageNet16-120']
+  # datasets = ['cifar10', 'cifar100', 'ImageNet16-120']
+  if search_space == 'tss':
+    datasets = ['cifar10-T20000', 'cifar100-T40000', 'ImageNet16-120-T120000']
+  elif search_space == 'sss':
+    datasets = ['cifar10-T20000', 'cifar100-T40000', 'ImageNet16-120-T60000']
+  else:
+    raise ValueError('Unknown search space: {:}'.format(search_space))
   for dataset, ax in zip(datasets, axs):
     sub_plot_fn(ax, dataset)
     print('sub-plot {:} on {:} done.'.format(dataset, search_space))
@@ -121,13 +136,12 @@ def visualize_curve(api, vis_save_dir, search_space, max_time):
 
 
 if __name__ == '__main__':
-  parser = argparse.ArgumentParser(description='NATS-Bench', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
   parser.add_argument('--save_dir',     type=str,   default='output/vis-nas-bench/nas-algos', help='Folder to save checkpoints and log.')
   parser.add_argument('--search_space', type=str,   choices=['tss', 'sss'], help='Choose the search space.')
-  parser.add_argument('--max_time',     type=float, default=20000, help='The maximum time budget.')
   args = parser.parse_args()
 
   save_dir = Path(args.save_dir)
 
   api = create(None, args.search_space, fast_mode=True, verbose=False)
-  visualize_curve(api, save_dir, args.search_space, args.max_time)
+  visualize_curve(api, save_dir, args.search_space)

exps/NATS-algos/bohb.py

@@ -167,7 +167,7 @@ if __name__ == '__main__':
   parser.add_argument('--rand_seed',          type=int,  default=-1, help='manual seed')
   args = parser.parse_args()
   
-  api = create(None, args.search_space, fast_mode=True, verbose=False)
+  api = create(None, args.search_space, fast_mode=False, verbose=False)
 
   args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space),
                                '{:}-T{:}'.format(args.dataset, args.time_budget), 'BOHB')

exps/NATS-algos/run-all.sh

@@ -14,55 +14,24 @@ alg_type=$1
 
 if [ "$alg_type" == "mul" ]; then
   # datasets="cifar10 cifar100 ImageNet16-120"
+  run_four_algorithms(){
+    dataset=$1
+    search_space=$2
+    time_budget=$3
+    python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
+    python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
+    python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
+    python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
+  }
   # The topology search space
-  dataset="cifar10"
+  run_four_algorithms "cifar10"        "tss" "20000"
-  search_space="tss"
+  run_four_algorithms "cifar100"       "tss" "40000"
-  time_budget="20000"
+  run_four_algorithms "ImageNet16-120" "tss" "120000"
-  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
-  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
-  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
-  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
-
-  dataset="cifar100"
-  search_space="tss"
-  time_budget="40000"
-  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
-  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
-  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
-  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
-
-  dataset="ImageNet16-120"
-  search_space="tss"
-  time_budget="120000"
-  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
-  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
-  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space}
-  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
 
   # The size search space
-  dataset="cifar10"
+  run_four_algorithms "cifar10"        "sss" "20000"
-  search_space="sss"
+  run_four_algorithms "cifar100"       "sss" "40000"
-  time_budget="20000"
+  run_four_algorithms "ImageNet16-120" "sss" "60000"
-  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
-  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
-  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
-  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
-
-  dataset="cifar100"
-  search_space="sss"
-  time_budget="40000"
-  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
-  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
-  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
-  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
-
-  dataset="ImageNet16-120"
-  search_space="tss"
-  time_budget="60000"
-  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
-  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
-  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
-  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
   # python exps/experimental/vis-bench-algos.py --search_space tss
   # python exps/experimental/vis-bench-algos.py --search_space sss
 else