Reformulate via black

exps/NATS-Bench/draw-correlations.py

@@ -10,81 +10,88 @@ import numpy as np
 from typing import List, Text, Dict, Any
 from shutil import copyfile
 from collections import defaultdict, OrderedDict
-from copy    import deepcopy
+from copy import deepcopy
 from pathlib import Path
 import matplotlib
 import seaborn as sns
-matplotlib.use('agg')
+
+matplotlib.use("agg")
 import matplotlib.pyplot as plt
 import matplotlib.ticker as ticker
 
-lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve()
-if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
+lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve()
+if str(lib_dir) not in sys.path:
+    sys.path.insert(0, str(lib_dir))
 from config_utils import dict2config, load_config
 from nats_bench import create
 from log_utils import time_string
 
 
 def get_valid_test_acc(api, arch, dataset):
-  is_size_space = api.search_space_name == 'size'
-  if dataset == 'cifar10':
-      xinfo = api.get_more_info(arch, dataset=dataset, hp=90 if is_size_space else 200, is_random=False)
-      test_acc = xinfo['test-accuracy']
-      xinfo = api.get_more_info(arch, dataset='cifar10-valid', hp=90 if is_size_space else 200, is_random=False)
-      valid_acc = xinfo['valid-accuracy']
-  else:
-      xinfo = api.get_more_info(arch, dataset=dataset, hp=90 if is_size_space else 200, is_random=False)
-      valid_acc = xinfo['valid-accuracy']
-      test_acc = xinfo['test-accuracy']
-  return valid_acc, test_acc, 'validation = {:.2f}, test = {:.2f}\n'.format(valid_acc, test_acc)
+    is_size_space = api.search_space_name == "size"
+    if dataset == "cifar10":
+        xinfo = api.get_more_info(arch, dataset=dataset, hp=90 if is_size_space else 200, is_random=False)
+        test_acc = xinfo["test-accuracy"]
+        xinfo = api.get_more_info(arch, dataset="cifar10-valid", hp=90 if is_size_space else 200, is_random=False)
+        valid_acc = xinfo["valid-accuracy"]
+    else:
+        xinfo = api.get_more_info(arch, dataset=dataset, hp=90 if is_size_space else 200, is_random=False)
+        valid_acc = xinfo["valid-accuracy"]
+        test_acc = xinfo["test-accuracy"]
+    return valid_acc, test_acc, "validation = {:.2f}, test = {:.2f}\n".format(valid_acc, test_acc)
 
 
 def compute_kendalltau(vectori, vectorj):
-  # indexes = list(range(len(vectori)))
-  # rank_1 = sorted(indexes, key=lambda i: vectori[i])
-  # rank_2 = sorted(indexes, key=lambda i: vectorj[i])
-  # import pdb; pdb.set_trace()
-  coef, p = scipy.stats.kendalltau(vectori, vectorj)
-  return coef
+    # indexes = list(range(len(vectori)))
+    # rank_1 = sorted(indexes, key=lambda i: vectori[i])
+    # rank_2 = sorted(indexes, key=lambda i: vectorj[i])
+    # import pdb; pdb.set_trace()
+    coef, p = scipy.stats.kendalltau(vectori, vectorj)
+    return coef
 
 
 def compute_spearmanr(vectori, vectorj):
-  coef, p = scipy.stats.spearmanr(vectori, vectorj)
-  return coef
+    coef, p = scipy.stats.spearmanr(vectori, vectorj)
+    return coef
 
 
-if __name__ == '__main__':
-  parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument('--save_dir',     type=str,   default='output/vis-nas-bench/nas-algos', help='Folder to save checkpoints and log.')
-  parser.add_argument('--search_space', type=str,   choices=['tss', 'sss'], help='Choose the search space.')
-  args = parser.parse_args()
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--save_dir", type=str, default="output/vis-nas-bench/nas-algos", help="Folder to save checkpoints and log."
+    )
+    parser.add_argument("--search_space", type=str, choices=["tss", "sss"], help="Choose the search space.")
+    args = parser.parse_args()
 
-  save_dir = Path(args.save_dir)
+    save_dir = Path(args.save_dir)
 
-  api = create(None, 'tss', fast_mode=True, verbose=False)
-  indexes = list(range(1, 10000, 300))
-  scores_1 = []
-  scores_2 = []
-  for index in indexes:
-    valid_acc, test_acc, _ = get_valid_test_acc(api, index, 'cifar10')
-    scores_1.append(valid_acc)
-    scores_2.append(test_acc)
-  correlation = compute_kendalltau(scores_1, scores_2)
-  print('The kendall tau correlation of {:} samples : {:}'.format(len(indexes), correlation))
-  correlation = compute_spearmanr(scores_1, scores_2)
-  print('The spearmanr correlation of {:} samples : {:}'.format(len(indexes), correlation))
-  # scores_1 = ['{:.2f}'.format(x) for x in scores_1]
-  # scores_2 = ['{:.2f}'.format(x) for x in scores_2]
-  # print(', '.join(scores_1))
-  # print(', '.join(scores_2))
+    api = create(None, "tss", fast_mode=True, verbose=False)
+    indexes = list(range(1, 10000, 300))
+    scores_1 = []
+    scores_2 = []
+    for index in indexes:
+        valid_acc, test_acc, _ = get_valid_test_acc(api, index, "cifar10")
+        scores_1.append(valid_acc)
+        scores_2.append(test_acc)
+    correlation = compute_kendalltau(scores_1, scores_2)
+    print("The kendall tau correlation of {:} samples : {:}".format(len(indexes), correlation))
+    correlation = compute_spearmanr(scores_1, scores_2)
+    print("The spearmanr correlation of {:} samples : {:}".format(len(indexes), correlation))
+    # scores_1 = ['{:.2f}'.format(x) for x in scores_1]
+    # scores_2 = ['{:.2f}'.format(x) for x in scores_2]
+    # print(', '.join(scores_1))
+    # print(', '.join(scores_2))
 
-  dpi, width, height = 250, 1000, 1000
-  figsize = width / float(dpi), height / float(dpi)
-  LabelSize, LegendFontsize = 14, 14
+    dpi, width, height = 250, 1000, 1000
+    figsize = width / float(dpi), height / float(dpi)
+    LabelSize, LegendFontsize = 14, 14
 
-  fig, ax = plt.subplots(1, 1, figsize=figsize)
-  ax.scatter(scores_1, scores_2 , marker='^', s=0.5, c='tab:green', alpha=0.8)
+    fig, ax = plt.subplots(1, 1, figsize=figsize)
+    ax.scatter(scores_1, scores_2, marker="^", s=0.5, c="tab:green", alpha=0.8)
 
-  save_path = '/Users/xuanyidong/Desktop/test-temp-rank.png'
-  fig.savefig(save_path, dpi=dpi, bbox_inches='tight', format='png')
-  plt.close('all')
+    save_path = "/Users/xuanyidong/Desktop/test-temp-rank.png"
+    fig.savefig(save_path, dpi=dpi, bbox_inches="tight", format="png")
+    plt.close("all")