diff --git a/exps/NATS-Bench/draw-fig6.py b/exps/NATS-Bench/draw-fig6.py index 7be9086..432f38a 100644 --- a/exps/NATS-Bench/draw-fig6.py +++ b/exps/NATS-Bench/draw-fig6.py @@ -33,7 +33,7 @@ def fetch_data(root_dir='./output/search', search_space='tss', dataset=None): alg2name['REA'] = 'R-EA-SS3' alg2name['REINFORCE'] = 'REINFORCE-0.01' alg2name['RANDOM'] = 'RANDOM' - alg2name['BOHB'] = 'BOHB' + # alg2name['BOHB'] = 'BOHB' for alg, name in alg2name.items(): alg2path[alg] = os.path.join(ss_dir, dataset, name, 'results.pth') assert os.path.isfile(alg2path[alg]), 'invalid path : {:}'.format(alg2path[alg]) @@ -76,11 +76,18 @@ y_max_s = {('cifar10', 'tss'): 94.5, ('ImageNet16-120', 'tss'): 44, ('ImageNet16-120', 'sss'): 46} +x_axis_s = {('cifar10', 'tss'): 200, + ('cifar10', 'sss'): 200, + ('cifar100', 'tss'): 400, + ('cifar100', 'sss'): 400, + ('ImageNet16-120', 'tss'): 1200, + ('ImageNet16-120', 'sss'): 600} + name2label = {'cifar10': 'CIFAR-10', 'cifar100': 'CIFAR-100', 'ImageNet16-120': 'ImageNet-16-120'} -def visualize_curve(api, vis_save_dir, search_space, max_time): +def visualize_curve(api, vis_save_dir, search_space): vis_save_dir = vis_save_dir.resolve() vis_save_dir.mkdir(parents=True, exist_ok=True) @@ -89,28 +96,36 @@ def visualize_curve(api, vis_save_dir, search_space, max_time): LabelSize, LegendFontsize = 16, 16 def sub_plot_fn(ax, dataset): + xdataset, max_time = dataset.split('-T') alg2data = fetch_data(search_space=search_space, dataset=dataset) alg2accuracies = OrderedDict() total_tickets = 150 - time_tickets = [float(i) / total_tickets * max_time for i in range(total_tickets)] + time_tickets = [float(i) / total_tickets * int(max_time) for i in range(total_tickets)] colors = ['b', 'g', 'c', 'm', 'y'] - ax.set_xlim(0, 200) - ax.set_ylim(y_min_s[(dataset, search_space)], y_max_s[(dataset, search_space)]) + ax.set_xlim(0, x_axis_s[(xdataset, search_space)]) + ax.set_ylim(y_min_s[(xdataset, search_space)], + y_max_s[(xdataset, search_space)]) for idx, (alg, data) in enumerate(alg2data.items()): - print('plot alg : {:}'.format(alg)) + print('{:} plot alg : {:}'.format(time_string(), alg)) accuracies = [] for ticket in time_tickets: - accuracy = query_performance(api, data, dataset, ticket) + accuracy = query_performance(api, data, xdataset, ticket) accuracies.append(accuracy) alg2accuracies[alg] = accuracies ax.plot([x/100 for x in time_tickets], accuracies, c=colors[idx], label='{:}'.format(alg)) ax.set_xlabel('Estimated wall-clock time (1e2 seconds)', fontsize=LabelSize) - ax.set_ylabel('Test accuracy on {:}'.format(name2label[dataset]), fontsize=LabelSize) - ax.set_title('Searching results on {:}'.format(name2label[dataset]), fontsize=LabelSize+4) + ax.set_ylabel('Test accuracy on {:}'.format(name2label[xdataset]), fontsize=LabelSize) + ax.set_title('Searching results on {:}'.format(name2label[xdataset]), fontsize=LabelSize+4) ax.legend(loc=4, fontsize=LegendFontsize) fig, axs = plt.subplots(1, 3, figsize=figsize) - datasets = ['cifar10', 'cifar100', 'ImageNet16-120'] + # datasets = ['cifar10', 'cifar100', 'ImageNet16-120'] + if search_space == 'tss': + datasets = ['cifar10-T20000', 'cifar100-T40000', 'ImageNet16-120-T120000'] + elif search_space == 'sss': + datasets = ['cifar10-T20000', 'cifar100-T40000', 'ImageNet16-120-T60000'] + else: + raise ValueError('Unknown search space: {:}'.format(search_space)) for dataset, ax in zip(datasets, axs): sub_plot_fn(ax, dataset) print('sub-plot {:} on {:} done.'.format(dataset, search_space)) @@ -121,13 +136,12 @@ def visualize_curve(api, vis_save_dir, search_space, max_time): if __name__ == '__main__': - parser = argparse.ArgumentParser(description='NATS-Bench', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--save_dir', type=str, default='output/vis-nas-bench/nas-algos', help='Folder to save checkpoints and log.') parser.add_argument('--search_space', type=str, choices=['tss', 'sss'], help='Choose the search space.') - parser.add_argument('--max_time', type=float, default=20000, help='The maximum time budget.') args = parser.parse_args() save_dir = Path(args.save_dir) api = create(None, args.search_space, fast_mode=True, verbose=False) - visualize_curve(api, save_dir, args.search_space, args.max_time) + visualize_curve(api, save_dir, args.search_space) diff --git a/exps/NATS-algos/bohb.py b/exps/NATS-algos/bohb.py index 0d9479f..0d8af60 100644 --- a/exps/NATS-algos/bohb.py +++ b/exps/NATS-algos/bohb.py @@ -167,7 +167,7 @@ if __name__ == '__main__': parser.add_argument('--rand_seed', type=int, default=-1, help='manual seed') args = parser.parse_args() - api = create(None, args.search_space, fast_mode=True, verbose=False) + api = create(None, args.search_space, fast_mode=False, verbose=False) args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space), '{:}-T{:}'.format(args.dataset, args.time_budget), 'BOHB') diff --git a/exps/NATS-algos/run-all.sh b/exps/NATS-algos/run-all.sh index d845cff..d24f23f 100644 --- a/exps/NATS-algos/run-all.sh +++ b/exps/NATS-algos/run-all.sh @@ -14,55 +14,24 @@ alg_type=$1 if [ "$alg_type" == "mul" ]; then # datasets="cifar10 cifar100 ImageNet16-120" + run_four_algorithms(){ + dataset=$1 + search_space=$2 + time_budget=$3 + python ./exps/NATS-algos/reinforce.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01 + python ./exps/NATS-algos/regularized_ea.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3 + python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} + python ./exps/NATS-algos/bohb.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 + } # The topology search space - dataset="cifar10" - search_space="tss" - time_budget="20000" - python ./exps/NATS-algos/reinforce.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01 - python ./exps/NATS-algos/regularized_ea.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3 - python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} - python ./exps/NATS-algos/bohb.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 - - dataset="cifar100" - search_space="tss" - time_budget="40000" - python ./exps/NATS-algos/reinforce.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01 - python ./exps/NATS-algos/regularized_ea.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3 - python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} - python ./exps/NATS-algos/bohb.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 - - dataset="ImageNet16-120" - search_space="tss" - time_budget="120000" - python ./exps/NATS-algos/reinforce.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01 - python ./exps/NATS-algos/regularized_ea.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3 - python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} - python ./exps/NATS-algos/bohb.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 + run_four_algorithms "cifar10" "tss" "20000" + run_four_algorithms "cifar100" "tss" "40000" + run_four_algorithms "ImageNet16-120" "tss" "120000" # The size search space - dataset="cifar10" - search_space="sss" - time_budget="20000" - python ./exps/NATS-algos/reinforce.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01 - python ./exps/NATS-algos/regularized_ea.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3 - python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} - python ./exps/NATS-algos/bohb.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 - - dataset="cifar100" - search_space="sss" - time_budget="40000" - python ./exps/NATS-algos/reinforce.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01 - python ./exps/NATS-algos/regularized_ea.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3 - python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} - python ./exps/NATS-algos/bohb.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 - - dataset="ImageNet16-120" - search_space="tss" - time_budget="60000" - python ./exps/NATS-algos/reinforce.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01 - python ./exps/NATS-algos/regularized_ea.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3 - python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} - python ./exps/NATS-algos/bohb.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3 + run_four_algorithms "cifar10" "sss" "20000" + run_four_algorithms "cifar100" "sss" "40000" + run_four_algorithms "ImageNet16-120" "sss" "60000" # python exps/experimental/vis-bench-algos.py --search_space tss # python exps/experimental/vis-bench-algos.py --search_space sss else