Fix/config param

compare.py

@@ -0,0 +1,30 @@
+import json
+
+accuracies = {}
+
+try:
+    with open('/tmp/nas-metrics.json') as f:
+        nas = json.load(f)
+        print("Metrics for Neural Architecture Search: ", nas)
+    accuracies['nas_acc'] = [float(i['value']) for i in nas if i['name'] == 'accuracy'][0]
+except RuntimeError as e:
+    print("Error occurred while reading metrics for NAS: ", e)
+
+try:
+    with open('/tmp/hyperop-metrics.json') as f:
+        hyper = json.load(f)
+        print("Metrics for hyper parameter optimization: ", hyper)
+    accuracies['hyper_acc'] = [float(i['value']) for i in hyper if i['name'] == 'accuracy'][0]
+except RuntimeError as e:
+    print("Error occurred while reading metrics for hyperparameter optimization: ", e)
+
+try:
+    with open('/tmp/singlemodel-metrics.json') as f:
+        fm = json.load(f)
+        print("Metrics for model trained with fixed parameters: ", fm)
+    accuracies['fm_acc'] = [float(i['value']) for i in fm if i['name'] == 'accuracy'][0]
+except RuntimeError as e:
+    print("Error occurred while reading metrics for fixed-param model: ", e)
+
+max_acc_name = max(accuracies, key=accuracies.get)
+print("Maximum accuracy was {} for {}".format(max(accuracies.values()), max_acc_name))

examples/nas/enas-tf/search.py

@@ -5,7 +5,7 @@
 from tensorflow.keras.losses import Reduction, SparseCategoricalCrossentropy
 from tensorflow.keras.optimizers import SGD
 
-from nni.nas.tensorflow import enas
+from nni.algorithms.nas.tensorflow import enas
 
 import datasets
 from macro import GeneralNetwork

examples/nas/enas/datasets.py

@@ -3,9 +3,23 @@
 
 from torchvision import transforms
 from torchvision.datasets import CIFAR10
+from torchvision.datasets import ImageFolder
 
+def get_custom_dataset(train_dir, valid_dir):
+    """ Load custom classification dataset using ImageFolder.
+        The train and test directory should have sub directories with name equals to label names.
 
-def get_dataset(cls):
+    """
+    transform = transforms.Compose([
+        transforms.Resize((32, 32)),
+        transforms.ToTensor()
+    ])
+    train_dataset = ImageFolder(root=train_dir, transform=transform)
+    valid_dataset = ImageFolder(root=valid_dir, transform=transform)
+    return train_dataset, valid_dataset
+
+
+def get_dataset(cls, train_dir=None, valid_data=None):
     MEAN = [0.49139968, 0.48215827, 0.44653124]
     STD = [0.24703233, 0.24348505, 0.26158768]
     transf = [
@@ -23,6 +37,8 @@ def get_dataset(cls):
     if cls == "cifar10":
         dataset_train = CIFAR10(root="./data", train=True, download=True, transform=train_transform)
         dataset_valid = CIFAR10(root="./data", train=False, download=True, transform=valid_transform)
+    elif cls == "custom_classification":
+        dataset_train, dataset_valid = get_custom_dataset(train_dir, valid_data)
     else:
         raise NotImplementedError
     return dataset_train, dataset_valid

examples/nas/enas/search.py

@@ -2,17 +2,16 @@
 # Licensed under the MIT license.
 
 import logging
-import time
 from argparse import ArgumentParser
-
+import json
 import torch
 import torch.nn as nn
 
 import datasets
 from macro import GeneralNetwork
 from micro import MicroNetwork
-from nni.nas.pytorch import enas
-from nni.nas.pytorch.callbacks import (ArchitectureCheckpoint,
+from nni.algorithms.nas.pytorch import enas
+from nni.nas.pytorch.callbacks import (ArchitectureCheckpoint, ModelCheckpoint,
                                        LRSchedulerCallback)
 from utils import accuracy, reward_accuracy
 
@@ -21,21 +20,35 @@
 
 if __name__ == "__main__":
     parser = ArgumentParser("enas")
-    parser.add_argument("--batch-size", default=128, type=int)
+    # parser.add_argument("--batch-size", default=128, type=int)
     parser.add_argument("--log-frequency", default=10, type=int)
-    parser.add_argument("--search-for", choices=["macro", "micro"], default="macro")
-    parser.add_argument("--epochs", default=None, type=int, help="Number of epochs (default: macro 310, micro 150)")
-    parser.add_argument("--visualization", default=False, action="store_true")
+    parser.add_argument("--num-classes", default=2, type=int)
+    parser.add_argument("--dataset", default="cifar10", choices=["cifar10", "custom_classification"])
+    # parser.add_argument("--search-for", choices=["macro", "micro"], default="macro")
+    # parser.add_argument("--epochs", default=None, type=int, help="Number of epochs (default: macro 310, micro 150)")
+    parser.add_argument("--visualization", default=True, action="store_true")
+    parser.add_argument("--train-data-dir", default="/home/savan/Documents/train_data", help="train dataset for classification")
+    parser.add_argument("--valid-data-dir", default="/home/savan/Documents/test_data", help="validation dataset for classification")
+    parser.add_argument("--config", default="batch-size=128 \n search-for=macro \n epochs=30")
     args = parser.parse_args()
 
-    dataset_train, dataset_valid = datasets.get_dataset("cifar10")
-    if args.search_for == "macro":
-        model = GeneralNetwork()
-        num_epochs = args.epochs or 310
+    extras = args.config.split("\n")
+    print("nas extras", extras)
+    extras_processed = [i.split("#")[0].replace(" ","") for i in extras if i]
+    print("nas extra processed", extras_processed)
+    config = {i.split('=')[0]:i.split('=')[1] for i in extras_processed}
+    print("nas config", config)
+    config.update(vars(args))
+    args = config
+
+    dataset_train, dataset_valid = datasets.get_dataset(args['dataset'], train_dir=args['train_data_dir'], valid_data=args['valid_data_dir'])
+    if args['search_for'] == "macro":
+        model = GeneralNetwork(num_classes=int(args['num_classes']))
+        num_epochs = int(args['epochs']) or 310
         mutator = None
-    elif args.search_for == "micro":
-        model = MicroNetwork(num_layers=6, out_channels=20, num_nodes=5, dropout_rate=0.1, use_aux_heads=True)
-        num_epochs = args.epochs or 150
+    elif args['search_for'] == "micro":
+        model = MicroNetwork(num_layers=6, out_channels=20, num_nodes=5, dropout_rate=0.1, num_classes=int(args['num_classes']), use_aux_heads=True)
+        num_epochs = int(args['epochs']) or 150
         mutator = enas.EnasMutator(model, tanh_constant=1.1, cell_exit_extra_step=True)
     else:
         raise AssertionError
@@ -49,13 +62,16 @@
                                metrics=accuracy,
                                reward_function=reward_accuracy,
                                optimizer=optimizer,
-                               callbacks=[LRSchedulerCallback(lr_scheduler), ArchitectureCheckpoint("./checkpoints")],
-                               batch_size=args.batch_size,
+                               callbacks=[LRSchedulerCallback(lr_scheduler), ArchitectureCheckpoint("/mnt/output"), ModelCheckpoint("/mnt/output")],
+                               batch_size=int(args['batch_size']),
                                num_epochs=num_epochs,
                                dataset_train=dataset_train,
                                dataset_valid=dataset_valid,
-                               log_frequency=args.log_frequency,
+                               log_frequency=args['log_frequency'],
                                mutator=mutator)
-    if args.visualization:
+    if args['visualization']:
         trainer.enable_visualization()
     trainer.train()
+    metrics = [{'name':'accuracy', 'value':round(trainer.val_model_summary['acc1'].avg, 2)}, {'name':'loss', 'value':round(trainer.val_model_summary['loss'].avg,2)}]
+    with open('/tmp/sys-metrics.json', 'w') as f:
+        json.dump(metrics, f)

examples/trials/mnist-tfv2/config.yml

@@ -12,6 +12,6 @@ tuner:
     classArgs:
         optimize_mode: maximize  # choices: maximize, minimize
 trial:
-  command: python3 mnist.py
+  command: python mnist.py
   codeDir: .
   gpuNum: 0

examples/trials/pytorch-classifier/config.yml

@@ -0,0 +1,21 @@
+authorName: default
+experimentName: pytorch_classifier
+trialConcurrency: 1
+maxExecDuration: 10h
+maxTrialNum: 1
+#choice: local, remote, pai
+trainingServicePlatform: local
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python3 main.py
+  codeDir: .
+  gpuNum: 0

examples/trials/pytorch-classifier/config_aml.yml

@@ -0,0 +1,25 @@
+authorName: default
+experimentName: pytorch_classifier
+trialConcurrency: 1
+maxExecDuration: 10h
+maxTrialNum: 10
+trainingServicePlatform: aml
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python3 main.py
+  codeDir: .
+  image: msranni/nni
+amlConfig:
+  subscriptionId: ${replace_to_your_subscriptionId}
+  resourceGroup: ${replace_to_your_resourceGroup}
+  workspaceName: ${replace_to_your_workspaceName}
+  computeTarget: ${replace_to_your_computeTarget}

examples/trials/pytorch-classifier/config_assessor.yml

@@ -0,0 +1,27 @@
+authorName: default
+experimentName: pytorch_classifier
+trialConcurrency: 1
+maxExecDuration: 10h
+maxTrialNum: 50
+#choice: local, remote
+trainingServicePlatform: local
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+assessor:
+  #choice: Medianstop, Curvefitting
+  builtinAssessorName: Curvefitting
+  classArgs:
+    epoch_num: 20
+    threshold: 0.9
+trial:
+  command: python3 main.py
+  codeDir: .
+  gpuNum: 0

examples/trials/pytorch-classifier/config_frameworkcontroller.yml

@@ -0,0 +1,40 @@
+authorName: default
+experimentName: pytorch_classifier
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+#choice: local, remote, pai, kubeflow
+trainingServicePlatform: frameworkcontroller
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+trial:
+  codeDir: .
+  taskRoles:
+    - name: worker
+      taskNum: 1
+      command: python3 main.py
+      gpuNum: 1
+      cpuNum: 1
+      memoryMB: 8192
+      image: msranni/nni:latest
+      frameworkAttemptCompletionPolicy:
+        minFailedTaskCount: 1
+        minSucceededTaskCount: 1
+frameworkcontrollerConfig:
+  storage: nfs
+  nfs:
+    # Your NFS server IP, like 10.10.10.10
+    server: {your_nfs_server_ip}
+    # Your NFS server export path, like /var/nfs/nni
+    path: {your_nfs_server_export_path}

examples/trials/pytorch-classifier/config_kubeflow.yml

@@ -0,0 +1,32 @@
+authorName: default
+experimentName: pytorch_classifier
+trialConcurrency: 1
+maxExecDuration: 10h
+maxTrialNum: 1
+#choice: local, remote, pai, kubeflow
+trainingServicePlatform: kubeflow
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  codeDir: .
+  worker:
+    replicas: 1
+    command: python3 main.py
+    gpuNum: 0
+    cpuNum: 1
+    memoryMB: 8192
+    image: msranni/nni:latest
+kubeflowConfig:
+  operator: tf-operator
+  apiVersion: v1alpha2
+  storage: nfs
+  nfs:
+    server: 10.10.10.10
+    path: /var/nfs/general

examples/trials/pytorch-classifier/config_pai.yml

@@ -0,0 +1,35 @@
+authorName: default
+experimentName: pytorch_classifier
+trialConcurrency: 1
+maxExecDuration: 10h
+maxTrialNum: 10
+#choice: local, remote, pai
+trainingServicePlatform: pai
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python3 main.py
+  codeDir: .
+  gpuNum: 0
+  cpuNum: 1
+  memoryMB: 8196
+  #The docker image to run nni job on pai
+  image: msranni/nni:latest
+  nniManagerNFSMountPath: {replace_to_your_nfs_mount_path}
+  containerNFSMountPath: {replace_to_your_container_mount_path}
+  paiStorageConfigName: {replace_to_your_storage_config_name}
+paiConfig:
+  #The username to login pai
+  userName: username
+  #The token to login pai
+  token: token
+  #The host of restful server of pai
+  host: 10.10.10.10

examples/trials/pytorch-classifier/config_windows.yml

@@ -0,0 +1,21 @@
+authorName: default
+experimentName: pytorch_classifier
+trialConcurrency: 1
+maxExecDuration: 10h
+maxTrialNum: 10
+#choice: local, remote, pai
+trainingServicePlatform: local
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python main.py
+  codeDir: .
+  gpuNum: 0