Merge remote-tracking branch 'origin/release_05'

Pilot1/Attn/attn.py

@@ -8,8 +8,6 @@
 import numpy as np
 
 file_path = os.path.dirname(os.path.realpath(__file__))
-lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
-sys.path.append(lib_path2)
 
 import candle
 

Pilot1/Attn/attn_abstention_keras2.py

@@ -8,7 +8,7 @@
 import tensorflow as tf
 
 from tensorflow.keras import backend as K
-from tensorflow.keras.models import model_from_json, model_from_yaml
+from tensorflow.keras.models import model_from_json
 from tensorflow.keras.callbacks import ModelCheckpoint, CSVLogger, ReduceLROnPlateau, EarlyStopping, TensorBoard
 
 from sklearn.utils.class_weight import compute_class_weight
@@ -214,7 +214,7 @@ def run(params):
 
     # Try class weight and abstention classifier
     y_integers = np.argmax(Y_train, axis=1)
-    class_weights = compute_class_weight('balanced', np.unique(y_integers), y_integers)
+    class_weights = compute_class_weight(class_weight='balanced', classes=np.unique(y_integers), y=y_integers)
     d_class_weights = dict(enumerate(class_weights))
 
     print('X_train shape:', X_train.shape)
@@ -444,12 +444,6 @@ def save_and_test_saved_model(params, model, root_fname, nb_classes, alpha, mask
     with open(params['save_path'] + root_fname + '.model.json', "w") as json_file:
         json_file.write(model_json)
 
-    # serialize model to YAML
-    model_yaml = model.to_yaml()
-    with open(params['save_path'] + root_fname + '.model.yaml', "w") as yaml_file:
-
-        yaml_file.write(model_yaml)
-
     # serialize weights to HDF5
     model.save_weights(params['save_path'] + root_fname + '.model.h5')
     print("Saved model to disk")
@@ -460,18 +454,8 @@ def save_and_test_saved_model(params, model, root_fname, nb_classes, alpha, mask
     json_file.close()
     loaded_model_json = model_from_json(loaded_model_json)
 
-    # load yaml and create model
-    yaml_file = open(params['save_path'] + root_fname + '.model.yaml', 'r')
-    loaded_model_yaml = yaml_file.read()
-    yaml_file.close()
-    loaded_model_yaml = model_from_yaml(loaded_model_yaml)
-    # yaml.load(input, Loader=yaml.FullLoader)
-
     # load weights into new model
     loaded_model_json.load_weights(params['save_path'] + root_fname + '.model.h5')
-    # input = params['save_path'] + root_fname +  '.model.h5'
-    # loaded_model_json.load(input, Loader=yaml.FullLoader)
-    # print("Loaded json model from disk")
 
     # evaluate json loaded model on test data
     loaded_model_json.compile(loss=candle.abstention_loss(alpha, mask), optimizer='SGD', metrics=[candle.abstention_acc_metric(nb_classes)])
@@ -480,27 +464,17 @@ def save_and_test_saved_model(params, model, root_fname, nb_classes, alpha, mask
     print('json Validation abstention accuracy:', score_json[1])
     print("json %s: %.2f%%" % (loaded_model_json.metrics_names[1], score_json[1] * 100))
 
-    # load weights into new model
-    loaded_model_yaml.load_weights(params['save_path'] + root_fname + '.model.h5')
-    print("Loaded yaml model from disk")
-    # evaluate yaml loaded model on test data
-    loaded_model_yaml.compile(loss=candle.abstention_loss(alpha, mask), optimizer='SGD', metrics=[candle.abstention_acc_metric(nb_classes)])
-    score_yaml = loaded_model_yaml.evaluate(X_test, Y_test, verbose=0)
-    print('yaml Validation abstention loss:', score_yaml[0])
-    print('yaml Validation abstention accuracy:', score_yaml[1])
-    print("yaml %s: %.2f%%" % (loaded_model_yaml.metrics_names[1], score_yaml[1] * 100))
-
     # predict using loaded yaml model on test and training data
-    predict_yaml_train = loaded_model_yaml.predict(X_train)
-    predict_yaml_test = loaded_model_yaml.predict(X_test)
-    print('Yaml_train_shape:', predict_yaml_train.shape)
-    print('Yaml_test_shape:', predict_yaml_test.shape)
-    predict_yaml_train_classes = np.argmax(predict_yaml_train, axis=1)
-    predict_yaml_test_classes = np.argmax(predict_yaml_test, axis=1)
-    np.savetxt(params['save_path'] + root_fname + '_predict_yaml_train.csv', predict_yaml_train, delimiter=",", fmt="%.3f")
-    np.savetxt(params['save_path'] + root_fname + '_predict_yaml_test.csv', predict_yaml_test, delimiter=",", fmt="%.3f")
-    np.savetxt(params['save_path'] + root_fname + '_predict_yaml_train_classes.csv', predict_yaml_train_classes, delimiter=",", fmt="%d")
-    np.savetxt(params['save_path'] + root_fname + '_predict_yaml_test_classes.csv', predict_yaml_test_classes, delimiter=",", fmt="%d")
+    predict_train = loaded_model_json.predict(X_train)
+    predict_test = loaded_model_json.predict(X_test)
+    print('train_shape:', predict_train.shape)
+    print('test_shape:', predict_test.shape)
+    predict_train_classes = np.argmax(predict_train, axis=1)
+    predict_test_classes = np.argmax(predict_test, axis=1)
+    np.savetxt(params['save_path'] + root_fname + '_predict_train.csv', predict_train, delimiter=",", fmt="%.3f")
+    np.savetxt(params['save_path'] + root_fname + '_predict_test.csv', predict_test, delimiter=",", fmt="%.3f")
+    np.savetxt(params['save_path'] + root_fname + '_predict_train_classes.csv', predict_train_classes, delimiter=",", fmt="%d")
+    np.savetxt(params['save_path'] + root_fname + '_predict_test_classes.csv', predict_test_classes, delimiter=",", fmt="%d")
 
 
 def main():

Pilot1/Attn/attn_baseline_keras2.py

@@ -10,7 +10,7 @@
 from tensorflow.keras import backend as K
 
 from tensorflow.keras.layers import Input, Dense, Dropout, BatchNormalization
-from tensorflow.keras.models import Model, model_from_json, model_from_yaml
+from tensorflow.keras.models import Model, model_from_json
 from tensorflow.keras.utils import to_categorical
 
 from tensorflow.keras.callbacks import Callback, ModelCheckpoint, CSVLogger, ReduceLROnPlateau, EarlyStopping, TensorBoard
@@ -197,7 +197,7 @@ def run(params):
     Y_val = to_categorical(Y_val, nb_classes)
 
     y_integers = np.argmax(Y_train, axis=1)
-    class_weights = compute_class_weight('balanced', np.unique(y_integers), y_integers)
+    class_weights = compute_class_weight(class_weight='balanced', classes=np.unique(y_integers), y=y_integers)
     d_class_weights = dict(enumerate(class_weights))
 
     print('X_train shape:', X_train.shape)
@@ -363,11 +363,6 @@ def save_and_test_saved_model(params, model, root_fname, X_train, X_test, Y_test
     with open(params['save_path'] + root_fname + ".model.json", "w") as json_file:
         json_file.write(model_json)
 
-    # serialize model to YAML
-    model_yaml = model.to_yaml()
-    with open(params['save_path'] + root_fname + ".model.yaml", "w") as yaml_file:
-        yaml_file.write(model_yaml)
-
     # serialize weights to HDF5
     model.save_weights(params['save_path'] + root_fname + ".model.h5")
     print("Saved model to disk")
@@ -378,12 +373,6 @@ def save_and_test_saved_model(params, model, root_fname, X_train, X_test, Y_test
     json_file.close()
     loaded_model_json = model_from_json(loaded_model_json)
 
-    # load yaml and create model
-    yaml_file = open(params['save_path'] + root_fname + '.model.yaml', 'r')
-    loaded_model_yaml = yaml_file.read()
-    yaml_file.close()
-    loaded_model_yaml = model_from_yaml(loaded_model_yaml)
-
     # load weights into new model
     loaded_model_json.load_weights(params['save_path'] + root_fname + ".model.h5")
     print("Loaded json model from disk")
@@ -397,30 +386,19 @@ def save_and_test_saved_model(params, model, root_fname, X_train, X_test, Y_test
 
     print("json %s: %.2f%%" % (loaded_model_json.metrics_names[1], score_json[1] * 100))
 
-    # load weights into new model
-    loaded_model_yaml.load_weights(params['save_path'] + root_fname + ".model.h5")
-    print("Loaded yaml model from disk")
-
-    # evaluate loaded model on test data
-    loaded_model_yaml.compile(loss='binary_crossentropy', optimizer=params['optimizer'], metrics=['accuracy'])
-    score_yaml = loaded_model_yaml.evaluate(X_test, Y_test, verbose=0)
-    print('yaml Validation loss:', score_yaml[0])
-    print('yaml Validation accuracy:', score_yaml[1])
-    print("yaml %s: %.2f%%" % (loaded_model_yaml.metrics_names[1], score_yaml[1] * 100))
-
-    # predict using loaded yaml model on test and training data
-    predict_yaml_train = loaded_model_yaml.predict(X_train)
-    predict_yaml_test = loaded_model_yaml.predict(X_test)
-    print('Yaml_train_shape:', predict_yaml_train.shape)
-    print('Yaml_test_shape:', predict_yaml_test.shape)
-
-    predict_yaml_train_classes = np.argmax(predict_yaml_train, axis=1)
-    predict_yaml_test_classes = np.argmax(predict_yaml_test, axis=1)
-    np.savetxt(params['save_path'] + root_fname + "_predict_yaml_train.csv", predict_yaml_train, delimiter=",", fmt="%.3f")
-    np.savetxt(params['save_path'] + root_fname + "_predict_yaml_test.csv", predict_yaml_test, delimiter=",", fmt="%.3f")
-
-    np.savetxt(params['save_path'] + root_fname + "_predict_yaml_train_classes.csv", predict_yaml_train_classes, delimiter=",", fmt="%d")
-    np.savetxt(params['save_path'] + root_fname + "_predict_yaml_test_classes.csv", predict_yaml_test_classes, delimiter=",", fmt="%d")
+    # predict using loaded model on test and training data
+    predict_train = loaded_model_json.predict(X_train)
+    predict_test = loaded_model_json.predict(X_test)
+    print('train_shape:', predict_train.shape)
+    print('test_shape:', predict_test.shape)
+
+    predict_train_classes = np.argmax(predict_train, axis=1)
+    predict_test_classes = np.argmax(predict_test, axis=1)
+    np.savetxt(params['save_path'] + root_fname + "_predict_train.csv", predict_train, delimiter=",", fmt="%.3f")
+    np.savetxt(params['save_path'] + root_fname + "_predict_test.csv", predict_test, delimiter=",", fmt="%.3f")
+
+    np.savetxt(params['save_path'] + root_fname + "_predict_train_classes.csv", predict_train_classes, delimiter=",", fmt="%d")
+    np.savetxt(params['save_path'] + root_fname + "_predict_test_classes.csv", predict_test_classes, delimiter=",", fmt="%d")
 
 
 def main():

Pilot1/Attn/test.sh

@@ -1,3 +1,4 @@
+set -x
 python attn_baseline_keras2.py
-# python attn_abstention_keras2.py --epochs 1
+python attn_abstention_keras2.py --epochs 1
 python attn_bin_working_jan7_h5.py --in ../../Data/Pilot1/top_21_1fold_001.h5 --ep 1 --save_dir "./save"

Pilot1/Combo/NCI60.py

@@ -2,7 +2,6 @@
 
 import collections
 import os
-import sys
 
 import numpy as np
 import pandas as pd
@@ -14,8 +13,6 @@
 from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler
 
 file_path = os.path.dirname(os.path.realpath(__file__))
-lib_path = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
-sys.path.append(lib_path)
 
 import candle
 
@@ -286,7 +283,7 @@ def load_drug_set_descriptors(drug_set='ALMANAC', ncols=None, scaling='std', add
     # df1['NAME'] = df1['NAME'].map(lambda x: x[4:])
     df1.rename(columns={'NAME': 'Drug'}, inplace=True)
 
-    df2 = df.drop('NAME', 1)
+    df2 = df.drop('NAME', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('dragon7.')
 
@@ -336,7 +333,7 @@ def load_drug_descriptors_new(ncols=None, scaling='std', add_prefix=True):
     # df1['NAME'] = df1['NAME'].map(lambda x: x[4:])
     df1.rename(columns={'NAME': 'Drug'}, inplace=True)
 
-    df2 = df.drop('NAME', 1)
+    df2 = df.drop('NAME', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('dragon7.')
 
@@ -383,7 +380,7 @@ def load_drug_descriptors(ncols=None, scaling='std', add_prefix=True):
     df1['NAME'] = df1['NAME'].map(lambda x: x[4:])
     df1.rename(columns={'NAME': 'NSC'}, inplace=True)
 
-    df2 = df.drop('NAME', 1)
+    df2 = df.drop('NAME', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('dragon7.')
 
@@ -427,7 +424,7 @@ def load_drug_descriptors_old(ncols=None, scaling='std', add_prefix=True):
     df1 = pd.DataFrame(df.loc[:, 'NAME'].astype(int).astype(str))
     df1.rename(columns={'NAME': 'NSC'}, inplace=True)
 
-    df2 = df.drop('NAME', 1)
+    df2 = df.drop('NAME', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('dragon7.')
 
@@ -489,7 +486,7 @@ def load_sample_rnaseq(ncols=None, scaling='std', add_prefix=True, use_landmark_
 
     df1 = df['Sample']
 
-    df2 = df.drop('Sample', 1)
+    df2 = df.drop('Sample', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('rnaseq.')
 
@@ -541,7 +538,7 @@ def load_cell_expression_rnaseq(ncols=None, scaling='std', add_prefix=True, use_
     df1 = df['CELLNAME']
     df1 = df1.map(lambda x: x.replace(':', '.'))
 
-    df2 = df.drop('CELLNAME', 1)
+    df2 = df.drop('CELLNAME', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('rnaseq.')
 
@@ -589,7 +586,7 @@ def load_cell_expression_u133p2(ncols=None, scaling='std', add_prefix=True, use_
     df1 = df['CELLNAME']
     df1 = df1.map(lambda x: x.replace(':', '.'))
 
-    df2 = df.drop('CELLNAME', 1)
+    df2 = df.drop('CELLNAME', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('expr.')
 
@@ -639,7 +636,7 @@ def load_cell_expression_5platform(ncols=None, scaling='std', add_prefix=True, u
     df1 = df['CellLine']
     df1.name = 'CELLNAME'
 
-    df2 = df.drop('CellLine', 1)
+    df2 = df.drop('CellLine', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('expr_5p.')
 
@@ -680,7 +677,7 @@ def load_cell_mirna(ncols=None, scaling='std', add_prefix=True):
     df1 = df['CellLine']
     df1.name = 'CELLNAME'
 
-    df2 = df.drop('CellLine', 1)
+    df2 = df.drop('CellLine', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('mRNA.')
 
@@ -775,7 +772,7 @@ def load_drug_autoencoded_AG(ncols=None, scaling='std', add_prefix=True):
         global_cache[path] = df
 
     df1 = pd.DataFrame(df.loc[:, 'NSC'].astype(int).astype(str))
-    df2 = df.drop('NSC', 1)
+    df2 = df.drop('NSC', axis=1)
     if add_prefix:
         df2 = df2.add_prefix('smiles_latent_AG.')
 

Pilot1/Combo/combo.py

@@ -1,12 +1,9 @@
 from __future__ import print_function
 
 import os
-import sys
 import logging
 
 file_path = os.path.dirname(os.path.realpath(__file__))
-lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
-sys.path.append(lib_path2)
 
 import candle
 

Pilot1/Combo/combo_baseline_keras2.py

@@ -4,6 +4,7 @@
 
 import collections
 import logging
+import sys
 import os
 import threading
 
@@ -46,7 +47,7 @@ def set_up_logger(logfile, verbose):
     fh.setFormatter(logging.Formatter("[%(asctime)s %(process)d] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"))
     fh.setLevel(logging.DEBUG)
 
-    sh = logging.StreamHandler()
+    sh = logging.StreamHandler(sys.stdout)
     sh.setFormatter(logging.Formatter(''))
     sh.setLevel(logging.DEBUG if verbose else logging.INFO)
 

Pilot1/Combo/combo_dose.py

@@ -6,6 +6,7 @@
 import collections
 import logging
 import os
+import sys
 import threading
 
 import numpy as np
@@ -49,7 +50,7 @@ def set_up_logger(logfile, verbose):
     fh.setFormatter(logging.Formatter("[%(asctime)s %(process)d] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"))
     fh.setLevel(logging.DEBUG)
 
-    sh = logging.StreamHandler()
+    sh = logging.StreamHandler(sys.stdout)
     sh.setFormatter(logging.Formatter(''))
     sh.setLevel(logging.DEBUG if verbose else logging.INFO)
 

Pilot1/Combo/test.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 
+set -x
+
 python combo_baseline_keras2.py --use_landmark_genes True --warmup_lr True --reduce_lr True -z 256 --epochs 4
-python infer.py --sample_set NCIPDM --drug_set ALMANAC --use_landmark_genes -m ./save/combo.A=relu.B=256.E=10.O=adam.LR=None.CF=e.DF=d.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.model.h5 -w ./save/combo.A=relu.B=256.E=10.O=adam.LR=None.CF=e.DF=d.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.weights.h5 --epochs 4
+python infer.py --sample_set NCIPDM --drug_set ALMANAC --use_landmark_genes -m ./save/combo.A=relu.B=256.E=4.O=adam.LR=None.CF=e.DF=d.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.model.h5 -w ./save/combo.A=relu.B=256.E=4.O=adam.LR=None.CF=e.DF=d.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.weights.h5 
 
 # Need to revisit combo_dose.py and infer_dose.py
 # python combo_dose.py --use_landmark_genes True -z 256

Pilot1/NT3/abstain_functions.py

@@ -45,6 +45,9 @@
      'nargs': '+',
      'type': int,
      'help': 'list of names corresponding to each task to use'},
+    {'name': 'cf_noise',
+     'type': str,
+     'help': 'input file with cf noise'}
 ]