Skip to content

Commit

Permalink
Small pep8 fixes.
Browse files Browse the repository at this point in the history
  • Loading branch information
Brandon Amos committed Oct 13, 2015
1 parent fcc5b54 commit 5893377
Showing 1 changed file with 69 additions and 41 deletions.
110 changes: 69 additions & 41 deletions evaluation/lfw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@

from scipy import arange


def main():
parser = argparse.ArgumentParser()
parser.add_argument('--workDir', type=str, default='reps')
Expand All @@ -46,9 +47,10 @@ def main():

print("Loading embeddings.")
fname = "{}/labels.csv".format(args.workDir)
paths = pd.read_csv(fname, header=None).as_matrix()[:,1]
paths = map(os.path.basename, paths) # Get the filename.
paths = map(lambda path: os.path.splitext(path)[0], paths) # Remove the extension.
paths = pd.read_csv(fname, header=None).as_matrix()[:, 1]
paths = map(os.path.basename, paths) # Get the filename.
# Remove the extension.
paths = map(lambda path: os.path.splitext(path)[0], paths)
fname = "{}/reps.csv".format(args.workDir)
rawEmbeddings = pd.read_csv(fname, header=None).as_matrix()
embeddings = dict(zip(*[paths, rawEmbeddings]))
Expand All @@ -57,6 +59,7 @@ def main():
classifyExp(args.workDir, pairs, embeddings)
plotClassifyExp(args.workDir)


def loadPairs(pairsFname):
print(" + Reading pairs.")
pairs = []
Expand All @@ -67,14 +70,15 @@ def loadPairs(pairsFname):
assert(len(pairs) == 6000)
return np.array(pairs)


def getEmbeddings(pair, embeddings):
if len(pair) == 3:
name1 = "{}_{}".format(pair[0],pair[1].zfill(4))
name2 = "{}_{}".format(pair[0],pair[2].zfill(4))
name1 = "{}_{}".format(pair[0], pair[1].zfill(4))
name2 = "{}_{}".format(pair[0], pair[2].zfill(4))
actual_same = True
elif len(pair) == 4:
name1 = "{}_{}".format(pair[0],pair[1].zfill(4))
name2 = "{}_{}".format(pair[2],pair[3].zfill(4))
name1 = "{}_{}".format(pair[0], pair[1].zfill(4))
name2 = "{}_{}".format(pair[2], pair[3].zfill(4))
actual_same = False
else:
raise Exception(
Expand All @@ -83,39 +87,52 @@ def getEmbeddings(pair, embeddings):
(x1, x2) = (embeddings[name1], embeddings[name2])
return (x1, x2, actual_same)


def writeROC(fname, thresholds, embeddings, pairsTest):
with open(fname, "w") as f:
with open(fname, "w") as f:
f.write("threshold,tp,tn,fp,fn,tpr,fpr\n")
tp=tn=fp=fn=0
tp = tn = fp = fn = 0
for threshold in thresholds:
tp=tn=fp=fn=0
tp = tn = fp = fn = 0
for pair in pairsTest:
(x1, x2, actual_same) = getEmbeddings(pair, embeddings)
diff = x1-x2
diff = x1 - x2
dist = np.dot(diff.T, diff)
predict_same = dist < threshold

if predict_same and actual_same: tp += 1
elif predict_same and not actual_same: fp += 1
elif not predict_same and not actual_same: tn += 1
elif not predict_same and actual_same: fn += 1

if tp+fn == 0: tpr = 0
else: tpr = float(tp)/float(tp+fn)
if fp+tn == 0: fpr = 0
else: fpr = float(fp)/float(fp+tn)
f.write(",".join([str(x) for x in [threshold,tp,tn,fp,fn,tpr,fpr]]))
if predict_same and actual_same:
tp += 1
elif predict_same and not actual_same:
fp += 1
elif not predict_same and not actual_same:
tn += 1
elif not predict_same and actual_same:
fn += 1

if tp + fn == 0:
tpr = 0
else:
tpr = float(tp) / float(tp + fn)
if fp + tn == 0:
fpr = 0
else:
fpr = float(fp) / float(fp + tn)
f.write(",".join([str(x)
for x in [threshold, tp, tn, fp, fn, tpr, fpr]]))
f.write("\n")
if tpr == 1.0 and fpr == 1.0:
# No further improvements.
f.write(",".join([str(x) for x in [4.0,tp,tn,fp,fn,tpr,fpr]]))
f.write(",".join([str(x)
for x in [4.0, tp, tn, fp, fn, tpr, fpr]]))
return


def evalThresholdAccuracy(embeddings, pairs, threshold):
y_true = []; y_predict = []
y_true = []
y_predict = []
for pair in pairs:
(x1, x2, actual_same) = getEmbeddings(pair, embeddings)
diff = x1-x2
diff = x1 - x2
dist = np.dot(diff.T, diff)
predict_same = dist < threshold
y_predict.append(predict_same)
Expand All @@ -126,6 +143,7 @@ def evalThresholdAccuracy(embeddings, pairs, threshold):
accuracy = accuracy_score(y_true, y_predict)
return accuracy


def findBestThreshold(thresholds, embeddings, pairsTrain):
bestThresh = bestThreshAcc = 0
for threshold in thresholds:
Expand All @@ -138,10 +156,11 @@ def findBestThreshold(thresholds, embeddings, pairsTrain):
return bestThresh
return bestThresh


def classifyExp(workDir, pairs, embeddings):
print(" + Computing accuracy.")
folds = KFold(n=6000, n_folds=10, shuffle=False)
thresholds = arange(0,4,0.01)
thresholds = arange(0, 4, 0.01)

if os.path.exists("{}/accuracies.txt".format(workDir)):
print("{}/accuracies.txt already exists. Skipping processing.".format(workDir))
Expand All @@ -153,43 +172,48 @@ def classifyExp(workDir, pairs, embeddings):
fname = "{}/l2-roc.fold-{}.csv".format(workDir, idx)
writeROC(fname, thresholds, embeddings, pairs[test])

bestThresh = findBestThreshold(thresholds, embeddings, pairs[train])
accuracy = evalThresholdAccuracy(embeddings, pairs[test], bestThresh)
bestThresh = findBestThreshold(
thresholds, embeddings, pairs[train])
accuracy = evalThresholdAccuracy(
embeddings, pairs[test], bestThresh)
accuracies.append(accuracy)
f.write('{}, {:0.2f}, {:0.2f}\n'.format(idx, bestThresh, accuracy))
f.write('{}, {:0.2f}, {:0.2f}\n'.format(
idx, bestThresh, accuracy))
f.write('\navg, {:0.4f} +/- {:0.4f}\n'.format(np.mean(accuracies),
np.std(accuracies)))


def getAUC(fprs, tprs):
sortedFprs, sortedTprs = zip(*sorted(zip(*(fprs,tprs))))
sortedFprs, sortedTprs = zip(*sorted(zip(*(fprs, tprs))))
sortedFprs = list(sortedFprs)
sortedTprs = list(sortedTprs)
if sortedFprs[-1] != 1.0:
sortedFprs.append(1.0)
sortedTprs.append(sortedTprs[-1])
return np.trapz(sortedTprs, sortedFprs)


def plotClassifyExp(workDir):
print("Plotting.")

fig, ax = plt.subplots(1,1)
fig, ax = plt.subplots(1, 1)

fs = []
for i in range(10):
rocData = pd.read_csv("{}/l2-roc.fold-{}.csv".format(workDir, i))
fs.append(interp1d(rocData['fpr'], rocData['tpr']))
x = np.linspace(0,1,1000)
x = np.linspace(0, 1, 1000)
fnFoldPlot, = plt.plot(x, fs[-1](x), color='grey', alpha=0.5)

openbrData = pd.read_csv("comparisons/openbr.v1.1.0.DET.csv")
openbrData['Y'] = 1-openbrData['Y']
openbrData['Y'] = 1 - openbrData['Y']
# brPlot = openbrData.plot(x='X', y='Y', legend=True, ax=ax)
brPlot, = plt.plot(openbrData['X'], openbrData['Y'])
brAUC = getAUC(openbrData['X'], openbrData['Y'])

fprs = []; tprs = []
for fpr in np.linspace(0,1,1000):
fprs = []
tprs = []
for fpr in np.linspace(0, 1, 1000):
tpr = 0.0
for f in fs:
v = f(fpr)
Expand All @@ -202,20 +226,24 @@ def plotClassifyExp(workDir):
fnMeanPlot, = plt.plot(fprs, tprs)
fnAUC = getAUC(fprs, tprs)

humanData = pd.read_table("comparisons/kumar_human_crop.txt", header=None, sep=' ')
humanData = pd.read_table(
"comparisons/kumar_human_crop.txt", header=None, sep=' ')
humanPlot, = plt.plot(humanData[1], humanData[0])
humanAUC = getAUC(humanData[1], humanData[0])

deepfaceData = pd.read_table("comparisons/deepface_ensemble.txt", header=None, sep=' ')
deepfaceData = pd.read_table(
"comparisons/deepface_ensemble.txt", header=None, sep=' ')
dfPlot, = plt.plot(deepfaceData[1], deepfaceData[0], '--',
alpha=0.75)
deepfaceAUC = getAUC(deepfaceData[1], deepfaceData[0])

baiduData = pd.read_table("comparisons/BaiduIDLFinal.TPFP", header=None, sep=' ')
baiduData = pd.read_table(
"comparisons/BaiduIDLFinal.TPFP", header=None, sep=' ')
bPlot, = plt.plot(baiduData[1], baiduData[0])
baiduAUC = getAUC(baiduData[1], baiduData[0])

eigData = pd.read_table("comparisons/eigenfaces-original-roc.txt", header=None, sep=' ')
eigData = pd.read_table(
"comparisons/eigenfaces-original-roc.txt", header=None, sep=' ')
eigPlot, = plt.plot(eigData[1], eigData[0])
eigAUC = getAUC(eigData[1], eigData[0])

Expand All @@ -229,17 +257,17 @@ def plotClassifyExp(workDir):
'OpenFace nn4.v1 folds'],
loc='lower right')

plt.plot([0,1], color='k', linestyle=':')
plt.plot([0, 1], color='k', linestyle=':')

plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
# plt.ylim(ymin=0,ymax=1)
plt.xlim(xmin=0,xmax=1)
plt.xlim(xmin=0, xmax=1)

plt.grid(b=True, which='major', color='k', linestyle='-')
plt.grid(b=True, which='minor', color='k', linestyle='-', alpha=0.2)
plt.minorticks_on()
fig.savefig(os.path.join(workDir, "roc.pdf"))

if __name__=='__main__':
if __name__ == '__main__':
main()

0 comments on commit 5893377

Please sign in to comment.