-
Notifications
You must be signed in to change notification settings - Fork 7
/
f1.py
153 lines (124 loc) · 5.75 KB
/
f1.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
"""
Competition metric, modified from https://www.kaggle.com/anokas/kuzushiji-modified-f-1-score
Python equivalent of the Kuzushiji competition metric (https://www.kaggle.com/c/kuzushiji-recognition/)
Kaggle's backend uses a C# implementation of the same metric. This version is
provided for convenience only; in the event of any discrepancies the C# implementation
is the master version.
Tested on Python 3.6 with numpy 1.16.4 and pandas 0.24.2.
Usage: python f1.py --sub_path [submission.csv] --solution_path [groundtruth.csv]
"""
import multiprocessing
import sys
import numpy as np
import pandas as pd
from functools import partial
def score_page(preds, truth, detection_only=False):
"""
Scores a single page.
Args:
preds: prediction string of labels and center points.
truth: ground truth string of labels and bounding boxes.
Returns:
True/false positive and false negative counts for the page
"""
tp = 0
fp = 0
fn = 0
truth_indices = {
'label': 0,
'X': 1,
'Y': 2,
'Width': 3,
'Height': 4
}
preds_indices = {
'label': 0,
'X': 1,
'Y': 2
}
if pd.isna(truth) and pd.isna(preds):
return {'tp': tp, 'fp': fp, 'fn': fn}
if pd.isna(truth):
fp += len(preds.split(' ')) // len(preds_indices)
return {'tp': tp, 'fp': fp, 'fn': fn}
if pd.isna(preds):
fn += len(truth.split(' ')) // len(truth_indices)
return {'tp': tp, 'fp': fp, 'fn': fn}
truth = truth.split(' ')
if len(truth) % len(truth_indices) != 0:
raise ValueError('Malformed solution string')
truth_label = np.array(truth[truth_indices['label']::len(truth_indices)])
truth_xmin = np.array(truth[truth_indices['X']::len(truth_indices)]).astype(float)
truth_ymin = np.array(truth[truth_indices['Y']::len(truth_indices)]).astype(float)
truth_xmax = truth_xmin + np.array(truth[truth_indices['Width']::len(truth_indices)]).astype(float)
truth_ymax = truth_ymin + np.array(truth[truth_indices['Height']::len(truth_indices)]).astype(float)
preds = preds.split(' ')
if len(preds) % len(preds_indices) != 0:
raise ValueError('Malformed prediction string')
preds_label = np.array(preds[preds_indices['label']::len(preds_indices)])
preds_x = np.array(preds[preds_indices['X']::len(preds_indices)]).astype(float)
preds_y = np.array(preds[preds_indices['Y']::len(preds_indices)]).astype(float)
preds_unused = np.ones(len(preds_label)).astype(bool)
for xmin, xmax, ymin, ymax, label in zip(truth_xmin, truth_xmax, truth_ymin, truth_ymax, truth_label):
# Matching = point inside box & character same & prediction not already used
matching = (xmin < preds_x) & (xmax > preds_x) & (ymin < preds_y) & (ymax > preds_y) & ((preds_label == label) | detection_only) & preds_unused
if matching.sum() == 0:
fn += 1
else:
tp += 1
preds_unused[np.argmax(matching)] = False
fp += preds_unused.sum()
return {'tp': tp, 'fp': fp, 'fn': fn}
def kuzushiji_f1(sub, solution, detection_only=False):
"""
Calculates the competition metric.
Args:
sub: submissions, as a Pandas dataframe
solution: solution, as a Pandas dataframe
detection_only: return what f1 score would be if classification was 100% accurate
Returns:
f1 score
"""
sub = sub.rename(columns={'rowId': 'image_id', 'PredictionString': 'labels'})
solution = solution.rename(columns={'rowId': 'image_id', 'PredictionString': 'labels'})
if not all(sub['image_id'].values == solution['image_id'].values):
raise ValueError("Submission image id codes don't match solution")
pool = multiprocessing.Pool()
results = pool.starmap(partial(score_page, detection_only=detection_only), zip(sub['labels'].values, solution['labels'].values))
pool.close()
pool.join()
tp = sum([x['tp'] for x in results])
fp = sum([x['fp'] for x in results])
fn = sum([x['fn'] for x in results])
if (tp + fp) == 0 or (tp + fn) == 0:
return 0
precision = tp / (tp + fp)
recall = tp / (tp + fn)
if precision > 0 and recall > 0:
f1 = (2 * precision * recall) / (precision + recall)
else:
f1 = 0
return f1
# This function takes in a filename of an image, and the labels in the string format given in train.csv, and returns an image containing the bounding boxes and characters annotated
def visualize_training_data(image_fn, preds, labels):
print("PO")
return "POOO"
from PIL import Image, ImageDraw, ImageFont
font = ImageFont.truetype('./NotoSansCJKjp-Regular.otf', fontsize, encoding='utf-8')
# Convert annotation string to array
labels = np.array(labels.split(' ')).reshape(-1, 5)
# Read image
imsource = Image.open(image_fn).convert('RGBA')
bbox_canvas = Image.new('RGBA', imsource.size)
char_canvas = Image.new('RGBA', imsource.size)
bbox_draw = ImageDraw.Draw(bbox_canvas) # Separate canvases for boxes and chars so a box doesn't cut off a character
char_draw = ImageDraw.Draw(char_canvas)
for codepoint, x, y, w, h in labels:
x, y, w, h = int(x), int(y), int(w), int(h)
char = unicode_map[codepoint] # Convert codepoint to actual unicode character
# Draw bounding box around character, and unicode character next to it
bbox_draw.rectangle((x, y, x+w, y+h), fill=(255, 255, 255, 0), outline=(255, 0, 0, 255))
char_draw.text((x + w + fontsize/4, y + h/2 - fontsize), char, fill=(0, 0, 255, 255), font=font)
imsource = Image.alpha_composite(Image.alpha_composite(imsource, bbox_canvas), char_canvas)
imsource = imsource.convert("RGB") # Remove alpha for saving in jpg format.
return np.asarray(imsource)