forked from amazon-science/bias-bounties
-
Notifications
You must be signed in to change notification settings - Fork 0
/
verifier.py
122 lines (93 loc) · 3.65 KB
/
verifier.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
from sklearn import metrics
import numpy as np
def verify(curr_model, test_x, test_y, h, g, alpha=0.001):
"""
Updates the curr_model model object to incorporate the (h,g).
Inputs:
curr_model: model object that is to be updated
holdout_data: data to test the proposed new model on
h_t: new model
g_t: function from X -> {0,1} which returns 1 if x is in identified group and 0 else.
Return: None
"""
# pull the x and y values that belong to g
indices = test_x.apply(g, axis=1) == 1
xs = test_x[indices]
ys = test_y[indices]
# get predicted ys from current model and proposed h
curr_model_preds = xs.apply(curr_model.predict, axis=1)
h_preds = h(xs)
# measure the error of current model and proposed h
curr_model_error = metrics.zero_one_loss(ys, curr_model_preds)
h_error = metrics.zero_one_loss(ys, h_preds)
# determine if (g,h) should be accepted or not
group_weight = sum(indices) / float(len(test_x))
improvement = curr_model_error - h_error
if group_weight * improvement >= alpha:
return True
return False
def is_proposed_group_good(curr_model, test_x, test_y, h, g):
"""
Checks that the group error of g on h isn't worse than g on fun.
Doesn't worry about weight of group
Inputs:
curr_model: model object that is to be updated
holdout_data: data to test the proposed new model on
h_t: new model
g_t: function from X -> {0,1} which returns 1 if x is in identified group and 0 else.
Return: None
"""
# pull the x and y values that belong to g
indices = test_x.apply(g, axis=1) == 1
xs = test_x[indices]
ys = test_y[indices]
# get predicted ys from current model and proposed h
curr_model_preds = xs.apply(curr_model.predict, axis=1)
h_preds = h(xs)
# measure the error of current model and proposed h
curr_model_error = metrics.zero_one_loss(ys, curr_model_preds)
h_error = metrics.zero_one_loss(ys, h_preds)
print("Error of current model on proposed group: %s" % curr_model_error)
print("Error of h trained on proposed group: %s" % h_error)
if h_error >= curr_model_error:
return False
else:
return True
def is_proposed_group_good_csc(curr_model, test_x, test_y, h, g):
"""
Checks that the group error of g on h isn't worse than g on fun.
Doesn't worry about weight of group
Inputs:
curr_model: model object that is to be updated
holdout_data: data to test the proposed new model on
h_t: new model
g_t: function from X -> {0,1} which returns 1 if x is in identified group and 0 else.
Return: None
"""
# pull the x and y values that belong to g
indices = test_x.apply(g, axis=1) == 1
xs = test_x[indices]
ys = test_y[indices]
# get predicted ys from current model and proposed h
curr_model_preds = xs.apply(curr_model.predict, axis=1)
# reshaping to mesh with how h takes inputs
def _h(x):
_x = np.array(x).reshape(1, -1)
return h(_x)[0]
h_preds = xs.apply(_h, axis=1)
# measure the error of current model and proposed h
curr_model_error = metrics.zero_one_loss(ys, curr_model_preds)
h_error = metrics.zero_one_loss(ys, h_preds)
print("Error of current model on proposed group: %s" % curr_model_error)
print("Error of h trained on proposed group: %s" % h_error)
if h_error >= curr_model_error:
return False
else:
return True
def check_group_sizes(test_x, group):
# Returns True if the group has more than 0 elements in test_x
indices = test_x.apply(group, axis=1) == 1
if sum(indices) >= 1:
return True
else:
return False