-
Notifications
You must be signed in to change notification settings - Fork 1
/
matchups.py-txt
152 lines (119 loc) · 4.95 KB
/
matchups.py-txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
# Matches and Elo ratings for optimizers against synthetic objective functions
from timemachines.skaters.optimization import optimal_r
from timemachines import OPTIMIZERS
from timemachines.optimizers.odious import optimizer_name
from timemachines.skaters.evaluation import EVALUATORS
from timemachines import SKATERS
from microprediction import MicroWriter
import time
import random
import json
import os
import numpy as np
from pprint import pprint
try:
from config_private import DOOMSDAY_STOAT as write_key
except:
write_key = os.environ.get('WRITE_KEY')
mw = MicroWriter(write_key=write_key)
print(mw.animal)
MATCHUPS_DIR = './matchups'
def random_json_file_name():
return ''.join([random.choice('abcdef1234567890') for _ in range(12)]) + '.json'
def ensure_dir(path):
if not os.path.exists(path):
os.makedirs(path)
ensure_dir(MATCHUPS_DIR)
def random_optimizer_matchup():
""" Field test two optimizers and assign a win/loss/draw """
white,black = np.random.choice(OPTIMIZERS,2,replace=False)
f = random.choice(SKATERS)
print('Skater is '+f.__name__)
evaluator = random.choice(EVALUATORS)
names = mw.get_stream_names()
n_lagged = 0
n = 2000
while n_lagged<n:
name = random.choice(names)
ys = list(reversed(mw.get_lagged_values(name=name, count=2000)))
n_lagged = len(ys)
n_trials = random.choice([20,40,80])
n_dim = random.choice([2,3])
n_burn = min(n-50,100)
# White plays
white_valid = False
print('White is '+white.__name__)
white_n = n_trials
while white_valid is False:
print('Starting White optimization')
start_time = time.time()
best_val_white, best_x_white, count_white = optimize(f=f, ys=ys, n_trials=white_n, n_dim=n_dim, n_burn=n_burn, optimizer=white, evaluator=evaluator, with_count=True)
white_elapsed = time.time()-start_time
white_valid = count_white < 2*n_trials+10
white_n = int( n_trials / 2 )
report = {'white': optimizer_name(white),
'black': optimizer_name(black),
'epoch_time':time.time(),
'f': f.__name__,
'name': name,
'stream_url': 'https://www.microprediction.org/stream_dashboard.html?stream=' + name.replace('.json', ''),
'white_name':'optimizer_elo_' + optimizer_name(white) + '.json',
'black_name':'optimizer_elo_' + optimizer_name(black) + '.json',
'white_stream_url': 'https://www.microprediction.org/stream_dashboard.html?stream=optimizer_elo_' + optimizer_name(white) + '.json',
'black_stream_url': 'https://www.microprediction.org/stream_dashboard.html?stream=optimizer_elo_' + optimizer_name(
black) + '.json',
'evaluator': evaluator.__name__,
'n': n, 'n_trials': n_trials, 'n_dim': n_dim, 'n_burn': n_burn,
'white_elapsed':white_elapsed,
'best_val_white':best_val_white,
'count_white':count_white,
}
pprint(report)
# Black plays
print('Black is ' + black.__name__)
start_time = time.time()
best_val_black, best_x_black, count_black = optimize(f=f, ys=ys, n_trials=n_trials, n_dim=n_dim, n_burn=n_burn, optimizer=black, evaluator=evaluator, with_count=True)
black_elapsed = time.time()-start_time
valid_white = count_white <= 1.2*n_trials
valid_black = count_black <= 1.2*count_white
tol = 1e-3 * (abs(best_val_white) + abs(best_val_black))
if valid_white and valid_black:
if best_val_white < best_val_black - tol:
points = 1.0
elif best_val_black < best_val_white - tol:
points = 0.0
else:
points = 0.5
else:
points = None
report.update({'points':points,'valid_white':valid_white,'valid_black':valid_black,'white_elapsed':white_elapsed,'black_elapsed':black_elapsed})
return report
def random_optimizer_matchup_and_elo_update():
report = random_optimizer_matchup()
white_name = report['white_name']
black_name = report['black_name']
# Initialize Elo rating if they don't exist
names = mw.get_stream_names()
if len(names)>100:
for name in [white_name,black_name]:
if white_name not in names:
mw.set(name=name,value=2000)
else:
raise Exception('Cannot get streams')
white_elo = float(mw.get_current_value(name=white_name))
black_elo = float(mw.get_current_value(name=black_name))
d = black_elo-white_elo
e = elo_expected(d=d,f=400)
w = report['points']-e
K = 16
white_new_elo = white_elo + K*e
black_new_elo = black_elo - K*e
mw.set(name=white_name,value=white_new_elo)
mw.set(name=black_name,value=black_new_elo)
match_report_log = MATCHUPS_DIR + os.path.sep + random_json_file_name()
with open(match_report_log, 'wt') as fpt:
json.dump(report, fpt)
return report
if __name__=='__main__':
report=random_optimizer_matchup()
pprint(report)