Skip to content

E
elo-rating
Commit 8cb5bfd0 by songxinkai
Options

Enable to run on MLUs, and add some analysis tools.

parent e3cc7fb5
Showing with 253 additions and 77 deletions
ELO_client.py 100644 → 100755
#!/usr/bin/python
# coding=utf-8
import os
from PK import pk
import time
import random
import re
import sys
commdir='share' #共享目录
def cd(path):
......@@ -12,11 +15,9 @@ def choose_game():
dir=os.listdir(cd('comm'))
random.shuffle(dir)
for d in dir:
res=re.match(r'(\d+)_(\d+)\.init',d)
if res:
black=res.group(1)
white=res.group(2)
name=black+'_'+white
if d.endswith('.init'):
black, white = d.replace('.init', '').split('-')[0:2]
name=black+'-'+white
try:
src=cd('comm/'+d)
dst=cd('comm/'+name+'.running')
......@@ -28,21 +29,23 @@ def choose_game():
return None
def main():
time.sleep(random.uniform(3,6))
mlu_id = int(sys.argv[1])
assert(mlu_id < 4 and mlu_id >= 0)
# time.sleep(random.uniform(3,6))
while not os.path.exists(cd('stop.txt')):
if os.path.exists(cd('pause.txt')):
time.sleep(10)
time.sleep(0.1)
continue
game=choose_game()
if game:
winrate=pk(game[0],game[1])
name='comm/'+'_'.join([game[0],game[1],str(winrate)])+'.finish'
winrate=pk(game[0],game[1], 1, mlu_id)
name='comm/'+'-'.join([game[0],game[1],str(winrate)])+'.finish'
os.system('touch '+cd(name))
os.remove(cd('comm/'+game[0]+'_'+game[1]+'.running'))
try:
os.remove(cd('comm/'+game[0]+'-'+game[1]+'.running'))
except:
continue
else:
time.sleep(1)
time.sleep(0.1)
if __name__ == '__main__':
main()
ELo_server.py → ELO_server.py
# coding=utf-8
from elorating import EloRatingSystem
import numpy as np
import random
......@@ -8,13 +9,13 @@ import time
commdir='share' #共享目录
def cd(path):
return os.path.join(commdir,path)
elo=EloRatingSystem(recordSaveFile='gameRecord.txt') #初始化
save_step = 1000
def askYes(str):
ans=''
while True:
ans=input(str)
ans=raw_input(str)
ans=ans.lower()
if ans=='y' or ans=='yes':
return True
......@@ -52,7 +53,7 @@ def main():
commandList=[r'play (\d+)','summary','stop','save']
print('commandList:\n '+'\n '.join(commandList))
ans=input('>>')
ans=raw_input('>>')
while True:
if ans=='stop':
os.system('touch '+cd('stop.txt'))
......@@ -60,18 +61,17 @@ def main():
elif ans=='summary':
elo.summary()
elif ans=='save':
elo.save(saveFile='output.txt',sortByRank=False)
elo.save(saveFile='output.txt',sortByRank=True)
else:
matchRes=re.match(r'play (\d+)',ans)
if matchRes:
num=matchRes.group(1)
play(int(num))
print('commandList:\n '+'\n '.join(commandList))
ans=input('>>')
ans=raw_input('>>')
def play(total_game):
max_game=500
max_game=800
complete_game=0
pause_flag='pause.txt'
if os.path.exists(cd(pause_flag)):
......@@ -95,58 +95,63 @@ def play(total_game):
i=0
while i<num:
black,white=choose()
name=black+'_'+white
name=black+'-'+white
if os.path.exists(cd('comm/'+name+'.init')) or os.path.exists(cd('comm/'+name+'.running')) or os.path.exists(cd('comm/'+name+'.complete')):
continue
os.system('touch '+cd('comm/'+name+'.init'))
i+=1
def read():
def read(dir):
#读取完成的比赛记录
count=0
dir=os.listdir(cd('comm'))
for d in dir:
res=re.match(r'(\d+)_(\d+)_(\d+)\.finish',d)
if res:
black=res.group(1)
white=res.group(2)
winrate=int(res.group(3))
# res=re.match(r'(\d+)-(\d+)-(\d+)\.finish',d)
if d.endswith('.finish'):
black, white, winrate = d.replace(".finish", '').split('-')[0:3]
winrate = int(float(winrate))
# black=res.group(1)
# white=res.group(2)
# winrate=int(res.group(3))
elo.playgame(black,white,winrate)
os.remove(cd(os.path.join('comm',d)))
count+=1
return count
def exist_game():
def exist_game(dir):
count=0
dir=os.listdir(cd('comm'))
for d in dir:
res=re.match(r'(\d+)_(\d+)\.running',d)
if res:
count+=1
continue
res=re.match(r'(\d+)_(\d+)\.init',d)
if res:
count+=1
if d.endswith('running') or d.endswith('init'):
count += 1
continue
# res=re.match(r'(\d+)-(\d+)\.running',d)
# if res:
# count+=1
# continue
# res=re.match(r'(\d+)-(\d+)\.init',d)
# if res:
# count+=1
# continue
return count
idx=0
while complete_game<total_game:
complete_game+=read()
num=exist_game()
create_game(min(max_game-num,total_game-complete_game-num+10))
if idx==5:
print('{}/{}'.format(complete_game,total_game))
idx=0
elo.save(saveFile='output.txt',sortByRank=True)
tstart = time.time()
save_step_id = 0
while True:
dir=os.listdir(cd('comm'))
complete_game += read(dir)
num = exist_game(dir)
if complete_game < total_game:
create_game(min(max_game - num, total_game - complete_game - num + 10))
elif num == 0:
break
if (complete_game >= save_step_id * save_step):
print('{}/{} games completed, taking {}s'.format(complete_game,total_game, time.time() - tstart))
if (save_step_id > 0):
elo.save(saveFile='output.txt',sortByRank=True)
save_step_id = complete_game / save_step + 1
idx+=1
time.sleep(1)
elo.save(saveFile='output.txt',sortByRank=True)
os.system('touch '+cd('pause.txt')) #通知client先暂停行动
if __name__ == '__main__':
main()
PK.py
# coding=utf-8
import random
import time
import os
import commands as cmd
pj_dir = "/share/sxk_mlu_alphago"
data_dir = "/share/data_sxk"
model_dir = "/share/elo-rating/models/"
def get_cmd(num_games, mlu, model1, model2, rollout, c_puct):
cmd_str = ""
cmd_str += "%s " % (os.path.join(pj_dir, "build", "pk"))
cmd_str += "--eval_game_num %d " %(num_games)
cmd_str += "--mlu_list_0 %d "%(mlu)
cmd_str += "--mlu_list_1 %d "%(mlu)
cmd_str += "--config_path_0 %s "%(os.path.join(pj_dir, "etc", "9_pk_offline.conf"))
cmd_str += "--config_path_1 %s "%(os.path.join(pj_dir, "etc", "9_pk_offline.conf"))
cmd_str += "--model_path_0 %s "%(model1)
cmd_str += "--model_path_1 %s "%(model2)
cmd_str += "--rollout_0 %d "%(rollout)
cmd_str += "--rollout_1 %d "%(rollout)
cmd_str += "--rollouts_0 %s "%("-1")
cmd_str += "--rollouts_1 %s "%("-1")
cmd_str += "--c_puct_0 %f "%(c_puct)
cmd_str += "--c_puct_1 %f "%(c_puct)
# TODO: check this
# cmd_str += "--result_path %s "%("-1")
return cmd_str
def pk(model1, model2, num_games = 1, mlu_id = 0):
rollout = 400
c_puct = 0.8
model1 = model_dir + model1
model2 = model_dir + model2
cmd_str = get_cmd(num_games, mlu_id, model1, model2, rollout, c_puct)
s, o = cmd.getstatusoutput(cmd_str)
win = 0
assert(s == 0)
win = 1 - float(o.strip().split(":")[-1])
print("%s vs %s, win_rate = %f" % (model1.split('/')[-1], model2.split('/')[-1], win))
return win
if __name__ == "__main__":
model1 = "R1600_M1600_600000"
model2 = "R400_M400_180000"
res = pk(model1, model2, 100, 3)
#测试用pk函数
def pk(model1,model2,num_games=1):
time.sleep(random.uniform(2,4))
winrate=1-0.5**(abs(int(model1)-int(model2))/1) #假设新模型以一定概率战胜旧模型
res=random.choices([1,0.5,0],weights=[winrate,0,1-winrate],k=1)
if int(model1)>int(model2):
return res[0]
else:
return 1-res[0]
\ No newline at end of file
# def pk(model1,model2,num_games=1):
# time.sleep(random.uniform(2,4))
# winrate=1-0.5**(abs(int(model1)-int(model2))/1) #假设新模型以一定概率战胜旧模型
# res=random.choices([1,0.5,0],weights=[winrate,0,1-winrate],k=1)
# if int(model1)>int(model2):
# return res[0]
# else:
# return 1-res[0]
TEST_start_clients.py
# coding=utf-8
import os
import time
......
analysis.py 0 → 100644
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
model={}
with open('bak/output.txt_bak', 'r') as f:
lines = f.readlines()
for l in lines:
if l.startswith('R'):
l = l.strip()
model[l.split()[0]] = float(l.split()[1])
def getInfo(k):
ks = k.split('_')
r = int(ks[0].lstrip('R'))
m = int(ks[1].lstrip('M'))
i = int(ks[2])
return r, m, i
def cmpKey(k1, k2):
k1 = k1[0]
k2 = k2[0]
r1, m1, i1 = getInfo(k1)
r2, m2, i2 = getInfo(k2)
if (r1 != r2):
return r1 - r2
elif (m1 != m2):
return m1 - m2
else:
return i1 - i2
smodel = model.items()
smodel.sort(cmp = cmpKey)
print(smodel)
def getPlot(mp, r, m):
x = []
y = []
for k, v in mp:
kr, km, ki = getInfo(k)
if km == m and kr == r:
x.append(ki)
y.append(v)
return x, y
def getStr(r, m):
return "R%d_M%d" % (r, m)
routines = [[400, 400],[400, 1600],[1600,400],[1600, 1600]]
colors = ['red', 'blue', 'black', 'yellow', 'green', 'cyan', 'slateblue', 'peru', 'slategrey']
for i in range(len(routines)):
plt.plot(*getPlot(smodel, *routines[i]), color = colors[i], linestyle='--', label=getStr(*routines[i]))
# plt.plot(*getPlot(smodel, *routines[0]), color = colors[0], linestyle='--', label=getStr(*routines[0]))
# plt.plot(*getPlot(smodel, *routines[1]), color = colors[1], linestyle='--', label=getStr(*routines[1]))
# plt.plot(*getPlot(smodel, *routines[2]), color = colors[2], linestyle='--', label=getStr(*routines[2]))
# plt.plot(*getPlot(smodel, *routines[3]), color = colors[3], linestyle='--', label=getStr(*routines[3]))
plt.legend()
plt.show()
plt.savefig('figure.png')
# x_data = ['2011','2012','2013','2014','2015','2016','2017']
# y_data = [58000,60200,63000,71000,84000,90500,107000]
# y_data2 = [52000,54200,51500,58300,56800,59500,62700]
#
# plt.plot(x_data,y_data,color='red',linewidth=2.0,linestyle='--')
# plt.plot(x_data,y_data2,color='blue',linewidth=3.0,linestyle='-.')
# plt.show()
# table = Texttable()
# table.add_rows()
# print(table.draw())
# def getRoutine(k):
# k = k[0]
# return k.split('_')[0] + '_' + k.split('_')[1]
# rm = {}
# for k in smodel:
# krm = getRoutine(k)
# if krm not in rm:
# rm[krm] = list()
# rm[krm].append(k[1])
# print(rm)
# print(max_iter)
clean.sh 0 → 100755
rm -rf share/*
mkdir share/comm
elorating.py
#!/usr/bin/env python3
# coding=utf-8
# -*- coding: utf-8 -*-
#----------------------------------------------
......@@ -20,6 +21,7 @@
import numpy as np
import random
import os
from PK import pk
def main():
models=[str(i) for i in range (0,50000,1000)]
......@@ -39,24 +41,25 @@ def play(models,total_game=5000):
players=elo.getPlayers(sortByRank=True) #选择分数相近的作为比赛对手
black=random.choice(players).name
matchList=elo.getNearPlayer(black)
white = []
if matchList:
white=random.choice(matchList)
else:
continue #找不到对手，谁都打不过，或者谁都打不过
num=1 #比赛局数
num=4 #比赛局数
winrate=pk(black,white,num)
elo.playgame(black,white,winrate,numgames=num) #得到胜率后时用elo.playgame
elo.summary()
#测试用pk函数
def pk(model1,model2,num_games):
winrate=1-0.5**(abs(int(model1)-int(model2))/1000) #假设新模型以一定概率战胜旧模型
res=random.choices([1,0.5,0],weights=[winrate,0,1-winrate],k=1)
if int(model1)>int(model2):
return res[0]
else:
return 1-res[0]
# def pk(model1,model2,num_games):
# winrate=1-0.5**(abs(int(model1)-int(model2))/1000) #假设新模型以一定概率战胜旧模型
# res=random.choices([1,0.5,0],weights=[winrate,0,1-winrate],k=1)
# if int(model1)>int(model2):
# return res[0]
# else:
# return 1-res[0]
#以上是测试代码
......@@ -76,7 +79,7 @@ class EloRatingSystem:
WIN=1
LOSS=0
DRAW=0.5
def __init__(self,base_rating=1500,recordSaveFile='gameRecord.txt'):
def __init__(self,base_rating=1500.0,recordSaveFile='gameRecord.txt'):
self.base_rating=base_rating
self.players={}
......@@ -101,7 +104,6 @@ class EloRatingSystem:
return False
return True
def playgame(self,p1,p2,winrate,numgames=1,online=True):
'''
result [int]:
......@@ -165,7 +167,7 @@ class EloRatingSystem:
player2.rating=0
if online:
#print('{}\'s rating:{}->{} ; {}\'s rating:{}->{}'.format(p1,R1,player1.rating,p2,R2,player2.rating))
# print('winrate: {}. {}\'s rating:{}->{} ; {}\'s rating:{}->{}'.format(winrate, p1,R1,player1.rating,p2,R2,player2.rating))
pass
def readRecords(self,filename):
......@@ -190,17 +192,17 @@ class EloRatingSystem:
players.sort(key=sortKey,reverse=True)
print('{:<20s}{:<10s}{:<10s}{:<10s}{:<10s}'.format('model','rating','games','wins','losses'))
print('{:<60s}{:<10s}{:<10s}{:<10s}{:<10s}'.format('model','rating','games','wins','losses'))
for p in players:
print('{:<20s}{:<10.2f}{:<10d}{:<10d}{:<10d}'.format(str(p.name),p.rating,p.num_game,p.wins,p.losses))
print('{:<60s}{:<10.2f}{:<10d}{:<10d}{:<10d}'.format(str(p.name),p.rating,p.num_game,p.wins,p.losses))
def save(self,saveFile='elo_rating_output.txt',sortByRank=False):
players=self.getPlayers(sortByRank)
with open(saveFile,'w') as f:
f.write('{:<20s}{:<10s}{:<10s}{:<10s}{:<10s}\n'.format('model','rating','games','wins','losses'))
f.write('{:<60s}{:<10s}{:<10s}{:<10s}{:<10s}\n'.format('model','rating','games','wins','losses'))
for p in players:
f.write('{:<20s}{:<10.2f}{:<10d}{:<10d}{:<10d}\n'.format(str(p.name),p.rating,p.num_game,p.wins,p.losses))
f.write('{:<60s}{:<10.2f}{:<10d}{:<10d}{:<10d}\n'.format(str(p.name),p.rating,p.num_game,p.wins,p.losses))
def getPlayers(self,sortByRank=False):
......
getModels.py 0 → 100644
import os
step_delta = 10000
models_file = "../models.txt"
routines = ["R1600_M1600", "R400_M1600", "R400_M400", "R1600_M400", "R1600_M100"]
# pwd = "/share/go9_data/routines/R1600_M1600/test1/caffemodel/received_9b/"
# prefix = "R1600_M1600_"
def getModels(pwd, prefix):
models = []
with open(pwd + "time_record", 'r') as f:
lines = f.readlines()
for l in lines:
l = l.strip().split(':')[0]
if "cambricon" in l:
step_iter = int(l.split('.')[0].split('_')[-1])
if step_iter % step_delta == 0:
m_name = prefix + str(step_iter)
os.system("ln -s %s %s" %(pwd + l, m_name))
models.append(m_name)
with open(models_file, 'a+') as f:
for m in models:
f.write(m + "\n")
def getPwd(routine):
pwd = "/share/go9_data/routines/" + routine + "/test1/caffemodel/received_9b/"
prefix = routine + "_"
return pwd, prefix
if __name__ == "__main__":
for routine in routines:
pwd, prefix = getPwd(routine)
getModels(pwd, prefix)
start_client.sh 0 → 100755
cd /share/elo-rating
./ELO_client.py 0 & ./ELO_client.py 1 & ./ELO_client.py 2 & ./ELO_client.py 3
start_server.sh 0 → 100755
cd /share/elo-rating
python ELO_server.py
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment