-
Notifications
You must be signed in to change notification settings - Fork 0
/
inter-rater.py
33 lines (26 loc) · 1.27 KB
/
inter-rater.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
import krippendorff
import pandas as pd
import numpy as np
class KrippendorffAlpha():
def __init__(self, task, to_eval):
self.task = task
self.to_eval = to_eval
def calculate_alpha(self):
df_h1 = pd.read_excel("human-evaluation/human/annotator-1.xlsx", self.task)
df_h2 = pd.read_excel("human-evaluation/human/annotator-2.xlsx", self.task)
a1 = df_h1[["Rank M1", "Rank M2", "Rank M3", "Rank M4"]].to_numpy()
a2 = df_h2[["Rank M1", "Rank M2", "Rank M3", "Rank M4"]].to_numpy()
x = [np.array(a1).flatten(), np.array(a2).flatten()]
df_gpt = None
df_llama = None
if self.to_eval == "gpt" or self.to_eval == "all":
df_gpt = pd.read_csv(f"human-evaluation/gpt/ranks/{self.task}.csv")
a3 = df_gpt[["M1", "M2", "M3", "M4"]].to_numpy()
x.append(np.array(a3).flatten())
if self.to_eval == "llama" or self.to_eval == "all":
df_llama = pd.read_csv(f"human-evaluation/llama/ranks/{self.task}.csv")
a4 = df_llama[["M1", "M2", "M3", "M4"]].to_numpy()
x.append(np.array(a4).flatten())
return krippendorff.alpha(reliability_data=x, level_of_measurement='ordinal')
k = KrippendorffAlpha("summarization", "llama")
print(k.calculate_alpha())