forked from CharlieMat/PivotCVAE
-
Notifications
You must be signed in to change notification settings - Fork 0
/
analysis.py
123 lines (114 loc) · 5.13 KB
/
analysis.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
import torch
import torch.nn.functional as F
def get_coverage(slates, N):
"""
Item coverage for give slates
@input:
- slates: list of slates
- N: the total number of items in D
"""
return len(torch.unique(slates)) * 1.0 / N
def get_ILS(slates, embeds, normalize = False):
"""
Intra-List Similarity, diversity can be calculated as (1 - ILS)
@input:
- slates: list of slates
- embeds: nn.Embedding for all possible items
"""
assert slates.shape[1] == 5
# obtain embeddings for all items
emb = F.normalize(embeds(slates), p = 2, dim = 2)
# calculate similarities for each pair of items in each slate
sims = torch.bmm(emb, emb.transpose(1,2)).reshape(slates.shape[0], -1)
# if normalize:
# sims /= torch.max(sims)
# take the average for each slate
sims = (torch.sum(sims, dim = 1) - slates.shape[1]) / (slates.shape[1] * (slates.shape[1] - 1))
return sims
def get_hit_rate(recommendation, candidates, labels):
'''
@input:
- recommendation: (N * #recommendation)
- candidates: (N * #candidates)
- labels: 0-1 label matrix of shape (N * #candidates)
@output:
- hitRateAtK: [hitrate for k in range(#recommendation)]
- hitRateAtK: (N * #recommendation)
'''
hitAtK = torch.zeros(recommendation.shape[0], recommendation.shape[1] + 1, device = recommendation.device).to(torch.uint8)
for k in range(recommendation.shape[1]):
found = (candidates.t() == recommendation[:,k]).to(torch.float)
hit = torch.sum(found.t() * labels, dim = 1)
hitAtK[:,k+1] = (hit > 0) | hitAtK[:,k]
hitAtK = hitAtK[:,1:].to(torch.float)
hitRateAtK = torch.mean(hitAtK,dim=0)
return hitRateAtK, hitAtK
def get_recall(recommendation, candidates, labels):
'''
@input:
- recommendation: (N * #recommendation)
- candidates: (N * #candidates)
- labels: 0-1 label matrix of shape (N * #candidates)
@output:
- recallAtK: [hitrate for k in range(#recommendation)]
- recallAtK: (N * #recommendation)
'''
recallAtK = torch.zeros(recommendation.shape[0], recommendation.shape[1] + 1, device = recommendation.device)
for k in range(recommendation.shape[1]):
found = (candidates.t() == recommendation[:,k]).to(torch.float)
recall = torch.sum(found.t() * labels, dim = 1)
recallAtK[:,k+1] = recall + recallAtK[:,k]
recallAtK = recallAtK[:,1:].to(torch.float)
recallAtK = (recallAtK.t() / (torch.sum(labels, dim=1) + 1e-3)).t()
return recallAtK
def get_rank_matrix(gt_score_matrix, candidate_score_matrix):
'''
@input:
- gt_score_matrix: (N * slateSize * #ground truth items)
- candidate_score_matrix: (N * slateSize * #candidate)
'''
rank = torch.zeros_like(gt_score_matrix, device = gt_score_matrix.device).to(torch.long)
for i in range(gt_score_matrix.shape[-1]):
for j in range(candidate_score_matrix.shape[-1]):
# print(rank[:,:,i])
rank[:,:,i] += (candidate_score_matrix[:,:,j] > gt_score_matrix[:,:,i]).to(torch.long)
# print(rank[:,:,i])
# input()
return rank
def get_positional_recall(gt_score_matrix, candidate_score_matrix, labels):
'''
@input:
- gt_score_matrix: (N * slateSize * #ground truth items)
- candidate_score_matrix: (N * slateSize * #candidate)
- labels: (N * #ground truth items)
'''
precall = torch.zeros_like(candidate_score_matrix, device = candidate_score_matrix.device).to(torch.float)
# rank matrix: (N * slateSize * #ground truth items)
rankMatrix = get_rank_matrix(gt_score_matrix, candidate_score_matrix)
# print(rankMatrix == 0)
# print(precal[:,:,0])
precall[:,:,0] = torch.sum((rankMatrix == 0).transpose(0,1).to(torch.float) * labels.to(torch.float), dim = 2).t()
# print(precal[:,:,0])
for i in range(precall.shape[-1]-1):
found = (rankMatrix == (i+1)).transpose(0,1).to(torch.float) * labels.to(torch.float)
precall[:,:,i+1] = precall[:,:,i] + torch.sum(found, dim = 2).t()
denom = 1.0 / (torch.sum(labels, dim = 1) + 1e-3)
precall = (precall.transpose(0,2) * denom).transpose(0,2)
return precall, torch.mean(precall, dim=0)
def get_positional_mrr(gt_score_matrix, candidate_score_matrix, labels):
'''
@input:
- gt_score_matrix: (N * slateSize * #ground truth items)
- candidate_score_matrix: (N * slateSize * #candidate)
- labels: (N * #ground truth items)
'''
pmrr = torch.zeros_like(candidate_score_matrix, device = candidate_score_matrix.device).to(torch.float)
# rank matrix: (N * slateSize * #ground truth items)
rankMatrix = get_rank_matrix(gt_score_matrix, candidate_score_matrix)
precall[:,:,0] = torch.sum((rankMatrix == 0).transpose(0,1).to(torch.float) * labels.to(torch.float), dim = 2).t()
for i in range(precall.shape[-1]-1):
found = (rankMatrix == (i+1)).transpose(0,1).to(torch.float) * labels.to(torch.float)
precall[:,:,i+1] = precall[:,:,i] + torch.sum(found, dim = 2).t()
denom = 1.0 / (torch.sum(labels, dim = 1) + 1e-3)
precall = (precall.transpose(0,2) * denom).transpose(0,2)
return precall, torch.mean(precall, dim=0)