blob: 6954cf393e2e31c7130d4a9a8e665799d68d683c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
|
import torch
from torch import nn, Tensor
from torch.nn import functional as F
class InfoNCELoss(nn.Module):
def __init__(self, temp=0.01):
super().__init__()
self.temp = temp
def forward(self, feat1: Tensor, feat2: Tensor) -> tuple[Tensor, Tensor]:
bz = feat1.size(0)
feat1_norm = F.normalize(feat1)
feat2_norm = F.normalize(feat2)
logits = feat1_norm @ feat2_norm.T
pos_logits_mask = torch.eye(bz, dtype=torch.bool)
pos_logits = logits[pos_logits_mask].unsqueeze(-1)
neg_logits = logits[~pos_logits_mask].view(bz, -1)
# Put the positive at first (0-th) and maximize its likelihood
logits = torch.cat([pos_logits, neg_logits], dim=1)
labels = torch.zeros(bz, dtype=torch.long, device=feat1.device)
loss_contra = F.cross_entropy(logits / self.temp, labels)
acc_contra = (logits.argmax(dim=1) == labels).float().mean()
return loss_contra, acc_contra
|
