aboutsummaryrefslogtreecommitdiff
path: root/simclr/models.py
blob: 689e0d3bda08e2042eb36074cee9acff359fc05c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
from pathlib import Path

import sys
import torch
from torch import nn, Tensor
from torchvision.models.resnet import BasicBlock, ResNet

path = str(Path(Path(__file__).parent.absolute()).parent.absolute())
sys.path.insert(0, path)

from supervised.models import CIFARViTTiny


# TODO Make a SimCLR base class

class CIFARSimCLRResNet50(ResNet):
    def __init__(self, hid_dim, out_dim=128, pretrain=True):
        super(CIFARSimCLRResNet50, self).__init__(
            block=BasicBlock, layers=[3, 4, 6, 3], num_classes=hid_dim
        )
        self.pretrain = pretrain
        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        if pretrain:
            self.projector = nn.Sequential(
                nn.Linear(hid_dim, hid_dim),
                nn.ReLU(inplace=True),
                nn.Linear(hid_dim, out_dim),
            )

    def backbone(self, x: Tensor) -> Tensor:
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        x = self.avgpool(x)
        x = torch.flatten(x, 1)
        x = self.fc(x)

        return x

    def forward(self, x: Tensor) -> Tensor:
        h = self.backbone(x)
        if self.pretrain:
            z = self.projector(h)
            return z
        else:
            return h


class CIFARSimCLRViTTiny(CIFARViTTiny):
    def __init__(self, hid_dim, out_dim=128, pretrain=True):
        super().__init__(num_classes=hid_dim)

        self.pretrain = pretrain
        if pretrain:
            self.projector = nn.Sequential(
                nn.Linear(hid_dim, hid_dim),
                nn.ReLU(inplace=True),
                nn.Linear(hid_dim, out_dim),
            )

    def forward(self, x: torch.Tensor) -> Tensor:
        h = super(CIFARSimCLRViTTiny, self).forward(x)
        if self.pretrain:
            z = self.projector(h)
            return z
        else:
            return h


class ImageNetSimCLRResNet50(ResNet):
    def __init__(self, hid_dim, out_dim=128, pretrain=True):
        super(ImageNetSimCLRResNet50, self).__init__(
            block=BasicBlock, layers=[3, 4, 6, 3], num_classes=hid_dim
        )
        self.pretrain = pretrain
        if pretrain:
            self.projector = nn.Sequential(
                nn.Linear(hid_dim, hid_dim),
                nn.ReLU(inplace=True),
                nn.Linear(hid_dim, out_dim),
            )

    def forward(self, x: Tensor) -> Tensor:
        h = super(ImageNetSimCLRResNet50, self).forward(x)
        if self.pretrain:
            z = self.projector(h)
            return z
        else:
            return h