From 6f278a962d70e90ac530f5723e198c7c356e8297 Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Sat, 9 Jan 2021 20:53:29 +0800 Subject: Fix NaN when separate sum is zero --- utils/triplet_loss.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/triplet_loss.py b/utils/triplet_loss.py index 1d63a0e..1899cc9 100644 --- a/utils/triplet_loss.py +++ b/utils/triplet_loss.py @@ -30,8 +30,9 @@ class BatchAllTripletLoss(nn.Module): all_loss = F.relu(self.margin + positive_negative_dist).view(p, -1) # Non-zero parted mean - parted_loss_mean = all_loss.sum(1) / (all_loss != 0).sum(1) - parted_loss_mean[parted_loss_mean == float('Inf')] = 0 + non_zero_counts = (all_loss != 0).sum(1) + parted_loss_mean = all_loss.sum(1) / non_zero_counts + parted_loss_mean[non_zero_counts == 0] = 0 loss = parted_loss_mean.sum() return loss -- cgit v1.2.3 From 62f14a6ef0d902b9ffd4e57427a40663e2e5c2ad Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Sat, 9 Jan 2021 21:02:34 +0800 Subject: Change auto-encoder input in evaluation --- models/auto_encoder.py | 2 +- models/rgb_part_net.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/models/auto_encoder.py b/models/auto_encoder.py index 7c1f7ef..5e7558b 100644 --- a/models/auto_encoder.py +++ b/models/auto_encoder.py @@ -128,7 +128,7 @@ class AutoEncoder(nn.Module): BasicLinear(f_c_dim, num_class) ) - def forward(self, x_c1_t1, x_c1_t2, x_c2_t2, y=None): + def forward(self, x_c1_t2, x_c1_t1=None, x_c2_t2=None, y=None): # x_c1_t2 is the frame for later module (f_a_c1_t2, f_c_c1_t2, f_p_c1_t2) = self.encoder(x_c1_t2) diff --git a/models/rgb_part_net.py b/models/rgb_part_net.py index 3037da0..456695d 100644 --- a/models/rgb_part_net.py +++ b/models/rgb_part_net.py @@ -51,10 +51,12 @@ class RGBPartNet(nn.Module): def fc(self, x): return x @ self.fc_mat - def forward(self, x_c1, x_c2, y=None): + def forward(self, x_c1, x_c2=None, y=None): # Step 0: Swap batch_size and time dimensions for next step # n, t, c, h, w - x_c1, x_c2 = x_c1.transpose(0, 1), x_c2.transpose(0, 1) + x_c1 = x_c1.transpose(0, 1) + if self.training: + x_c2 = x_c2.transpose(0, 1) # Step 1: Disentanglement # t, n, c, h, w @@ -84,7 +86,7 @@ class RGBPartNet(nn.Module): else: return x - def _disentangle(self, x_c1, x_c2, y): + def _disentangle(self, x_c1, x_c2=None, y=None): num_frames = len(x_c1) # Decoded canonical features and Pose images x_c_c1, x_p_c1 = [], [] @@ -94,7 +96,7 @@ class RGBPartNet(nn.Module): xrecon_loss, cano_cons_loss = [], [] for t2 in range(num_frames): t1 = random.randrange(num_frames) - output = self.ae(x_c1[t1], x_c1[t2], x_c2[t2], y) + output = self.ae(x_c1[t2], x_c1[t1], x_c2[t2], y) (x_c1_t2, f_p_t2, losses) = output # Decoded features or image @@ -127,8 +129,7 @@ class RGBPartNet(nn.Module): else: # evaluating for t2 in range(num_frames): - t1 = random.randrange(num_frames) - x_c1_t2 = self.ae(x_c1[t1], x_c1[t2], x_c2[t2]) + x_c1_t2 = self.ae(x_c1[t2]) # Decoded features or image (x_c_c1_t2, x_p_c1_t2) = x_c1_t2 # Canonical Features for HPM -- cgit v1.2.3 From de911a563fc503114559d7e0e7f710db090cec0d Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Sat, 9 Jan 2021 21:54:10 +0800 Subject: Add prototype predict function --- models/model.py | 59 +++++++++++++++++++++++++++++++++++++++++++++----- models/rgb_part_net.py | 2 +- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/models/model.py b/models/model.py index 54f3441..ba29ede 100644 --- a/models/model.py +++ b/models/model.py @@ -8,6 +8,7 @@ import torch.optim as optim from torch.utils.data import DataLoader from torch.utils.data.dataloader import default_collate from torch.utils.tensorboard import SummaryWriter +from tqdm import tqdm from models.rgb_part_net import RGBPartNet from utils.configuration import DataloaderConfiguration, \ @@ -88,10 +89,7 @@ class Model: self.scheduler = optim.lr_scheduler.StepLR(self.optimizer, 500, 0.9) self.writer = SummaryWriter(self.log_name) - if not self.disable_acc: - if torch.cuda.device_count() > 1: - self.rgb_pn = nn.DataParallel(self.rgb_pn) - self.rgb_pn = self.rgb_pn.to(self.device) + self._accelerate() self.rgb_pn.train() # Init weights at first iter @@ -143,6 +141,57 @@ class Model: self.writer.close() break + def _accelerate(self): + if not self.disable_acc: + if torch.cuda.device_count() > 1: + self.rgb_pn = nn.DataParallel(self.rgb_pn) + self.rgb_pn = self.rgb_pn.to(self.device) + + def predict( + self, + iter_: int, + dataset_config: DatasetConfiguration, + dataloader_config: DataloaderConfiguration, + ): + self.is_train = False + dataset = self._parse_dataset_config(dataset_config) + dataloader = self._parse_dataloader_config(dataset, dataloader_config) + hp = self.hp.copy() + _, _ = hp.pop('lr'), hp.pop('betas') + dataset_name = dataset_config.get('name', 'CASIA-B') + if dataset_name == 'CASIA-B': + self.rgb_pn = RGBPartNet(124 - self.train_size, + self.in_channels, + **hp) + elif dataset_name == 'FVG': + # TODO + pass + else: + raise ValueError('Invalid dataset: {0}'.format(dataset_name)) + + self._accelerate() + + self.rgb_pn.eval() + # Load checkpoint at iter_ + self.curr_iter = iter_ + checkpoint = torch.load(self.checkpoint_name) + self.rgb_pn.load_state_dict(checkpoint['model_state_dict']) + + labels, conditions, views, features = [], [], [], [] + for sample in tqdm(dataloader, desc='Transforming', unit='clips'): + label, condition, view, clip = sample.values() + feature = self.rgb_pn(clip).detach().cpu().numpy() + labels.append(label) + conditions.append(condition) + views.append(view) + features.append(feature) + labels = np.asarray(labels) + conditions = np.asarray(conditions) + views = np.asarray(views) + features = np.asarray(features) + + # TODO Implement evaluation function here + @staticmethod def init_weights(m): if isinstance(m, nn.modules.conv._ConvNd): @@ -167,7 +216,7 @@ class Model: ) self.log_name = '_'.join((self.log_name, self._dataset_sig)) config: dict = dataset_config.copy() - name = config.pop('name') + name = config.pop('name', 'CASIA-B') if name == 'CASIA-B': return CASIAB(**config, is_train=self.is_train) elif name == 'FVG': diff --git a/models/rgb_part_net.py b/models/rgb_part_net.py index 456695d..f39b40b 100644 --- a/models/rgb_part_net.py +++ b/models/rgb_part_net.py @@ -84,7 +84,7 @@ class RGBPartNet(nn.Module): loss = torch.sum(torch.stack(losses)) return loss, [loss.item() for loss in losses] else: - return x + return x.unsqueeze(1).view(-1) def _disentangle(self, x_c1, x_c2=None, y=None): num_frames = len(x_c1) -- cgit v1.2.3 From d30cf2cb280e83e4a4abe1e9c2abdbba17d903a3 Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Sun, 10 Jan 2021 19:54:42 +0800 Subject: Make predict function transform samples different conditions in a single shot --- models/model.py | 142 ++++++++++++++++++++++++++++++++++++++++---------------- test/model.py | 4 +- train.py | 12 ++--- 3 files changed, 110 insertions(+), 48 deletions(-) diff --git a/models/model.py b/models/model.py index ba29ede..cabfa97 100644 --- a/models/model.py +++ b/models/model.py @@ -14,7 +14,7 @@ from models.rgb_part_net import RGBPartNet from utils.configuration import DataloaderConfiguration, \ HyperparameterConfiguration, DatasetConfiguration, ModelConfiguration, \ SystemConfiguration -from utils.dataset import CASIAB +from utils.dataset import CASIAB, ClipConditions, ClipViews, ClipClasses from utils.sampler import TripletSampler @@ -57,21 +57,30 @@ class Model: self._hp_sig: str = self._make_signature(self.hp) self._dataset_sig: str = 'undefined' self._log_sig: str = '_'.join((self._model_sig, self._hp_sig)) - self.log_name: str = os.path.join(self.log_dir, self._log_sig) + self._log_name: str = os.path.join(self.log_dir, self._log_sig) self.rgb_pn: Optional[RGBPartNet] = None self.optimizer: Optional[optim.Adam] = None self.scheduler: Optional[optim.lr_scheduler.StepLR] = None self.writer: Optional[SummaryWriter] = None + self.CASIAB_GALLERY_SELECTOR = { + 'selector': {'conditions': ClipConditions({r'nm-0[1-4]'})} + } + self.CASIAB_PROBE_SELECTORS = { + 'nm': {'selector': {'conditions': ClipConditions({r'nm-0[5-6]'})}}, + 'bg': {'selector': {'conditions': ClipConditions({r'bg-0[1-2]'})}}, + 'cl': {'selector': {'conditions': ClipConditions({r'cl-0[1-2]'})}}, + } + @property - def signature(self) -> str: + def _signature(self) -> str: return '_'.join((self._model_sig, str(self.curr_iter), self._hp_sig, self._dataset_sig, str(self.pr), str(self.k))) @property - def checkpoint_name(self) -> str: - return os.path.join(self.checkpoint_dir, self.signature) + def _checkpoint_name(self) -> str: + return os.path.join(self.checkpoint_dir, self._signature) def fit( self, @@ -87,8 +96,8 @@ class Model: self.rgb_pn = RGBPartNet(self.train_size, self.in_channels, **hp) self.optimizer = optim.Adam(self.rgb_pn.parameters(), lr, betas) self.scheduler = optim.lr_scheduler.StepLR(self.optimizer, 500, 0.9) - self.writer = SummaryWriter(self.log_name) - + self.writer = SummaryWriter(self._log_name) + # Try to accelerate computation using CUDA or others self._accelerate() self.rgb_pn.train() @@ -96,7 +105,7 @@ class Model: if self.curr_iter == 0: self.rgb_pn.apply(self.init_weights) else: # Load saved state dicts - checkpoint = torch.load(self.checkpoint_name) + checkpoint = torch.load(self._checkpoint_name) iter_, loss = checkpoint['iter'], checkpoint['loss'] print('{0:5d} loss: {1:.3f}'.format(iter_, loss)) self.rgb_pn.load_state_dict(checkpoint['model_state_dict']) @@ -135,7 +144,7 @@ class Model: 'model_state_dict': self.rgb_pn.state_dict(), 'optim_state_dict': self.optimizer.state_dict(), 'loss': loss, - }, self.checkpoint_name) + }, self._checkpoint_name) if self.curr_iter == self.total_iter: self.writer.close() @@ -151,46 +160,98 @@ class Model: self, iter_: int, dataset_config: DatasetConfiguration, + dataset_selectors: dict[ + str, dict[str, Union[ClipClasses, ClipConditions, ClipViews]] + ], dataloader_config: DataloaderConfiguration, ): self.is_train = False - dataset = self._parse_dataset_config(dataset_config) - dataloader = self._parse_dataloader_config(dataset, dataloader_config) + # Split gallery and probe dataset + gallery_dataloader, probe_dataloaders = self._split_gallery_probe( + dataset_config, dataloader_config + ) + # Get pretrained models at iter_ + checkpoints = self._load_pretrained( + iter_, dataset_config, dataset_selectors + ) + # Init models hp = self.hp.copy() - _, _ = hp.pop('lr'), hp.pop('betas') + hp.pop('lr'), hp.pop('betas') + self.rgb_pn = RGBPartNet(ae_in_channels=self.in_channels, **hp) + # Try to accelerate computation using CUDA or others + self._accelerate() + + self.rgb_pn.eval() + gallery_samples, probe_samples = [], {} + + # Gallery + self.rgb_pn.load_state_dict(torch.load(list(checkpoints.values())[0])) + for sample in tqdm(gallery_dataloader, + desc='Transforming gallery', unit='clips'): + clip = sample.pop('clip').to(self.device) + feature = self.rgb_pn(clip).detach().cpu() + gallery_samples.append({**sample, **{'feature': feature}}) + gallery_samples = default_collate(gallery_samples) + + # Probe + for (name, dataloader) in probe_dataloaders.items(): + self.rgb_pn.load_state_dict(torch.load(checkpoints[name])) + probe_samples[name] = [] + for sample in tqdm(dataloader, + desc=f'Transforming probe {name}', unit='clips'): + clip = sample.pop('clip').to(self.device) + feature = self.rgb_pn(clip) + probe_samples[name].append({**sample, **{'feature': feature}}) + for (k, v) in probe_samples.items(): + probe_samples[k] = default_collate(v) + + # TODO Implement evaluation function here + + def _load_pretrained( + self, + iter_: int, + dataset_config: DatasetConfiguration, + dataset_selectors: dict[ + str, dict[str, Union[ClipClasses, ClipConditions, ClipViews]] + ] + ) -> dict[str, str]: + checkpoints = {} + self.curr_iter = iter_ + for (k, v) in dataset_selectors.items(): + self._dataset_sig = self._make_signature( + dict(**dataset_config, **v), + popped_keys=['root_dir', 'cache_on'] + ) + checkpoints[k] = self._checkpoint_name + return checkpoints + + def _split_gallery_probe( + self, + dataset_config: DatasetConfiguration, + dataloader_config: DataloaderConfiguration, + ) -> tuple[DataLoader, dict[str: DataLoader]]: dataset_name = dataset_config.get('name', 'CASIA-B') if dataset_name == 'CASIA-B': - self.rgb_pn = RGBPartNet(124 - self.train_size, - self.in_channels, - **hp) + gallery_dataset = self._parse_dataset_config( + dict(**dataset_config, **self.CASIAB_GALLERY_SELECTOR) + ) + gallery_dataloader = self._parse_dataloader_config( + gallery_dataset, dataloader_config + ) + probe_datasets = {k: self._parse_dataset_config( + dict(**dataset_config, **v) + ) for (k, v) in self.CASIAB_PROBE_SELECTORS.items()} + probe_dataloaders = {k: self._parse_dataloader_config( + v, dataloader_config + ) for (k, v) in probe_datasets.items()} elif dataset_name == 'FVG': # TODO - pass + gallery_dataloader = None + probe_dataloaders = None else: raise ValueError('Invalid dataset: {0}'.format(dataset_name)) - self._accelerate() - - self.rgb_pn.eval() - # Load checkpoint at iter_ - self.curr_iter = iter_ - checkpoint = torch.load(self.checkpoint_name) - self.rgb_pn.load_state_dict(checkpoint['model_state_dict']) - - labels, conditions, views, features = [], [], [], [] - for sample in tqdm(dataloader, desc='Transforming', unit='clips'): - label, condition, view, clip = sample.values() - feature = self.rgb_pn(clip).detach().cpu().numpy() - labels.append(label) - conditions.append(condition) - views.append(view) - features.append(feature) - labels = np.asarray(labels) - conditions = np.asarray(conditions) - views = np.asarray(views) - features = np.asarray(features) - - # TODO Implement evaluation function here + return gallery_dataloader, probe_dataloaders @staticmethod def init_weights(m): @@ -214,7 +275,7 @@ class Model: dataset_config, popped_keys=['root_dir', 'cache_on'] ) - self.log_name = '_'.join((self.log_name, self._dataset_sig)) + self._log_name = '_'.join((self._log_name, self._dataset_sig)) config: dict = dataset_config.copy() name = config.pop('name', 'CASIA-B') if name == 'CASIA-B': @@ -232,7 +293,8 @@ class Model: config: dict = dataloader_config.copy() if self.is_train: (self.pr, self.k) = config.pop('batch_size') - self.log_name = '_'.join((self.log_name, str(self.pr), str(self.k))) + self._log_name = '_'.join( + (self._log_name, str(self.pr), str(self.k))) triplet_sampler = TripletSampler(dataset, (self.pr, self.k)) return DataLoader(dataset, batch_sampler=triplet_sampler, diff --git a/test/model.py b/test/model.py index 472c210..f679908 100644 --- a/test/model.py +++ b/test/model.py @@ -11,11 +11,11 @@ def test_default_signature(): model = Model(conf['system'], conf['model'], conf['hyperparameter']) casiab = model._parse_dataset_config(conf['dataset']) model._parse_dataloader_config(casiab, conf['dataloader']) - assert model.log_name == os.path.join( + assert model._log_name == os.path.join( 'runs', 'logs', 'RGB-GaitPart_80000_64_128_128_64_1_2_4_True_True_32_5_' '3_3_3_3_3_2_1_1_1_1_1_0_2_3_4_16_256_0.2_0.0001_0.9_' '0.999_CASIA-B_74_30_15_3_64_32_8_16') - assert model.signature == ('RGB-GaitPart_80000_0_64_128_128_64_1_2_4_True_' + assert model._signature == ('RGB-GaitPart_80000_0_64_128_128_64_1_2_4_True_' 'True_32_5_3_3_3_3_3_2_1_1_1_1_1_0_2_3_4_16_256_' '0.2_0.0001_0.9_0.999_CASIA-B_74_30_15_3_64_32_' '8_16') diff --git a/train.py b/train.py index d921839..cdb2fb0 100644 --- a/train.py +++ b/train.py @@ -12,12 +12,12 @@ if CUDA_VISIBLE_DEVICES: model = Model(config['system'], config['model'], config['hyperparameter']) # 3 models for different conditions -dataset_selectors = [ - {'conditions': ClipConditions({r'nm-0\d'})}, - {'conditions': ClipConditions({r'nm-0\d', r'bg-0\d'})}, - {'conditions': ClipConditions({r'nm-0\d', r'cl-0\d'})}, -] -for selector in dataset_selectors: +dataset_selectors = { + 'nm': {'conditions': ClipConditions({r'nm-0\d'})}, + 'bg': {'conditions': ClipConditions({r'nm-0\d', r'bg-0\d'})}, + 'cl': {'conditions': ClipConditions({r'nm-0\d', r'cl-0\d'})}, +} +for selector in dataset_selectors.values(): model.fit( dict(**config['dataset'], **{'selector': selector}), config['dataloader'] -- cgit v1.2.3 From 7188d71b2b6faf3da527c8d0ade9a32ec4893dc5 Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Mon, 11 Jan 2021 21:15:58 +0800 Subject: Implement evaluator --- models/model.py | 83 ++++++++++++++++++++++++++++++++++++++++++++------- utils/triplet_loss.py | 4 +-- 2 files changed, 75 insertions(+), 12 deletions(-) diff --git a/models/model.py b/models/model.py index cabfa97..456c2f1 100644 --- a/models/model.py +++ b/models/model.py @@ -4,6 +4,7 @@ from typing import Union, Optional import numpy as np import torch import torch.nn as nn +import torch.nn.functional as F import torch.optim as optim from torch.utils.data import DataLoader from torch.utils.data.dataloader import default_collate @@ -53,6 +54,9 @@ class Model: self.pr: Optional[int] = None self.k: Optional[int] = None + self._gallery_dataset_meta: Optional[dict[str, list]] = None + self._probe_datasets_meta: Optional[dict[str, dict[str, list]]] = None + self._model_sig: str = self._make_signature(self.meta, ['restore_iter']) self._hp_sig: str = self._make_signature(self.hp) self._dataset_sig: str = 'undefined' @@ -164,7 +168,7 @@ class Model: str, dict[str, Union[ClipClasses, ClipConditions, ClipViews]] ], dataloader_config: DataloaderConfiguration, - ): + ) -> dict[str, torch.Tensor]: self.is_train = False # Split gallery and probe dataset gallery_dataloader, probe_dataloaders = self._split_gallery_probe( @@ -189,7 +193,7 @@ class Model: for sample in tqdm(gallery_dataloader, desc='Transforming gallery', unit='clips'): clip = sample.pop('clip').to(self.device) - feature = self.rgb_pn(clip).detach().cpu() + feature = self.rgb_pn(clip).detach() gallery_samples.append({**sample, **{'feature': feature}}) gallery_samples = default_collate(gallery_samples) @@ -200,12 +204,60 @@ class Model: for sample in tqdm(dataloader, desc=f'Transforming probe {name}', unit='clips'): clip = sample.pop('clip').to(self.device) - feature = self.rgb_pn(clip) + feature = self.rgb_pn(clip).detach() probe_samples[name].append({**sample, **{'feature': feature}}) for (k, v) in probe_samples.items(): probe_samples[k] = default_collate(v) - # TODO Implement evaluation function here + return self._evaluate(gallery_samples, probe_samples) + + def _evaluate( + self, + gallery_samples: dict[str, Union[list[str], torch.Tensor]], + probe_samples: dict[str, dict[str, Union[list[str], torch.Tensor]]], + num_ranks: int = 5 + ) -> dict[str, torch.Tensor]: + probe_conditions = self._probe_datasets_meta.keys() + gallery_views_meta = self._gallery_dataset_meta['views'] + probe_views_meta = list(self._probe_datasets_meta.values())[0]['views'] + accuracy = { + condition: torch.empty( + len(gallery_views_meta), len(probe_views_meta), num_ranks + ) + for condition in self._probe_datasets_meta.keys() + } + + (labels_g, _, views_g, features_g) = gallery_samples.values() + views_g = np.asarray(views_g) + for (v_g_i, view_g) in enumerate(gallery_views_meta): + gallery_view_mask = (views_g == view_g) + f_g = features_g[gallery_view_mask] + y_g = labels_g[gallery_view_mask] + for condition in probe_conditions: + probe_samples_c = probe_samples[condition] + accuracy_c = accuracy[condition] + (labels_p, _, views_p, features_p) = probe_samples_c.values() + views_p = np.asarray(views_p) + for (v_p_i, view_p) in enumerate(probe_views_meta): + probe_view_mask = (views_p == view_g) + f_p = features_p[probe_view_mask] + y_p = labels_p[probe_view_mask] + # Euclidean distance + f_g_squared_sum = torch.sum(f_g ** 2, dim=1).unsqueeze(1) + f_p_squared_sum = torch.sum(f_p ** 2, dim=1).unsqueeze(0) + f_g_times_f_p_sum = f_g @ f_p.T + dist = torch.sqrt(F.relu( + f_g_squared_sum - 2*f_g_times_f_p_sum + f_p_squared_sum + )) + # Ranked accuracy + rank_mask = dist.argsort(1)[:, :num_ranks] + positive_mat = torch.eq(y_p.unsqueeze(1), + y_g[rank_mask]).cumsum(1).gt(0) + positive_counts = positive_mat.sum(0) + total_counts, _ = dist.size() + accuracy_c[v_g_i, v_p_i, :] = positive_counts / total_counts + + return accuracy def _load_pretrained( self, @@ -235,15 +287,26 @@ class Model: gallery_dataset = self._parse_dataset_config( dict(**dataset_config, **self.CASIAB_GALLERY_SELECTOR) ) + self._gallery_dataset_meta = gallery_dataset.metadata gallery_dataloader = self._parse_dataloader_config( gallery_dataset, dataloader_config ) - probe_datasets = {k: self._parse_dataset_config( - dict(**dataset_config, **v) - ) for (k, v) in self.CASIAB_PROBE_SELECTORS.items()} - probe_dataloaders = {k: self._parse_dataloader_config( - v, dataloader_config - ) for (k, v) in probe_datasets.items()} + probe_datasets = { + condition: self._parse_dataset_config( + dict(**dataset_config, **selector) + ) + for (condition, selector) in self.CASIAB_PROBE_SELECTORS.items() + } + self._probe_datasets_meta = { + condition: dataset.metadata + for (condition, dataset) in probe_datasets.items() + } + probe_dataloaders = { + condtion: self._parse_dataloader_config( + dataset, dataloader_config + ) + for (condtion, dataset) in probe_datasets.items() + } elif dataset_name == 'FVG': # TODO gallery_dataloader = None diff --git a/utils/triplet_loss.py b/utils/triplet_loss.py index 1899cc9..8c143d6 100644 --- a/utils/triplet_loss.py +++ b/utils/triplet_loss.py @@ -15,8 +15,8 @@ class BatchAllTripletLoss(nn.Module): # Euclidean distance p x n x n x_squared_sum = torch.sum(x ** 2, dim=2) - x1_squared_sum = x_squared_sum.unsqueeze(1) - x2_squared_sum = x_squared_sum.unsqueeze(2) + x1_squared_sum = x_squared_sum.unsqueeze(2) + x2_squared_sum = x_squared_sum.unsqueeze(1) x1_times_x2_sum = x @ x.transpose(1, 2) dist = torch.sqrt( F.relu(x1_squared_sum - 2 * x1_times_x2_sum + x2_squared_sum) -- cgit v1.2.3 From 72a53806746bc7ffa2f3939721e34b5cfdb7330a Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Mon, 11 Jan 2021 23:59:30 +0800 Subject: Add evaluation script, code review and fix some bugs 1. Add new `train_all` method for one shot calling 2. Print time used in 1k iterations 3. Correct label dimension in predict function 4. Transpose distance matrix for convenient indexing 5. Sort dictionary before generate signature 6. Extract visible CUDA setting function --- eval.py | 16 ++++++++++++++ models/model.py | 67 +++++++++++++++++++++++++++++++++++++++++++-------------- train.py | 16 ++++---------- utils/misc.py | 10 +++++++++ 4 files changed, 81 insertions(+), 28 deletions(-) create mode 100644 eval.py create mode 100644 utils/misc.py diff --git a/eval.py b/eval.py new file mode 100644 index 0000000..fee4ab9 --- /dev/null +++ b/eval.py @@ -0,0 +1,16 @@ +from config import config +from models import Model +from utils.dataset import ClipConditions +from utils.misc import set_visible_cuda + +set_visible_cuda(config['system']) +model = Model(config['system'], config['model'], config['hyperparameter']) + +dataset_selectors = { + 'nm': {'conditions': ClipConditions({r'nm-0\d'})}, + 'bg': {'conditions': ClipConditions({r'nm-0\d', r'bg-0\d'})}, + 'cl': {'conditions': ClipConditions({r'nm-0\d', r'cl-0\d'})}, +} + +accuracy = model.predict_all(config['model']['total_iter'], config['dataset'], + dataset_selectors, config['dataloader']) diff --git a/models/model.py b/models/model.py index 456c2f1..b343f86 100644 --- a/models/model.py +++ b/models/model.py @@ -1,4 +1,5 @@ import os +from datetime import datetime from typing import Union, Optional import numpy as np @@ -86,6 +87,21 @@ class Model: def _checkpoint_name(self) -> str: return os.path.join(self.checkpoint_dir, self._signature) + def fit_all( + self, + dataset_config: DatasetConfiguration, + dataset_selectors: dict[ + str, dict[str, Union[ClipClasses, ClipConditions, ClipViews]] + ], + dataloader_config: DataloaderConfiguration, + ): + for (condition, selector) in dataset_selectors.items(): + print(f'Training model {condition} ...') + self.fit( + dict(**dataset_config, **{'selector': selector}), + dataloader_config + ) + def fit( self, dataset_config: DatasetConfiguration, @@ -115,6 +131,8 @@ class Model: self.rgb_pn.load_state_dict(checkpoint['model_state_dict']) self.optimizer.load_state_dict(checkpoint['optim_state_dict']) + # Training start + start_time = datetime.now() for (batch_c1, batch_c2) in dataloader: self.curr_iter += 1 # Zero the parameter gradients @@ -137,7 +155,7 @@ class Model: ], metrics)), self.curr_iter) if self.curr_iter % 100 == 0: - print('{0:5d} loss: {1:.3f}'.format(self.curr_iter, loss), + print('{0:5d} loss: {1:6.3f}'.format(self.curr_iter, loss), '(xrecon = {:f}, pose_sim = {:f},' ' cano_cons = {:f}, ba_trip = {:f})'.format(*metrics), 'lr:', self.scheduler.get_last_lr()[0]) @@ -149,8 +167,11 @@ class Model: 'optim_state_dict': self.optimizer.state_dict(), 'loss': loss, }, self._checkpoint_name) + print(datetime.now() - start_time, 'used') + start_time = datetime.now() if self.curr_iter == self.total_iter: + self.curr_iter = 0 self.writer.close() break @@ -160,7 +181,7 @@ class Model: self.rgb_pn = nn.DataParallel(self.rgb_pn) self.rgb_pn = self.rgb_pn.to(self.device) - def predict( + def predict_all( self, iter_: int, dataset_config: DatasetConfiguration, @@ -189,23 +210,36 @@ class Model: gallery_samples, probe_samples = [], {} # Gallery - self.rgb_pn.load_state_dict(torch.load(list(checkpoints.values())[0])) + checkpoint = torch.load(list(checkpoints.values())[0]) + self.rgb_pn.load_state_dict(checkpoint['model_state_dict']) for sample in tqdm(gallery_dataloader, desc='Transforming gallery', unit='clips'): + label = sample.pop('label').item() clip = sample.pop('clip').to(self.device) feature = self.rgb_pn(clip).detach() - gallery_samples.append({**sample, **{'feature': feature}}) + gallery_samples.append({ + **{'label': label}, + **sample, + **{'feature': feature} + }) gallery_samples = default_collate(gallery_samples) # Probe - for (name, dataloader) in probe_dataloaders.items(): - self.rgb_pn.load_state_dict(torch.load(checkpoints[name])) - probe_samples[name] = [] + for (condition, dataloader) in probe_dataloaders.items(): + checkpoint = torch.load(checkpoints[condition]) + self.rgb_pn.load_state_dict(checkpoint['model_state_dict']) + probe_samples[condition] = [] for sample in tqdm(dataloader, - desc=f'Transforming probe {name}', unit='clips'): + desc=f'Transforming probe {condition}', + unit='clips'): + label = sample.pop('label').item() clip = sample.pop('clip').to(self.device) feature = self.rgb_pn(clip).detach() - probe_samples[name].append({**sample, **{'feature': feature}}) + probe_samples[condition].append({ + **{'label': label}, + **sample, + **{'feature': feature} + }) for (k, v) in probe_samples.items(): probe_samples[k] = default_collate(v) @@ -243,11 +277,11 @@ class Model: f_p = features_p[probe_view_mask] y_p = labels_p[probe_view_mask] # Euclidean distance - f_g_squared_sum = torch.sum(f_g ** 2, dim=1).unsqueeze(1) - f_p_squared_sum = torch.sum(f_p ** 2, dim=1).unsqueeze(0) - f_g_times_f_p_sum = f_g @ f_p.T + f_p_squared_sum = torch.sum(f_p ** 2, dim=1).unsqueeze(1) + f_g_squared_sum = torch.sum(f_g ** 2, dim=1).unsqueeze(0) + f_p_times_f_g_sum = f_p @ f_g.T dist = torch.sqrt(F.relu( - f_g_squared_sum - 2*f_g_times_f_p_sum + f_p_squared_sum + f_p_squared_sum - 2*f_p_times_f_g_sum + f_g_squared_sum )) # Ranked accuracy rank_mask = dist.argsort(1)[:, :num_ranks] @@ -354,8 +388,8 @@ class Model: dataloader_config: DataloaderConfiguration ) -> DataLoader: config: dict = dataloader_config.copy() + (self.pr, self.k) = config.pop('batch_size') if self.is_train: - (self.pr, self.k) = config.pop('batch_size') self._log_name = '_'.join( (self._log_name, str(self.pr), str(self.k))) triplet_sampler = TripletSampler(dataset, (self.pr, self.k)) @@ -364,7 +398,6 @@ class Model: collate_fn=self._batch_splitter, **config) else: # is_test - config.pop('batch_size') return DataLoader(dataset, **config) def _batch_splitter( @@ -399,8 +432,10 @@ class Model: for v in values: if isinstance(v, str): strings.append(v) - elif isinstance(v, (tuple, list, set)): + elif isinstance(v, (tuple, list)): strings.append(self._gen_sig(v)) + elif isinstance(v, set): + strings.append(self._gen_sig(sorted(list(v)))) elif isinstance(v, dict): strings.append(self._gen_sig(list(v.values()))) else: diff --git a/train.py b/train.py index cdb2fb0..d91dcd0 100644 --- a/train.py +++ b/train.py @@ -1,14 +1,9 @@ -import os - from config import config from models import Model from utils.dataset import ClipConditions +from utils.misc import set_visible_cuda -# Set environment variable CUDA device(s) -CUDA_VISIBLE_DEVICES = config['system'].get('CUDA_VISIBLE_DEVICES', None) -if CUDA_VISIBLE_DEVICES: - os.environ['CUDA_VISIBLE_DEVICES'] = CUDA_VISIBLE_DEVICES - +set_visible_cuda(config['system']) model = Model(config['system'], config['model'], config['hyperparameter']) # 3 models for different conditions @@ -17,8 +12,5 @@ dataset_selectors = { 'bg': {'conditions': ClipConditions({r'nm-0\d', r'bg-0\d'})}, 'cl': {'conditions': ClipConditions({r'nm-0\d', r'cl-0\d'})}, } -for selector in dataset_selectors.values(): - model.fit( - dict(**config['dataset'], **{'selector': selector}), - config['dataloader'] - ) + +model.fit_all(config['dataset'], dataset_selectors, config['dataloader']) diff --git a/utils/misc.py b/utils/misc.py new file mode 100644 index 0000000..b850830 --- /dev/null +++ b/utils/misc.py @@ -0,0 +1,10 @@ +import os + +from utils.configuration import SystemConfiguration + + +def set_visible_cuda(config: SystemConfiguration): + """Set environment variable CUDA device(s)""" + CUDA_VISIBLE_DEVICES = config.get('CUDA_VISIBLE_DEVICES', None) + if CUDA_VISIBLE_DEVICES: + os.environ['CUDA_VISIBLE_DEVICES'] = CUDA_VISIBLE_DEVICES -- cgit v1.2.3 From 35a7218c7b445409829d3725b10c3b7d3cb51541 Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Tue, 12 Jan 2021 09:43:28 +0800 Subject: Bump up version for tqdm --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bcc6597..deaa0d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ torch~=1.7.1 torchvision~=0.8.0a0+ecf4e9c numpy~=1.19.4 -tqdm~=4.55.0 +tqdm~=4.56.0 Pillow~=8.1.0 scikit-learn~=0.23.2 \ No newline at end of file -- cgit v1.2.3 From 1ec023951862a5318527fecb2ff1d45305045543 Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Tue, 12 Jan 2021 11:10:59 +0800 Subject: Typo correct in evaluate function --- models/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/model.py b/models/model.py index b343f86..c6f6e57 100644 --- a/models/model.py +++ b/models/model.py @@ -273,7 +273,7 @@ class Model: (labels_p, _, views_p, features_p) = probe_samples_c.values() views_p = np.asarray(views_p) for (v_p_i, view_p) in enumerate(probe_views_meta): - probe_view_mask = (views_p == view_g) + probe_view_mask = (views_p == view_p) f_p = features_p[probe_view_mask] y_p = labels_p[probe_view_mask] # Euclidean distance -- cgit v1.2.3 From 36cf502afe9b93efe31c244030270b0a62e644b8 Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Tue, 12 Jan 2021 11:13:39 +0800 Subject: Print result after evaluation --- eval.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/eval.py b/eval.py index fee4ab9..7b68220 100644 --- a/eval.py +++ b/eval.py @@ -1,3 +1,5 @@ +import numpy as np + from config import config from models import Model from utils.dataset import ClipConditions @@ -14,3 +16,15 @@ dataset_selectors = { accuracy = model.predict_all(config['model']['total_iter'], config['dataset'], dataset_selectors, config['dataloader']) +rank = 5 +np.set_printoptions(formatter={'float': '{:5.2f}'.format}) +for n in range(rank): + print(f'===Rank-{n + 1} Accuracy===') + for (condition, accuracy_c) in accuracy.items(): + acc_excl_identical_view = accuracy_c[:, :, n].fill_diagonal_(0) + num_gallery_views = (acc_excl_identical_view != 0).sum() + acc_each_angle = acc_excl_identical_view.sum(0) / num_gallery_views + print('{0}: {1} mean: {2:5.2f}'.format( + condition, acc_each_angle.cpu().numpy() * 100, + acc_each_angle.mean() * 100) + ) -- cgit v1.2.3