From 7b5ff0a8dd2406af74776f31b1e2afabf9fadacd Mon Sep 17 00:00:00 2001 From: Jordan Gong Date: Fri, 22 Jan 2021 18:12:41 +0800 Subject: Handle unexpected restore iter 1. Skip finished model before load it 2. Raise error when restore iter is greater than total iter --- models/model.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/models/model.py b/models/model.py index 3b54363..1a62bae 100644 --- a/models/model.py +++ b/models/model.py @@ -116,6 +116,13 @@ class Model: self.curr_iters, self.total_iters, dataset_selectors.items() ): print(f'Training model {condition} ...') + # Skip finished model + if curr_iter == total_iter: + continue + # Check invalid restore iter + elif curr_iter > total_iter: + raise ValueError("Restore iter '{}' should less than total " + "iter '{}'".format(curr_iter, total_iter)) self.curr_iter = curr_iter self.total_iter = total_iter self.fit( -- cgit v1.2.3