-
Notifications
You must be signed in to change notification settings - Fork 57
Improve checkpointing parameter validation #611
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
528c1bf
5d0829a
1d47860
17958d9
747fe3b
125e300
f34b2c9
fe2e7a9
005d8b2
b3b927c
0cde11d
7fcaae5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -60,9 +60,34 @@ def __init__( | |||||||||||||||||||||||||||||
| self.checkpoint_timesteps = checkpoint_timesteps | ||||||||||||||||||||||||||||||
| self.checkpoint_file = checkpoint_file | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| if self.checkpoint_file is not None: | ||||||||||||||||||||||||||||||
| if self.checkpoint_interval is None and self.checkpoint_timesteps is None: | ||||||||||||||||||||||||||||||
| raise ValueError("One of checkpoint_interval or checkpoint_timesteps must be set when checkpoint_file is set.") | ||||||||||||||||||||||||||||||
| self.validate() | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| def _validate_checkpoint_options(self): | ||||||||||||||||||||||||||||||
| # Checkpointing parameters are set | ||||||||||||||||||||||||||||||
| if self.checkpoint_interval is not None or self.checkpoint_timesteps is not None: | ||||||||||||||||||||||||||||||
| # No checkpoint file set | ||||||||||||||||||||||||||||||
| if self.checkpoint_file is None: | ||||||||||||||||||||||||||||||
| raise ValueError("`checkpoint_file` must be set when `checkpoint_interval` or `checkpoint_timesteps` is set.") | ||||||||||||||||||||||||||||||
| # Both checkpointing parameters are set | ||||||||||||||||||||||||||||||
| if self.checkpoint_interval is not None and self.checkpoint_timesteps is not None: | ||||||||||||||||||||||||||||||
| raise ValueError("`checkpoint_interval` and `checkpoint_timesteps` cannot be set at the same time.") | ||||||||||||||||||||||||||||||
| # Checkpoint file is set but no checkpointing parameters are set | ||||||||||||||||||||||||||||||
| if self.checkpoint_file is not None and self.checkpoint_interval is None and self.checkpoint_timesteps is None: | ||||||||||||||||||||||||||||||
| raise ValueError("`checkpoint_interval` or `checkpoint_timesteps` must be set when `checkpoint_file` is set.") | ||||||||||||||||||||||||||||||
|
Comment on lines
+70
to
+76
|
||||||||||||||||||||||||||||||
| raise ValueError("`checkpoint_file` must be set when `checkpoint_interval` or `checkpoint_timesteps` is set.") | |
| # Both checkpointing parameters are set | |
| if self.checkpoint_interval is not None and self.checkpoint_timesteps is not None: | |
| raise ValueError("`checkpoint_interval` and `checkpoint_timesteps` cannot be set at the same time.") | |
| # Checkpoint file is set but no checkpointing parameters are set | |
| if self.checkpoint_file is not None and self.checkpoint_interval is None and self.checkpoint_timesteps is None: | |
| raise ValueError("`checkpoint_interval` or `checkpoint_timesteps` must be set when `checkpoint_file` is set.") | |
| raise ValueError("'checkpoint_file' must be set when 'checkpoint_interval' or 'checkpoint_timesteps' is set.") | |
| # Both checkpointing parameters are set | |
| if self.checkpoint_interval is not None and self.checkpoint_timesteps is not None: | |
| raise ValueError("'checkpoint_interval' and 'checkpoint_timesteps' cannot be set at the same time.") | |
| # Checkpoint file is set but no checkpointing parameters are set | |
| if self.checkpoint_file is not None and self.checkpoint_interval is None and self.checkpoint_timesteps is None: | |
| raise ValueError("'checkpoint_interval' or 'checkpoint_timesteps' must be set when 'checkpoint_file' is set.") |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
checkpoint_interval setter has the same bool-as-int coercion issue: checkpoint_interval = True would silently be stored as 1 second.
| @checkpoint_interval.setter | |
| def checkpoint_interval(self, value: Optional[int]): | |
| if value is not None: | |
| if not isinstance(value, int) or value < 0: | |
| raise ValueError("Checkpoint interval must be a positive integer") | |
| if not isinstance(value, int) or value <= 0: | |
| raise ValueError("Checkpoint interval must be a positive integer in seconds") | |
| self._checkpoint_interval = value | |
| @checkpoint_interval.setter | |
| def checkpoint_interval(self, value: Optional[int]): | |
| if value is not None: | |
| if not isinstance(value, int) or isinstance(value, bool) or value <= 0: | |
| raise ValueError("Checkpoint interval must be a positive integer in seconds") | |
| self._checkpoint_interval = value |
Uh oh!
There was an error while loading. Please reload this page.