import argparse import pytorch_lightning as pl import torch from typing import Dict, Any OPTIMIZER = "AdamW" LR = 5e-5 LOSS = "cross_entropy" ONE_CYCLE_TOTAL_STEPS = 100 class Config(dict): def __getattr__(self, name): return self.get(name) def __setattr__(self, name, val): self[name] = val class BaseLitModel(pl.LightningModule): """ Generic PyTorch-Lightning class that must be initialized with a PyTorch module. """ def __init__(self, model, args: argparse.Namespace = None): super().__init__() self.model = model self.args = Config(vars(args)) if args is not None else {} optimizer = self.args.get("optimizer", OPTIMIZER) self.optimizer_class = getattr(torch.optim, optimizer) self.lr = self.args.get("lr", LR) @staticmethod def add_to_argparse(parser): parser.add_argument("--optimizer", type=str, default=OPTIMIZER, help="optimizer class from torch.optim") parser.add_argument("--lr", type=float, default=LR) parser.add_argument("--weight_decay", type=float, default=0.01) return parser def configure_optimizers(self): optimizer = self.optimizer_class(self.parameters(), lr=self.lr) if self.one_cycle_max_lr is None: return optimizer scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, max_lr=self.one_cycle_max_lr, total_steps=self.one_cycle_total_steps) return {"optimizer": optimizer, "lr_scheduler": scheduler, "monitor": "val_loss"} def forward(self, x): return self.model(x) def training_step(self, batch, batch_idx): # pylint: disable=unused-argument x, y = batch logits = self(x) loss = self.loss_fn(logits, y) self.log("train_loss", loss) self.train_acc(logits, y) self.log("train_acc", self.train_acc, on_step=False, on_epoch=True) return loss def validation_step(self, batch, batch_idx): # pylint: disable=unused-argument x, y = batch logits = self(x) loss = self.loss_fn(logits, y) self.log("val_loss", loss, prog_bar=True) self.val_acc(logits, y) self.log("val_acc", self.val_acc, on_step=False, on_epoch=True, prog_bar=True) def test_step(self, batch, batch_idx): # pylint: disable=unused-argument x, y = batch logits = self(x) self.test_acc(logits, y) self.log("test_acc", self.test_acc, on_step=False, on_epoch=True) @property def num_training_steps(self) -> int: """Total training steps inferred from datamodule and devices.""" if isinstance(self.trainer.limit_train_batches, int) and self.trainer.limit_train_batches != 0: dataset_size = self.trainer.limit_train_batches elif isinstance(self.trainer.limit_train_batches, float): # limit_train_batches is a percentage of batches dataset_size = len(self.trainer.datamodule.train_dataloader()) dataset_size = int(dataset_size * self.trainer.limit_train_batches) else: dataset_size = len(self.trainer.datamodule.train_dataloader()) num_devices = max(1, self.trainer.num_gpus, self.trainer.num_processes) if self.trainer.tpu_cores: num_devices = max(num_devices, self.trainer.tpu_cores) effective_batch_size = self.trainer.accumulate_grad_batches * num_devices max_estimated_steps = (dataset_size // effective_batch_size) * self.trainer.max_epochs if self.trainer.max_steps and self.trainer.max_steps < max_estimated_steps: return self.trainer.max_steps return max_estimated_steps