Optimizer got an empty parameter list

1

Hello I’m a new user of pytorch and pytorch lightning and I’m facing the error mentioned in the title of the post : " ValueError: optimizer got an empty parameter list "

This is the code I’m using :

class MyClassifier(pl.LightningModule):
  def __init__(self, n_classes: int, steps_per_epoch= None, n_epochs=None):
    super().__init__()
    
    self.config = AutoConfig.from_pretrained(MODEL_NAME),
    self.model = AutoModel.from_pretrained(MODEL_NAME,return_dict=True,config=self.config),
    self.classifier = nn.Linear(self.config[0].hidden_size,n_classes),
    self.steps_per_epoch = steps_per_epoch,
    self.n_epochs = n_epochs,
    self.criterion = nn.BCELoss()

  def forward(self, input_ids, attention_mask, labels=None):
    output = self.model(input_ids, attention_mask = attention_mask)
    output = self.classifier(output.pooler_output)
    output = torch.sigmoid(output)

    loss = 0
    
    if labels is not None:
      loss = self.criterion(output,labels)
    return loss,output

  def training_step(self,batch, batch_ids):
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    loss,outputs =  self(input_ids,attention_mask,labels)
    self.log("train_loss", loss, prog_bar = True, logger=True)
    return {"loss":loss, "predictions":outputs,"labels":labels}


  def validation_step(self,batch, batch_ids):
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    loss,outputs =  self(input_ids,attention_mask,labels)
    self.log("val_loss", loss, prog_bar = True, logger=True)
    return loss

  def test_step(self,batch, batch_ids):
    input_ids = batch['input_ids']
    attention_mask = batch['attention_mask']
    labels = batch['labels']
    loss,outputs =  self(input_ids,attention_mask,labels)
    self.log("test_loss", loss, prog_bar = True, logger=True)
    return loss
  
  def training_epoch_end(self, outputs):
    labels = []
    predictions = []

    for output in outputs:
      for out_labels in output["labels"].detach().cpu():
        labels.append(out_labels)
      for out_predictions in output["predictions"].detach().cpu():
        predictions.append(out_predictions)
      labels = torch.stack(labels)
      predictions = torch.stack(predictions)

      for i,name in enumerate(LABELS_COLS):
        roc_score = auroc(predictions[:,i],labels[:,i])
        self.logger.experiment.add_scalar(f"{name}_roc_auc/Train",roc_score,self.current_epoch)

  def configure_optimizers(self):
    optimizer = AdamW(self.parameters(),lr=2e-5)
    warmup_steps = self.steps_per_epoch // 3 
    total_steps = self.steps_per_epoch * self.n_epochs - warmup_steps

    scheduler = get_linear_schedule_with_warmup(
        optimizer,
        warmup_steps,
        total_steps
    )
    return [optimizer], [scheduler]


model = MyClassifier(n_classes=len(LABELS_COLS), 
                           steps_per_epoch=len(train_df) // BATCH_SIZE,
                           n_epochs = N_EPOCHS)

trainer = pl.Trainer(max_epochs=N_EPOCHS, gpus=1, progress_bar_refresh_rate=30)

trainer.fit(model,data_module)

I figured that when I print list(model.parameters()) it’s empty but I don’t know why.

Can anyone assist me please ?

2

you have a comma (,) here at the end which converts it to tuple so nn.Module won’t consider it under self.parameters().

3

Hi,

I have the same issue but the solution eludes me.

import torch
import pytorch_lightning as pl
import torch.nn as nn
import torch.nn.functional as F
from torch import optim
import utils
from pytorch_lightning.loggers import TensorBoardLogger
from torchmetrics import Accuracy
from pytorch_lightning import Callback

bn_momentum = 0.3


class LitS10CustomResNet(pl.LightningModule):
    def __int__(self):
        super().__int__()

        self.train_acc = Accuracy(task="multiclass", num_classes=10)
        self.test_acc = Accuracy(task="multiclass", num_classes=10)

        self.prep = nn.Sequential(
            nn.Conv2d(3, 64, 3, padding=1, bias=False),
            nn.BatchNorm2d(64, momentum=bn_momentum),
            nn.ReLU()
        )

        self.layer1 = nn.Sequential(
            nn.Conv2d(64, 128, 3, padding=1, bias=False),
            nn.MaxPool2d(2, 2),
            nn.BatchNorm2d(128, momentum=bn_momentum),
            nn.ReLU()
        )

        self.res1 = nn.Sequential(
            nn.Conv2d(128, 128, 3, padding=1, bias=False),
            nn.BatchNorm2d(128, momentum=bn_momentum),
            nn.ReLU(),
            nn.Conv2d(128, 128, 3, padding=1, bias=False),
            nn.BatchNorm2d(128, momentum=bn_momentum),
            nn.ReLU()
        )

        self.layer2 = nn.Sequential(
            nn.Conv2d(128, 256, 3, padding=1, bias=False),
            nn.MaxPool2d(2, 2),
            nn.BatchNorm2d(256, momentum=bn_momentum),
            nn.ReLU()
        )

        self.layer3 = nn.Sequential(
            nn.Conv2d(256, 512, 3, padding=1, bias=False),
            nn.MaxPool2d(2, 2),
            nn.BatchNorm2d(512, momentum=bn_momentum),
            nn.ReLU()
        )

        self.res3 = nn.Sequential(
            nn.Conv2d(512, 512, 3, padding=1, bias=False),
            nn.BatchNorm2d(512, momentum=bn_momentum),
            nn.ReLU(),
            nn.Conv2d(512, 512, 3, padding=1, bias=False),
            nn.BatchNorm2d(512, momentum=bn_momentum),
            nn.ReLU()
        )

        self.final_max = nn.MaxPool2d(4)
        self.fc = nn.Linear(in_features=512, out_features=10, bias=False)

    def forward(self, x):
        """
        The `forward` pass, used during predict/inference in Pytorch Lightning
        """
        prep = self.prep(x)

        layer1 = self.layer1(prep)
        res1 = self.res1(layer1)
        layer1 = layer1 + res1

        layer2 = self.layer2(layer1)
        layer3 = self.layer3(layer2)
        res3 = self.res3(layer3)
        layer3 = layer3 + res3

        max = self.final_max(layer3)
        out = max.view(max.size(0), -1)

        fc = self.fc(out)

        out = fc.view(-1, 10)

        return out

    def training_step(self, batch, batch_idx):
        """
        Lightning automates the training loop for you and manages all the associated components such as:
        epoch and batch tracking, optimizers and schedulers, and metric reduction. As a user,
        you just need to define how your model behaves with a batch of training data within the training_step().
        When using Lightning, simply override the training_step() method which takes the current batch and the batch_idx
        as arguments.
        """
        x, y = batch
        preds = self(x)
        loss = F.cross_entropy(preds, y)
        self.train_acc(preds, y)
        self.log(name="train_loss", value=loss, logger=True, on_step=True, on_epoch=True)
        self.log("train_acc", self.train_acc, logger=True, on_step=False, on_epoch=True)
        return loss

    def test_step(self, batch, batch_idx):
        x, y = batch
        preds = self(x)
        test_loss = F.cross_entropy(preds, y)
        self.test_acc(preds, y)
        self.log("test_loss", test_loss, logger=True, on_step=False, on_epoch=True)
        self.log('test_acc', self.test_acc, logger=True, on_step=False, on_epoch=True)

    def configure_optimizers(self):
        optimizer = optim.Adam(self.parameters(), lr=1e-3)
        return optimizer


class MyCallback(Callback):
    def on_fit_start(self, trainer, pl_module) -> None:
        print("Calling `Fit` now...")

    def on_train_start(self, trainer, pl_module) -> None:
        print("Starting training...")

    def on_train_end(self, trainer, pl_module):
        print("Training is ending")


model = LitS10CustomResNet()

train_set, test_set, mean, sdev = utils.get_train_test_datasets(data="cifar10",
                                                                model_name="resnet",
                                                                lr_scheduler="onecycle",
                                                                cutout_prob=0.5)

# data loaders on data sets
train_loader = torch.utils.data.DataLoader(dataset=train_set)

test_loader = torch.utils.data.DataLoader(test_set)

tb_logger = TensorBoardLogger(save_dir="../logs/")

trainer = pl.Trainer(fast_dev_run=True)
trainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=test_loader)

# trainer = pl.Trainer(auto_lr_find=True,
#                      auto_scale_batch_size="power",
#                      logger=tb_logger,
#                      callbacks=[MyCallback()],
#                      benchmark=True)
#
# # Find the batch size
# trainer.tune(model)
# trainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=test_loader)

Please help me on this.