The loss of deep convolutional SNN cannot be reduced

[EliteSasuke]

• snntorch version:0.4.10
• Python version:3.7
• Operating System:win11

Description

I designed a SNN with four convolutional layers, but the loss per epoch (average of 128 minibatch losses) is always 1.386, and the four-class classification accuracy is always 25%. I found that the output layer neurons did not fire any spikes at all time steps, so I Increase the decay rate beta and lower the threshold. Although this makes the loss curve a little choppy, the accuracy is still 25%.

What I Did

#CSNN model
import torch.nn as nn
import snntorch as snn
from snntorch import surrogate
import torch.nn.functional as F
spike_grad = surrogate.fast_sigmoid(slope=25)
beta = 0.95
batch_size=128
class Net(nn.Module):
    def __init__(self):
        super().__init__()
        # Initialize layers
        self.conv1 = nn.Conv2d(1, 12, 5)
        self.lif1 = snn.Leaky(beta=beta, spike_grad=spike_grad,threshold=0.3)
        self.conv2 = nn.Conv2d(12, 12, 5)
        self.lif2 = snn.Leaky(beta=beta, spike_grad=spike_grad,threshold=0.3)
        self.conv3 = nn.Conv2d(12, 64, 3)
        self.lif3 = snn.Leaky(beta=beta, spike_grad=spike_grad,threshold=0)
        self.conv4 = nn.Conv2d(64, 64, 3)
        self.lif4 = snn.Leaky(beta=beta, spike_grad=spike_grad,threshold=0)
        self.fc1 = nn.Linear(1024, 4)
        self.lif5 = snn.Leaky(beta=beta, spike_grad=spike_grad,threshold=0)

    def forward(self, x):

        # Initialize hidden states and outputs at t=0
        mem1 = self.lif1.init_leaky()
        mem2 = self.lif2.init_leaky()
        mem3 = self.lif3.init_leaky()
        mem4 = self.lif4.init_leaky()
        mem5 = self.lif5.init_leaky()
        cur1 = self.conv1(x)
        spk1, mem1 = self.lif1(cur1, mem1)
        cur2 = F.max_pool2d(self.conv2(spk1), 2)
        spk2, mem2 = self.lif2(cur2, mem2)
        cur3 = self.conv3(spk2)
        spk3, mem3 = self.lif3(cur3, mem3)
        cur4 = F.max_pool2d(self.conv4(spk3), 2)
        spk4, mem4 = self.lif4(cur4, mem4)
        cur5 = self.fc1(spk4.view(batch_size, -1))
        spk5, mem5 = self.lif5(cur5, mem5)

        return spk5, mem5

#main
def forward_pass(net, num_steps, data):
  mem_rec = []
  spk_rec = []
  sutils.reset(net)  # resets hidden states for all LIF neurons in net

  for step in range(num_steps):
      spk_out, mem_out = net(data)
      spk_rec.append(spk_out)
      mem_rec.append(mem_out)

  return torch.stack(spk_rec), torch.stack(mem_rec)

def train(epoch,device):
    print('\nEpoch: %d' % epoch)
    net.train()
    train_loss = 0
    correct = 0
    total = 0

    loss = 0

    # confusion = np.zeros((4, 4))

    for batch_idx, (inputs, targets) in enumerate(train_loader):  # trainloader
        # target = targets
        inputs, targets = inputs.to(device), targets.to(device)

        targets = targets.long()
        spk_rec, _ = forward_pass(net, num_steps, inputs)
        _, predicted = spk_rec.sum(dim=0).max(1)
        loss_val = loss_fn(spk_rec, targets)
        optimizer.zero_grad()
        loss_val.backward()
        optimizer.step()

        train_loss += loss_val.item()
        correct += (predicted == targets).sum().item()
        total += targets.size(0)

        progress_bar(batch_idx, len(train_loader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
        loss = train_loss/(batch_idx+1)

        
        if batch_idx + 1 == len(trainloader):
            pass
      #      save_results(epoch, history)
    result_train = 100.*correct/total
    
    return result_train, loss

def test(epoch,device):
    global best_acc
    net.eval()
    test_loss = 0
    correct = 0
    total = 0

    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(test_loader):

            # target = targets

            inputs, targets = inputs.to(device), targets.to(device)

            targets = targets.long()

            spk_rec, _ = forward_pass(net, num_steps, inputs)
            _, predicted = spk_rec.sum(dim=0).max(1)
            loss_val = loss_fn(spk_rec, targets)
            test_loss += loss_val.item()
            total += targets.size(0)
            correct += (predicted == targets).sum().item()


           
            progress_bar(batch_idx, len(test_loader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                         % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))

            loss = test_loss / (batch_idx + 1)
           

            if batch_idx == 0:
                total_target = targets
                total_predicted = predicted
            else:
                total_target = torch.cat((total_target, targets), dim=0)
                total_predicted = torch.cat((total_predicted, predicted), dim=0)


    # Save checkpoint.
    confusion = confusion_matrix(total_target.data.cpu().numpy(), total_predicted.data.cpu().numpy())


    acc = 100.*correct/total
    if acc > best_acc:
        print('Saving..')
        state = {
            'net': net.state_dict(),
            'acc': acc,
            'epoch': epoch,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/ckpt.pth')
        best_acc = acc

    # print("test_loss = ", loss)
    return acc, confusion, loss

if __name__ == '__main__':
    dtype=torch.float
    loss_fn = SF.ce_rate_loss()
    spike_grad = surrogate.fast_sigmoid(slope=25)
    beta = 0.95
    num_steps = 50
    lr = 0.01
    epoch_num = 50
    bat_size_train = 128
    bat_size_test = 128
    transform_train = transforms.Compose([
        transforms.Grayscale(num_output_channels=1),
        transforms.Resize([32,32]),
        transforms.ToTensor(),
        transforms.Normalize((0,), (1,)),
        # transforms.RandomHorizontalFlip(),
     ])
    transform_test = transforms.Compose([
        transforms.Grayscale(num_output_channels=1),
        transforms.Resize([32, 32]),
        transforms.ToTensor(),
        transforms.Normalize((0,), (1,)),
        # transforms.RandomHorizontalFlip(),
     ])
    net = Net()
    net = net.to(device)


    optimizer = optim.Adam(net.parameters(), lr=lr,betas=(0.9,0.999))
    #scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)

    for epoch in range(start_epoch, start_epoch+epoch_num):

        train_result, train_loss = train(epoch, device=device)

        test_result, test_confusion, test_loss = test(epoch, device=device)

        train_loss_list.append(train_loss)
        test_loss_list.append(test_loss)

        train_acc_list.append(train_result)
        test_acc_list.append(test_result)

The above is the accuracy curve, the orange dotted line represents the test set, and the blue solid line represents the training set.

Above is the loss curve.

The above is a data sample. After I convert it into a grayscale image, the background is white pixels and the trails is black pixels. Wouldn't it be better if the background were black pixels and the trails were white pixels?

[jeshraghian]

Converting into a discussion.

Some suggestions:

• setting threshold=0 seems like a bad idea. Low enough to trigger spikes, but above 0 should be the sweet spot.
• Don't set spike_grad, the default surrogate gradient is usually the best option
• consider skip connections

[EliteSasuke]

Thanks for your reply, I took your suggestion but it doesn't seem to be working. I used a new model which is a fully connected network with only one hidden layer.

class FCN(nn.Module):
    def __init__(self):
        super().__init__()
        # Initialize layers
        self.fc1 = nn.Linear(1024, 1024)
        self.lif1 = snn.Leaky(beta=beta)
        self.fc2 = nn.Linear(1024, 4)
        self.lif2 = snn.Leaky(beta=beta)

    def forward(self, x):

        # Initialize hidden states and outputs at t=0
        mem1 = self.lif1.init_leaky()
        mem2 = self.lif2.init_leaky()
        cur1 = self.fc1(x)
        spk1, mem1 = self.lif1(cur1, mem1)
        cur2 = self.fc2(spk1)
        spk2, mem2 = self.lif2(cur2, mem2)

        return spk2, mem2

I checked the membrane potentials of hidden layer neurons and they are mostly negative and cannot reach the threshold.Below is the hidden layer neuron membrane potential at time step 50 (I set batchsize to 1 for easy viewing).

Is my problem related to the input data?And how should I set the appropriate membrane potential threshold for neurons?

---

Update on June 23rd：

I tried to use the FCN model above to classify the cifar10 dataset, but I encountered the same problem of loss not decreasing.