#ignore-index

A simplified version of the book's collate function for instruction fine-tuning. What does the last line print?

pad_token_id, ignore_index = 50256, -100
batch = [[0, 1, 2, 3, 4], [5, 6], [7, 8, 9]]           # token IDs of 3 examples
batch_max_length = max(len(item) + 1 for item in batch)
targets_lst = []
for item in batch:
    padded = item + [pad_token_id] * (batch_max_length - len(item))
    targets = torch.tensor(padded[1:])                  # shift by one
    indices = torch.nonzero(targets == pad_token_id).squeeze()
    if indices.numel() > 1:
        targets[indices[1:]] = ignore_index             # keep the first pad token
    targets_lst.append(targets)
targets_tensor = torch.stack(targets_lst)
print((targets_tensor == ignore_index).sum().item())