Sequential Data and Recurrent Neural Networks

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import seaborn as snss
import pandas as pd
plt.rcParams['xtick.direction'] = 'in'
plt.rcParams['ytick.direction'] = 'in'
plt.rcParams['axes.grid'] = True
plt.rcParams['axes.axisbelow'] = True
plt.rcParams['figure.dpi'] = 150

# Check available hardware
import torch
if torch.cuda.is_available():
    print(f'GPU is available, using GPU {torch.cuda.get_device_name(0)}')
    device = torch.device('cuda')
elif torch.mps.is_available():
    print(f'Apple Metal Performance Shaders framework is available, using {torch.device("mps")}')
    device = torch.device('mps')
else:
    print(f'Hardware acceleration not available, using CPU')
    device = torch.device('cpu')

GPU is available, using GPU Tesla T4

Example with toy sequences¶

x_seq = torch.FloatTensor([[1,], [2,], [3,]])
x_seq = x_seq.repeat(1, 2)
print(f"x_seq has shape {x_seq.shape} which means it is a sequence of {x_seq.shape[0]} {x_seq.shape[1]}-dimensional vectors")

x_seq has shape torch.Size([3, 2]) which means it is a sequence of 3 2-dimensional vectors

An RNN has a hidden state $h^{(t)}$ which receives two inputs:

the previous hidden state $h^{(t-1)}$
the current input $x^{(t)}$

import torch.nn as nn
import torch.nn.functional as F

class RNN_Cell(nn.Module):
    def __init__(self, input_size=2, hidden_size=4):
        super().__init__()

        self.hidden_size = hidden_size

        # Weight and bias connecting previous hidden state
        self.w_hh = nn.Parameter(torch.randn(size=(hidden_size, hidden_size)))
        self.b_hh = nn.Parameter(torch.zeros(hidden_size))
        
        # Weight and bias connecting inputs
        self.w_ih = nn.Parameter(torch.randn(size=(hidden_size, input_size)))
        self.b_ih = nn.Parameter(torch.zeros(hidden_size))

    def forward(self, x):
        """ x has size [B, t, c]
        """
        B, T, C = x.shape
        ht = torch.zeros([B, self.hidden_size], device=x.device, dtype=x.dtype)
        ot = torch.zeros([B, T, self.hidden_size], device=x.device, dtype=x.dtype)
        for t in range(T):
            xt = x[:, t, :]
            print(f'At t = {t}')
            print(f'\tInput = {xt.numpy()}')
            zt = xt @ self.w_ih.T + self.b_ih + \
                 ht @ self.w_hh.T + self.b_hh
            ht = F.tanh(zt)
            print(f'\tHidden = {ht.detach().numpy()}')
            ot[:, t, :] = ht

        return ot, ht
            
torch_RNN = nn.RNN(input_size=2, hidden_size=4, batch_first=True)
print("Ground truth")
print("------------")
output, hidden = torch_RNN(x_seq[None])
print(f"Output is of size {output.shape}")
print(output)
print()
print(f"Last hidden is of size {hidden.shape}")
print(hidden)

print()
print("Our implementation")
print("------------------")
np.set_printoptions(precision=4)
cell = RNN_Cell()
with torch.no_grad():
    cell.w_hh[:] = torch_RNN.weight_hh_l0
    cell.b_hh[:] = torch_RNN.bias_hh_l0
    cell.w_ih[:] = torch_RNN.weight_ih_l0
    cell.b_ih[:] = torch_RNN.bias_ih_l0

output, hidden = cell(x_seq[None])

print()
print(f"Output is of size {output.shape}")
print(output)
print()
print(f"Last hidden is of size {hidden.shape}")
print(hidden)

Ground truth
------------
Output is of size torch.Size([1, 3, 4])
tensor([[[-0.2209, -0.5565, -0.5489,  0.4516],
         [ 0.4324, -0.7515, -0.8411,  0.7951],
         [ 0.8690, -0.8406, -0.9448,  0.8712]]], grad_fn=<TransposeBackward1>)

Last hidden is of size torch.Size([1, 1, 4])
tensor([[[ 0.8690, -0.8406, -0.9448,  0.8712]]], grad_fn=<StackBackward0>)

Our implementation
------------------
At t = 0
	Input = [[1. 1.]]
	Hidden = [[-0.2209 -0.5565 -0.5489  0.4516]]
At t = 1
	Input = [[2. 2.]]
	Hidden = [[ 0.4324 -0.7515 -0.8411  0.7951]]
At t = 2
	Input = [[3. 3.]]
	Hidden = [[ 0.869  -0.8406 -0.9448  0.8712]]

Output is of size torch.Size([1, 3, 4])
tensor([[[-0.2209, -0.5565, -0.5489,  0.4516],
         [ 0.4324, -0.7515, -0.8411,  0.7951],
         [ 0.8690, -0.8406, -0.9448,  0.8712]]], grad_fn=<CopySlices>)

Last hidden is of size torch.Size([1, 4])
tensor([[ 0.8690, -0.8406, -0.9448,  0.8712]], grad_fn=<TanhBackward0>)

Classification example: IMDB sentiment database¶

!pip install pyarrow==15.0.2 datasets==4.0.0 transformers==5.0.0 numpy==1.26.4

Requirement already satisfied: pyarrow==15.0.2 in /opt/conda/lib/python3.11/site-packages (15.0.2)
Collecting datasets==4.0.0
  Using cached datasets-4.0.0-py3-none-any.whl.metadata (19 kB)
Collecting transformers==5.0.0
  Using cached transformers-5.0.0-py3-none-any.whl.metadata (37 kB)
Requirement already satisfied: numpy==1.26.4 in /opt/conda/lib/python3.11/site-packages (1.26.4)
Requirement already satisfied: filelock in /opt/conda/lib/python3.11/site-packages (from datasets==4.0.0) (3.13.1)
Requirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/conda/lib/python3.11/site-packages (from datasets==4.0.0) (0.3.8)
Requirement already satisfied: pandas in /opt/conda/lib/python3.11/site-packages (from datasets==4.0.0) (2.2.2)
Collecting requests>=2.32.2 (from datasets==4.0.0)
  Using cached requests-2.34.2-py3-none-any.whl.metadata (4.8 kB)
Collecting tqdm>=4.66.3 (from datasets==4.0.0)
  Using cached tqdm-4.70.1-py3-none-any.whl.metadata (57 kB)
Collecting xxhash (from datasets==4.0.0)
  Downloading xxhash-4.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (17 kB)
Collecting multiprocess<0.70.17 (from datasets==4.0.0)
  Using cached multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)
Requirement already satisfied: fsspec<=2025.3.0,>=2023.1.0 in /opt/conda/lib/python3.11/site-packages (from fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0) (2024.3.1)
Collecting huggingface-hub>=0.24.0 (from datasets==4.0.0)
  Using cached huggingface_hub-2.2.0-py3-none-any.whl.metadata (16 kB)
Requirement already satisfied: packaging in /opt/conda/lib/python3.11/site-packages (from datasets==4.0.0) (24.0)
Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.11/site-packages (from datasets==4.0.0) (6.0.1)
  Using cached huggingface_hub-1.33.0-py3-none-any.whl.metadata (16 kB)
Collecting regex!=2019.12.17 (from transformers==5.0.0)
  Using cached regex-2026.9.29-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (40 kB)
Collecting tokenizers<=0.23.0,>=0.22.0 (from transformers==5.0.0)
  Using cached tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.3 kB)
Collecting typer-slim (from transformers==5.0.0)
  Using cached typer_slim-0.24.0-py3-none-any.whl.metadata (4.2 kB)
Collecting safetensors>=0.4.3 (from transformers==5.0.0)
  Using cached safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.2 kB)
Collecting aiohttp!=4.0.0a0,!=4.0.0a1 (from fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0)
  Using cached aiohttp-3.14.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (8.3 kB)
Collecting click<9.0.0,>=8.4.2 (from huggingface-hub>=0.24.0->datasets==4.0.0)
  Using cached click-8.5.0-py3-none-any.whl.metadata (2.6 kB)
Collecting hf-xet<2.0.0,>=1.6.0 (from huggingface-hub>=0.24.0->datasets==4.0.0)
  Using cached hf_xet-1.7.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (4.9 kB)
Requirement already satisfied: httpx<1,>=0.23.0 in /opt/conda/lib/python3.11/site-packages (from huggingface-hub>=0.24.0->datasets==4.0.0) (0.27.0)
Requirement already satisfied: typing-extensions>=4.1.0 in /opt/conda/lib/python3.11/site-packages (from huggingface-hub>=0.24.0->datasets==4.0.0) (4.11.0)
Requirement already satisfied: charset_normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests>=2.32.2->datasets==4.0.0) (3.3.2)
Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.11/site-packages (from requests>=2.32.2->datasets==4.0.0) (3.6)
Requirement already satisfied: urllib3<3,>=1.26 in /opt/conda/lib/python3.11/site-packages (from requests>=2.32.2->datasets==4.0.0) (2.2.1)
Requirement already satisfied: certifi>=2023.5.7 in /opt/conda/lib/python3.11/site-packages (from requests>=2.32.2->datasets==4.0.0) (2024.2.2)
Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.11/site-packages (from pandas->datasets==4.0.0) (2.9.0)
Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.11/site-packages (from pandas->datasets==4.0.0) (2024.1)
Requirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.11/site-packages (from pandas->datasets==4.0.0) (2024.1)
Collecting typer>=0.24.0 (from typer-slim->transformers==5.0.0)
  Using cached typer-0.27.3-py3-none-any.whl.metadata (16 kB)
Collecting aiohappyeyeballs>=2.5.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0)
  Using cached aiohappyeyeballs-2.7.1-py3-none-any.whl.metadata (5.9 kB)
Collecting aiosignal>=1.4.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0)
  Using cached aiosignal-1.4.0-py3-none-any.whl.metadata (3.7 kB)
Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0) (23.2.0)
Collecting frozenlist>=1.1.1 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0)
  Using cached frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl.metadata (20 kB)
Collecting multidict<8.0,>=4.5 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0)
  Using cached multidict-7.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (6.2 kB)
Collecting propcache>=0.2.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0)
  Using cached propcache-0.5.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (25 kB)
Collecting yarl<2.0,>=1.25.1 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.3.0,>=2023.1.0->datasets==4.0.0)
  Using cached yarl-1.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (111 kB)
Requirement already satisfied: anyio in /opt/conda/lib/python3.11/site-packages (from httpx<1,>=0.23.0->huggingface-hub>=0.24.0->datasets==4.0.0) (4.3.0)
Requirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.11/site-packages (from httpx<1,>=0.23.0->huggingface-hub>=0.24.0->datasets==4.0.0) (1.0.5)
Requirement already satisfied: sniffio in /opt/conda/lib/python3.11/site-packages (from httpx<1,>=0.23.0->huggingface-hub>=0.24.0->datasets==4.0.0) (1.3.1)
Requirement already satisfied: h11<0.15,>=0.13 in /opt/conda/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->huggingface-hub>=0.24.0->datasets==4.0.0) (0.14.0)
Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets==4.0.0) (1.16.0)
Collecting shellingham>=1.3.0 (from typer>=0.24.0->typer-slim->transformers==5.0.0)
  Using cached shellingham-1.5.4-py2.py3-none-any.whl.metadata (3.5 kB)
Collecting rich>=13.8.0 (from typer>=0.24.0->typer-slim->transformers==5.0.0)
  Using cached rich-15.0.0-py3-none-any.whl.metadata (18 kB)
Collecting annotated-doc>=0.0.2 (from typer>=0.24.0->typer-slim->transformers==5.0.0)
  Using cached annotated_doc-0.0.5-py3-none-any.whl.metadata (6.5 kB)
Collecting markdown-it-py>=2.2.0 (from rich>=13.8.0->typer>=0.24.0->typer-slim->transformers==5.0.0)
  Using cached markdown_it_py-4.2.0-py3-none-any.whl.metadata (7.4 kB)
Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.11/site-packages (from rich>=13.8.0->typer>=0.24.0->typer-slim->transformers==5.0.0) (2.17.2)
Collecting mdurl~=0.1 (from markdown-it-py>=2.2.0->rich>=13.8.0->typer>=0.24.0->typer-slim->transformers==5.0.0)
  Using cached mdurl-0.1.2-py3-none-any.whl.metadata (1.6 kB)
Using cached datasets-4.0.0-py3-none-any.whl (494 kB)
Using cached transformers-5.0.0-py3-none-any.whl (10.1 MB)
Using cached huggingface_hub-1.33.0-py3-none-any.whl (846 kB)
Using cached multiprocess-0.70.16-py311-none-any.whl (143 kB)
Using cached regex-2026.9.29-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (804 kB)
Using cached requests-2.34.2-py3-none-any.whl (73 kB)
Using cached safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (516 kB)
Using cached tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)
Using cached tqdm-4.70.1-py3-none-any.whl (80 kB)
Using cached typer_slim-0.24.0-py3-none-any.whl (3.4 kB)
Downloading xxhash-4.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (260 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 261.0/261.0 kB 10.1 MB/s eta 0:00:00
Using cached aiohttp-3.14.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (1.9 MB)
Using cached click-8.5.0-py3-none-any.whl (125 kB)
Using cached hf_xet-1.7.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (4.2 MB)
Using cached typer-0.27.3-py3-none-any.whl (123 kB)
Using cached aiohappyeyeballs-2.7.1-py3-none-any.whl (15 kB)
Using cached aiosignal-1.4.0-py3-none-any.whl (7.5 kB)
Using cached annotated_doc-0.0.5-py3-none-any.whl (5.3 kB)
Using cached frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl (231 kB)
Using cached multidict-7.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (480 kB)
Using cached propcache-0.5.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (235 kB)
Using cached rich-15.0.0-py3-none-any.whl (310 kB)
Using cached shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)
Using cached yarl-1.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (117 kB)
Using cached markdown_it_py-4.2.0-py3-none-any.whl (91 kB)
Using cached mdurl-0.1.2-py3-none-any.whl (10.0 kB)
Installing collected packages: xxhash, tqdm, shellingham, safetensors, requests, regex, propcache, multiprocess, multidict, mdurl, hf-xet, frozenlist, click, annotated-doc, aiohappyeyeballs, yarl, markdown-it-py, aiosignal, rich, huggingface-hub, aiohttp, typer, tokenizers, typer-slim, datasets, transformers
  Attempting uninstall: tqdm
    Found existing installation: tqdm 4.66.2
    Uninstalling tqdm-4.66.2:
      Successfully uninstalled tqdm-4.66.2
  Attempting uninstall: requests
    Found existing installation: requests 2.31.0
    Uninstalling requests-2.31.0:
      Successfully uninstalled requests-2.31.0
  Attempting uninstall: click
    Found existing installation: click 8.1.7
    Uninstalling click-8.1.7:
      Successfully uninstalled click-8.1.7
Successfully installed aiohappyeyeballs-2.7.1 aiohttp-3.14.5 aiosignal-1.4.0 annotated-doc-0.0.5 click-8.5.0 datasets-4.0.0 frozenlist-1.8.0 hf-xet-1.7.0 huggingface-hub-1.33.0 markdown-it-py-4.2.0 mdurl-0.1.2 multidict-7.1.0 multiprocess-0.70.16 propcache-0.5.4 regex-2026.9.29 requests-2.34.2 rich-15.0.0 safetensors-0.8.0 shellingham-1.5.4 tokenizers-0.22.2 tqdm-4.70.1 transformers-5.0.0 typer-0.27.3 typer-slim-0.24.0 xxhash-4.0.2 yarl-1.25.1

from datasets import load_dataset

imdb_dataset = load_dataset("stanfordnlp/imdb")

Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.

imdb_dataset

DatasetDict({
    train: Dataset({
        features: ['text', 'label'],
        num_rows: 25000
    })
    test: Dataset({
        features: ['text', 'label'],
        num_rows: 25000
    })
    unsupervised: Dataset({
        features: ['text', 'label'],
        num_rows: 50000
    })
})

from torch.utils.data import Dataset, random_split, DataLoader

class IMDBDataset(Dataset):
    def __init__(self, hf_data):
        self.hf_data = hf_data

    def __len__(self):
        return len(self.hf_data)

    def __getitem__(self, idx):
        return {
            'text': self.hf_data[idx]['text'],
            'label': self.hf_data[idx]['label']
        }

train_val_dataset = IMDBDataset(imdb_dataset['train'])
train_dataset, val_dataset = random_split(train_val_dataset, lengths=[0.8, 0.2])

loader = DataLoader(train_dataset, batch_size=1, shuffle=True)
batch = next(iter(loader))
print(batch['text'], batch['label'])

['Ying, a Chinese girl who speaks Czech, invited us to screening of a Czech movie (with English subtitles) in the Department of Visual and Environmental Studies (VES). It was the first time I saw Samotá&#345;i (Loners, 2000) and it was pretty good.<br /><br />Much like in many other Czech movies, the seven central characters seem to have a pretty difficult, dirty life; the web indicates that this theme was popular among the U.S. movies in the early 1990s. Their relationships are breaking up, combining, and recombining. Another typical feature of the Czech movies is that neither of the characters is designed to be a universally negative one and neither of them is a permanently positive character either. Also, you can see how the characters judge the features of others depending on the context; that\'s a very realistic feature of the movie\'s psychological analysis.<br /><br />Ond&#345;ej is a talented and married young surgeon who has two daughters. Nevertheless, you learn that he has only studied neurobiology to prove how much he loved another woman, Hanka. He is so obsessed that he repeatedly dresses up as a plumber to get into Hanka\'s parents\' house - a house that he repeatedly burns.<br /><br />Meanwhile, Hanka has a very mixed relationship with her parents. She just decides - by tossing up a coin - to break up with Petr who works in a private radio station. Hanka does not view her parents\' bourgeois life as a good example but seems rather unsuccessful in creating a better environment. But she is a very flexible figure, as far as the type of her boyfriends go.<br /><br />For a while, Hanka seems to have serious plans with Jakub, an innocent drug addict whose memory seems to be rather devastated by the drugs. However, the friends from his band inform Jakub that he already has another girlfriend. Hanka is disappointed and returns to her parents.<br /><br />When Hanka and Petr break up, it is organized by Robert, a matchmaker who also works for a travel agency where his job is to show the life of ordinary Czech people to Japanese tourists. Robert - who also provides Jakub with marijuana - is never serious about anything and he usually sleeps with many different women; eventually, his mother dies in a hospital and he has his own ways to deal with the depression.<br /><br />Vesna (a Slavic word for "Spring") who came to Prague from Macedonia works as a barmaid - and you won\'t learn whether she came to Czechia in order to see her dad or UFOs. She seems pretty confused but sometimes helps the other characters from their problems.<br /><br />Petr works in the radio station and he is the only one who likes his job - a job that he eventually loses. He announces to his audience that he broke up with Hanka - which is how Ond&#345;ej learns about the news that make him very happy.<br /><br />Finally, Ond&#345;ej\'s wife Lenka is always ready to forgive him and stabilize their marriage - even after Ond&#345;ej asks a magician to make him disappear so that Ond&#345;ej can try to capture Hanka again. (The magician pays his debt because he is a brother of a victim of an important car accident - Jakub and Hanka bring the victim to the hospital and Ond&#345;ej saves his life.) Lenka also works for the travel agency - as a translator - and eventually she has to translate some hysterical scenes for 20 or so Japanese tourists who are shooting their movies during Hanka parents\' dinner.<br /><br />The seven characters interact in interesting and exciting ways that would be natural if Prague were smaller by four orders of magnitude. Given the actual size of the Czech capital, it looks a bit unlikely that all these events would take place among seven people, but it is fun.'] tensor([1])

train_val_dataset = imdb_dataset['train']
train_dataset, val_dataset = random_split(train_val_dataset, lengths=[0.8, 0.2])
test_dataset = imdb_dataset['test']

loader = DataLoader(train_dataset, batch_size=1, shuffle=True)
batch = next(iter(loader))
print(batch['text'], batch['label'])

['I went into this film with expectations, from the hype, that it would be insightful and uplifting. Certainly something more than a cheap promotional for the band "Wilco."<br /><br />Instead we get a lot of moping and whining about "the process," a dishonorable and no doubt one-sided portrayal of one band members who was kicked out by the prima donna lead singer/songwriter, a gut-wrenching confession by the fallen member\'s friend -- for like 18 years -- saying the "friendship had run its course," and this whiny, uncompelling story about how one record label "hurt their feelings" by dumping them, only so that the band could immediately get 50 offers from other labels (oh, the tension...not!) They tried their best to make it look like it was a strain, but I suspect it was all smoke and mirrors to generate a tragedy that didn\'t exist. This doesn\'t even take into account the long stretches where we get many of their newest songs shoved at us in full without any storyline, insight or even a decent job at cinematography. The strained attempts at emotional sincerity or reasonable perspective on life made me sick to watch.<br /><br />From the film, this band sounds like a bunch of vile little babies who poke around to find a voice they don\'t have and think they\'re some kind of guardians for the art of music, which they most definitely are not. And I thought the music sucked, and I couldn\'t even understand the lyrics due to the mumbling style of the lead singer.<br /><br />I give it a 2/10.'] tensor([0])

Preprocessing each sample¶

Tokenization¶

import re
def tokenize(sentence):
    # Remove HTML tags - anything that starts with <, ends with >, and doesn't contain a > inside
    sentence = re.sub('<[^>]*>', '', sentence)

    # Emoticon processing
    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)', sentence.lower())

    # Standardize all whitespace and append emoticons to make sentiment processing easier
    sentence = re.sub('[\W]+', ' ', sentence.lower()) + ' '.join(emoticons).replace('-', '')

    return sentence.split()

text = '  your  :-) text   here <br/> :-)'
tokenize(text)

['your', 'text', 'here', ':)', ':)']

tokenize(batch['text'][0])

['i',
 'went',
 'into',
 'this',
 'film',
 'with',
 'expectations',
 'from',
 'the',
 'hype',
 'that',
 'it',
 'would',
 'be',
 'insightful',
 'and',
 'uplifting',
 'certainly',
 'something',
 'more',
 'than',
 'a',
 'cheap',
 'promotional',
 'for',
 'the',
 'band',
 'wilco',
 'instead',
 'we',
 'get',
 'a',
 'lot',
 'of',
 'moping',
 'and',
 'whining',
 'about',
 'the',
 'process',
 'a',
 'dishonorable',
 'and',
 'no',
 'doubt',
 'one',
 'sided',
 'portrayal',
 'of',
 'one',
 'band',
 'members',
 'who',
 'was',
 'kicked',
 'out',
 'by',
 'the',
 'prima',
 'donna',
 'lead',
 'singer',
 'songwriter',
 'a',
 'gut',
 'wrenching',
 'confession',
 'by',
 'the',
 'fallen',
 'member',
 's',
 'friend',
 'for',
 'like',
 '18',
 'years',
 'saying',
 'the',
 'friendship',
 'had',
 'run',
 'its',
 'course',
 'and',
 'this',
 'whiny',
 'uncompelling',
 'story',
 'about',
 'how',
 'one',
 'record',
 'label',
 'hurt',
 'their',
 'feelings',
 'by',
 'dumping',
 'them',
 'only',
 'so',
 'that',
 'the',
 'band',
 'could',
 'immediately',
 'get',
 '50',
 'offers',
 'from',
 'other',
 'labels',
 'oh',
 'the',
 'tension',
 'not',
 'they',
 'tried',
 'their',
 'best',
 'to',
 'make',
 'it',
 'look',
 'like',
 'it',
 'was',
 'a',
 'strain',
 'but',
 'i',
 'suspect',
 'it',
 'was',
 'all',
 'smoke',
 'and',
 'mirrors',
 'to',
 'generate',
 'a',
 'tragedy',
 'that',
 'didn',
 't',
 'exist',
 'this',
 'doesn',
 't',
 'even',
 'take',
 'into',
 'account',
 'the',
 'long',
 'stretches',
 'where',
 'we',
 'get',
 'many',
 'of',
 'their',
 'newest',
 'songs',
 'shoved',
 'at',
 'us',
 'in',
 'full',
 'without',
 'any',
 'storyline',
 'insight',
 'or',
 'even',
 'a',
 'decent',
 'job',
 'at',
 'cinematography',
 'the',
 'strained',
 'attempts',
 'at',
 'emotional',
 'sincerity',
 'or',
 'reasonable',
 'perspective',
 'on',
 'life',
 'made',
 'me',
 'sick',
 'to',
 'watch',
 'from',
 'the',
 'film',
 'this',
 'band',
 'sounds',
 'like',
 'a',
 'bunch',
 'of',
 'vile',
 'little',
 'babies',
 'who',
 'poke',
 'around',
 'to',
 'find',
 'a',
 'voice',
 'they',
 'don',
 't',
 'have',
 'and',
 'think',
 'they',
 're',
 'some',
 'kind',
 'of',
 'guardians',
 'for',
 'the',
 'art',
 'of',
 'music',
 'which',
 'they',
 'most',
 'definitely',
 'are',
 'not',
 'and',
 'i',
 'thought',
 'the',
 'music',
 'sucked',
 'and',
 'i',
 'couldn',
 't',
 'even',
 'understand',
 'the',
 'lyrics',
 'due',
 'to',
 'the',
 'mumbling',
 'style',
 'of',
 'the',
 'lead',
 'singer',
 'i',
 'give',
 'it',
 'a',
 '2',
 '10']

Vocabulary building¶

from collections import Counter
from tqdm import tqdm

def build_vocab_counts(dataset):
    vocab = Counter()
    for sample in tqdm(dataset):
        tokens = tokenize(sample['text'])
        vocab.update(tokens)
    return vocab

vocab_counts = build_vocab_counts(train_val_dataset)

100%|██████████| 25000/25000 [00:04<00:00, 5024.98it/s]

sorted(vocab_counts.items(), key = lambda x: x[1], reverse=True)[-20:]

[('pommies', 1),
 ('flamin', 1),
 ('abos', 1),
 ('yacca', 1),
 ('bonser', 1),
 ('donger', 1),
 ('sheilas', 1),
 ('everage', 1),
 ('poofters', 1),
 ('quantas', 1),
 ('gyppos', 1),
 ('earls', 1),
 ('rickmansworth', 1),
 ('chundering', 1),
 ('euphemisms', 1),
 ('covington', 1),
 ('snacka', 1),
 ('fitzgibbon', 1),
 ('dunny', 1),
 ('grossest', 1)]

vocab = {
    '<pad>': 0,
    '<unk>': 1, 
}
for idx, (token, count) in enumerate(sorted(vocab_counts.items(), key=lambda x : x[1], reverse=True)):
    vocab[token] = idx + 2

print(len(vocab))

Sequence encoding¶

def encode_tokens(tokens):
    return [ vocab.get(token, vocab['<unk>']) for token in tokens]

encode_tokens(tokenize(batch['text'][0]))

Putting it all together¶

def collate_fn(batch):
    """ This will be what DataLoader uses to process all of the samples together
    """
    texts, labels, lengths = [], [], []
    for sample in batch:
        tokens = tokenize(sample['text'])
        encoded = encode_tokens(tokens)
        texts.append(torch.tensor(encoded))
        labels.append(sample['label'])
        lengths.append(len(encoded))

    labels = torch.tensor(labels)
    lengths = torch.tensor(lengths)

    # Pad each sequence to have a common shape
    texts = nn.utils.rnn.pad_sequence(texts, batch_first=True)

    return texts, labels, lengths

loader = DataLoader(train_val_dataset, batch_size=2, shuffle=False, 
                    collate_fn=collate_fn)

batch = next(iter(loader))

print('Text batch:', batch[0].shape, batch[0].dtype)
print('Label batch:', batch[1].shape, batch[1].dtype)
print('Lengths batch:', batch[2])

Text batch: torch.Size([2, 291]) torch.int64
Label batch: torch.Size([2]) torch.int64
Lengths batch: tensor([291, 228])

# What does a padded sequence look like?
batch[0]

tensor([[   10,  1607,    10,   244,  1991,  4099,    38,    60,   370,  1109,
            86,     5,    31,     2,  7095,    12,  3397,     8,    54,     8,
            14,    84,   620,     9,  6811,    10,    82,   553,    12,    32,
            84,     8,    14, 19102,    34,  1190,    13,  9266,    46,     8,
           124,   792,     6,  2519,    11,   663,  1617,   111,     4,   334,
             5,   107,  1184,  3091,    10,    65,    68,     6,    66,    11,
            16,   541,     2,   112,     7,  3999,   187,     4,   185,  3924,
           452,  1425,   766,  4335,    36,   489,     6,   839,   286,    53,
            51,    44,   110,     9,   833,    53,   489,     6,  1134,    41,
         11898,     6,   229,    49,   432,     5,   650,    23,    48,     2,
           841, 26535,   196,    44,   802,   985,  1334,   140,    15,     2,
          2550,   324,     3,  1504,  1334,     9,     2,  2335,  1618,     9,
           199,  2246,  7277,     3,  1956, 19103,     5, 20134,    44,    67,
          4718,    23,  2406,    53,    47,   379,    17,    41,   452,  1708,
          8185,     3,  1004,   335,    48,  1085,    71,    44,    10,   244,
          1991,  4099,     7,    12,  1434,   155,   593,    11,    14,  1184,
          7815,    65,     2,   379,     3,   996,   137,    26,   171,     3,
           228,   199,    59,    93,     8,    13,    24,   323,    39,    49,
          6472,    91,  4374,   136,    60, 22658,   330,   169,     8,  1597,
             9,   619,   379,     3,   996,    26,     4,   669, 10084,     9,
          3924,   428,    59, 14256,  4449,  4672,    67,  1515,     6,    50,
           152,   409,   299,  1748,    68,   379,   137,     9,    27,   107,
            10,    80, 12581,     2,  1035,    16,     2,   191,    12,   100,
           379,   613,     9,     2,    20,     7,   613,    16,  1598,  4934,
           248,    73,    43,     6,  1442,    77,     3,    95,   275,     6,
            29,   613,     9,  7815,  2235,     9,   835,    10,   244,  1991,
          4099,     7,     4,    50,    20,    16,   253,  1788,     6,  2074,
             2,  3531,     3, 17376,    58,  5268,  1431,     5,  3924,   428,
            19,    65,    11,    20,   151,    21,    28,    74,     5,     4,
           112],
        [   10,   244,  1991,  4099,     7,     4, 17377,     3,  1949,  8621,
          2412,     8,   151,    21,   547,    48,    30,    13,   985,  2685,
            26,    86,    11,    20,    51,   975,    29,   621,   607,    23,
           100,   640,    15,    16,     2,  2294,    12,  6155,   895,   996,
             7,    35,  8073, 10781,  2745,    12,   216,    21,   282,    10,
           138,   108,  1408,  1142,   107,    17,   895,   996,  2474,    33,
            64,  1457,    49,  8958,  2685,    19,   118,    26,     2,  1408,
          1142,   107,    17, 11899, 47042,     3, 18192, 37902,  1268,    86,
            33,    89,    21,  1772,     2,   172,   268,    16,   147,  2136,
          1877,   288, 47043,  7003,     9,     2, 11285,    19,    24,     4,
         47044,     9,  1668,     3,   147,  1949,  2698,    99,    39,     2,
          1938,  5224,     9,    62,    69,   154,  1899,     6,     2,  2123,
             5,  3198, 32679,    13, 16606,  2568,    19,    24,     4,  6717,
             5,  4881,  4719,    23,  7701, 24389,   161,  2569,    42, 13784,
          1395,  1260,     9,  2281,     5,   996,     2,  2983, 21292,   143,
           192,    83,  2640,    30, 24390,   573, 29110,  1463,   199,   335,
             3,   356,    40,    26,    58, 11562,    23,  2434,    54,  1485,
           733,  2503,     3,     2,   172,   560,    29,   304,    16,     4,
           125,     9,   191,    22,  1214,   380,    21,    66,   659, 11562,
             9,    35,   296,    20,     9,   231,   347,     5,  1497,    42,
          3733, 15321,    11,  6718,  1395,  1260,     7,   328,     4,  1395,
          1260,    73,    35,  3398,  2265,  1244,     6,   214,     6,  1297,
         18193,    17,     2, 22659,     5,   356,    13,  2330,     0,     0,
             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
             0]])

Checking that it works¶

train_loader = DataLoader(train_dataset, batch_size=32, num_workers=2, shuffle=True, collate_fn=collate_fn)
val_loader = DataLoader(val_dataset, batch_size=32, num_workers=2, shuffle=False, collate_fn=collate_fn)
test_loader = DataLoader(test_dataset, batch_size=32, num_workers=2, shuffle=False, collate_fn=collate_fn)

from tqdm import tqdm
for batch in tqdm(train_loader):
    a = len(batch[0])
for batch in tqdm(val_loader):
    a = len(batch[0])
for batch in tqdm(test_loader):
    a = len(batch[1])

100%|██████████| 625/625 [00:04<00:00, 139.08it/s]
100%|██████████| 157/157 [00:01<00:00, 86.71it/s] 
100%|██████████| 782/782 [00:05<00:00, 154.61it/s]

The neural network part¶

# The embedding layer
# Role: convert the tokens (index of each word in the vocabulary) into a float-valued vector. 
# This is trainable. Ideally, similar words ("happy" "ecstatic" "joyful") get mapped to similar embeddings
#
# This is a lookup table. We don't apply activations. It just makes the network able to process the words easier

import torch.nn as nn

embedding = nn.Embedding(num_embeddings=len(vocab),
                         embedding_dim=5,
                         padding_idx=0)

embedding(batch[0])

tensor([[[ 0.2479,  1.2932, -0.6358,  0.4109, -1.1808],
         [ 1.1749,  1.3072,  0.1997, -0.7165,  0.1751],
         [-1.6811,  0.3445,  0.6239, -1.6965, -0.1272],
         ...,
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]],

        [[-0.6953, -1.1784, -1.2069,  0.3019,  0.7441],
         [ 1.1450, -0.4860,  0.8701, -0.3199,  0.2850],
         [-0.6267,  0.3853,  2.8689,  0.9056,  0.1998],
         ...,
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]],

        [[-0.7870, -1.4343, -0.6233, -0.3136,  0.1094],
         [ 1.3517, -0.2219,  0.4292, -1.5721, -0.9920],
         [ 0.0471, -0.7046, -0.2849,  0.5618,  1.4956],
         ...,
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]],

        ...,

        [[ 1.8067,  0.6434, -0.7601,  0.8410, -0.8968],
         [ 1.1534,  0.3719,  1.3020,  1.4731, -0.1524],
         [ 1.2299, -1.5774, -0.7618, -0.1498,  0.4041],
         ...,
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]],

        [[-1.2845, -0.1079,  1.5384,  1.3041, -0.6044],
         [-1.0643, -0.9902,  1.0545, -0.3694, -0.2472],
         [ 0.3569, -0.3388, -0.5320, -0.8539, -1.3272],
         ...,
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]],

        [[ 1.8067,  0.6434, -0.7601,  0.8410, -0.8968],
         [ 0.3559, -0.0247, -1.4254,  0.6853,  0.6276],
         [-0.7870, -1.4343, -0.6233, -0.3136,  0.1094],
         ...,
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]]],
       grad_fn=<EmbeddingBackward0>)

# Embedding turns this from a list of integers into a list of vectors
embedding(batch[0]).shape

torch.Size([8, 770, 5])

# Packing the sequence is useful for numerical efficiency. The elements from each batch element are interleaved
# to let them be processed without wasting computation on padding elements
nn.utils.rnn.pack_padded_sequence(embedding(batch[0]), batch[2], enforce_sorted=False, batch_first=True)

PackedSequence(data=tensor([[ 0.2139, -1.0111,  0.7977,  0.2365,  0.2288],
        [-0.7870, -1.4343, -0.6233, -0.3136,  0.1094],
        [-1.2845, -0.1079,  1.5384,  1.3041, -0.6044],
        ...,
        [ 0.7079, -1.8732, -0.2077,  0.0625, -0.9782],
        [ 2.2620,  0.6611, -0.3340,  1.2712, -1.3557],
        [ 1.4973, -0.2092, -0.9083, -0.6872, -0.5027]],
       grad_fn=<PackPaddedSequenceBackward0>), batch_sizes=tensor([8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
        8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
        8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
        8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
        8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 7, 7,
        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5,
        5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
        4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
        4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3,
        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
        3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1]), sorted_indices=tensor([3, 2, 6, 5, 1, 7, 4, 0]), unsorted_indices=tensor([7, 4, 1, 0, 6, 3, 2, 5]))

import torch.nn.functional as F

class RNN_Model(nn.Module):
    def __init__(self, vocab_size, embedding_dim, rnn_size, fc_size):
        super().__init__()

        self.embedding = nn.Embedding(
            num_embeddings = vocab_size,
            embedding_dim = embedding_dim,
            padding_idx=0)

        self.rnn = nn.RNN(embedding_dim, rnn_size, batch_first=True)
        self.fc1 = nn.Linear(rnn_size, fc_size)
        self.fc2 = nn.Linear(fc_size, 2)

    def forward(self, text, lengths):
        x = self.embedding(text)
        x = nn.utils.rnn.pack_padded_sequence(
            x, lengths, enforce_sorted=False, batch_first=True)
        x, h = self.rnn(x)

        x = h[-1, :, :] # -1 to collect the last layer output

        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        return x

torch.manual_seed(42)
model = RNN_Model(
    vocab_size=len(vocab),
    embedding_dim=32,
    rnn_size=128,
    fc_size=128,
)
model

RNN_Model(
  (embedding): Embedding(75979, 32, padding_idx=0)
  (rnn): RNN(32, 128, batch_first=True)
  (fc1): Linear(in_features=128, out_features=128, bias=True)
  (fc2): Linear(in_features=128, out_features=2, bias=True)
)

import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam
from torch.utils.data import DataLoader, random_split
from tqdm.notebook import tqdm, trange

def train(model, optimizer, loss_func, train_loader, val_loader, num_epochs=10):
    model = model.to(device)

    train_loss = []
    train_acc = []
    val_loss = []
    val_acc = []

    pbar_epoch = trange(num_epochs)
    for ii in pbar_epoch:
        model.train()
        train_loss_ = 0.
        train_correct_ = 0.
        for xb, yb, lengths in tqdm(train_loader, leave=False):
            # Send data to device
            xb = xb.to(device)
            yb = yb.to(device)

            # Make a prediction, compute the loss
            y_pred = model(xb, lengths.cpu().numpy())
            loss = loss_func(y_pred, yb)

            # Backpropagate, apply gradient updates, zero gradients
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()

            # Logging
            train_loss_ += loss.item() * yb.shape[0]
            train_correct_ += (torch.argmax(y_pred, dim=1) == yb).float().sum().item()
            
        train_loss.append(train_loss_ / len(train_loader.dataset))
        train_acc.append(train_correct_ / len(train_loader.dataset))

        model.eval()
        with torch.no_grad():
            val_loss_ = 0.
            val_correct_ = 0.
            for xb, yb, lengths in tqdm(val_loader, leave=False):
                # Send data to device
                xb = xb.to(device)
                yb = yb.to(device)
                
                # Make a prediction, compute the loss
                y_pred = model(xb, lengths.cpu().numpy())
                loss = loss_func(y_pred, yb)

                # Logging
                val_loss_ += loss.item() * yb.shape[0]
                val_correct_ += (torch.argmax(y_pred, dim=1) == yb).float().sum().item()
            val_loss.append(val_loss_ / len(val_loader.dataset))
            val_acc.append(val_correct_ / len(val_loader.dataset))

        pbar_epoch.set_postfix(
            epoch=f"{ii+1:03d} / {num_epochs:03d}",
            train_loss=f"{train_loss[-1]:.3f}",
            train_acc=f"{train_acc[-1]:.3f}",
            val_loss=f"{val_loss[-1]:.3f}",
            val_acc=f"{val_acc[-1]:.3f}")

    return model, (train_loss, train_acc, val_loss, val_acc)

# Define a loss function and optimizer
loss = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
model, model_stats = train(model, optimizer, loss, train_loader, val_loader, num_epochs=5)

fig, ax = plt.subplots(1, 2, sharex=True, figsize=(5, 2.5))

ax[0].plot(model_stats[0], color='black', label='Train')
ax[0].plot(model_stats[2], color='red', label='Validation')
ax[0].set(xlabel='Epoch', ylabel='Loss')

ax[1].plot(model_stats[1], color='black', label='Train')
ax[1].plot(model_stats[3], color='red', label='Validation')
ax[1].set(xlabel='Epoch', ylabel='Accuracy')

fig.legend(*ax[1].get_legend_handles_labels(), 
           loc='center left', bbox_to_anchor=[1, 0.5], framealpha=0)

plt.tight_layout()

model.eval()
with torch.no_grad():
    test_correct_ = 0.
    for xb, yb, lengths in tqdm(test_loader,leave=False):
        xb = xb.to(device)
        yb = yb.to(device)
        y_pred = model(xb, lengths.cpu().numpy())
        y_pred = torch.argmax(y_pred, dim=1)
        test_correct_ += (y_pred == yb).float().sum().item()
    test_acc = test_correct_ / len(test_loader.dataset)

print(f"Test accuracy: {test_acc:.3f}")

Test accuracy: 0.647

# Classify a single example
idx = np.random.choice(len(test_dataset))
sample = test_dataset[idx]
text = sample['text']
print('Comment:', text)
print()
print('True sentiment:', sample['label'])

# Convert comment with tokenizer, etc.
model.eval()
batch = collate_fn([sample,])
with torch.no_grad():
    y_pred = model(batch[0].to(device), batch[2])
    y_pred = F.softmax(y_pred, dim=1)

print('Predicted probability:', y_pred[0].cpu().numpy())

Comment: A young theater actress reluctantly accepts her first major part in a staged play as Lady Macbeth thanks to the mishap of the production's diva falling in front of a moving car. A success winning her accolades, Betty(Christina Marsillach)finds instead horror as the one responsible for getting her to this point terrorizes her in cruel ways.<br /><br />First her stage manager boyfriend, Stefano(William McNamara)is viciously stabbed with a knife while she has to watch, rope-tied to a pillar in his uncle's vast mansion, keeping her eyes from closing thanks to needles scotch-taped under her eyes, with pricking a result if she blinks. Allowing her to escape, Betty again finds herself in this unfortunate position, when a specific gold chain is found on her Lady Macbeth wardrobe that had been torn to shreds by the madman after killing a series of crows which escaped from their cage in the equipment room, by the clothing designer who is first knocked to the floor by an iron and subsequently stabbed heinously by a pair of scissors(..to cap off this nasty scene, the chain falls into her throat with the killer having to cut open her throat to find it; while not specifically elaborating this act, Dario uses the ripping sound of the scissors for optimum effect with the camera often retreating back to the victim's dead face). Having nowhere to turn, Inspector Alan Santini(Urbano Barberini, playing him cold and bland at Dario's request)promises to catch the psycho as Betty relies on her few remaining friends for comfort, the theater director Marco(Ian Charleson, of GHANDI & CHARIOTS OF FIRE who would later die of AIDS;A sad footnote, Dario revealed in an interview that Charleson informed him at the end of the shoot that he was HIV positive)attempting the stage after a successful horror movie career despite being rejected by critics, and pal Mira(Daria Nicolodi, Dario's former squeeze), her agent and confident. But, the serial killer is quite driven and a showdown between them will, of course, occur in the theater as Marco has added an interesting change to the production using the crows at his disposal.<br /><br />I think this is Dario at his most savage & nihilistic. Although he has certainly made later gruesome films(..such as his Masters of Horror entries and SLEEPLESS would suggest), this film really ups the ante in pure violence towards the victims of the psychopath. His method of forcing Betty to watch was admittedly a gag by Dario regarding the type of audience who like to look away from the more horrific parts of horror movies. I found myself rubbing my eyes every time she just has to blink(..brilliantly, Dario shoots the pricking from point-of-view achieving a tormenting effect from our perspective as if we were the ones with the scotch taped needles holding our eyes open). I like how Dario will show the widened eyes of Betty, horrified at what she's being forced into watching, as little blood tears down the needles when she has no choice but blink. The photographic work of Ronnie Taylor is impeccable, such as the crow's point-of-view shot in the theater at the end as it searches for the killer. Or, when the camera "travels" through rooms in the theater following the killer who wishes to see his muse from a box seat. Or, the dream sequence where we are taken into Betty's memory of an event regarding her mother's death by a certain killer, donning the same mask and gloves as the one causing her trauma at present. The highlight, in my opinion, is the peephole bullet-fire sequence, masterly staged by all involved as the camera follows a bullet which shoots through the eye of a victim, exploding from the back of her head, going through a telephone Betty planned to use to call for help. We even get a crow pecking the eyeball from the killer(..to add to this vicious scene, the crow is shown with the eyeball rolling around in it's beak). Only aspect I didn't care for, often pointed out by naysayers of the film, is the "final" ending which I personally felt was a bit unnecessary, but I guess Dario wanted to point out that Betty was indeed not like her mother, a woman with sadomasochistic tendencies which, in a twist, relate to why the killer torments our heroine. The rock music used during the violent scenes didn't bother me, because I felt that those moments of wicked graphic attacks needed a jarring thud which heavy metal can often provide.

True sentiment: 1
Predicted probability: [0.5691 0.4309]

Implementing an LSTM cell¶

x_seq = torch.FloatTensor([[1,], [2,], [3,]])
x_seq = x_seq.repeat(1, 2)
print(f"x_seq has shape {x_seq.shape} which means it is a sequence of {x_seq.shape[0]} {x_seq.shape[1]}-dimensional vectors")

x_seq has shape torch.Size([3, 2]) which means it is a sequence of 3 2-dimensional vectors

An LSTM has a hidden state $h^{(t)}$ and a cell state $c^{(t)}$ . It contains a set of gating functions that allow the model to input/update ( $i^{(t)}$ ), forget ( $f^{(t)}$ ), and output ( $o^{(t)}$ )

Each gate ( $i^{(t)},f^{(t)},o^{(t)}$ each receive inputs from the previous hidden state $h^{(t-1)}$ and the current input $x^{(t)}$

The gating functions are given by

\begin{align} i^{(t)} &= \sigma\left( W_{ii} x^{(t)} + b_{ii} + W_{hi} h^{(t-1)} + b_{hi} \right) \\ f^{(t)} &= \sigma\left( W_{if} x^{(t)} + b_{if} + W_{hf} h^{(t-1)} + b_{hf} \right) \\ o^{(t)} &= \sigma\left( W_{io} x^{(t)} + b_{io} + W_{ho} h^{(t-1)} + b_{ho} \right) \\ \end{align}

There is also a cell input $g^{(t)}$

\begin{align} g^{(t)} &= \tanh \left( W_{ig} x^{(t)} + b_{ig} + W_{hg} h^{(t-1)} + b_{hg} \right) \\ \end{align}

The cell state $c^{(t)}$ uses the forget gate to determine whether to discard the previous cell state $c^{(t-1)}$ and the input gate to determine whether to accept the new input $g^{(t)}$

\begin{align} c^{(t)} &= f^{(t)} c^{(t-1)} + i^{(t)} g^{(t)} \end{align}

The hidden state of the LSTM is then the product of the activated cell state and the output gate.

\begin{align} h^{(t)} &= o^{(t)} \tanh c^{(t)} \end{align}

import torch.nn as nn
import torch.nn.functional as F

class LSTM_Cell(nn.Module):
    def __init__(self, input_size=2, hidden_size=4):
        super().__init__()

        self.hidden_size = hidden_size

        # Input gate weight and biases
        self.w_ii = nn.Parameter(torch.randn(size=(hidden_size, input_size)))
        self.b_ii = nn.Parameter(torch.zeros(hidden_size))

        self.w_ih = nn.Parameter(torch.randn(size=(hidden_size, hidden_size)))
        self.b_ih = nn.Parameter(torch.zeros(hidden_size))

        # Forget gate weight and biases
        self.w_fi = nn.Parameter(torch.randn(size=(hidden_size, input_size)))
        self.b_fi = nn.Parameter(torch.zeros(hidden_size))

        self.w_fh = nn.Parameter(torch.randn(size=(hidden_size, hidden_size)))
        self.b_fh = nn.Parameter(torch.zeros(hidden_size))

        # Cell input weight and biases
        self.w_gi = nn.Parameter(torch.randn(size=(hidden_size, input_size)))
        self.b_gi = nn.Parameter(torch.zeros(hidden_size))

        self.w_gh = nn.Parameter(torch.randn(size=(hidden_size, hidden_size)))
        self.b_gh = nn.Parameter(torch.zeros(hidden_size))
        
        # Output gate weight and biases
        self.w_oi = nn.Parameter(torch.randn(size=(hidden_size, input_size)))
        self.b_oi = nn.Parameter(torch.zeros(hidden_size))

        self.w_oh = nn.Parameter(torch.randn(size=(hidden_size, hidden_size)))
        self.b_oh = nn.Parameter(torch.zeros(hidden_size))
        
    def forward(self, x):
        """ x has size [B, t, c]
        """
        B, T, C = x.shape
        ht = torch.zeros([B, self.hidden_size], dtype=x.dtype, device=x.device)
        ct = torch.zeros([B, self.hidden_size], dtype=x.dtype, device=x.device)

        outputs = torch.zeros([B, T, self.hidden_size], dtype=x.dtype, device=x.device)
        for t in range(T):
            xt = x[:, t, :]
            print(f'At t = {t}')
            print(f'\tInput = {xt.numpy()}')

            # Compute the input and forget gates so I can do the cell update
            it = F.sigmoid(
                xt @ self.w_ii.T + self.b_ii +
                ht @ self.w_ih.T + self.b_ih
            )
            ft = F.sigmoid(
                xt @ self.w_fi.T + self.b_fi +
                ht @ self.w_fh.T + self.b_fh
            )

            # Compute my cell update information
            gt = F.tanh(
                xt @ self.w_gi.T + self.b_gi +
                ht @ self.w_gh.T + self.b_gh
            )

            ct = ft * ct + it * gt
            print(f'\tCell = {ct.detach().cpu().numpy()}')

            # How much of the memory cell do I output?
            ot = F.sigmoid(
                xt @ self.w_oi.T + self.b_oi +
                ht @ self.w_oh.T + self.b_oh
            )
            ht = ot * F.tanh(ct)
            print(f'\tHidden = {ht.detach().cpu().numpy()}')
            outputs[:, t, :] = ht

        return output, (ht, ct)


torch_LSTM = nn.LSTM(input_size=2, hidden_size=4, batch_first=True)
print("Ground truth")
print("------------")
output, (hidden, cell) = torch_LSTM(x_seq[None])
print(f"Output is of size {output.shape}")
print(output)
print()
print(f"Last hidden is of size {hidden.shape}")
print(hidden)
print()
print(f"Last cell is of size {cell.shape}")
print(cell)

print()
print("Our implementation")
print("------------------")
np.set_printoptions(precision=4)
lstm = LSTM_Cell()
with torch.no_grad():
    lstm.w_ii[:] = torch_LSTM.weight_ih_l0[0:4]
    lstm.b_ii[:] = torch_LSTM.bias_ih_l0[0:4]
    lstm.w_ih[:] = torch_LSTM.weight_hh_l0[0:4]
    lstm.b_ih[:] = torch_LSTM.bias_hh_l0[0:4]

    lstm.w_fi[:] = torch_LSTM.weight_ih_l0[4:8]
    lstm.b_fi[:] = torch_LSTM.bias_ih_l0[4:8]
    lstm.w_fh[:] = torch_LSTM.weight_hh_l0[4:8]
    lstm.b_fh[:] = torch_LSTM.bias_hh_l0[4:8]

    lstm.w_gi[:] = torch_LSTM.weight_ih_l0[8:12]
    lstm.b_gi[:] = torch_LSTM.bias_ih_l0[8:12]
    lstm.w_gh[:] = torch_LSTM.weight_hh_l0[8:12]
    lstm.b_gh[:] = torch_LSTM.bias_hh_l0[8:12]

    lstm.w_oi[:] = torch_LSTM.weight_ih_l0[12:16]
    lstm.b_oi[:] = torch_LSTM.bias_ih_l0[12:16]
    lstm.w_oh[:] = torch_LSTM.weight_hh_l0[12:16]
    lstm.b_oh[:] = torch_LSTM.bias_hh_l0[12:16]

output, (hidden, cell) = lstm(x_seq[None])

print()
print(f"Output is of size {output.shape}")
print(output)
print()
print(f"Last hidden is of size {hidden.shape}")
print(hidden)
print()
print(f"Last cell is of size {cell.shape}")
print(cell)

Ground truth
------------
Output is of size torch.Size([1, 3, 4])
tensor([[[-0.0251, -0.0371, -0.1357,  0.0286],
         [-0.1091, -0.0206, -0.2578, -0.0543],
         [-0.1423, -0.0065, -0.3741, -0.0532]]], grad_fn=<TransposeBackward0>)

Last hidden is of size torch.Size([1, 1, 4])
tensor([[[-0.1423, -0.0065, -0.3741, -0.0532]]], grad_fn=<StackBackward0>)

Last cell is of size torch.Size([1, 1, 4])
tensor([[[-0.7170, -0.0729, -0.7173, -0.1713]]], grad_fn=<StackBackward0>)

Our implementation
------------------
At t = 0
	Input = [[1. 1.]]
	Cell = [[-0.086  -0.1008 -0.2536  0.061 ]]
	Hidden = [[-0.0251 -0.0371 -0.1357  0.0286]]
At t = 1
	Input = [[2. 2.]]
	Cell = [[-0.4374 -0.105  -0.4843 -0.138 ]]
	Hidden = [[-0.1091 -0.0206 -0.2578 -0.0543]]
At t = 2
	Input = [[3. 3.]]
	Cell = [[-0.717  -0.0729 -0.7173 -0.1713]]
	Hidden = [[-0.1423 -0.0065 -0.3741 -0.0532]]

Output is of size torch.Size([1, 3, 4])
tensor([[[-0.0251, -0.0371, -0.1357,  0.0286],
         [-0.1091, -0.0206, -0.2578, -0.0543],
         [-0.1423, -0.0065, -0.3741, -0.0532]]], grad_fn=<TransposeBackward0>)

Last hidden is of size torch.Size([1, 4])
tensor([[-0.1423, -0.0065, -0.3741, -0.0532]], grad_fn=<MulBackward0>)

Last cell is of size torch.Size([1, 4])
tensor([[-0.7170, -0.0729, -0.7173, -0.1713]], grad_fn=<AddBackward0>)

import torch.nn.functional as F

class LSTM_Model(nn.Module):
    def __init__(self, vocab_size, embedding_dim, rnn_size, fc_size):
        super().__init__()
        self.embedding = nn.Embedding(num_embeddings=vocab_size,
                                      embedding_dim=embedding_dim,
                                      padding_idx=0)
        self.lstm = nn.LSTM(embedding_dim, rnn_size, batch_first=True)
        self.fc1 = nn.Linear(rnn_size, fc_size)
        self.fc2 = nn.Linear(fc_size, 2) 

    def forward(self, text, lengths):
        x = self.embedding(text)
        x = nn.utils.rnn.pack_padded_sequence(x, lengths, enforce_sorted=False, batch_first=True)
        x, (h, c) = self.lstm(x)

        x = h[-1, :, :]
        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        return x


torch.manual_seed(42)
model = LSTM_Model(
    vocab_size=len(vocab),# tokenizer.vocab_size,
    embedding_dim=32,
    rnn_size=128,
    fc_size=128
)
model

LSTM_Model(
  (embedding): Embedding(75979, 32, padding_idx=0)
  (lstm): LSTM(32, 128, batch_first=True)
  (fc1): Linear(in_features=128, out_features=128, bias=True)
  (fc2): Linear(in_features=128, out_features=2, bias=True)
)

# Define a loss function and optimizer
loss = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
model, model_stats = train(model, optimizer, loss, train_loader, val_loader, num_epochs=5)

model.eval()
with torch.no_grad():
    test_correct_ = 0.
    for xb, yb, lengths in tqdm(test_loader,leave=False):
        xb = xb.to(device)
        yb = yb.to(device)
        y_pred = model(xb, lengths.cpu().numpy())
        y_pred = torch.argmax(y_pred, dim=1)
        test_correct_ += (y_pred == yb).float().sum().item()
    test_acc = test_correct_ / len(test_loader.dataset)

print(f"Test accuracy: {test_acc:.3f}")

Test accuracy: 0.815

# Classify a single example
idx = np.random.choice(len(test_dataset))
sample = test_dataset[idx]
text = sample['text']
print('Comment:', text)
print()
print('True sentiment:', sample['label'])

# Convert comment with tokenizer, etc.
model.eval()
batch = collate_fn([sample,])
with torch.no_grad():
    y_pred = model(batch[0].to(device), batch[2])
    y_pred = F.softmax(y_pred, dim=1)

print('Predicted probability:', y_pred[0].cpu().numpy())

Comment: I have just finished watching this film for the first time, and I must say that I am very impressed.<br /><br />How bleak. How full of despair. How nightmarish. Incredible.<br /><br />Visually stunning, several scenes are embedded in my mind...the first appearance of the phantom carriage...the soul of David Holm as it rises from his corpse...his spirit on his knees, pleading.<br /><br />This film takes a simple story-that of the ghostly driver of the phantom carriage, doomed to collect the souls of the dead for a year-brings it into the present setting of the film and then uses flashbacks as a means to explain how David Holm ends up in his predicament.<br /><br />I would love to see this released on DVD so that more might see it. Everyone should.

True sentiment: 1
Predicted probability: [0.01 0.99]