Model statistics
This commit is contained in:
Thastertyn committed
1 parent
0bae8deb42
commit
859bdd19c3
7 files changed
+782
-143
No files matched your search
+229
-138
@@ -4,12 +4,27 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Import data"
|
||||
"# Omega\n",
|
||||
"Prediction of next key to be pressed using Multilayer Perceptron"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Import and load data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Import all required modules"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": 34,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -19,18 +34,32 @@
|
||||
"from torch.utils.data import DataLoader, TensorDataset, random_split"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Load data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 35,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"data = np.load(\"./data.npy\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Define contstants describing the dataset and other useful information"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 36,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -62,7 +91,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 37,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -80,7 +109,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 38,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -90,19 +119,47 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 39,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"train_loader = DataLoader(train_set, batch_size=128, shuffle=True)\n",
|
||||
"test_loader = DataLoader(test_set, batch_size=128)"
|
||||
"train_loader = DataLoader(train_set, batch_size=1024, shuffle=True)\n",
|
||||
"test_loader = DataLoader(test_set, batch_size=1024)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 40,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"learning_rates = [1e-4, 5e-4, 1e-3, 5e-3, 1e-2, 5e-2]\n",
|
||||
"activation_layers = [nn.ReLU, nn.GELU]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Train on data"
|
||||
"## Model and training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"To find the best model for MLP, combinations of hyperparams are defined. \n",
|
||||
"This includes **activation layers** and **learning rates**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 32,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from itertools import product\n",
|
||||
"all_activation_combinations = list(product(activation_layers, repeat=len(activation_layers)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -110,17 +167,24 @@
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 66,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class MLP(nn.Module):\n",
|
||||
" def __init__(self):\n",
|
||||
" def __init__(self, activation_layers: list):\n",
|
||||
" super().__init__()\n",
|
||||
" self.net = nn.Sequential(\n",
|
||||
" nn.Embedding(num_embeddings=VOCAB_SIZE, embedding_dim=EMBEDDING_DIM),\n",
|
||||
" nn.Flatten(),\n",
|
||||
" nn.Linear(CONTEXT_SIZE * EMBEDDING_DIM, 256),\n",
|
||||
" nn.ReLU(),\n",
|
||||
" activation_layers[0](),\n",
|
||||
" nn.Linear(256, 128),\n",
|
||||
" nn.ReLU(),\n",
|
||||
" activation_layers[1](),\n",
|
||||
" nn.Linear(128, OUTPUT_SIZE)\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
@@ -130,22 +194,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 42,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Using device: cpu\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/thastertyn/Code/Skola/4-rocnik/programove-vybaveni/omega/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:129: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:109.)\n",
|
||||
" return torch._C._cuda_getDeviceCount() > 0\n"
|
||||
"Using device: cuda\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -156,71 +212,50 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 55,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# model = MLP().to(device)\n",
|
||||
"model = None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Test all the activation_layer combinations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 65,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = MLP().to(device)\n",
|
||||
"optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n",
|
||||
"\n",
|
||||
"criterion = nn.CrossEntropyLoss()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 71,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1, Loss: 4068.5562\n",
|
||||
"Epoch 2, Loss: 3446.1109\n",
|
||||
"Epoch 3, Loss: 3260.1651\n",
|
||||
"Epoch 4, Loss: 3165.0248\n",
|
||||
"Epoch 5, Loss: 3101.6501\n",
|
||||
"Epoch 6, Loss: 3054.4113\n",
|
||||
"Epoch 7, Loss: 3021.7103\n",
|
||||
"Epoch 8, Loss: 2994.6145\n",
|
||||
"Epoch 9, Loss: 2973.1683\n",
|
||||
"Epoch 10, Loss: 2955.0090\n",
|
||||
"Epoch 11, Loss: 2940.0807\n",
|
||||
"Epoch 12, Loss: 2928.2814\n",
|
||||
"Epoch 13, Loss: 2916.9362\n",
|
||||
"Epoch 14, Loss: 2905.9567\n",
|
||||
"Epoch 15, Loss: 2897.3687\n",
|
||||
"Epoch 16, Loss: 2890.6869\n",
|
||||
"Epoch 17, Loss: 2882.7104\n",
|
||||
"Epoch 18, Loss: 2876.6815\n",
|
||||
"Epoch 19, Loss: 2870.7298\n",
|
||||
"Epoch 20, Loss: 2865.6343\n",
|
||||
"Epoch 21, Loss: 2860.5506\n",
|
||||
"Epoch 22, Loss: 2856.7977\n",
|
||||
"Epoch 23, Loss: 2852.8814\n",
|
||||
"Epoch 24, Loss: 2847.7687\n",
|
||||
"Epoch 25, Loss: 2846.0855\n",
|
||||
"Epoch 26, Loss: 2842.2640\n",
|
||||
"Epoch 27, Loss: 2838.4780\n",
|
||||
"Epoch 28, Loss: 2836.9773\n",
|
||||
"Epoch 29, Loss: 2833.8416\n",
|
||||
"Epoch 30, Loss: 2830.5508\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for epoch in range(30):\n",
|
||||
" model.train()\n",
|
||||
" total_loss = 0\n",
|
||||
" for batch_X, batch_y in train_loader:\n",
|
||||
" batch_X, batch_y = batch_X.to(device), batch_y.to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" output = model(batch_X)\n",
|
||||
" loss = criterion(output, batch_y)\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" total_loss += loss.item()\n",
|
||||
" print(f\"Epoch {epoch+1}, Loss: {total_loss:.4f}\")"
|
||||
"def train_model(model, optimizer):\n",
|
||||
" for epoch in range(30):\n",
|
||||
" model.train()\n",
|
||||
" total_loss = 0\n",
|
||||
" for batch_X, batch_y in train_loader:\n",
|
||||
" batch_X, batch_y = batch_X.to(device), batch_y.to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" output = model(batch_X)\n",
|
||||
" loss = criterion(output, batch_y)\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" total_loss += loss.item()\n",
|
||||
" # print(f\"Epoch {epoch+1}, Loss: {total_loss:.4f}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -232,73 +267,89 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 70,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def test_model(model) -> tuple[float]:\n",
|
||||
" model.eval()\n",
|
||||
" correct_top1 = 0\n",
|
||||
" correct_top3 = 0\n",
|
||||
" correct_top5 = 0\n",
|
||||
" total = 0\n",
|
||||
"\n",
|
||||
" with torch.no_grad():\n",
|
||||
" for batch_X, batch_y in test_loader:\n",
|
||||
" batch_X, batch_y = batch_X.to(device), batch_y.to(device)\n",
|
||||
" outputs = model(batch_X)\n",
|
||||
"\n",
|
||||
" _, top_preds = outputs.topk(5, dim=1)\n",
|
||||
"\n",
|
||||
" for true, top5 in zip(batch_y, top_preds):\n",
|
||||
" total += 1\n",
|
||||
" if true == top5[0]:\n",
|
||||
" correct_top1 += 1\n",
|
||||
" if true in top5[:3]:\n",
|
||||
" correct_top3 += 1\n",
|
||||
" if true in top5:\n",
|
||||
" correct_top5 += 1\n",
|
||||
"\n",
|
||||
" top1_acc = correct_top1 / total\n",
|
||||
" top3_acc = correct_top3 / total\n",
|
||||
" top5_acc = correct_top5 / total\n",
|
||||
"\n",
|
||||
" return (top1_acc, top3_acc, top5_acc)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 72,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Top-1 Accuracy: 52.77%\n",
|
||||
"Top-3 Accuracy: 74.39%\n",
|
||||
"Top-5 Accuracy: 83.37%\n"
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.0001 had success of (0.44952931636286714, 0.6824383880407573, 0.788915135916511)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.0005 had success of (0.5080210132919649, 0.7299298381694461, 0.8241018227973064)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.001 had success of (0.5215950357860593, 0.7354299615696506, 0.826111483270458)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.005 had success of (0.5230758382399605, 0.7383563092761697, 0.8298840038077777)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.01 had success of (0.5206783485526919, 0.7364171632055847, 0.8278390861333428)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.05 had success of (0.12682015301625357, 0.29884003807777737, 0.45160949123858546)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.0001 had success of (0.44251313330747805, 0.6765504354264359, 0.7860240454112752)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.0005 had success of (0.5103127313753835, 0.7293304657476289, 0.8237492507844727)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.001 had success of (0.5211366921693756, 0.7379332228607693, 0.8288968021718436)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.005 had success of (0.5246271550964284, 0.739942883333921, 0.8305538906321617)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.01 had success of (0.5214892641822092, 0.7391319677044036, 0.8297077178013609)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.05 had success of (0.1655325600253852, 0.3544759017029228, 0.495469449635088)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.0001 had success of (0.44706131227303175, 0.6806755279765893, 0.7906427387793957)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.0005 had success of (0.5120050770369848, 0.7312343546169305, 0.8229735923562388)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.001 had success of (0.5179282868525896, 0.7381800232697528, 0.8289673165744104)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.005 had success of (0.5234636674540775, 0.7421640870147728, 0.8307654338398618)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.01 had success of (0.5197264041180412, 0.7384268236787364, 0.8286500017628601)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.05 had success of (0.12551563656876918, 0.29757077883157634, 0.45034023199238443)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.0001 had success of (0.4493530303564503, 0.683284560871558, 0.7907837675845292)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.0005 had success of (0.5151077107499207, 0.733808130310616, 0.8255121108486408)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.001 had success of (0.5195148609103409, 0.7389204244967035, 0.8294961745936608)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.005 had success of (0.5214892641822092, 0.7401896837429045, 0.8302365758206114)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.01 had success of (0.5198674329231746, 0.7398371117300708, 0.8258294256601911)\n",
|
||||
"Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.05 had success of (0.3762648520960406, 0.6283538412720798, 0.7500617001022459)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.eval()\n",
|
||||
"correct_top1 = 0\n",
|
||||
"correct_top3 = 0\n",
|
||||
"correct_top5 = 0\n",
|
||||
"total = 0\n",
|
||||
"\n",
|
||||
"with torch.no_grad():\n",
|
||||
" for batch_X, batch_y in test_loader:\n",
|
||||
" batch_X, batch_y = batch_X.to(device), batch_y.to(device)\n",
|
||||
" outputs = model(batch_X) # shape: [batch_size, 26]\n",
|
||||
"\n",
|
||||
" # Get top-5 predictions\n",
|
||||
" _, top_preds = outputs.topk(5, dim=1) # shape: [batch_size, 5]\n",
|
||||
"\n",
|
||||
" for true, top5 in zip(batch_y, top_preds):\n",
|
||||
" total += 1\n",
|
||||
" if true == top5[0]:\n",
|
||||
" correct_top1 += 1\n",
|
||||
" if true in top5[:3]:\n",
|
||||
" correct_top3 += 1\n",
|
||||
" if true in top5:\n",
|
||||
" correct_top5 += 1\n",
|
||||
"\n",
|
||||
"top1_acc = correct_top1 / total\n",
|
||||
"top3_acc = correct_top3 / total\n",
|
||||
"top5_acc = correct_top5 / total\n",
|
||||
"\n",
|
||||
"print(f\"Top-1 Accuracy: {top1_acc * 100:.2f}%\")\n",
|
||||
"print(f\"Top-3 Accuracy: {top3_acc * 100:.2f}%\")\n",
|
||||
"print(f\"Top-5 Accuracy: {top5_acc * 100:.2f}%\")\n"
|
||||
"for activation_layer_combination in all_activation_combinations:\n",
|
||||
" for learning_rate in learning_rates:\n",
|
||||
" model = MLP(activation_layer_combination).to(device)\n",
|
||||
" optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
|
||||
" train_model(model, optimizer)\n",
|
||||
" results = test_model(model)\n",
|
||||
" print(\"Model with activation layers\", activation_layer_combination, \"and learning rate\", learning_rate, \"had success of\", results)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 38,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"torch.save(model.state_dict(), \"mlp_weights.pth\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"torch.save(model, \"mlp_full_model.pth\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -338,25 +389,65 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"execution_count": 24,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"A: 0.4302\n",
|
||||
"T: 0.2897\n",
|
||||
"E: 0.1538\n",
|
||||
"I: 0.0905\n",
|
||||
"C: 0.0159\n"
|
||||
"I: 89.74 %\n",
|
||||
"N: 4.42 %\n",
|
||||
"Y: 1.88 %\n",
|
||||
"M: 1.51 %\n",
|
||||
"B: 0.90 %\n",
|
||||
"E: 0.65 %\n",
|
||||
"G: 0.21 %\n",
|
||||
"R: 0.16 %\n",
|
||||
"L: 0.15 %\n",
|
||||
"O: 0.13 %\n",
|
||||
"C: 0.09 %\n",
|
||||
"U: 0.08 %\n",
|
||||
"A: 0.05 %\n",
|
||||
"V: 0.02 %\n",
|
||||
"S: 0.01 %\n",
|
||||
"F: 0.00 %\n",
|
||||
"H: 0.00 %\n",
|
||||
"T: 0.00 %\n",
|
||||
"W: 0.00 %\n",
|
||||
"P: 0.00 %\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"preds = predict_next_chars(model, \"doors\")\n",
|
||||
"preds = predict_next_chars(model, \"susta\", top_k=20)\n",
|
||||
"for char, prob in preds:\n",
|
||||
" print(f\"{char.upper()}: {prob:.4f}\")\n"
|
||||
" print(f\"{char.upper()}: {(prob * 100):.2f} %\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Model saving"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"torch.save(model, \"mlp_full_model.pth\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 38,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"torch.save(model.state_dict(), \"mlp_weights.pth\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
(ReLU,ReLU) 0.0001 (0.44952931636286714,0.6824383880407573,0.788915135916511)
|
||||
(ReLU,ReLU) 0.0005 (0.5080210132919649,0.7299298381694461,0.8241018227973064)
|
||||
(ReLU,ReLU) 0.001 (0.5215950357860593,0.7354299615696506,0.826111483270458)
|
||||
(ReLU,ReLU) 0.005 (0.5230758382399605,0.7383563092761697,0.8298840038077777)
|
||||
(ReLU,ReLU) 0.01 (0.5206783485526919,0.7364171632055847,0.8278390861333428)
|
||||
(ReLU,ReLU) 0.05 (0.12682015301625357,0.29884003807777737,0.45160949123858546)
|
||||
(ReLU,GELU) 0.0001 (0.44251313330747805,0.6765504354264359,0.7860240454112752)
|
||||
(ReLU,GELU) 0.0005 (0.5103127313753835,0.7293304657476289,0.8237492507844727)
|
||||
(ReLU,GELU) 0.001 (0.5211366921693756,0.7379332228607693,0.8288968021718436)
|
||||
(ReLU,GELU) 0.005 (0.5246271550964284,0.739942883333921,0.8305538906321617)
|
||||
(ReLU,GELU) 0.01 (0.5214892641822092,0.7391319677044036,0.8297077178013609)
|
||||
(ReLU,GELU) 0.05 (0.1655325600253852,0.3544759017029228,0.495469449635088)
|
||||
(GELU,ReLU) 0.0001 (0.44706131227303175,0.6806755279765893,0.7906427387793957)
|
||||
(GELU,ReLU) 0.0005 (0.5120050770369848,0.7312343546169305,0.8229735923562388)
|
||||
(GELU,ReLU) 0.001 (0.5179282868525896,0.7381800232697528,0.8289673165744104)
|
||||
(GELU,ReLU) 0.005 (0.5234636674540775,0.7421640870147728,0.8307654338398618)
|
||||
(GELU,ReLU) 0.01 (0.5197264041180412,0.7384268236787364,0.8286500017628601)
|
||||
(GELU,ReLU) 0.05 (0.12551563656876918,0.29757077883157634,0.45034023199238443)
|
||||
(GELU,GELU) 0.0001 (0.4493530303564503,0.683284560871558,0.7907837675845292)
|
||||
(GELU,GELU) 0.0005 (0.5151077107499207,0.733808130310616,0.8255121108486408)
|
||||
(GELU,GELU) 0.001 (0.5195148609103409,0.7389204244967035,0.8294961745936608)
|
||||
(GELU,GELU) 0.005 (0.5214892641822092,0.7401896837429045,0.8302365758206114)
|
||||
(GELU,GELU) 0.01 (0.5198674329231746,0.7398371117300708,0.8258294256601911)
|
||||
(GELU,GELU) 0.05 (0.3762648520960406,0.6283538412720798,0.7500617001022459)
|
||||
@@ -0,0 +1,24 @@
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.0001 had success of (0.44952931636286714, 0.6824383880407573, 0.788915135916511)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.0005 had success of (0.5080210132919649, 0.7299298381694461, 0.8241018227973064)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.001 had success of (0.5215950357860593, 0.7354299615696506, 0.826111483270458)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.005 had success of (0.5230758382399605, 0.7383563092761697, 0.8298840038077777)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.01 had success of (0.5206783485526919, 0.7364171632055847, 0.8278390861333428)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.05 had success of (0.12682015301625357, 0.29884003807777737, 0.45160949123858546)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.0001 had success of (0.44251313330747805, 0.6765504354264359, 0.7860240454112752)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.0005 had success of (0.5103127313753835, 0.7293304657476289, 0.8237492507844727)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.001 had success of (0.5211366921693756, 0.7379332228607693, 0.8288968021718436)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.005 had success of (0.5246271550964284, 0.739942883333921, 0.8305538906321617)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.01 had success of (0.5214892641822092, 0.7391319677044036, 0.8297077178013609)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.ReLU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.05 had success of (0.1655325600253852, 0.3544759017029228, 0.495469449635088)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.0001 had success of (0.44706131227303175, 0.6806755279765893, 0.7906427387793957)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.0005 had success of (0.5120050770369848, 0.7312343546169305, 0.8229735923562388)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.001 had success of (0.5179282868525896, 0.7381800232697528, 0.8289673165744104)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.005 had success of (0.5234636674540775, 0.7421640870147728, 0.8307654338398618)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.01 had success of (0.5197264041180412, 0.7384268236787364, 0.8286500017628601)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.ReLU'>) and learning rate 0.05 had success of (0.12551563656876918, 0.29757077883157634, 0.45034023199238443)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.0001 had success of (0.4493530303564503, 0.683284560871558, 0.7907837675845292)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.0005 had success of (0.5151077107499207, 0.733808130310616, 0.8255121108486408)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.001 had success of (0.5195148609103409, 0.7389204244967035, 0.8294961745936608)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.005 had success of (0.5214892641822092, 0.7401896837429045, 0.8302365758206114)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.01 had success of (0.5198674329231746, 0.7398371117300708, 0.8258294256601911)
|
||||
Model with activation layers (<class 'torch.nn.modules.activation.GELU'>, <class 'torch.nn.modules.activation.GELU'>) and learning rate 0.05 had success of (0.3762648520960406, 0.6283538412720798, 0.7500617001022459)
|
||||
@@ -0,0 +1,60 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import re
|
||||
|
||||
# Load and parse raw data from a text file
|
||||
def parse_raw_data(file_path):
|
||||
pattern = re.compile(r"\((.*?)\)\s+([\d.]+)\s+\(([\d.eE+,-]+)\)")
|
||||
data = []
|
||||
with open(file_path, "r") as f:
|
||||
for line in f:
|
||||
match = pattern.search(line)
|
||||
if match:
|
||||
activation_combo = match.group(1)
|
||||
learning_rate = float(match.group(2))
|
||||
top_accuracies = tuple(map(float, match.group(3).split(",")))
|
||||
data.append((activation_combo, learning_rate, top_accuracies))
|
||||
return data
|
||||
|
||||
# Replace this with your actual path
|
||||
file_path = "./parsed.txt"
|
||||
data = parse_raw_data(file_path)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(data, columns=["activation_combo", "learning_rate", "accuracy"])
|
||||
df[["top1", "top3", "top5"]] = pd.DataFrame(df["accuracy"].tolist(), index=df.index)
|
||||
|
||||
# Unique sorted learning rates and activation combos
|
||||
learning_rates = sorted(df["learning_rate"].unique())
|
||||
activation_combos = df["activation_combo"].unique()
|
||||
|
||||
# Settings for bar positions
|
||||
bar_width = 0.2
|
||||
x = np.arange(len(learning_rates))
|
||||
|
||||
# Start plotting
|
||||
fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(14, 16), sharex=True)
|
||||
|
||||
# Function to draw bars
|
||||
def plot_bars(ax, column, title, ylabel):
|
||||
for i, combo in enumerate(activation_combos):
|
||||
combo_data = df[df["activation_combo"] == combo].sort_values("learning_rate")
|
||||
ax.bar(x + i * bar_width, combo_data[column], width=bar_width, label=combo)
|
||||
ax.set_ylabel(ylabel)
|
||||
ax.set_title(title)
|
||||
ax.legend(title="Activation Combo")
|
||||
|
||||
# Plot each accuracy type
|
||||
plot_bars(ax1, "top1", "Top-1 Accuracy", "Top-1 Accuracy")
|
||||
plot_bars(ax2, "top3", "Top-3 Accuracy", "Top-3 Accuracy")
|
||||
plot_bars(ax3, "top5", "Top-5 Accuracy", "Top-5 Accuracy")
|
||||
|
||||
# X-axis ticks for learning rates
|
||||
ax3.set_xticks(x + bar_width * (len(activation_combos) - 1) / 2)
|
||||
ax3.set_xticklabels([str(lr) for lr in learning_rates])
|
||||
ax3.set_xlabel("Learning Rate")
|
||||
|
||||
# Final layout
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
Reference in new issue
Block a user