notebooks/demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "described-saying",
   "metadata": {},
   "source": [
    "# DEMO RUN DOWN\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "refined-texas",
   "metadata": {},
   "source": [
    "### Install dependencies\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "african-inclusion",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: dpkt>=1.9.4 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 1)) (1.9.4)\n",
      "Requirement already satisfied: h5py>=3.1.0 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 2)) (3.1.0)\n",
      "Requirement already satisfied: scipy>=1.5.4 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 3)) (1.5.4)\n",
      "Requirement already satisfied: tqdm>=4.54.0 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 4)) (4.54.0)\n",
      "Requirement already satisfied: rich>=9.10.0 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 5)) (9.10.0)\n",
      "Requirement already satisfied: numpy>=1.19.4 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 6)) (1.19.4)\n",
      "Requirement already satisfied: tensorboard>=2.3.0 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 7)) (2.3.0)\n",
      "Requirement already satisfied: scikit-learn>=0.23.2 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 8)) (0.23.2)\n",
      "Requirement already satisfied: spatialentropy>=0.0.4 in /home/geb/.local/lib/python3.7/site-packages (from -r ../requirements.txt (line 9)) (0.0.4)\n",
      "Requirement already satisfied: cached-property; python_version < \"3.8\" in /home/geb/.local/lib/python3.7/site-packages (from h5py>=3.1.0->-r ../requirements.txt (line 2)) (1.5.2)\n",
      "Requirement already satisfied: typing-extensions<4.0.0,>=3.7.4 in /home/geb/.local/lib/python3.7/site-packages (from rich>=9.10.0->-r ../requirements.txt (line 5)) (3.7.4.3)\n",
      "Requirement already satisfied: colorama<0.5.0,>=0.4.0 in /home/geb/.local/lib/python3.7/site-packages (from rich>=9.10.0->-r ../requirements.txt (line 5)) (0.4.4)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.6.0 in /home/geb/.local/lib/python3.7/site-packages (from rich>=9.10.0->-r ../requirements.txt (line 5)) (2.7.4)\n",
      "Requirement already satisfied: commonmark<0.10.0,>=0.9.0 in /home/geb/.local/lib/python3.7/site-packages (from rich>=9.10.0->-r ../requirements.txt (line 5)) (0.9.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (3.3.3)\n",
      "Requirement already satisfied: protobuf>=3.6.0 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (3.14.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (50.3.2)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (0.4.2)\n",
      "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/lib/python3/dist-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (0.32.3)\n",
      "Requirement already satisfied: grpcio>=1.24.3 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (1.33.2)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (1.0.1)\n",
      "Requirement already satisfied: absl-py>=0.4 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (0.11.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/lib/python3/dist-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (1.12.0)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (1.23.0)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/geb/.local/lib/python3.7/site-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (1.7.0)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/lib/python3/dist-packages (from tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (2.21.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /home/geb/.local/lib/python3.7/site-packages (from scikit-learn>=0.23.2->-r ../requirements.txt (line 8)) (0.17.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/geb/.local/lib/python3.7/site-packages (from scikit-learn>=0.23.2->-r ../requirements.txt (line 8)) (2.1.0)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /home/geb/.local/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (3.1.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/geb/.local/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (1.3.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/geb/.local/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (0.2.8)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /home/geb/.local/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (4.1.1)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.5\" in /home/geb/.local/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (4.6)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/geb/.local/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (3.4.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /home/geb/.local/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (3.1.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/geb/.local/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard>=2.3.0->-r ../requirements.txt (line 7)) (0.4.8)\n"
     ]
    }
   ],
   "source": [
    "!python3 -m pip install -r ../requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bizarre-soundtrack",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[sudo] password for geb: Sorry, try again.\n",
      "[sudo] password for geb: \n",
      "sudo: no password was provided\n",
      "sudo: 1 incorrect password attempt\n"
     ]
    }
   ],
   "source": [
    "!echo PASSWORD |sudo -S apt-get install tshark"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "floral-substitute",
   "metadata": {},
   "source": [
    "### Download dummy data\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "specific-thumb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-02-14 20:55:36--  https://www.wireshark.org/download/automated/captures/randpkt-2020-09-06-16170.pcap\n",
      "Resolving www.wireshark.org (www.wireshark.org)... 104.26.11.240, 104.26.10.240, 172.67.75.39, ...\n",
      "Connecting to www.wireshark.org (www.wireshark.org)|104.26.11.240|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 5098521 (4.9M) [application/vnd.tcpdump.pcap]\n",
      "Saving to: ‘vali.pcap’\n",
      "\n",
      "vali.pcap           100%[===================>]   4.86M  4.16MB/s    in 1.2s    \n",
      "\n",
      "2021-02-14 20:55:38 (4.16 MB/s) - ‘vali.pcap’ saved [5098521/5098521]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget -O vali.pcap https://www.wireshark.org/download/automated/captures/randpkt-2020-09-06-16170.pcap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "celtic-eleven",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-02-14 20:55:38--  https://www.wireshark.org/download/automated/captures/randpkt-2020-04-02-31746.pcap\n",
      "Resolving www.wireshark.org (www.wireshark.org)... 104.26.10.240, 104.26.11.240, 172.67.75.39, ...\n",
      "Connecting to www.wireshark.org (www.wireshark.org)|104.26.10.240|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 5138207 (4.9M) [application/vnd.tcpdump.pcap]\n",
      "Saving to: ‘train.pcap’\n",
      "\n",
      "train.pcap          100%[===================>]   4.90M  5.45MB/s    in 0.9s    \n",
      "\n",
      "2021-02-14 20:55:40 (5.45 MB/s) - ‘train.pcap’ saved [5138207/5138207]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget -O train.pcap https://www.wireshark.org/download/automated/captures/randpkt-2020-04-02-31746.pcap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "blond-tradition",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-02-14 20:55:51--  https://www.wireshark.org/download/automated/captures/randpkt-2020-03-04-18423.pcap\n",
      "Resolving www.wireshark.org (www.wireshark.org)... 172.67.75.39, 104.26.10.240, 104.26.11.240, ...\n",
      "Connecting to www.wireshark.org (www.wireshark.org)|172.67.75.39|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 5056322 (4.8M) [application/vnd.tcpdump.pcap]\n",
      "Saving to: ‘fit.pcap’\n",
      "\n",
      "fit.pcap            100%[===================>]   4.82M  4.89MB/s    in 1.0s    \n",
      "\n",
      "2021-02-14 20:55:52 (4.89 MB/s) - ‘fit.pcap’ saved [5056322/5056322]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget -O fit.pcap https://www.wireshark.org/download/automated/captures/randpkt-2020-03-04-18423.pcap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "polish-attitude",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-02-14 20:56:00--  https://www.wireshark.org/download/automated/captures/randpkt-2020-03-04-18423.pcap\n",
      "Resolving www.wireshark.org (www.wireshark.org)... 172.67.75.39, 104.26.11.240, 104.26.10.240, ...\n",
      "Connecting to www.wireshark.org (www.wireshark.org)|172.67.75.39|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 5056322 (4.8M) [application/vnd.tcpdump.pcap]\n",
      "Saving to: ‘predict.pcap’\n",
      "\n",
      "predict.pcap        100%[===================>]   4.82M  5.01MB/s    in 1.0s    \n",
      "\n",
      "2021-02-14 20:56:02 (5.01 MB/s) - ‘predict.pcap’ saved [5056322/5056322]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget -O predict.pcap https://www.wireshark.org/download/automated/captures/randpkt-2020-03-04-18423.pcap"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "traditional-finnish",
   "metadata": {},
   "source": [
    "### Convert data\n",
    "\n",
    "preprocess (-m byte focus) data for training\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "chronic-preserve",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\u001b+\u001b] \u001bbyte\u001b preprocessing\n",
      "[\u001b*\u001b] \u001bcalculating\u001b maximum number of packets...\n",
      "[\u001b*\u001b] saving to \u001b/home/geb/Workspace/gits/thesis/test/vali\u001b\n",
      "[\u001b*\u001b] starting \u001b2\u001b consumer threads\n",
      "100%|###################################################| 5000/5000 [00:01<00:00, 3007.57 packets/s]\n",
      "[\u001b*\u001b] merge parts...\n",
      "100%|########################| 2423/2423 [00:01<00:00, 2222.41 keys/s, part=0/1]\n",
      "------------------------------\n",
      "[!] extracted \u001b4,900\u001b 2D byte fragments (a 1024 bytes) from 5,000 packets\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/pcap2ds.py -p vali.pcap -o vali -m byte --chunk 1024 --threads 2 --force"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "indirect-transparency",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\u001b+\u001b] \u001bbyte\u001b preprocessing\n",
      "[\u001b*\u001b] \u001bcalculating\u001b maximum number of packets...\n",
      "[\u001b*\u001b] saving to \u001b/home/geb/Workspace/gits/thesis/test/train\u001b\n",
      "[\u001b*\u001b] starting \u001b2\u001b consumer threads\n",
      "100%|###################################################| 5000/5000 [00:01<00:00, 2710.23 packets/s]\n",
      "[\u001b*\u001b] merge parts...\n",
      "100%|########################| 2465/2465 [00:00<00:00, 2593.38 keys/s, part=0/1]\n",
      "------------------------------\n",
      "[!] extracted \u001b4,939\u001b 2D byte fragments (a 1024 bytes) from 5,000 packets\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/pcap2ds.py -p train.pcap -o train -m byte --chunk 1024 --threads 2 --force"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "patient-falls",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\u001b+\u001b] \u001bbyte\u001b preprocessing\n",
      "[\u001b*\u001b] \u001bcalculating\u001b maximum number of packets...\n",
      "[\u001b*\u001b] saving to \u001b/home/geb/Workspace/gits/thesis/test/fit\u001b\n",
      "[\u001b*\u001b] starting \u001b2\u001b consumer threads\n",
      "100%|###################################################| 5000/5000 [00:01<00:00, 3213.97 packets/s]\n",
      "[\u001b*\u001b] merge parts...\n",
      "100%|########################| 2427/2427 [00:00<00:00, 3094.33 keys/s, part=0/1]\n",
      "------------------------------\n",
      "[!] extracted \u001b4,859\u001b 2D byte fragments (a 1024 bytes) from 5,000 packets\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/pcap2ds.py -p fit.pcap -o fit -m byte --chunk 1024 --threads 2 --force"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "angry-administration",
   "metadata": {},
   "source": [
    "## *PCAPAE* API warapper \n",
    "\n",
    "trains model on -[t]rain set for 4 epochs, validates model using -[v]alidation data\n",
    "\n",
    "(add --cuda to train on GPU)\n",
    "\n",
    "\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "reserved-image",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[!] NO GPU support enabled\n",
      "[*] tensorboard is running at http://localhost:9001\n",
      "[*] fitting data to compressor\n",
      "[*] building the GRU network\n",
      "[*] logging to -> file:///home/geb/Workspace/gits/thesis/test/runs/Darius22 experiment tag: Darius22\n",
      "\n",
      "[*] Hyperparameters\n",
      "\t[§] train=train\n",
      "\t[§] vali=vali\n",
      "\t[§] fit=\n",
      "\t[§] predict=\n",
      "\t[§] model=\n",
      "\t[§] batch_size=128\n",
      "\t[§] learn_rate=0.001\n",
      "\t[§] finput=1\n",
      "\t[§] foutput=0\n",
      "\t[§] optim=adamW\n",
      "\t[§] clipping=10.0\n",
      "\t[§] fraction=1\n",
      "\t[§] workers=0\n",
      "\t[§] loss=MSE\n",
      "\t[§] scheduler=cycle\n",
      "\t[§] epochs=4\n",
      "\t[§] cell=GRU\n",
      "\t[§] seed=1994\n",
      "\t[§] noTensorboard=False\n",
      "\t[§] cuda=False\n",
      "\t[§] verbose=True\n",
      "\t[§] cache=False\n",
      "\t[§] retrain=False\n",
      "\t[§] name=None\n",
      "\t[§] OCSVM=False\n",
      "\t[§] ignoreLables=False\n",
      "\t[§] encoder_params=convGRU_encoder_params_cpu_32\n",
      "\t[§] encoder_params=convGRU_decoder_params_cpu_32\n",
      "\t[§] loss function=MSE\n",
      "\t[§] grad clip=10.0\n",
      "\t[§] optimizer=adamW\n",
      "\n",
      "[*] generating dataset loaders using 100% of the data\n",
      "[*] loading /home/geb/Workspace/gits/thesis/test/train/train_pcap.hdf5 for training\n",
      "[*] data shape (4939, 1024)\n",
      "[*] loading /home/geb/Workspace/gits/thesis/test/vali/vali_pcap.hdf5 for training\n",
      "[*] data shape (4900, 1024)\n",
      "[*] training for 4 with batch size of 128\n",
      "[*] using MSELoss() loss criterion\n",
      "[*] using AdamW gradient optimizer with:\n",
      "\tlr=0.001\n",
      "\tbetas=(0.9, 0.999)\n",
      "\teps=1e-08\n",
      "\tweight_decay=0.01\n",
      "\tamsgrad=False\n",
      "[*] using cycle scheduler with:\n",
      "\ttotal_steps=152\n",
      "\tstep_size_up=44.6\n",
      "\tstep_size_down=106.4\n",
      "\tanneal_func=<bound method OneCycleLR._annealing_cos of <torch.optim.lr_scheduler.OneCycleLR object at 0x7ffb6ae0a358>>\n",
      "\tcycle_momentum=True\n",
      "\tuse_beta1=True\n",
      "\tbase_lrs=[4e-05]\n",
      "\tlast_epoch=0\n",
      "\t_step_count=1\n",
      "\tverbose=False\n",
      "\t_get_lr_called_within_step=False\n",
      "\t_last_lr=[3.9999999999999996e-05]\n",
      "[*] learn rate scheduling starting at 0.001\n",
      "[*] gradient clipping value 10.0\n",
      "[*] using early stopping with patience 7\n",
      "100%|█| 38/38 [00:19<00:00,  1.98batch/s, epoch=0, loss=0.000813, lr=0.000040, a\n",
      "[*] validation loss record 0.000813 at epoch: 0                                 \n",
      "100%|█| 38/38 [00:16<00:00,  2.25batch/s, epoch=1, loss=0.000742, lr=0.000041, a\n",
      "[*] validation loss record 0.000743 at epoch: 1                                 \n",
      "100%|█| 38/38 [00:17<00:00,  2.17batch/s, epoch=2, loss=0.000685, lr=0.000045, a\n",
      "[*] validation loss record 0.000685 at epoch: 2                                 \n",
      "100%|█| 38/38 [00:17<00:00,  2.13batch/s, epoch=3, loss=0.000635, lr=0.000051, a\n",
      "[*] validation loss record 0.000635 at epoch: 3                                 \n",
      "100%|█| 38/38 [00:16<00:00,  2.27batch/s, epoch=4, loss=0.000589, lr=0.000059, a\n",
      "[*] validation loss record 0.000590 at epoch: 4                                 \n",
      "\n",
      "[$] trainig completed... 'file:///home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22'\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/main.py -t train -v vali --verbose --epochs 4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "severe-recognition",
   "metadata": {},
   "source": [
    "## retrain example\n",
    "provide --[m]odel path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "specialized-tutorial",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[!] NO GPU support enabled\n",
      "[*] tensorboard is running at http://localhost:9001\n",
      "[*] fitting data to compressor\n",
      "[*] building the GRU network\n",
      "[*] loading model -> /home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22/LAST_checkpoint_3_0.000635.pth.tar\n",
      "\n",
      "[*] Hyperparameters\n",
      "\t[§] train=train\n",
      "\t[§] vali=vali\n",
      "\t[§] fit=\n",
      "\t[§] predict=\n",
      "\t[§] model=/home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22/LAST_checkpoint_3_0.000635.pth.tar\n",
      "\t[§] batch_size=128\n",
      "\t[§] learn_rate=0.001\n",
      "\t[§] finput=1\n",
      "\t[§] foutput=0\n",
      "\t[§] optim=adamW\n",
      "\t[§] clipping=10.0\n",
      "\t[§] fraction=1\n",
      "\t[§] workers=0\n",
      "\t[§] loss=MSE\n",
      "\t[§] scheduler=cycle\n",
      "\t[§] epochs=6\n",
      "\t[§] cell=GRU\n",
      "\t[§] seed=1994\n",
      "\t[§] noTensorboard=False\n",
      "\t[§] cuda=False\n",
      "\t[§] verbose=True\n",
      "\t[§] cache=False\n",
      "\t[§] retrain=True\n",
      "\t[§] name=None\n",
      "\t[§] OCSVM=False\n",
      "\t[§] ignoreLables=False\n",
      "\t[§] encoder_params=convGRU_encoder_params_cpu_32\n",
      "\t[§] encoder_params=convGRU_decoder_params_cpu_32\n",
      "\t[§] loss function=MSE\n",
      "\t[§] grad clip=10.0\n",
      "\t[§] optimizer=adamW\n",
      "\n",
      "[*] generating dataset loaders using 100% of the data\n",
      "[*] loading /home/geb/Workspace/gits/thesis/test/train/train_pcap.hdf5 for training\n",
      "[*] data shape (4939, 1024)\n",
      "[*] loading /home/geb/Workspace/gits/thesis/test/vali/vali_pcap.hdf5 for training\n",
      "[*] data shape (4900, 1024)\n",
      "[*] training for 3 with batch size of 128\n",
      "[*] using MSELoss() loss criterion\n",
      "[*] using AdamW gradient optimizer with:\n",
      "\tlr=0.001\n",
      "\tbetas=(0.9, 0.999)\n",
      "\teps=1e-08\n",
      "\tweight_decay=0.01\n",
      "\tamsgrad=False\n",
      "[*] using cycle scheduler with:\n",
      "\ttotal_steps=114\n",
      "\tstep_size_up=33.199999999999996\n",
      "\tstep_size_down=79.80000000000001\n",
      "\tanneal_func=<bound method OneCycleLR._annealing_cos of <torch.optim.lr_scheduler.OneCycleLR object at 0x7fbd37366a20>>\n",
      "\tcycle_momentum=True\n",
      "\tuse_beta1=True\n",
      "\tbase_lrs=[4e-05]\n",
      "\tlast_epoch=0\n",
      "\t_step_count=1\n",
      "\tverbose=False\n",
      "\t_get_lr_called_within_step=False\n",
      "\t_last_lr=[3.9999999999999996e-05]\n",
      "[*] learn rate scheduling starting at 0.001\n",
      "[*] gradient clipping value 10.0\n",
      "[*] using early stopping with patience 7\n",
      "100%|█| 38/38 [00:17<00:00,  2.20batch/s, epoch=3, loss=0.000594, lr=0.000040, a\n",
      "[*] validation loss record 0.000595 at epoch: 3                                 \n",
      "100%|█| 38/38 [00:16<00:00,  2.27batch/s, epoch=4, loss=0.000560, lr=0.000042, a\n",
      "[*] validation loss record 0.000561 at epoch: 4                                 \n",
      "100%|█| 38/38 [00:17<00:00,  2.23batch/s, epoch=5, loss=0.000528, lr=0.000049, a\n",
      "[*] validation loss record 0.000529 at epoch: 5                                 \n",
      "100%|█| 38/38 [00:16<00:00,  2.28batch/s, epoch=6, loss=0.000495, lr=0.000059, a\n",
      "[*] validation loss record 0.000495 at epoch: 6                                 \n",
      "\n",
      "[$] trainig completed... 'file:///home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22'\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/main.py -t train -v vali --verbose --epochs 6 --retrain --model /home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22/LAST_checkpoint_3_0.000635.pth.tar"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "third-asset",
   "metadata": {},
   "source": [
    "## convert data to reduced format using a trained model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "incomplete-pavilion",
   "metadata": {},
   "source": [
    "### generate dummy ground truth for 5000 packets (50%/50%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "athletic-reality",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "rm -f dummy_groundTruth.csv ; cat /dev/urandom | tr -dc '1[=d=]' | fold -w 1 | head -n 5000 | sed -r 's/[d]+/-1/g' >> dummy_groundTruth.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dress-extraction",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\u001b+\u001b] \u001bbyte\u001b preprocessing\n",
      "[\u001b*\u001b] \u001bremoving\u001b /home/geb/Workspace/gits/thesis/test/predict_data...\n",
      "[\u001b*\u001b] \u001bcalculating\u001b maximum number of packets...\n",
      "[\u001b*\u001b] saving to \u001b/home/geb/Workspace/gits/thesis/test/predict_data\u001b\n",
      "[\u001b*\u001b] starting \u001b2\u001b consumer threads\n",
      "100%|###################################################| 5000/5000 [00:03<00:00, 1457.13 packets/s]\n",
      "[\u001b*\u001b] merge parts...\n",
      "100%|########################| 2444/2444 [00:01<00:00, 2318.79 keys/s, part=0/1]\n",
      "------------------------------\n",
      "[!] extracted \u001b4,859\u001b 2D byte fragments (a 1024 bytes) from 5,000 packets\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/pcap2ds.py -p predict.pcap -o predict_data -m byte --chunk 1024 --threads 2 -g dummy_groundTruth.csv --force"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "geological-prevention",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[!] NO GPU support enabled\n",
      "[*] tensorboard is running at http://localhost:9001\n",
      "[*] transforming data with compressor\n",
      "[*] loading model -> /home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22/LAST_checkpoint_6_0.000495.pth.tar\n",
      "100%|████████████████████████████████| 4859/4859 [00:23<00:00, 210.92 packetX/s]\n",
      "[*] transformed shape: (4859, 64)\n",
      "[*] saving 'fit' to: 'file:///home/geb/Workspace/gits/thesis/test/runs/redu_data/Brandon41/' ...\n",
      "100%|████████████████████████████████| 4859/4859 [00:27<00:00, 175.19 packetX/s]\n",
      "[*] transformed shape: (4859, 64)\n",
      "[*] saving 'predict_data' to: 'file:///home/geb/Workspace/gits/thesis/test/runs/redu_data/Brandon41/' ...\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/main.py --model /home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22/LAST_checkpoint_6_0.000495.pth.tar --fit fit --predict predict_data --name Brandon41"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ethical-salem",
   "metadata": {},
   "source": [
    "### train OCSVM with reduced data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "desperate-twist",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[!] NO GPU support enabled\n",
      "[*] tensorboard is running at http://localhost:9001\n",
      "# load train data from /redu path\n",
      "# load predict data from /redu path /home/geb/Workspace/gits/thesis/test/runs/redu_data/Brandon41/predict_data_wGT\n",
      "[*] training OCSVM: {'cache_size': 200, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'rbf', 'max_iter': -1, 'nu': 0.5, 'shrinking': True, 'tol': 0.001, 'verbose': True}\n",
      "[*] fitting reduced data\n",
      ".\n",
      "Warning: using -h 0 may be faster\n",
      "*\n",
      "optimization finished, #iter = 1246\n",
      "obj = 546386.390342, rho = 491.905797\n",
      "nSV = 2441, nBSV = 2417\n",
      "[LibSVM][*] predicting reduced data\n",
      "+--------------------------------+\n",
      "|\u001b \u001b\u001bpred\\con\u001b\u001b \u001b|\u001b \u001b\u001b1       \u001b\u001b \u001b|\u001b \u001b\u001b-1      \u001b\u001b \u001b|\n",
      "|----------+----------+----------|\n",
      "| positive | \u001bTP: \u001b\u001b653\u001b  | \u001bFP: \u001b\u001b660\u001b  |\n",
      "| negative | \u001bFN: \u001b\u001b1779\u001b | \u001bTN: \u001b\u001b1767\u001b |\n",
      "+--------------------------------+\n",
      "+---------------------+\n",
      "|\u001b \u001b\u001bmetrics  \u001b\u001b \u001b|\u001b \u001b\u001bvalue  \u001b\u001b \u001b|\n",
      "|-----------+---------|\n",
      "| precision | \u001b0.72806\u001b |\n",
      "| recall    | \u001b0.49831\u001b |\n",
      "| F_1       | \u001b0.59166\u001b |\n",
      "+---------------------+\n",
      "[*] saving 'OCSVM_model.jlib' classifier to -> 'file:///home/geb/Workspace/gits/thesis/test/runs/save_model/Catherine55/AD/'\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/main.py --fit /home/geb/Workspace/gits/thesis/test/runs/redu_data/Brandon41/fit_compressed.npy --predict /home/geb/Workspace/gits/thesis/test/runs/redu_data/Brandon41/predict_data_wGT --OCSVM --verbose"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liked-progress",
   "metadata": {},
   "source": [
    "### predicting new data with a saved OCSVM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "prostate-reservation",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "rm -f snd_dummy_groundTruth.csv ; cat /dev/urandom | tr -dc '1[=d=]' | fold -w 1 | head -n 5000 | sed -r 's/[d]+/-1/g' >> snd_dummy_groundTruth.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "entitled-broadcast",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\u001b+\u001b] \u001bbyte\u001b preprocessing\n",
      "[\u001b*\u001b] \u001bcalculating\u001b maximum number of packets...\n",
      "[\u001b*\u001b] saving to \u001b/home/geb/Workspace/gits/thesis/test/snd_predict_data\u001b\n",
      "[\u001b*\u001b] starting \u001b2\u001b consumer threads\n",
      "100%|###################################################| 5000/5000 [00:03<00:00, 1410.78 packets/s]\n",
      "[\u001b*\u001b] merge parts...\n",
      "100%|########################| 2429/2429 [00:01<00:00, 2228.48 keys/s, part=0/1]\n",
      "------------------------------\n",
      "[!] extracted \u001b4,859\u001b 2D byte fragments (a 1024 bytes) from 5,000 packets\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/pcap2ds.py -p predict.pcap -o snd_predict_data -m byte --chunk 1024 --threads 2 -g snd_dummy_groundTruth.csv --force"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "alike-chapel",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[!] NO GPU support enabled\n",
      "[*] tensorboard is running at http://localhost:9001\n",
      "[*] loading model -> /home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22/best/checkpoint_6_0.000495.pth.tar\n",
      "100%|████████████████████████████████| 4859/4859 [00:27<00:00, 178.86 packetX/s]\n",
      "[*] transformed shape: (4859, 64)\n",
      "[*] saving 'snd_predict_data' to: 'file:///home/geb/Workspace/gits/thesis/test/runs/redu_data/NEW_DATA/' ...\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/main.py --model /home/geb/Workspace/gits/thesis/test/runs/save_model/Darius22/best/checkpoint_6_0.000495.pth.tar --predict snd_predict_data --name NEW_DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "german-music",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[!] NO GPU support enabled\n",
      "[*] tensorboard is running at http://localhost:9001\n",
      "# load predict data from /redu path /home/geb/Workspace/gits/thesis/test/runs/redu_data/NEW_DATA/snd_predict_data_wGT\n",
      "[*] training OCSVM: {'cache_size': 200, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'rbf', 'max_iter': -1, 'nu': 0.5, 'shrinking': True, 'tol': 0.001, 'verbose': True}\n",
      "[*] predicting reduced data\n",
      "+--------------------------------+\n",
      "|\u001b \u001b\u001bpred\\con\u001b\u001b \u001b|\u001b \u001b\u001b1       \u001b\u001b \u001b|\u001b \u001b\u001b-1      \u001b\u001b \u001b|\n",
      "|----------+----------+----------|\n",
      "| positive | \u001bTP: \u001b\u001b634\u001b  | \u001bFP: \u001b\u001b676\u001b  |\n",
      "| negative | \u001bFN: \u001b\u001b1798\u001b | \u001bTN: \u001b\u001b1751\u001b |\n",
      "+--------------------------------+\n",
      "+---------------------+\n",
      "|\u001b \u001b\u001bmetrics  \u001b\u001b \u001b|\u001b \u001b\u001bvalue  \u001b\u001b \u001b|\n",
      "|-----------+---------|\n",
      "| precision | \u001b0.72147\u001b |\n",
      "| recall    | \u001b0.49338\u001b |\n",
      "| F_1       | \u001b0.58601\u001b |\n",
      "+---------------------+\n"
     ]
    }
   ],
   "source": [
    "!python3 ../src/main.py --model /home/geb/Workspace/gits/thesis/test/runs/save_model/Catherine55/AD/OCSVM_model.jlib --predict /home/geb/Workspace/gits/thesis/test/runs/redu_data/NEW_DATA/snd_predict_data_wGT"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}