unicorn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Flash Evaluation on Unicorn Dataset:\n",
    "\n",
    "This notebook is dedicated to evaluating Flash on the Unicorn datasets, which are graph-level in nature. We employ Flash in graph-level detection mode to analyze this dataset effectively. Upon completion of the notebook execution, the results will be presented.\n",
    "\n",
    "## Dataset Access: \n",
    "- The Unicorn dataset can be accessed at the following link: [Unicorn Dataset](https://github.com/margoseltzer/shellshock-apt).\n",
    "\n",
    "## Data Parsing and Execution:\n",
    "- The script automatically downloads the parsed dataset.\n",
    "- To obtain the evaluation results, execute all cells within this notebook.\n",
    "\n",
    "## Model Training and Execution Flexibility:\n",
    "- By default, the notebook operates using pre-trained model weights.\n",
    "- Additionally, this notebook offers the flexibility to set parameters for training Graph Neural Networks (GNNs) and word2vec models from scratch.\n",
    "- You can then utilize these freshly trained models to conduct the evaluation. \n",
    "\n",
    "Follow these guidelines for a thorough and efficient analysis of the Unicorn datasets using Flash.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "F1op-CbyLuN4",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import torch\n",
    "from torch_geometric.data import Data\n",
    "import os\n",
    "import torch.nn.functional as F\n",
    "import json\n",
    "import warnings\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.manifold import TSNE\n",
    "warnings.filterwarnings('ignore')\n",
    "from torch_geometric.loader import NeighborLoader\n",
    "import multiprocessing\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gdown\n",
    "url = \"https://drive.google.com/file/d/1dmezgT9zQ-8ydHrXvJmxwJN-LjPtumbU/view\"\n",
    "gdown.download(url, quiet=False, use_cookies=False, fuzzy=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "\n",
    "def unzip_file(zip_path, extract_to):\n",
    "    with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n",
    "        zip_ref.extractall(extract_to)\n",
    "\n",
    "zip_path = 'unicorn.zip'\n",
    "extract_to = 'unicorn'\n",
    "unzip_file(zip_path, extract_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "Train_Gnn = False\n",
    "Train_Word2vec = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nM7KaeCbA_mQ",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "import gzip\n",
    "from sklearn.manifold import TSNE\n",
    "import json\n",
    "import copy\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os.path as osp\n",
    "import csv\n",
    "def show(str):\n",
    "\tprint (str + ' ' + time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime(time.time())))\n",
    "\n",
    "def parse_data():\n",
    "    for i in range(3):\n",
    "        os.system('tar -zxvf camflow-attack-' + str(i) + '.gz.tar')\n",
    "    for i in range(13):\n",
    "        os.system('tar -zxvf camflow-benign-' + str(i) + '.gz.tar')\n",
    "\n",
    "    os.system('rm error.log')\n",
    "    os.system('rm parse-error-camflow-*')\n",
    "    show('Start processing.')\n",
    "    for i in range(25):\n",
    "        show('Attack graph ' + str(i+125))\n",
    "        f = open('camflow-attack.txt.'+str(i), 'r')\n",
    "        fw = open('unicorn/'+str(i+125)+'.txt', 'w')\n",
    "        for line in f:\n",
    "                tempp = line.strip('\\n').split('\\t')\n",
    "                temp = []\n",
    "                temp.append(tempp[0])\n",
    "                temp.append(tempp[2].split(':')[0])\n",
    "                temp.append(tempp[1])\n",
    "                temp.append(tempp[2].split(':')[1])\n",
    "                temp.append(tempp[2].split(':')[2])\n",
    "                temp.append(tempp[2].split(':')[3])\n",
    "                fw.write(temp[0]+'\\t'+temp[1]+'\\t'+temp[2]+'\\t'+temp[3]+'\\t'+temp[4]+'\\t'+temp[5]+'\\n')\n",
    "        f.close()\n",
    "        fw.close()\n",
    "        os.system('rm camflow-attack.txt.' + str(i))\n",
    "\n",
    "    for i in range(125):\n",
    "        show('Benign graph ' + str(i))\n",
    "        f = open('camflow-normal.txt.'+str(i), 'r')\n",
    "        fw = open('unicorn/'+str(i)+'.txt', 'w')\n",
    "        for line in f:\n",
    "                tempp = line.strip('\\n').split('\\t')\n",
    "                temp = []\n",
    "                temp.append(tempp[0])\n",
    "                temp.append(tempp[2].split(':')[0])\n",
    "                temp.append(tempp[1])\n",
    "                temp.append(tempp[2].split(':')[1])\n",
    "                temp.append(tempp[2].split(':')[2])\n",
    "                temp.append(tempp[2].split(':')[3])\n",
    "                fw.write(temp[0]+'\\t'+temp[1]+'\\t'+temp[2]+'\\t'+temp[3]+'\\t'+temp[4]+'\\t'+temp[5]+'\\n')\n",
    "        f.close()\n",
    "        fw.close()\n",
    "        os.system('rm camflow-normal.txt.' + str(i))\n",
    "    show('Done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def prepare_graph(df):\n",
    "    def process_node(node, action, node_dict, label_dict, dummies, node_type):\n",
    "        node_dict.setdefault(node, []).append(action)\n",
    "        label_dict[node] = dummies.get(getattr(row, node_type), -1)  \n",
    "\n",
    "    nodes = {}\n",
    "    labels = {}\n",
    "    edges = []\n",
    "    dummies = {\n",
    "        \"7998762093665332071\": 0, \"14709879154498484854\": 1, \"10991425273196493354\": 2,\n",
    "        \"14871526952859113360\": 3, \"8771628573506871447\": 4, \"7877121489144997480\": 5,\n",
    "        \"17841021884467483934\": 6, \"7895447931126725167\": 7, \"15125250455093594050\": 8,\n",
    "        \"8664433583651064836\": 9, \"14377490526132269506\": 10, \"15554536683409451879\": 11,\n",
    "        \"8204541918505434145\": 12, \"14356114695140920775\": 13\n",
    "    }\n",
    "\n",
    "    for row in df.itertuples():\n",
    "        process_node(row.actorID, row.action, nodes, labels, dummies, 'actor_type')\n",
    "        process_node(row.objectID, row.action, nodes, labels, dummies, 'object')\n",
    "\n",
    "        edges.append((row.actorID, row.objectID))\n",
    "\n",
    "    features = [nodes[node] for node in nodes]\n",
    "    feat_labels = [labels[node] for node in nodes]\n",
    "    edge_index = [[], []]\n",
    "    for src, dst in edges:\n",
    "        src_index = list(nodes.keys()).index(src)\n",
    "        dst_index = list(nodes.keys()).index(dst)\n",
    "        edge_index[0].append(src_index)\n",
    "        edge_index[1].append(dst_index)\n",
    "\n",
    "    return features, feat_labels, edge_index, list(nodes.keys())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fmXWs1dKIzD8",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch_geometric.nn import GCNConv\n",
    "from torch_geometric.nn import SAGEConv, GATConv\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "\n",
    "class GCN(torch.nn.Module):\n",
    "    def __init__(self,in_channel,out_channel):\n",
    "        super().__init__()\n",
    "        self.conv1 = SAGEConv(in_channel, 32, normalize=True)\n",
    "        self.conv2 = SAGEConv(32, 20, normalize=True)\n",
    "        self.linear = nn.Linear(in_features=20,out_features=out_channel)\n",
    "\n",
    "    def forward(self, x, edge_index):\n",
    "        x = self.conv1(x, edge_index)\n",
    "        x = x.relu()\n",
    "        x = F.dropout(x, p=0.5, training=self.training)\n",
    "\n",
    "        x = self.conv2(x, edge_index)\n",
    "        x = self.linear(x)\n",
    "        return F.softmax(x, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3PCP6SXwZaif",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from gensim.models.callbacks import CallbackAny2Vec\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from multiprocessing import Pool\n",
    "from itertools import compress\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "class EpochSaver(CallbackAny2Vec):\n",
    "    '''Callback to save model after each epoch.'''\n",
    "\n",
    "    def __init__(self):\n",
    "        self.epoch = 0\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        model.save('trained_weights/unicorn/unicorn.model')\n",
    "        self.epoch += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "P8oBL8LFaeOf",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class EpochLogger(CallbackAny2Vec):\n",
    "    '''Callback to log information about training'''\n",
    "\n",
    "    def __init__(self):\n",
    "        self.epoch = 0\n",
    "\n",
    "    def on_epoch_begin(self, model):\n",
    "        print(\"Epoch #{} start\".format(self.epoch))\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        print(\"Epoch #{} end\".format(self.epoch))\n",
    "        self.epoch += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Se7Ei4tAapVj",
    "tags": []
   },
   "outputs": [],
   "source": [
    "logger = EpochLogger()\n",
    "saver = EpochSaver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Train_Word2vec:\n",
    "    comb_data = []\n",
    "    for i in range(20):\n",
    "        f = open(f\"unicorn/{i}.txt\")\n",
    "        data = f.read().split('\\n')\n",
    "        data = [line.split('\\t') for line in data]\n",
    "        comb_data = comb_data + data\n",
    "\n",
    "    df = pd.DataFrame (comb_data, columns = ['actorID', 'actor_type','objectID','object','action','timestamp'])\n",
    "    df.sort_values(by='timestamp', ascending=True,inplace=True)\n",
    "    df = df.dropna()\n",
    "    phrases,labels,edges,mapp = prepare_graph(df)\n",
    "    \n",
    "    word2vec = Word2Vec(sentences=phrases, vector_size=30, window=5, min_count=1, workers=8,epochs=300,callbacks=[saver,logger])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "p3TAi69zI1bO",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.utils import class_weight\n",
    "import torch.nn.functional as F\n",
    "from torch.nn import CrossEntropyLoss\n",
    "\n",
    "model = GCN(30,14).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Vn_pMyt5Jd-6",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import math\n",
    "\n",
    "class PositionalEncoder:\n",
    "\n",
    "    def __init__(self, d_model, max_len=100000):\n",
    "        position = torch.arange(max_len).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))\n",
    "        self.pe = torch.zeros(max_len, d_model)\n",
    "        self.pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        self.pe[:, 1::2] = torch.cos(position * div_term)\n",
    "\n",
    "    def embed(self, x):\n",
    "        return x + self.pe[:x.size(0)]\n",
    "\n",
    "def infer(document):\n",
    "    word_embeddings = [w2vmodel.wv[word] for word in document if word in  w2vmodel.wv]\n",
    "    \n",
    "    if not word_embeddings:\n",
    "        return np.zeros(20)\n",
    "\n",
    "    output_embedding = torch.tensor(word_embeddings, dtype=torch.float)\n",
    "    if len(document) < 100000:\n",
    "        output_embedding = encoder.embed(output_embedding)\n",
    "\n",
    "    output_embedding = output_embedding.detach().cpu().numpy()\n",
    "    return np.mean(output_embedding, axis=0)\n",
    "\n",
    "encoder = PositionalEncoder(30)\n",
    "w2vmodel = Word2Vec.load(\"trained_weights/unicorn/unicorn.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 689309,
     "status": "ok",
     "timestamp": 1673566932746,
     "user": {
      "displayName": "Mati Ur Rehman",
      "userId": "04281203290774044297"
     },
     "user_tz": 300
    },
    "id": "Gclj6HVL17lD",
    "outputId": "f60fadea-a7db-471f-defe-fee744f6ef25",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch_geometric import utils\n",
    "\n",
    "################################## Training Main Model #####################################\n",
    "if Train_Gnn:\n",
    "    for i in range(95):\n",
    "        f = open(f\"unicorn/{i}.txt\")\n",
    "        data = f.read().split('\\n')\n",
    "\n",
    "        data = [line.split('\\t') for line in data]\n",
    "        df = pd.DataFrame (data, columns = ['actorID', 'actor_type','objectID','object','action','timestamp'])\n",
    "        df.sort_values(by='timestamp', ascending=True,inplace=True)\n",
    "        df = df.dropna()\n",
    "        phrases,labels,edges,mapp = prepare_graph(df)\n",
    "\n",
    "        criterion = CrossEntropyLoss()\n",
    "\n",
    "        nodes = [infer(x) for x in phrases]\n",
    "        nodes = np.array(nodes)  \n",
    "\n",
    "        graph = Data(x=torch.tensor(nodes,dtype=torch.float).to(device),y=torch.tensor(labels,dtype=torch.long).to(device), edge_index=torch.tensor(edges,dtype=torch.long).to(device))\n",
    "        graph.n_id = torch.arange(graph.num_nodes)\n",
    "        mask = torch.tensor([True]*graph.num_nodes, dtype=torch.bool)\n",
    "\n",
    "        for m_n in range(20):\n",
    "            loader = NeighborLoader(graph, num_neighbors=[-1,-1], batch_size=5000,input_nodes=mask)\n",
    "            total_loss = 0\n",
    "            for subg in loader:\n",
    "                model.train()\n",
    "                optimizer.zero_grad() \n",
    "                out = model(subg.x, subg.edge_index) \n",
    "                loss = criterion(out, subg.y) \n",
    "                loss.backward() \n",
    "                optimizer.step()      \n",
    "                total_loss += loss.item() * subg.batch_size\n",
    "\n",
    "            loader = NeighborLoader(graph, num_neighbors=[-1,-1], batch_size=5000,input_nodes=mask)\n",
    "            for subg in loader:\n",
    "              model.eval()\n",
    "              out = model(subg.x, subg.edge_index)\n",
    "              sorted, indices = out.sort(dim=1,descending=True)\n",
    "              conf = (sorted[:,0] - sorted[:,1]) / sorted[:,0]\n",
    "              conf = (conf - conf.min()) / conf.max()\n",
    "              pred = indices[:,0]\n",
    "              cond = (pred == subg.y)\n",
    "              mask[subg.n_id[cond]] = False\n",
    "\n",
    "            print(f'Model# {m_n}. {mask.sum().item()} nodes still misclassified \\n')\n",
    "            torch.save(model.state_dict(), f'trained_weights/unicorn/unicorn{m_n}.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for i in range(95,98):\n",
    "    print(f\"Graph #: {i}\")\n",
    "    f = open(f\"unicorn/{i}.txt\")\n",
    "    data = f.read().split('\\n')\n",
    "\n",
    "    data = [line.split('\\t') for line in data]\n",
    "    df = pd.DataFrame (data, columns = ['actorID', 'actor_type','objectID','object','action','timestamp'])\n",
    "    df.sort_values(by='timestamp', ascending=True,inplace=True)\n",
    "    df = df.dropna()\n",
    "\n",
    "    phrases,labels,edges,mapp = prepare_graph(df)\n",
    "\n",
    "    nodes = [infer(x) for x in phrases]\n",
    "    nodes = np.array(nodes)  \n",
    "\n",
    "    graph = Data(x=torch.tensor(nodes,dtype=torch.float).to(device),y=torch.tensor(labels,dtype=torch.long).to(device), edge_index=torch.tensor(edges,dtype=torch.long).to(device))\n",
    "    graph.n_id = torch.arange(graph.num_nodes)\n",
    "    flag = torch.tensor([True]*graph.num_nodes, dtype=torch.bool)\n",
    "\n",
    "    for m_n in range(20):\n",
    "        model.load_state_dict(torch.load(f'trained_weights/unicorn/unicorn{m_n}.pth'))\n",
    "        model.eval()\n",
    "        out = model(graph.x, graph.edge_index)\n",
    "\n",
    "        sorted, indices = out.sort(dim=1,descending=True)\n",
    "        conf = (sorted[:,0] - sorted[:,1]) / sorted[:,0]\n",
    "        conf = (conf - conf.min()) / conf.max()\n",
    "\n",
    "        pred = indices[:,0]\n",
    "        cond = (pred == graph.y)\n",
    "        flag[graph.n_id[cond]] = torch.logical_and(flag[graph.n_id[cond]], torch.tensor([False]*len(flag[graph.n_id[cond]]), dtype=torch.bool))\n",
    "            \n",
    "    print(flag.sum().item(), (flag.sum().item() / len(flag))*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "thresh = 330"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "correct_benign = 0\n",
    "\n",
    "for i in range(100,125):\n",
    "    print(f\"Graph #: {i}\")\n",
    "    f = open(f\"unicorn/{i}.txt\")\n",
    "    data = f.read().split('\\n')\n",
    "\n",
    "    data = [line.split('\\t') for line in data]\n",
    "    df = pd.DataFrame (data, columns = ['actorID', 'actor_type','objectID','object','action','timestamp'])\n",
    "    df.sort_values(by='timestamp', ascending=True,inplace=True)\n",
    "    df = df.dropna()\n",
    "\n",
    "    phrases,labels,edges,mapp = prepare_graph(df)\n",
    "\n",
    "    nodes = [infer(x) for x in phrases]\n",
    "    nodes = np.array(nodes)  \n",
    "\n",
    "    graph = Data(x=torch.tensor(nodes,dtype=torch.float).to(device),y=torch.tensor(labels,dtype=torch.long).to(device), edge_index=torch.tensor(edges,dtype=torch.long).to(device))\n",
    "    graph.n_id = torch.arange(graph.num_nodes)\n",
    "    flag = torch.tensor([True]*graph.num_nodes, dtype=torch.bool)\n",
    "\n",
    "    for m_n in range(20):\n",
    "        model.load_state_dict(torch.load(f'trained_weights/unicorn/unicorn{m_n}.pth'))\n",
    "        model.eval()\n",
    "        out = model(graph.x, graph.edge_index)\n",
    "\n",
    "        sorted, indices = out.sort(dim=1,descending=True)\n",
    "        conf = (sorted[:,0] - sorted[:,1]) / sorted[:,0]\n",
    "        conf = (conf - conf.min()) / conf.max()\n",
    "\n",
    "        pred = indices[:,0]\n",
    "        cond = (pred == graph.y)\n",
    "        flag[graph.n_id[cond]] = torch.logical_and(flag[graph.n_id[cond]], torch.tensor([False]*len(flag[graph.n_id[cond]]), dtype=torch.bool))\n",
    "\n",
    "    if flag.sum().item() <= thresh:\n",
    "        correct_benign = correct_benign + 1\n",
    "            \n",
    "    print(flag.sum().item(), (flag.sum().item() / len(flag))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "correct_attack = 0\n",
    "\n",
    "for i in range(125,150):\n",
    "    print(f\"Graph #: {i}\")\n",
    "    f = open(f\"unicorn/{i}.txt\")\n",
    "    data = f.read().split('\\n')\n",
    "\n",
    "    data = [line.split('\\t') for line in data]\n",
    "    df = pd.DataFrame (data, columns = ['actorID', 'actor_type','objectID','object','action','timestamp'])\n",
    "    df.sort_values(by='timestamp', ascending=True,inplace=True)\n",
    "    df = df.dropna()\n",
    "    \n",
    "    phrases,labels,edges,mapp = prepare_graph(df)\n",
    "\n",
    "    nodes = [infer(x) for x in phrases]\n",
    "    nodes = np.array(nodes)  \n",
    "    \n",
    "    graph = Data(x=torch.tensor(nodes,dtype=torch.float).to(device),y=torch.tensor(labels,dtype=torch.long).to(device), edge_index=torch.tensor(edges,dtype=torch.long).to(device))\n",
    "    graph.n_id = torch.arange(graph.num_nodes)\n",
    "    flag = torch.tensor([True]*graph.num_nodes, dtype=torch.bool)\n",
    "\n",
    "    for m_n in range(20):\n",
    "        model.load_state_dict(torch.load(f'trained_weights/unicorn/unicorn{m_n}.pth'))\n",
    "        model.eval()\n",
    "        out = model(graph.x, graph.edge_index)\n",
    "\n",
    "        sorted, indices = out.sort(dim=1,descending=True)\n",
    "        conf = (sorted[:,0] - sorted[:,1]) / sorted[:,0]\n",
    "        conf = (conf - conf.min()) / conf.max()\n",
    "\n",
    "        pred = indices[:,0]\n",
    "        cond = (pred == graph.y)\n",
    "        flag[graph.n_id[cond]] = torch.logical_and(flag[graph.n_id[cond]], torch.tensor([False]*len(flag[graph.n_id[cond]]), dtype=torch.bool))\n",
    "\n",
    "    if  flag.sum().item() > thresh:\n",
    "        correct_attack = correct_attack + 1\n",
    "   \n",
    "    print(flag.sum().item(), (flag.sum().item() / len(flag))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TP = correct_attack\n",
    "FP = 25 - correct_benign\n",
    "TN = correct_benign\n",
    "FN = 25 - correct_attack\n",
    "\n",
    "FPR = FP / (FP + TN) if (FP + TN) > 0 else 0\n",
    "TPR = TP / (TP + FN) if (TP + FN) > 0 else 0\n",
    "\n",
    "print(f\"Number of True Positives (TP): {TP}\")\n",
    "print(f\"Number of False Positives (FP): {FP}\")\n",
    "print(f\"Number of False Negatives (FN): {FN}\")\n",
    "print(f\"Number of True Negatives (TN): {TN}\\n\")\n",
    "\n",
    "precision = TP / (TP + FP) if (TP + FP) > 0 else 0\n",
    "recall = TPR  \n",
    "print(f\"Precision: {precision}\")\n",
    "print(f\"Recall: {recall}\")\n",
    "\n",
    "fscore = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n",
    "print(f\"Fscore: {fscore}\\n\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "ASE 3.20.1",
   "language": "python",
   "name": "ase-3.20.1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}