experiments/describing_regions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de31a79b-50cc-4b9e-904e-95d2a1f3736a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append(\"../src\")\n",
    "from utils.metrics_hai import compute_metrics\n",
    "from utils.utils import loss_01\n",
    "\n",
    "logging.getLogger().setLevel(logging.INFO)\n",
    "import math\n",
    "\n",
    "# import os, json, cv2, random\n",
    "# import some common detectron2 utilities\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Some basic setup:\n",
    "# Setup detectron2 logger\n",
    "# import some common libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "import torch\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "from PIL import Image\n",
    "from sklearn.metrics import classification_report\n",
    "from tqdm import tqdm\n",
    "from transformers import CLIPModel, CLIPProcessor\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "print(torch.cuda.is_available())\n",
    "from datasets_hai.coco import *\n",
    "from describers.domino_describe import *\n",
    "from describers.itterative_describe import *\n",
    "from describers.seal_describe import *\n",
    "from utils.metrics_caption import *\n",
    "from utils.metrics_hai import *\n",
    "from utils.metrics_regions import *\n",
    "from utils.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32739169-8465-4bc5-a993-a4c80ba854c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# OPEN AI KEY\n",
    "keyfile = open(\"../keys.txt\", \"r\")\n",
    "# read the file\n",
    "key = keyfile.read()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4005c152-7180-44c0-9b19-4e42a7bc2be7",
   "metadata": {},
   "source": [
    "ms-coco dataset prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab0b95fe-6667-4da7-9527-39234c9dffc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_coco_captions = dset.CocoCaptions(root =\"../data/coco/val2017\",\n",
    "                        annFile =\"../data/coco/annotations/captions_val2017.json\",\n",
    "                        transform=transforms.Compose([transforms.PILToTensor(),pil_to_detectronformat]))\n",
    "captions = []\n",
    "for i in tqdm(range(len(validation_coco_captions))):\n",
    "    captions.append(validation_coco_captions[i][1][0])\n",
    "captions_clean = []\n",
    "for caption in captions:\n",
    "    caption = caption.replace(\"\\n\", \" \").replace(\"\\r\", \" \").replace('\"', '')\n",
    "    captions_clean.append(caption)\n",
    "captions = np.array(captions_clean)\n",
    "data_coco_storred = pickle.load(open(\"../data\" + \"/coco/data_coco_embs_preds.pkl\",\"rb\"))\n",
    "label_to_category_rcnn = data_coco_storred[\"label_to_category_rcnn\"]\n",
    "\n",
    "dataset = pickle.load(open(\"../data/cleaned_pkl/coco_dataset.pkl\",\"rb\"))\n",
    "\n",
    "metadata = [['' for _ in range(len(dataset.metadata[0]))] for __ in range(len(dataset.metadata))]\n",
    "for i in range(len(dataset.metadata)):\n",
    "    for j in range(len(dataset.metadata[0])):\n",
    "        if 'present' in dataset.metadata[i][j]:\n",
    "            metadata[i][j] =  \"present\"\n",
    "        else:\n",
    "            metadata[i][j] =  \"absent\"\n",
    "\n",
    "dataset.metadata = metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85c9c2b6-a60d-4c94-86ee-46d8ba585f72",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pre_instruction = \"I will provide you with a set of descriptions of points that belong to a region and a set of descriptions of point that do not belong to the region.\" + \\\n",
    "    \"Your task is to summarize the points inside the region in a concise and precise short sentence while making sure the summary contrast to points outside the region.\" + \\\n",
    "    \"Your one sentence summary should be able to allow a person to distinguish between points inside and outside the region while describing the region well.\" + \\\n",
    "    \"The summary should be a single word, it should be accurate, concise, distinguishing and precise.\" + \\\n",
    "    \"Example: \\n\" + \"inside the region: \\n two cows and two sheep grazing in a pasture. \\n the sheep is standing near a tree. \\n not in the region:  the cows are lying on the grass beside the water.\\n\" + \\\n",
    "    \"summary: sheep. \\n End of Example \\n\"\n",
    "\n",
    "pre_instruction_no_contrast = \"I will provide you with a set of descriptions of points that belong to a region.\" + \\\n",
    "    \"Your task is to summarize the points inside the region in a concise and precise short sentence .\" + \\\n",
    "    \"Your one sentence summary should be able to allow a person to distinguish  points inside the region while describing the region well.\" + \\\n",
    "    \"The summary should be a single word, it should be accurate, concise, distinguishing and precise.\" + \\\n",
    "    \"Example: \\n\" + \"inside the region: \\n two cows and two sheep grazing in a pasture. \\n the sheep is standing near a tree.\\n\" + \\\n",
    "    \"summary: sheep. \\n End of Example \\n\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "978ce861-90f0-4d72-b077-eb8cea446bb1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "unique_dim = []\n",
    "metrics_seal = []\n",
    "metrics_domino = []\n",
    "metrics_itt5 = []\n",
    "metrics_itt10 = []\n",
    "metrics_itt_nocontrast = []\n",
    "metrics_itt_1round = []\n",
    "metrics_itt_chat = []\n",
    "\n",
    "for i in range(73): # 73 is the limit\n",
    "    while True:\n",
    "        region_dims = random.sample(list(range(len(dataset.metadata[0]))),1)\n",
    "        if label_to_category_rcnn[region_dims[0]] in unique_dim:\n",
    "            continue\n",
    "        region_conds = ['present']#, 'absent']\n",
    "        cluster_scores = np.zeros((len(dataset.data_y), 2))\n",
    "        cluster_labels = np.zeros(len(dataset.data_y))\n",
    "        in_region_count = 0\n",
    "        index_inside = 0\n",
    "\n",
    "        for i in range(len(dataset.data_y)):\n",
    "            # check metadata condition\n",
    "\n",
    "            in_region = all([dataset.metadata[i][region_dims[j]] == region_conds[j] for j in range(len(region_dims))])\n",
    "            if in_region:\n",
    "                in_region_count += 1\n",
    "                index_inside = i\n",
    "                cluster_scores[i] = [0,1]\n",
    "                cluster_labels[i] = 1\n",
    "            else:\n",
    "                cluster_scores[i] = [1,0]\n",
    "                cluster_labels[i] = 0\n",
    "        in_region_present = 0 \n",
    "        for i in range(len(dataset.data_y)):\n",
    "            # check metadata condition\n",
    "            in_region = all([dataset.metadata[i][region_dims[j]] == region_conds[j] for j in range(1)])\n",
    "            if in_region:\n",
    "                in_region_present += 1\n",
    "                cluster_scores[i] = [0,1]\n",
    "                cluster_labels[i] = 1\n",
    "            else:\n",
    "                cluster_scores[i] = [1,0]\n",
    "                cluster_labels[i] = 0\n",
    "\n",
    "        if in_region_count >= 50:\n",
    "            #print(in_region_present)\n",
    "            #print(in_region_count)\n",
    "            embedding_inside = dataset.data_x[index_inside]\n",
    "            cosine_distances = pairwise_distances(embedding_inside.reshape(1, -1), dataset.data_x, metric='cosine')\n",
    "            cosine_distances = cosine_distances.reshape(-1)\n",
    "            # update cluter scores\n",
    "            #cluster_scores[:,0] = cosine_distances\n",
    "            #cluster_scores[:,1] = 1 - cosine_distances\n",
    "\n",
    "            break\n",
    "    cluster_labels = cluster_labels.astype(int)\n",
    "    unique_dim.append(label_to_category_rcnn[region_dims[0]])\n",
    "    print(len(unique_dim))\n",
    "    ground_truth1 = [label_to_category_rcnn[region_dims[0]]]# +\" \" + region_conds[0]] #+ \" and \" +  label_to_category_rcnn[region_dims[1]] + \" \" +  region_conds[1]]\n",
    "    print(ground_truth1)\n",
    "\n",
    "\n",
    "    domino_desriber = DOMINODescribe(captions, dataset.data_x, cluster_labels)\n",
    "    domino_des = domino_desriber.describe_region(1)\n",
    "    \n",
    "    seal_describer = SEALDescribe(captions, cluster_labels, 'object detection', key)\n",
    "    seal_des = seal_describer.describe_region(1)\n",
    "    \n",
    "    domino_metrics =  evaluate_captions(ground_truth1, [domino_des])\n",
    "    seal_metrics = evaluate_captions(ground_truth1, [seal_des[0]])\n",
    "    metrics_seal.append(seal_metrics)\n",
    "    metrics_domino.append(domino_metrics)\n",
    "    \n",
    "    # ORIGINAL VERSION 5\n",
    "    itt_desriber = IterativeRegionDescribe(captions, dataset.data_x, cluster_scores, cluster_labels, key, get_text_embeddings,\n",
    "                                           5,  initial_positive_set_size = 15, initial_negative_set_size = 5)\n",
    "    itt_desriber.pre_instruction = pre_instruction\n",
    "    itt_des = itt_desriber.describe_region(1)\n",
    "    itt_metrics = evaluate_captions(ground_truth1, [itt_des[0][-1]])\n",
    "    metrics_itt5.append(itt_metrics)\n",
    "    \n",
    "    # ORIGINAL VERSION 10 \n",
    "    itt_desriber = IterativeRegionDescribe(captions, dataset.data_x, cluster_scores, cluster_labels, key, get_text_embeddings,\n",
    "                                           10,  initial_positive_set_size = 15, initial_negative_set_size = 5)\n",
    "    itt_desriber.pre_instruction = pre_instruction\n",
    "    itt_des = itt_desriber.describe_region(1)\n",
    "    itt_metrics = evaluate_captions(ground_truth1, [itt_des[0][-1]])\n",
    "    metrics_itt10.append(itt_metrics)\n",
    "\n",
    "    # ONE SHOT NO CONTRASTING\n",
    "    itt_desriber = IterativeRegionDescribe(captions, dataset.data_x, cluster_scores, cluster_labels, key, get_text_embeddings,\n",
    "                                           0,  initial_positive_set_size = 20, initial_negative_set_size = 0)\n",
    "    itt_desriber.pre_instruction = pre_instruction_no_contrast\n",
    "    itt_des = itt_desriber.describe_region(1)\n",
    "    itt_metrics = evaluate_captions(ground_truth1, [itt_des[0][-1]])\n",
    "    metrics_itt_nocontrast.append(itt_metrics)\n",
    "    \n",
    "    # ONE SHOT  CONTRASTING\n",
    "    itt_desriber = IterativeRegionDescribe(captions, dataset.data_x, cluster_scores, cluster_labels, key, get_text_embeddings,\n",
    "                                           0,  initial_positive_set_size = 20, initial_negative_set_size = 10)\n",
    "    itt_desriber.pre_instruction = pre_instruction\n",
    "    itt_des = itt_desriber.describe_region(1)\n",
    "    itt_metrics = evaluate_captions(ground_truth1, [itt_des[0][-1]])\n",
    "    metrics_itt_1round.append(itt_metrics)\n",
    "    \n",
    "    # ORIGINAL VERSION CHAT\n",
    "    itt_desriber = IterativeRegionDescribe(captions, dataset.data_x, cluster_scores, cluster_labels, key, get_text_embeddings,\n",
    "                                           5,  initial_positive_set_size = 15, initial_negative_set_size = 5, chat_correct = True)\n",
    "    itt_desriber.pre_instruction = pre_instruction\n",
    "    itt_des = itt_desriber.describe_region(1)\n",
    "    itt_metrics = evaluate_captions(ground_truth1, [itt_des[0][-1]])\n",
    "    metrics_itt_chat.append(itt_metrics)\n",
    "    metrics_to_print = ['METEOR', 'sent-sim','ROUGE','SPICE']\n",
    "\n",
    "    for metric_print in  metrics_to_print:\n",
    "        print(f' domino {metric_print} {np.mean([x[metric_print] for x in metrics_domino])} {np.std([x[metric_print] for x in metrics_domino])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' seal {metric_print} {np.mean([x[metric_print] for x in metrics_seal])} {np.std([x[metric_print] for x in metrics_seal] )/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt5 {metric_print} {np.mean([x[metric_print] for x in metrics_itt5])} {np.std([x[metric_print] for x in metrics_itt5])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt10 {metric_print} {np.mean([x[metric_print] for x in metrics_itt10])} {np.std([x[metric_print] for x in metrics_itt10])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt no contrast {metric_print} {np.mean([x[metric_print] for x in metrics_itt_nocontrast])} {np.std([x[metric_print] for x in metrics_itt_nocontrast])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt 1 round {metric_print} {np.mean([x[metric_print] for x in metrics_itt_1round])} {np.std([x[metric_print] for x in metrics_itt_1round])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt chat {metric_print} {np.mean([x[metric_print] for x in metrics_itt_chat])} {np.std([x[metric_print] for x in metrics_itt_chat])/np.sqrt(len(metrics_seal))}')\n",
    "        print('\\n')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2c881bc-4262-4a84-9656-1fed6cf88257",
   "metadata": {},
   "outputs": [],
   "source": [
    "    for metric_print in  metrics_to_print:\n",
    "        print(f' domino {metric_print} {np.mean([x[metric_print] for x in metrics_domino])} {np.std([x[metric_print] for x in metrics_domino])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' seal {metric_print} {np.mean([x[metric_print] for x in metrics_seal])} {np.std([x[metric_print] for x in metrics_seal] )/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt5 {metric_print} {np.mean([x[metric_print] for x in metrics_itt5])} {np.std([x[metric_print] for x in metrics_itt5])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt10 {metric_print} {np.mean([x[metric_print] for x in metrics_itt10])} {np.std([x[metric_print] for x in metrics_itt10])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt no contrast {metric_print} {np.mean([x[metric_print] for x in metrics_itt_nocontrast])} {np.std([x[metric_print] for x in metrics_itt_nocontrast])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt 1 round {metric_print} {np.mean([x[metric_print] for x in metrics_itt_1round])} {np.std([x[metric_print] for x in metrics_itt_1round])/np.sqrt(len(metrics_seal))}')\n",
    "        print(f' itt chat {metric_print} {np.mean([x[metric_print] for x in metrics_itt_chat])} {np.std([x[metric_print] for x in metrics_itt_chat])/np.sqrt(len(metrics_seal))}')\n",
    "        print('\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:teach_vision] *",
   "language": "python",
   "name": "conda-env-teach_vision-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}