OpenCV_ORB_homography.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "OpenCV_ORB_homography.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "2ntFmXu88ySs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import cv2 as cv\n",
        "import cv2\n",
        "from matplotlib import pyplot as plt\n",
        "from google.colab.patches import cv2_imshow\n",
        "from google.colab import files\n",
        "from IPython.display import Image\n",
        "\n",
        "# command to remove files under the uploaded folder for uploading the same files would result in confusion sometimes\n",
        "#!rm *.jpg\n",
        "#!rm *.png\n",
        "#!ls\n",
        "\n",
        "uploaded = files.upload()\n",
        "# queryImage\n",
        "img1_rgb = cv.imread('box.jpg') # 0: loads image in grayscale mode         \n",
        "uploaded = files.upload()\n",
        "# trainImage\n",
        "img2_rgb = cv.imread('box_btm_right.jpg') # 0: loads image in grayscale mode\n",
        "\n",
        "# for panorama example\n",
        "img1_rgb_black = np.zeros_like(img1_rgb)\n",
        "img2_rgb = np.hstack((img1_rgb_black,img2_rgb))\n",
        "\n",
        "\n",
        "img1 = cv2.cvtColor(img1_rgb, cv2.COLOR_BGR2GRAY)\n",
        "img2 = cv2.cvtColor(img2_rgb, cv2.COLOR_BGR2GRAY)\n",
        "\n",
        "cv2_imshow(img1_rgb)\n",
        "#cv2_imshow(img1_rgb_black)\n",
        "cv2_imshow(img2_rgb)\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "F5leTeEE9dkQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "###\n",
        "def drawMatches_self(img1, kp1, img2, kp2, matches):\n",
        "\n",
        "    # Create a new output image that concatenates the two images together    \n",
        "    rows1, cols1 = img1.shape    \n",
        "    rows2, cols2 = img2.shape\n",
        "    \n",
        "    out = np.zeros((max([rows1,rows2]),cols1+cols2,3), dtype='uint8')\n",
        "\n",
        "    # Place the first image in the form of RGB to the left\n",
        "    out[:rows1,:cols1] = np.dstack([img1, img1, img1])\n",
        "\n",
        "    # Place the next image in the form of RGB to the right of the 1st image\n",
        "    out[:rows2,cols1:] = np.dstack([img2, img2, img2])\n",
        "    \n",
        "    \n",
        "    # For each correspondence point, draw circles and connect a line between them\n",
        "    for mat in matches:\n",
        "\n",
        "        # Get the matching keypoints for each of the images\n",
        "        img1_idx = mat.queryIdx\n",
        "        img2_idx = mat.trainIdx\n",
        "                \n",
        "        # x - columns\n",
        "        # y - rows\n",
        "        (x1,y1) = kp1[img1_idx].pt\n",
        "        (x2,y2) = kp2[img2_idx].pt        \n",
        "\n",
        "        # random color for each pair of correspondence\n",
        "        a = np.random.randint(0,256)\n",
        "        b = np.random.randint(0,256)\n",
        "        c = np.random.randint(0,256)\n",
        "\n",
        "        # draw circles\n",
        "        cv2.circle(out, (int(np.round(x1)),int(np.round(y1))), 2, (a, b, c), 5)      # random color with radius = 1\n",
        "        cv2.circle(out, (int(np.round(x2)+cols1),int(np.round(y2))), 2, (a, b, c), 5)\n",
        "\n",
        "        # Draw a line in-between each pair of correspondence                \n",
        "        cv2.line(out, (int(np.round(x1)),int(np.round(y1))), (int(np.round(x2)+cols1),int(np.round(y2))), (a, b, c), 1) # random color with thickness = 1\n",
        "\n",
        "    # Also return the image if you'd like a copy\n",
        "    return out\n",
        "\n",
        "###\n",
        "\n",
        "\n",
        "\n",
        "# Initiate SIFT detector\n",
        "#sift = cv.xfeatures2d.SIFT_create()\n",
        "\n",
        "# find the keypoints and descriptors with SIFT\n",
        "#kp1, des1 = sift.detectAndCompute(img1,None)\n",
        "#kp2, des2 = sift.detectAndCompute(img2,None)\n",
        "\n",
        "# Initiate ORB detector\n",
        "orb = cv.ORB_create()\n",
        "# find the keypoints and descriptors with ORB\n",
        "kp1, des1 = orb.detectAndCompute(img1,None)\n",
        "kp2, des2 = orb.detectAndCompute(img2,None)\n",
        "\n",
        "# create BFMatcher object\n",
        "bf = cv.BFMatcher(cv.NORM_HAMMING, crossCheck=True)\n",
        "# Match descriptors.\n",
        "matches = bf.match(des1,des2)\n",
        "\n",
        "# Sort them in the order of their distance.\n",
        "matches = sorted(matches, key = lambda x:x.distance)\n",
        "\n",
        " \n",
        "\n",
        "\n",
        "fig=plt.figure(figsize=(15, 15))\n",
        "\n",
        "# Draw matches. # official implementation\n",
        "img3 = cv.drawMatches(img1,kp1,img2,kp2,matches[:50] ,None, flags=2, matchColor = (0,0,255))\n",
        "# Draw matches. self implementation for knowing the detail\n",
        "#img3 = drawMatches_self(img1,kp1,img2,kp2,matches[:10])\n",
        "\n",
        "plt.imshow(img3)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lrXz2wFu-Jk1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import random\n",
        "import math\n",
        "\n",
        "def homo_estimation(feature_correspondence_in_query_image, feature_correspondence_in_train_image, desired_num_correspondence_homo):\n",
        "  # A in AX=0\n",
        "  MatrixA = np.zeros((desired_num_correspondence_homo*2,9))\n",
        "  \n",
        "  # The Homography transformation matrix\n",
        "  HomoMat = np.zeros((3,3))\n",
        "  \n",
        "  rows, _ = feature_correspondence_in_query_image.shape\n",
        "  \n",
        "  rand_num = random.sample(range(rows), desired_num_correspondence_homo)\n",
        "  #print(rand_num)\n",
        "  \n",
        "  \n",
        "  feature_correspondence_in_query_image_randomly_picked = np.zeros((desired_num_correspondence_homo,2))\n",
        "  feature_correspondence_in_train_image_randomly_picked = np.zeros((desired_num_correspondence_homo,2))      \n",
        "  \n",
        "  for i in range(desired_num_correspondence_homo):\n",
        "    feature_correspondence_in_query_image_randomly_picked.itemset((i,0),feature_correspondence_in_query_image.item(rand_num[i],0))\n",
        "    feature_correspondence_in_query_image_randomly_picked.itemset((i,1),feature_correspondence_in_query_image.item(rand_num[i],1))\n",
        "    \n",
        "    feature_correspondence_in_train_image_randomly_picked.itemset((i,0),feature_correspondence_in_train_image.item(rand_num[i],0))\n",
        "    feature_correspondence_in_train_image_randomly_picked.itemset((i,1),feature_correspondence_in_train_image.item(rand_num[i],1))\n",
        "    \n",
        "  \n",
        "            \n",
        "  for i in range(desired_num_correspondence_homo):\n",
        "    MatrixA.itemset((i*2,0),-feature_correspondence_in_query_image_randomly_picked.item(i,0))\n",
        "    MatrixA.itemset((i*2,1),-feature_correspondence_in_query_image_randomly_picked.item(i,1))\n",
        "    MatrixA.itemset((i*2,2),-1.0)\n",
        "    MatrixA.itemset((i*2,3),0.0)\n",
        "    MatrixA.itemset((i*2,4),0.0)\n",
        "    MatrixA.itemset((i*2,5),0.0)\n",
        "    MatrixA.itemset((i*2,6),feature_correspondence_in_train_image_randomly_picked.item(i,0)*feature_correspondence_in_query_image_randomly_picked.item(i,0))\n",
        "    MatrixA.itemset((i*2,7),feature_correspondence_in_train_image_randomly_picked.item(i,0)*feature_correspondence_in_query_image_randomly_picked.item(i,1))\n",
        "    MatrixA.itemset((i*2,8),feature_correspondence_in_train_image_randomly_picked.item(i,0))\n",
        "\n",
        "    MatrixA.itemset((i*2+1,0),0.0)\n",
        "    MatrixA.itemset((i*2+1,1),0.0)\n",
        "    MatrixA.itemset((i*2+1,2),0.0)\n",
        "    MatrixA.itemset((i*2+1,3),-feature_correspondence_in_query_image_randomly_picked.item(i,0))\n",
        "    MatrixA.itemset((i*2+1,4),-feature_correspondence_in_query_image_randomly_picked.item(i,1))\n",
        "    MatrixA.itemset((i*2+1,5),-1.0)\n",
        "    MatrixA.itemset((i*2+1,6),feature_correspondence_in_train_image_randomly_picked.item(i,1)*feature_correspondence_in_query_image_randomly_picked.item(i,0))\n",
        "    MatrixA.itemset((i*2+1,7),feature_correspondence_in_train_image_randomly_picked.item(i,1)*feature_correspondence_in_query_image_randomly_picked.item(i,1))\n",
        "    MatrixA.itemset((i*2+1,8),feature_correspondence_in_train_image_randomly_picked.item(i,1))\n",
        "\n",
        "  #print(MatrixA)    \n",
        "  # performing SVD for estimating Homography matrix\n",
        "  u, s, vh = np.linalg.svd(MatrixA, full_matrices=True)\n",
        "\n",
        "  HomoMat.itemset((0,0),vh.item(8,0))\n",
        "  HomoMat.itemset((0,1),vh.item(8,1))\n",
        "  HomoMat.itemset((0,2),vh.item(8,2))\n",
        "\n",
        "  HomoMat.itemset((1,0),vh.item(8,3))\n",
        "  HomoMat.itemset((1,1),vh.item(8,4))\n",
        "  HomoMat.itemset((1,2),vh.item(8,5))\n",
        "\n",
        "  HomoMat.itemset((2,0),vh.item(8,6))\n",
        "  HomoMat.itemset((2,1),vh.item(8,7))\n",
        "  HomoMat.itemset((2,2),vh.item(8,8))\n",
        "  \n",
        "  #print(HomoMat)\n",
        "                    \n",
        "  return HomoMat     \n",
        "                    \n",
        "def transfer_error_estimation(HomoMat, feature_correspondence_in_query_image, feature_correspondence_in_train_image):\n",
        "    input_vector = np.array([[0],[0],[0]])\n",
        "    output_vector = np.array([[0],[0],[0]])\n",
        "    #print(type(output_vector))\n",
        "                    \n",
        "    feature_num, _ = feature_correspondence_in_query_image.shape                    \n",
        "    square_error = 0\n",
        "    \n",
        "    for i in range(feature_num):\n",
        "        input_vector.itemset((0,0), feature_correspondence_in_query_image.item(i,0))\n",
        "        input_vector.itemset((1,0), feature_correspondence_in_query_image.item(i,1))\n",
        "        input_vector.itemset((2,0), 1.0)                    \n",
        "  \n",
        "        output_vector = np.matmul(HomoMat,input_vector)            \n",
        "                    \n",
        "        output_vector.itemset((0,0), output_vector.item(0,0)/output_vector.item(2,0))\n",
        "        output_vector.itemset((1,0), output_vector.item(1,0)/output_vector.item(2,0))            \n",
        "        \n",
        "        #print(input_vector)\n",
        "        #print(output_vector)\n",
        "        \n",
        "                    \n",
        "        square_error = square_error + math.sqrt( (output_vector.item(0,0) - feature_correspondence_in_train_image.item(i,0) )**2 + ( output_vector.item(1,0) - feature_correspondence_in_train_image.item(i,1) )**2 )         \n",
        "                    \n",
        "    return square_error        \n",
        "\n",
        "def RANSAC_fun(kp1,kp2,matches, num_correspondence, desired_num_correspondence_homo, iteration_num):\n",
        "    \n",
        "    \n",
        "    \n",
        "    num_matches = len(matches)\n",
        "    feature_correspondence_in_query_image = np.zeros((num_matches,2))\n",
        "    feature_correspondence_in_train_image = np.zeros((num_matches,2))\n",
        "    \n",
        "    ## for debug only\n",
        "    #feature_correspondence_in_query_image = np.zeros((4,2))\n",
        "    #feature_correspondence_in_train_image = np.zeros((4,2))\n",
        "    ##\n",
        "    running_index = 0\n",
        "    \n",
        "    \n",
        "    \n",
        "    for mat in matches:\n",
        "        img1_idx = mat.queryIdx\n",
        "        img2_idx = mat.trainIdx\n",
        "                \n",
        "        # x - columns\n",
        "        # y - rows\n",
        "        (x1,y1) = kp1[img1_idx].pt\n",
        "        (x2,y2) = kp2[img2_idx].pt\n",
        "        #print('kp1[img1_idx].pt={}'.format(kp1[img1_idx].pt))\n",
        "        #print('kp2[img2_idx].pt={}'.format(kp2[img2_idx].pt))\n",
        "        feature_correspondence_in_query_image.itemset((running_index,0),x1)\n",
        "        feature_correspondence_in_query_image.itemset((running_index,1),y1)\n",
        "      \n",
        "        feature_correspondence_in_train_image.itemset((running_index,0),x2)\n",
        "        feature_correspondence_in_train_image.itemset((running_index,1),y2)\n",
        "      \n",
        "        running_index = running_index + 1\n",
        "    \n",
        "    \n",
        "    '''\n",
        "    ## for debug only\n",
        "    feature_correspondence_in_query_image.itemset((0,0),0)\n",
        "    feature_correspondence_in_query_image.itemset((0,1),0)\n",
        "    \n",
        "    feature_correspondence_in_query_image.itemset((1,0),324)\n",
        "    feature_correspondence_in_query_image.itemset((1,1),0)\n",
        "    \n",
        "    feature_correspondence_in_query_image.itemset((2,0),0)\n",
        "    feature_correspondence_in_query_image.itemset((2,1),223)\n",
        "    \n",
        "    feature_correspondence_in_query_image.itemset((3,0),324)\n",
        "    feature_correspondence_in_query_image.itemset((3,1),223)\n",
        "    \n",
        "    \n",
        "    feature_correspondence_in_train_image.itemset((0,0),118)\n",
        "    feature_correspondence_in_train_image.itemset((0,1),164)\n",
        "    \n",
        "    feature_correspondence_in_train_image.itemset((1,0),277)\n",
        "    feature_correspondence_in_train_image.itemset((1,1),180)\n",
        "    \n",
        "    feature_correspondence_in_train_image.itemset((2,0),92)\n",
        "    feature_correspondence_in_train_image.itemset((2,1),262)\n",
        "    \n",
        "    feature_correspondence_in_train_image.itemset((3,0),265)\n",
        "    feature_correspondence_in_train_image.itemset((3,1),294)\n",
        "    ##\n",
        "    '''\n",
        "    \n",
        "    iteration_index  = 0  \n",
        "    transfer_error_best = 100000\n",
        "    \n",
        "    while(iteration_index<iteration_num):\n",
        "    \n",
        "      HomoMat = homo_estimation(feature_correspondence_in_query_image, feature_correspondence_in_train_image, desired_num_correspondence_homo)\n",
        "    \n",
        "      transfer_error = transfer_error_estimation(HomoMat, feature_correspondence_in_query_image, feature_correspondence_in_train_image)\n",
        "      #print(transfer_error)\n",
        "      if transfer_error < transfer_error_best:\n",
        "        transfer_error_best =  transfer_error\n",
        "        best_HomoMat = np.copy(HomoMat)\n",
        "    \n",
        "      iteration_index = iteration_index + 1\n",
        "    \n",
        "    #print(transfer_error_best)\n",
        "    return best_HomoMat\n",
        "                    \n",
        "    #print(rand_num)\n",
        "#####################################    \n",
        "\n",
        "\n",
        "######################################\n",
        "  \n",
        "def CoordinateTransformation(SimTranMax,y,x):\n",
        "    input_vector = np.array([[0],[0],[0]])\n",
        "    output_vector = np.array([[0],[0],[0]])\n",
        "    #print(type(output_vector))\n",
        "    \n",
        "    input_vector.itemset((0,0), x)\n",
        "    input_vector.itemset((1,0), y)\n",
        "    input_vector.itemset((2,0), 1.0)\n",
        "    \n",
        "    output_vector = np.matmul(SimTranMax,input_vector)\n",
        "    \n",
        "    output_vector.itemset((0,0), output_vector.item(0,0)/output_vector.item(2,0))\n",
        "    output_vector.itemset((1,0), output_vector.item(1,0)/output_vector.item(2,0))\n",
        "    return output_vector\n",
        "  \n",
        "  \n",
        "  \n",
        "def ImageTransformation(TranMax,src,result_img):\n",
        "    \n",
        "    h, w ,_ = src.shape\n",
        "    h1, w1 ,_ = result_img.shape\n",
        "    # output image\n",
        "    #result_img = np.zeros_like(src)\n",
        "    \n",
        "    for i in range(h):\n",
        "        for j in range(w):\n",
        "        \n",
        "            pixel_R = src.item(i,j,0)\n",
        "            pixel_G = src.item(i,j,1)\n",
        "            pixel_B = src.item(i,j,2)           \n",
        "            \n",
        "            output_vector = CoordinateTransformation(TranMax,i,j)\n",
        "            \n",
        "            output_vector = output_vector.astype(int)\n",
        "            \n",
        "            output_vector_x = output_vector.item(0,0)\n",
        "            output_vector_y = output_vector.item(1,0)            \n",
        "        \n",
        "            if output_vector_x >= 0 and output_vector_x < w1 and output_vector_y >= 0 and output_vector_y < h1:\n",
        "                result_img.itemset((output_vector_y,output_vector_x,0),pixel_R)\n",
        "                result_img.itemset((output_vector_y,output_vector_x,1),pixel_G)\n",
        "                result_img.itemset((output_vector_y,output_vector_x,2),pixel_B)\n",
        "            \n",
        "    return result_img\n",
        "\n",
        "\n",
        "## 1. self-implemented\n",
        "\n",
        "## eliminate poor pairs\n",
        "GOOD_MATCH_PERCENT = 0.15\n",
        "numGoodMatches = int(len(matches) * GOOD_MATCH_PERCENT)\n",
        "matches_selected = matches[:numGoodMatches]\n",
        "  \n",
        "## the number of matches \n",
        "num_correspondence = len(matches_selected[:])\n",
        "\n",
        "## setting the number of feature correspondence to be fed into homography estimation\n",
        "desired_num_correspondence_homo = 8\n",
        "\n",
        "## setting the iteration number\n",
        "iteration_num = 1000\n",
        "\n",
        "best_HomoMat = RANSAC_fun(kp1,kp2,matches_selected, num_correspondence, desired_num_correspondence_homo, iteration_num)\n",
        "\n",
        "## make an RGB image from grayscale image\n",
        "# Place the first image in the form of RGB to the left\n",
        "#img1_rgb = np.dstack([img1, img1, img1])\n",
        "\n",
        "# Place the next image in the form of RGB to the right of the 1st image\n",
        "#img2_rgb = np.dstack([img2, img2, img2])\n",
        "\n",
        "## draw polygon for clarity\n",
        "#h,w = img1.shape[:2]\n",
        "#pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2) # add a new dimension by -1\n",
        "#dst = cv2.perspectiveTransform(pts,best_HomoMat)\n",
        "#img2_rgb = cv2.polylines(img2_rgb, [np.int32(dst)], True, (0,0,255), 5, cv2.LINE_AA)\n",
        "##\n",
        "\n",
        "result_img = ImageTransformation(best_HomoMat,img1_rgb,img2_rgb)  \n",
        "\n",
        "cv2_imshow(result_img)\n",
        "\n",
        "\n",
        "## concise implementation by calling OpenCV functions\n",
        "'''\n",
        "src_pts  = np.float32([kp1[m.queryIdx].pt for m in matches_selected]).reshape(-1,1,2)\n",
        "dst_pts  = np.float32([kp2[m.trainIdx].pt for m in matches_selected]).reshape(-1,1,2)\n",
        "\n",
        "## find homography matrix and do perspective transform\n",
        "M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)\n",
        "h,w = img1.shape[:2]\n",
        "pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)\n",
        "dst = cv2.perspectiveTransform(pts,M)\n",
        "\n",
        "## draw found regions\n",
        "img2 = cv2.polylines(img2, [np.int32(dst)], True, (0,0,255), 1, cv2.LINE_AA)\n",
        "cv2_imshow(img2)\n",
        "\n",
        "## draw match lines\n",
        "res = cv2.drawMatches(img1, kp1, img2, kp2, matches_selected[:20],None,flags=2)\n",
        "\n",
        "cv2_imshow(res)\n",
        "'''"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1dj4zhWiLwHN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}