코드제출용.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "82d6eea6",
   "metadata": {},
   "source": [
    "## 작업 환경\n",
    "- 로컬 (NVIDIA GeForce RTX 3080)\n",
    "- Google Colab Pro (gpu)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "053e3597",
   "metadata": {},
   "source": [
    "## 학습 방법\n",
    "- 사용 모델 : Efficientnet-b7\n",
    "- 5-fold로 나누어 학습하였고, 에포크 하나하나 모델을 저장하였습니다.\n",
    "- 폴드 별로 생성된 모델의 loss와 accuracy를 비교하면서 폴드당 모델을 11개씩 선정하여 직접 불러와 앙상블을 진행하였습니다.\n",
    "- Augmentation은 RandomFlip 등을 써서 테스트 해봤지만, 성능이 향상되지 않아 RandomRotation만 적용했습니다.\n",
    "- 29일 이후 동일한 모델의 에포크를 늘려 학습을 진행했고, 앙상블하는 모델의 개수를 늘려 스코어 향상시켰습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28581979",
   "metadata": {},
   "source": [
    "# GPU 사용여부 확인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c741f197",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CUDA 사용 가능 여부 : True\n",
      "현재 사용 device : cuda:0\n",
      "CUDA Index : 0\n",
      "GPU 이름 : NVIDIA GeForce RTX 3080\n",
      "GPU 개수 : 1\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "device = torch.device('cuda:0' if USE_CUDA else 'cpu')\n",
    "\n",
    "print('CUDA 사용 가능 여부 :', USE_CUDA)\n",
    "print('현재 사용 device :', device)\n",
    "print('CUDA Index :', torch.cuda.current_device())\n",
    "print('GPU 이름 :', torch.cuda.get_device_name())\n",
    "print('GPU 개수 :', torch.cuda.device_count())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f340d19",
   "metadata": {},
   "source": [
    "# Library Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0d31b631",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "from tqdm import tqdm\n",
    "import imutils\n",
    "import zipfile\n",
    "import os\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import gc\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as T\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from efficientnet_pytorch import EfficientNet\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5cc122b",
   "metadata": {},
   "source": [
    "# Dataset 구성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0596b28f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dirty_mnist_answer = pd.read_csv(\"dirty_mnist_2nd_answer.csv\")\n",
    "# dirty_mnist라는 디렉터리 속에 들어있는 파일들의 이름을\n",
    "# namelist라는 변수에 저장\n",
    "namelist = os.listdir('./dirty_mnist/')\n",
    "\n",
    "# numpy를 tensor로 변환하는 ToTensor 정의\n",
    "class ToTensor(object):\n",
    "    \"\"\"numpy array를 tensor(torch)로 변환합니다.\"\"\"\n",
    "    def __call__(self, sample):\n",
    "        image, label = sample['image'], sample['label']\n",
    "        # swap color axis because\n",
    "        # numpy image: H x W x C\n",
    "        # torch image: C X H X W\n",
    "        image = image.transpose((2, 0, 1))\n",
    "        return {'image': torch.FloatTensor(image),\n",
    "                'label': torch.FloatTensor(label)}\n",
    "# to_tensor 선언\n",
    "\n",
    "to_tensor = T.Compose([\n",
    "                      ToTensor()\n",
    "])\n",
    "\n",
    "# Augmentation는 RandomRotation를 사용\n",
    "augmentations = T.Compose([\n",
    "                           T.ToPILImage(),\n",
    "                           T.RandomRotation(40),\n",
    "                           T.ToTensor()\n",
    "                           \n",
    "])\n",
    "\n",
    "class DatasetMNIST(torch.utils.data.Dataset):\n",
    "    def __init__(self,\n",
    "                 dir_path,\n",
    "                 meta_df,\n",
    "                 transforms=to_tensor,#미리 선언한 to_tensor를 transforms로 받음\n",
    "                 augmentations=None):\n",
    "        \n",
    "        self.dir_path = dir_path # 데이터의 이미지가 저장된 디렉터리 경로\n",
    "        self.meta_df = meta_df # 데이터의 인덱스와 정답지가 들어있는 DataFrame\n",
    "\n",
    "        self.transforms = transforms# Transform\n",
    "        self.augmentations = augmentations # Augmentation\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.meta_df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        # 폴더 경로 + 이미지 이름 + .png => 파일의 경로\n",
    "        # 참고) \"12\".zfill(5) => 000012\n",
    "        #       \"146\".zfill(5) => 000145\n",
    "        # cv2.IMREAD_GRAYSCALE : png파일을 채널이 1개인 GRAYSCALE로 읽음\n",
    "        image = cv2.imread(self.dir_path +\\\n",
    "                           str(self.meta_df.iloc[index,0]).zfill(5) + '.png',\n",
    "                           cv2.IMREAD_GRAYSCALE)\n",
    "        # 0 ~ 255의 값을 갖고 크기가 (256,256)인 numpy array를\n",
    "        # 0 ~ 1 사이의 실수를 갖고 크기가 (256,256,1)인 numpy array로 변환\n",
    "        image = (image/255).astype('float32')[..., np.newaxis]\n",
    "\n",
    "        # 정답 numpy array생성(존재하면 1 없으면 0)\n",
    "        label = self.meta_df.iloc[index, 1:].values.astype('float')\n",
    "        sample = {'image': image, 'label': label}\n",
    "\n",
    "        # transform 적용\n",
    "        # numpy to tensor\n",
    "        if self.transforms:\n",
    "            sample = self.transforms(sample)\n",
    "        if self.augmentations:\n",
    "            sample['image'] = self.augmentations(sample['image'])\n",
    "        # sample 반환\n",
    "        return sample"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "badb9577",
   "metadata": {},
   "source": [
    "# 학습 모델 구성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1949e0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# nn.Module을 상속 받아 MultiLabelEfficientnet를 정의\n",
    "class MultiLabelEfficientnet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MultiLabelEfficientnet, self).__init__()\n",
    "        self.conv2d = nn.Conv2d(1, 3, 3, stride=1)\n",
    "        self.efficientnet = EfficientNet.from_pretrained('efficientnet-b7')\n",
    "        self.drop = nn.Dropout(p=0.2) # 일반화 성능 향상을 위해 dropout 레이어 추가\n",
    "        self.FC = nn.Linear(1000, 26)\n",
    "\n",
    "    def forward(self, x):\n",
    "       \n",
    "        x = F.silu(self.conv2d(x))\n",
    "\n",
    "        x = F.silu(self.efficientnet(x))\n",
    "\n",
    "        # 마지막 출력에 nn.Linear를 추가\n",
    "        # multilabel을 예측해야 하기 때문에\n",
    "        # softmax가 아닌 sigmoid를 적용\n",
    "        x = self.drop(x)\n",
    "        x = torch.sigmoid(self.FC(x))\n",
    "        return x\n",
    "# 모델 선언\n",
    "model = MultiLabelEfficientnet()\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb472260",
   "metadata": {},
   "source": [
    "# 학습"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "71424584",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[fold: 1]\n",
      "Loaded pretrained weights for efficientnet-b7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Epoch 0:   5%|█▎                         | 247/5000 [01:51<35:44,  2.22batch/s, train_acc=0.52, train_loss=0.698]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_17368\\14636745.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     81\u001b[0m                     \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m                     \u001b[1;31m# weight 갱신\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 83\u001b[1;33m                     \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m                     \u001b[1;31m# train accuracy 계산\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\venv37\\lib\\site-packages\\torch\\optim\\lr_scheduler.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     65\u001b[0m                 \u001b[0minstance\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_step_count\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m                 \u001b[0mwrapped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__get__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 67\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     68\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m             \u001b[1;31m# Note that the returned function here is no longer a bound method,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\venv37\\lib\\site-packages\\torch\\autograd\\grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     24\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 26\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     27\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mF\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\venv37\\lib\\site-packages\\torch\\optim\\adam.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, closure)\u001b[0m\n\u001b[0;32m    117\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'lr'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    118\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'weight_decay'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 119\u001b[1;33m                    \u001b[0mgroup\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'eps'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    120\u001b[0m                    )\n\u001b[0;32m    121\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\venv37\\lib\\site-packages\\torch\\optim\\functional.py\u001b[0m in \u001b[0;36madam\u001b[1;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps)\u001b[0m\n\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m         \u001b[1;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 86\u001b[1;33m         \u001b[0mexp_avg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     87\u001b[0m         \u001b[0mexp_avg_sq\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     88\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "# dirty_mnist_answer에서 train_idx와 val_idx를 생성\n",
    "for fold_index, (trn_idx, val_idx) in enumerate(kfold.split(dirty_mnist_answer),1):\n",
    "    print(f'[fold: {fold_index}]')\n",
    "    # cuda cache 초기화\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "    #train fold, validation fold 분할\n",
    "    train_answer = dirty_mnist_answer.iloc[trn_idx]\n",
    "    test_answer  = dirty_mnist_answer.iloc[val_idx]\n",
    "\n",
    "    #Dataset 정의\n",
    "    train_dataset = DatasetMNIST(\"dirty_mnist/\", train_answer, augmentations=augmentations)\n",
    "    valid_dataset = DatasetMNIST(\"dirty_mnist/\", test_answer)\n",
    "\n",
    "    #DataLoader 정의\n",
    "    train_data_loader = DataLoader(\n",
    "        train_dataset,\n",
    "        batch_size = 8,\n",
    "        shuffle = True,\n",
    "        num_workers = 0\n",
    "    )\n",
    "    valid_data_loader = DataLoader(\n",
    "        valid_dataset,\n",
    "        batch_size = 8,\n",
    "        shuffle = False,\n",
    "        num_workers = 0\n",
    "    )\n",
    "\n",
    "    # 모델 선언\n",
    "    model = MultiLabelEfficientnet()\n",
    "    \n",
    "    # cuda cache 초기화\n",
    "    torch.cuda.empty_cache()\n",
    "    gc.collect()\n",
    "    \n",
    "    model.to(device)# gpu에 모델 할당\n",
    "\n",
    "    # 훈련 옵션 설정\n",
    "    optimizer = torch.optim.Adam(model.parameters(),\n",
    "                                lr = 0.001)\n",
    "    lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,\n",
    "                                                step_size = 5,\n",
    "                                                gamma = 0.9)\n",
    "    criterion = torch.nn.BCELoss()\n",
    "    # 훈련 시작\n",
    "    valid_acc_max = 0\n",
    "    valid_loss_min = float(\"inf\")\n",
    "    for epoch in range(150):\n",
    "        # 1개 epoch 훈련\n",
    "        train_acc_list = []\n",
    "        train_loss_list = []\n",
    "        with tqdm(train_data_loader,#train_data_loader를 iterative하게 반환\n",
    "                total=train_data_loader.__len__(), # train_data_loader의 크기\n",
    "                unit=\"batch\") as train_bar:# 한번 반환하는 sample의 단위는 \"batch\"\n",
    "            for sample in train_bar:\n",
    "                train_bar.set_description(f\"Train Epoch {epoch}\")\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "                images, labels = sample['image'], sample['label']\n",
    "                # tensor를 gpu에 올리기 \n",
    "                images = images.to(device)\n",
    "                labels = labels.to(device)\n",
    "\n",
    "                # 모델의 dropout, batchnormalization를 train 모드로 설정\n",
    "                model.train()\n",
    "                # .forward()에서 중간 노드의 gradient를 계산\n",
    "                with torch.set_grad_enabled(True):\n",
    "                    # cuda cache 초기화\n",
    "                    torch.cuda.empty_cache()\n",
    "                    gc.collect()\n",
    "                    # 모델 예측\n",
    "                    probs  = model(images)\n",
    "                    # loss 계산\n",
    "                    loss = criterion(probs, labels)\n",
    "                    # 중간 노드의 gradient로\n",
    "                    # backpropagation을 적용하여\n",
    "                    # gradient 계산\n",
    "                    loss.backward()\n",
    "                    # weight 갱신\n",
    "                    optimizer.step()\n",
    "\n",
    "                    # train accuracy 계산\n",
    "                    probs  = probs.cpu().detach().numpy()\n",
    "                    labels = labels.cpu().detach().numpy()\n",
    "                    preds = probs > 0.5\n",
    "                    batch_acc = (labels == preds).mean()    \n",
    "                    train_acc_list.append(batch_acc)\n",
    "                    train_acc = np.mean(train_acc_list)\n",
    "                    train_loss_list.append(loss.item())\n",
    "                    train_loss = np.mean(train_loss_list)\n",
    "                # 현재 progress bar에 현재 미니배치의 loss 결과 출력\n",
    "                train_bar.set_postfix(train_loss= train_loss,\n",
    "                                      train_acc = train_acc)\n",
    "\n",
    "\n",
    "        # 1개 epoch학습 후 Validation 점수 계산\n",
    "        valid_acc_list = []\n",
    "        valid_loss_list = []\n",
    "        with tqdm(valid_data_loader,\n",
    "                total=valid_data_loader.__len__(),\n",
    "                unit=\"batch\") as valid_bar:\n",
    "            for sample in valid_bar:\n",
    "                valid_bar.set_description(f\"Valid Epoch {epoch}\")\n",
    "                optimizer.zero_grad()\n",
    "                images, labels = sample['image'], sample['label']\n",
    "                images = images.to(device)\n",
    "                labels = labels.to(device)\n",
    "\n",
    "                # 모델의 dropoupt, batchnormalization를 eval모드로 설정\n",
    "                model.eval()\n",
    "                # .forward()에서 중간 노드의 gradient를 계산\n",
    "                with torch.no_grad():\n",
    "                    # validation loss만을 계산\n",
    "                    probs  = model(images)\n",
    "                    valid_loss = criterion(probs, labels)\n",
    "\n",
    "                    # train accuracy 계산\n",
    "                    probs  = probs.cpu().detach().numpy()\n",
    "                    labels = labels.cpu().detach().numpy()\n",
    "                    preds = probs > 0.5\n",
    "                    batch_acc = (labels == preds).mean()\n",
    "                    valid_acc_list.append(batch_acc)\n",
    "                    valid_loss_list.append(valid_loss.item())\n",
    "                valid_acc = np.mean(valid_acc_list)\n",
    "                valid_loss = np.mean(valid_loss_list)\n",
    "                valid_bar.set_postfix(valid_loss = valid_loss,\n",
    "                                      valid_acc = valid_acc)\n",
    "\n",
    "        # Learning rate 조절\n",
    "        lr_scheduler.step()\n",
    "\n",
    "        # 모델 저장\n",
    "        if valid_loss_min > valid_loss:\n",
    "            valid_loss_min = valid_loss\n",
    "            best_model = model \n",
    "\n",
    "        # 에포크 별로 모델 저장\n",
    "        MODEL = \"efficientnetb7\"\n",
    "        # 모델을 저장할 경로\n",
    "        path = \"./models/\"\n",
    "        torch.save(best_model, f'{path}{MODEL}_{fold_index}_{valid_loss:2.4f}.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe451dea",
   "metadata": {},
   "source": [
    "# 앙상블 적용"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78bb4140",
   "metadata": {},
   "source": [
    "## 사용된 모델\n",
    "- 아래의 코드에서 보이듯이 폴드 당 11개씩의 모델을 사용하여 앙상블을 진행했습니다.\n",
    "- gpu 메모리 문제로 25개와 30개로 나누어 추론을 하고 나온 두 가지의 결과를 바탕으로 최종 제출 파일 생성하였습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "33d94678",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cuda cache 초기화\n",
    "torch.cuda.empty_cache()\n",
    "gc.collect()\n",
    "\n",
    "\n",
    "best_models=[] # 폴드별로 가장 validation loss가 낮은 모델 5개씩 저장\n",
    "\n",
    "best_models.append(torch.load('./models/efficientnetb7_1_0.1604.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_2_0.1511.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_3_0.1512.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_4_0.1411.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_5_0.1510.pth'))\n",
    "\n",
    "best_models.append(torch.load('./models/efficientnetb7_1_0.1610.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_2_0.1520.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_3_0.1550.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_4_0.1421.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_5_0.1547.pth'))\n",
    "\n",
    "best_models.append(torch.load('./models/efficientnetb7_1_0.1624.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_2_0.1532.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_3_0.1553.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_4_0.1430.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_5_0.1615.pth'))\n",
    "\n",
    "best_models.append(torch.load('./models/efficientnetb7_1_0.1647.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_2_0.1541.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_3_0.1560.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_4_0.1434.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_5_0.1620.pth'))\n",
    "\n",
    "best_models.append(torch.load('./models/efficientnetb7_1_0.1661.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_2_0.1554.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_3_0.1573.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_4_0.1435.pth'))\n",
    "best_models.append(torch.load('./models/efficientnetb7_5_0.1642.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fb571cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_models2=[] # 폴드별로 가장 validation loss가 낮은 모델 6개씩 저장\n",
    "\n",
    "best_models2.append(torch.load('./models/efficientnetb7_1_0.1662.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_2_0.1558.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_3_0.1602.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_4_0.1447.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_5_0.1643.pth'))\n",
    "\n",
    "best_models2.append(torch.load('./models/efficientnetb7_1_0.1664.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_2_0.1570.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_3_0.1608.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_4_0.1450.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_5_0.1683.pth'))\n",
    "\n",
    "best_models2.append(torch.load('./models/efficientnetb7_1_0.1671.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_2_0.1585.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_3_0.1609.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_4_0.1459.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_5_0.1706.pth'))\n",
    "\n",
    "best_models2.append(torch.load('./models/efficientnetb7_1_0.1675.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_2_0.1662.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_3_0.1655.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_4_0.1477.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_5_0.1707.pth'))\n",
    "\n",
    "best_models2.append(torch.load('./models/efficientnetb7_1_0.1681.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_2_0.1706.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_3_0.1661.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_4_0.1487.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_5_0.1715.pth'))\n",
    "\n",
    "best_models2.append(torch.load('./models/efficientnetb7_1_0.1720.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_2_0.1717.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_3_0.1667.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_4_0.1490.pth'))\n",
    "best_models2.append(torch.load('./models/efficientnetb7_5_0.1717.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "931cb348",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test Dataset 정의\n",
    "sample_submission = pd.read_csv(\"sample_submission.csv\")\n",
    "test_dataset = DatasetMNIST(\"test_dirty_mnist/\", sample_submission)\n",
    "batch_size = 32\n",
    "test_data_loader = DataLoader(\n",
    "    test_dataset,\n",
    "    batch_size = batch_size,\n",
    "    shuffle = False,\n",
    "    num_workers = 0,\n",
    "    drop_last = False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb6aa435",
   "metadata": {},
   "source": [
    "# 추론"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f7c8a7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cuda cache 초기화\n",
    "torch.cuda.empty_cache()\n",
    "gc.collect()\n",
    "\n",
    "predictions_list = []\n",
    "# 배치 단위로 추론\n",
    "prediction_df = pd.read_csv(\"sample_submission.csv\")\n",
    "\n",
    "for model in best_models:\n",
    "    # 0으로 채워진 array 생성\n",
    "    prediction_array = np.zeros([prediction_df.shape[0],\n",
    "                                 prediction_df.shape[1] -1])\n",
    "    for idx, sample in enumerate(test_data_loader):\n",
    "        with torch.no_grad():\n",
    "            # 추론\n",
    "            model.eval()\n",
    "            images = sample['image']\n",
    "            images = images.to(device)\n",
    "            probs  = model(images)\n",
    "            probs = probs.cpu().detach().numpy()\n",
    "            preds = (probs > 0.5)\n",
    "\n",
    "            # 예측 결과를 \n",
    "            # prediction_array에 입력\n",
    "            batch_index = batch_size * idx\n",
    "            prediction_array[batch_index: batch_index + images.shape[0],:]\\\n",
    "                         = preds.astype(int)\n",
    "                         \n",
    "    # 채널을 하나 추가하여 list에 append\n",
    "    predictions_list.append(prediction_array[...,np.newaxis])\n",
    "    \n",
    "predictions_array = np.concatenate(predictions_list, axis = 2)\n",
    "np.save('predictions_array',predictions_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4ab12b5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_list2 = []\n",
    "# 배치 단위로 추론\n",
    "prediction_df2 = pd.read_csv(\"sample_submission.csv\")\n",
    "\n",
    "for model in best_models2:\n",
    "    # 0으로 채워진 array 생성\n",
    "    prediction_array2 = np.zeros([prediction_df2.shape[0],\n",
    "                                 prediction_df2.shape[1] -1])\n",
    "    for idx, sample in enumerate(test_data_loader):\n",
    "        with torch.no_grad():\n",
    "            # 추론\n",
    "            model.eval()\n",
    "            images = sample['image']\n",
    "            images = images.to(device)\n",
    "            probs  = model(images)\n",
    "            probs = probs.cpu().detach().numpy()\n",
    "            preds = (probs > 0.5)\n",
    "\n",
    "            # 예측 결과를 \n",
    "            # prediction_array에 입력\n",
    "            batch_index = batch_size * idx\n",
    "            prediction_array2[batch_index: batch_index + images.shape[0],:]\\\n",
    "                         = preds.astype(int)\n",
    "                         \n",
    "    # 채널을 하나 추가하여 list에 append\n",
    "    predictions_list2.append(prediction_array2[...,np.newaxis])\n",
    "\n",
    "predictions_array2 = np.concatenate(predictions_list2, axis = 2)\n",
    "np.save('predictions_array2',predictions_array2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "07273c3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# array 불러오기\n",
    "predictions_array = np.load('predictions_array.npy')\n",
    "predictions_array2 = np.load('predictions_array2.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "63a331f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 1, ..., 1, 0, 1],\n",
       "       [0, 1, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 1, ..., 0, 0, 1],\n",
       "       ...,\n",
       "       [1, 0, 0, ..., 0, 0, 1],\n",
       "       [0, 0, 1, ..., 0, 0, 1],\n",
       "       [1, 1, 0, ..., 1, 1, 1]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_array_final = np.concatenate([predictions_array, predictions_array2], axis = 2)\n",
    "predictions_mean = predictions_array_final.mean(axis = 2)\n",
    "\n",
    "# 평균 값이 0.5보다 클 경우 1 작으면 0\n",
    "predictions_mean = (predictions_mean > 0.5) * 1\n",
    "predictions_mean"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea020500",
   "metadata": {},
   "source": [
    "# 제출파일 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2ba0785f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>f</th>\n",
       "      <th>g</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>...</th>\n",
       "      <th>q</th>\n",
       "      <th>r</th>\n",
       "      <th>s</th>\n",
       "      <th>t</th>\n",
       "      <th>u</th>\n",
       "      <th>v</th>\n",
       "      <th>w</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50001</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>50002</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50003</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50004</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995</th>\n",
       "      <td>54995</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4996</th>\n",
       "      <td>54996</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>54997</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4998</th>\n",
       "      <td>54998</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999</th>\n",
       "      <td>54999</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5000 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index  a  b  c  d  e  f  g  h  i  ...  q  r  s  t  u  v  w  x  y  z\n",
       "0     50000  1  0  1  0  1  1  0  1  1  ...  0  0  0  1  1  1  0  1  0  1\n",
       "1     50001  0  1  0  0  1  0  1  0  1  ...  0  1  0  0  1  1  0  0  0  0\n",
       "2     50002  0  0  1  1  1  0  1  0  1  ...  0  1  1  0  1  0  1  0  0  1\n",
       "3     50003  1  1  0  0  0  1  1  1  0  ...  0  0  1  1  0  0  0  1  0  1\n",
       "4     50004  0  0  1  0  1  1  0  0  0  ...  1  0  1  1  1  0  1  0  0  0\n",
       "...     ... .. .. .. .. .. .. .. .. ..  ... .. .. .. .. .. .. .. .. .. ..\n",
       "4995  54995  0  1  1  1  0  0  0  1  0  ...  0  1  0  1  0  1  1  0  1  0\n",
       "4996  54996  1  1  1  0  0  0  0  0  1  ...  0  0  1  0  0  1  0  1  0  1\n",
       "4997  54997  1  0  0  1  0  1  0  0  0  ...  0  0  0  1  1  1  1  0  0  1\n",
       "4998  54998  0  0  1  0  0  0  1  0  1  ...  1  1  0  1  0  0  1  0  0  1\n",
       "4999  54999  1  1  0  0  0  1  0  1  1  ...  0  1  1  0  0  0  1  1  1  1\n",
       "\n",
       "[5000 rows x 27 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission = pd.read_csv(\"sample_submission.csv\")\n",
    "sample_submission.iloc[:,1:] = predictions_mean\n",
    "sample_submission.to_csv(\"Lee_Woon_jae_save2.csv\", index = False)\n",
    "sample_submission"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}