exemplos/basic_use_pandas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import epmwebapi as epm\n",
    "import datetime as dt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Para não expor o usuário e senha do EPM, salvo essas informações numa variável de ambiente e busco via código"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "credentials = os.environ.get('user_and_password').split(';')\n",
    "user = credentials[0]\n",
    "password = credentials[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Leitura de dados brutos(Raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cria conexao\n",
    "epmConn = epm.EpmConnection('http://localhost:44333', 'http://localhost:44332', user, password)\n",
    "\n",
    "path = 'R80711_Wind_speed'\n",
    "bv = epmConn.getDataObjects(path)\n",
    "\n",
    "iniTime = dt.datetime(2016, 1, 1, 0, 0, 0, 0)\n",
    "endTime = iniTime + dt.timedelta(days=3)\n",
    "\n",
    "#cria objeto queryperiod\n",
    "queryPeriod = epm.QueryPeriod(iniTime, endTime)\n",
    "\n",
    "#aplica o objeto um historyReadRaw com o queryPeriod na basicVariable selecionada\n",
    "result = bv[path].historyReadRaw(queryPeriod)\n",
    "\n",
    "#fechando a conexão\n",
    "connection.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mostra o formato dos dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(432,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Criação do dataframe pandas\n",
    "corrige o problema: ValueError: Big-endian buffer not supported on little-endian compiler\n",
    "para ser possível transformar os dados vindos do EPM em um Dataframe Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_Quality = result[:]['Quality'].byteswap().newbyteorder()\n",
    "new_Timestamp = result[:]['Timestamp']\n",
    "new_Value = result[:]['Value'].byteswap().newbyteorder()\n",
    "\n",
    "d = {'Value':new_Value, 'Timestamp':new_Timestamp, 'Quality':new_Quality}\n",
    "df_original = pd.DataFrame(d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mostra o formato dos dados, para esse caso são 432 linhas por 3 colunas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(432, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Comparação entre valores em formato Numpy Array e Dataframe do Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numpy Array:\n",
      "[(6.14, datetime.datetime(2016, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 0)\n",
      " (5.75, datetime.datetime(2016, 1, 1, 0, 10, tzinfo=datetime.timezone.utc), 0)\n",
      " (5.71, datetime.datetime(2016, 1, 1, 0, 20, tzinfo=datetime.timezone.utc), 0)\n",
      " (6.07, datetime.datetime(2016, 1, 1, 0, 30, tzinfo=datetime.timezone.utc), 0)\n",
      " (5.75, datetime.datetime(2016, 1, 1, 0, 40, tzinfo=datetime.timezone.utc), 0)]\n",
      "\n",
      "\n",
      "Dataframe Pandas:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.14</td>\n",
       "      <td>2016-01-01 00:00:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.75</td>\n",
       "      <td>2016-01-01 00:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.71</td>\n",
       "      <td>2016-01-01 00:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.07</td>\n",
       "      <td>2016-01-01 00:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.75</td>\n",
       "      <td>2016-01-01 00:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Value                 Timestamp  Quality\n",
       "0   6.14 2016-01-01 00:00:00+00:00        0\n",
       "1   5.75 2016-01-01 00:10:00+00:00        0\n",
       "2   5.71 2016-01-01 00:20:00+00:00        0\n",
       "3   6.07 2016-01-01 00:30:00+00:00        0\n",
       "4   5.75 2016-01-01 00:40:00+00:00        0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Numpy Array:\")\n",
    "print(result[0:5])\n",
    "print(\"\\n\")\n",
    "print(\"Dataframe Pandas:\")\n",
    "df_original.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mostra o tipo dos dados em cada coluna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value                    float32\n",
       "Timestamp    datetime64[ns, UTC]\n",
       "Quality                    int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Gera estatísticas descritivas que resumem a tendência central, a dispersão e a forma da distribuição de um conjunto de dados, excluindo os valores NaN.\n",
    "\n",
    "#### Aplicado somente a dados numéricos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>432.000000</td>\n",
       "      <td>432.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.456759</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.637152</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.870000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.687500</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.490000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.270000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>11.180000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Value  Quality\n",
       "count  432.000000    432.0\n",
       "mean     6.456759      0.0\n",
       "std      1.637152      0.0\n",
       "min      0.870000      0.0\n",
       "25%      5.687500      0.0\n",
       "50%      6.490000      0.0\n",
       "75%      7.270000      0.0\n",
       "max     11.180000      0.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inclui todos os tipos de dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>432.000000</td>\n",
       "      <td>432</td>\n",
       "      <td>432.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>432</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-01 20:40:00+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.456759</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.637152</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.870000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.687500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.490000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.270000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>11.180000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Value                  Timestamp  Quality\n",
       "count   432.000000                        432    432.0\n",
       "unique         NaN                        432      NaN\n",
       "top            NaN  2016-01-01 20:40:00+00:00      NaN\n",
       "freq           NaN                          1      NaN\n",
       "mean      6.456759                        NaN      0.0\n",
       "std       1.637152                        NaN      0.0\n",
       "min       0.870000                        NaN      0.0\n",
       "25%       5.687500                        NaN      0.0\n",
       "50%       6.490000                        NaN      0.0\n",
       "75%       7.270000                        NaN      0.0\n",
       "max      11.180000                        NaN      0.0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original.describe(include='all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Somente os que são do tipo float32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>432.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.456759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.637152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.870000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.687500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.490000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.270000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>11.180000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Value\n",
       "count  432.000000\n",
       "mean     6.456759\n",
       "std      1.637152\n",
       "min      0.870000\n",
       "25%      5.687500\n",
       "50%      6.490000\n",
       "75%      7.270000\n",
       "max     11.180000"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original.describe(include=['float32'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "####                              REMOVENDO COLUNAS \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Boa prática ao realizar algumas alteração no dataframe é atribuir a outra variável\n",
    "#### axis= 1 - Colunas / axis = 0 - Linhas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>432.000000</td>\n",
       "      <td>432.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.456759</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.637152</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.870000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.687500</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.490000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.270000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>11.180000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Value  Quality\n",
       "count  432.000000    432.0\n",
       "mean     6.456759      0.0\n",
       "std      1.637152      0.0\n",
       "min      0.870000      0.0\n",
       "25%      5.687500      0.0\n",
       "50%      6.490000      0.0\n",
       "75%      7.270000      0.0\n",
       "max     11.180000      0.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sem_timestamp = df_original.drop(['Timestamp'],axis=1)\n",
    "\n",
    "df_sem_timestamp.describe(include='all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### REMOVENDO LINHAS "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#axis= 1 - Colunas / axis = 0 - Linhas\n",
    "df_sem_timestamp_v1 = df_sem_timestamp.drop([0,1],axis=0)\n",
    "\n",
    "df_sem_timestamp_v1.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ITERAÇÕES "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for indice, linha in df_original.iterrows():\n",
    "    print(indice, linha.Value , ' = ', linha.Timestamp)\n",
    "    if indice == 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ORDENAÇÃO \n",
    "\n",
    "#### Ordenando com dot notation retorna uma serie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = df_original.Value.sort_values(ascending=False)\n",
    "type(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Para ordernar direto o DF aplicar sort_values no DF\n",
    "b = df_original.sort_values(['Value'], ascending=False)\n",
    "type(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "  \n",
    "###                              Filtros Simples\n",
    "#### Usar a notação de colchetes para indexar ou para aplicar filtros de series booleanos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "booleanos = []\n",
    "for item in df_original.Value:\n",
    "    if item > 10:\n",
    "        booleanos.append(True)\n",
    "    else:\n",
    "        booleanos.append(False)\n",
    "        \n",
    "#mostra se os primeiros 10 valores possuem \"value\" mais que 10\n",
    "booleanos[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mostra que existem 13 dados com value maior que 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'booleanos' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-2879ed279fc6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdf_filtrado\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_original\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mbooleanos\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mdf_filtrado\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'booleanos' is not defined"
     ]
    }
   ],
   "source": [
    "df_filtrado = df_original[booleanos]\n",
    "df_filtrado.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Simplificando ou utilizando em List Comprehension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>10.33</td>\n",
       "      <td>2016-01-03 17:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>11.18</td>\n",
       "      <td>2016-01-03 17:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>11.14</td>\n",
       "      <td>2016-01-03 17:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>10.40</td>\n",
       "      <td>2016-01-03 18:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>10.76</td>\n",
       "      <td>2016-01-03 18:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>10.62</td>\n",
       "      <td>2016-01-03 19:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>10.36</td>\n",
       "      <td>2016-01-03 19:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>10.09</td>\n",
       "      <td>2016-01-03 20:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>410</th>\n",
       "      <td>10.32</td>\n",
       "      <td>2016-01-03 20:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>10.13</td>\n",
       "      <td>2016-01-03 20:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>10.26</td>\n",
       "      <td>2016-01-03 20:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>10.49</td>\n",
       "      <td>2016-01-03 20:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419</th>\n",
       "      <td>10.21</td>\n",
       "      <td>2016-01-03 21:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Value                 Timestamp  Quality\n",
       "393  10.33 2016-01-03 17:30:00+00:00        0\n",
       "394  11.18 2016-01-03 17:40:00+00:00        0\n",
       "395  11.14 2016-01-03 17:50:00+00:00        0\n",
       "397  10.40 2016-01-03 18:10:00+00:00        0\n",
       "398  10.76 2016-01-03 18:20:00+00:00        0\n",
       "403  10.62 2016-01-03 19:10:00+00:00        0\n",
       "404  10.36 2016-01-03 19:20:00+00:00        0\n",
       "409  10.09 2016-01-03 20:10:00+00:00        0\n",
       "410  10.32 2016-01-03 20:20:00+00:00        0\n",
       "411  10.13 2016-01-03 20:30:00+00:00        0\n",
       "412  10.26 2016-01-03 20:40:00+00:00        0\n",
       "413  10.49 2016-01-03 20:50:00+00:00        0\n",
       "419  10.21 2016-01-03 21:50:00+00:00        0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A é uma serie do pandas\n",
    "A = df_original.Value > 10\n",
    "\n",
    "#apresenta todos os dados maiores que 10\n",
    "df_original[A]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List Comprehension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_original[df_original.Value > 10].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Retorna somente a serie desejada"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "393   2016-01-03 17:30:00+00:00\n",
       "394   2016-01-03 17:40:00+00:00\n",
       "395   2016-01-03 17:50:00+00:00\n",
       "397   2016-01-03 18:10:00+00:00\n",
       "398   2016-01-03 18:20:00+00:00\n",
       "Name: Timestamp, dtype: datetime64[ns, UTC]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "serie_timestamp = df_original[df_original.Value > 10].Timestamp\n",
    "serie_timestamp.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Contudo, por boa prática, utilizarmos sempre o método LOC, que na verdade é definir uma condição e filtrar por labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#retorna dataframe\n",
    "df_filtrado = df_original.loc[df_original.Value > 10]\n",
    "df_filtrado.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#retorna uma serie\n",
    "serie_filtrada = df_original.loc[df_original.Value > 10, 'Timestamp']\n",
    "serie_filtrada.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###                              Filtros Multiplos\n",
    "#### Operador lógico E"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>10.33</td>\n",
       "      <td>2016-01-03 17:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>11.18</td>\n",
       "      <td>2016-01-03 17:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>11.14</td>\n",
       "      <td>2016-01-03 17:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>10.40</td>\n",
       "      <td>2016-01-03 18:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>10.76</td>\n",
       "      <td>2016-01-03 18:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Value                 Timestamp  Quality\n",
       "393  10.33 2016-01-03 17:30:00+00:00        0\n",
       "394  11.18 2016-01-03 17:40:00+00:00        0\n",
       "395  11.14 2016-01-03 17:50:00+00:00        0\n",
       "397  10.40 2016-01-03 18:10:00+00:00        0\n",
       "398  10.76 2016-01-03 18:20:00+00:00        0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original[(df_original.Value > 10) & (df_original.Quality == 0)].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Operador lógico OU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.14</td>\n",
       "      <td>2016-01-01 00:00:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.75</td>\n",
       "      <td>2016-01-01 00:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.71</td>\n",
       "      <td>2016-01-01 00:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.07</td>\n",
       "      <td>2016-01-01 00:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.75</td>\n",
       "      <td>2016-01-01 00:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Value                 Timestamp  Quality\n",
       "0   6.14 2016-01-01 00:00:00+00:00        0\n",
       "1   5.75 2016-01-01 00:10:00+00:00        0\n",
       "2   5.71 2016-01-01 00:20:00+00:00        0\n",
       "3   6.07 2016-01-01 00:30:00+00:00        0\n",
       "4   5.75 2016-01-01 00:40:00+00:00        0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original[(df_original.Value > 10) | (df_original.Quality == 0)].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Utilizando ISIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>6.0</td>\n",
       "      <td>2016-01-02 09:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Value                 Timestamp  Quality\n",
       "203    6.0 2016-01-02 09:50:00+00:00        0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original[df_original.Value.isin([6])].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Utilizando loc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>10.33</td>\n",
       "      <td>2016-01-03 17:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>11.18</td>\n",
       "      <td>2016-01-03 17:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>11.14</td>\n",
       "      <td>2016-01-03 17:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>10.40</td>\n",
       "      <td>2016-01-03 18:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>10.76</td>\n",
       "      <td>2016-01-03 18:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Value                 Timestamp  Quality\n",
       "393  10.33 2016-01-03 17:30:00+00:00        0\n",
       "394  11.18 2016-01-03 17:40:00+00:00        0\n",
       "395  11.14 2016-01-03 17:50:00+00:00        0\n",
       "397  10.40 2016-01-03 18:10:00+00:00        0\n",
       "398  10.76 2016-01-03 18:20:00+00:00        0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original.loc[(df_original.Value > 10) & (df_original.Quality == 0)].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "##                                      Eixos\n",
    "\n",
    "###                       axis 0 = linhas | axis 1 = colunas\n",
    "\n",
    "###                  eixo 0 das linhas trabalha de cima para baixo\n",
    "###                 eixo 1 das colunas trabalha da esquerda para direita\n",
    "\n",
    "#### Media das colunas numericas, ou seja, de cima para baixo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value      6.456759\n",
      "Quality    0.000000\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>432.000000</td>\n",
       "      <td>432.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.456759</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.637152</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.870000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.687500</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.490000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.270000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>11.180000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Value  Quality\n",
       "count  432.000000    432.0\n",
       "mean     6.456759      0.0\n",
       "std      1.637152      0.0\n",
       "min      0.870000      0.0\n",
       "25%      5.687500      0.0\n",
       "50%      6.490000      0.0\n",
       "75%      7.270000      0.0\n",
       "max     11.180000      0.0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 'index' é um alias para axis 0\n",
    "print(df_original.mean(axis=0))\n",
    "#conferir se a media estar certa\n",
    "df_original.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Media das linhas, ou seja, esquerda para direita"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3.070\n",
       "1    2.875\n",
       "2    2.855\n",
       "3    3.035\n",
       "4    2.875\n",
       "dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 'columns' é um alias para axis 1\n",
    "df_original.mean(axis=1).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### calculo para provar que 'df_original.mean(axis=1).head()' fez a média da linha somando as colunas numéricas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.069999933242798"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "media_linha = (df_original['Quality'][0] + df_original['Value'][0])/2\n",
    "media_linha"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### soma das colunas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    6.14\n",
       "1    5.75\n",
       "2    5.71\n",
       "3    6.07\n",
       "4    5.75\n",
       "dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soma = df_original.sum(axis=1)\n",
    "soma.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### soma das linhas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value      2789.320068\n",
       "Quality       0.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soma = df_original.sum(axis=0)\n",
    "soma.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###                              TIPANDO COLUNAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value                    float32\n",
       "Timestamp    datetime64[ns, UTC]\n",
       "Quality                    int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "      <th>Value_string</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.14</td>\n",
       "      <td>2016-01-01 00:00:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>6.139999866485596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.75</td>\n",
       "      <td>2016-01-01 00:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.71</td>\n",
       "      <td>2016-01-01 00:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.710000038146973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.07</td>\n",
       "      <td>2016-01-01 00:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>6.070000171661377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.75</td>\n",
       "      <td>2016-01-01 00:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Value                 Timestamp  Quality       Value_string\n",
       "0   6.14 2016-01-01 00:00:00+00:00        0  6.139999866485596\n",
       "1   5.75 2016-01-01 00:10:00+00:00        0               5.75\n",
       "2   5.71 2016-01-01 00:20:00+00:00        0  5.710000038146973\n",
       "3   6.07 2016-01-01 00:30:00+00:00        0  6.070000171661377\n",
       "4   5.75 2016-01-01 00:40:00+00:00        0               5.75"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = []\n",
    "\n",
    "for item in df_original.Value:\n",
    "    a = str(item)\n",
    "    strings.append(a)\n",
    "    \n",
    "df_nova_coluna = df_original\n",
    "df_nova_coluna['Value_string'] = pd.Series(strings)\n",
    "\n",
    "#Note que o DF df_original está com a coluna Value_string criada em df_nova_coluna, por isso no python é preciso copiar a lista\n",
    "df_original.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### voltando df_original para o padrão"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value                    float32\n",
       "Timestamp    datetime64[ns, UTC]\n",
       "Quality                    int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original.drop(['Value_string'], axis=1, inplace=True)\n",
    "df_original.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Copiando para outro DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value                       float32\n",
       "Timestamp       datetime64[ns, UTC]\n",
       "Quality                       int64\n",
       "Value_string                 object\n",
       "dtype: object"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna = df_original.copy()\n",
    "df_nova_coluna['Value_string'] = pd.Series(strings)\n",
    "df_nova_coluna.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Converter Value para string mostra informações extras, como é possivel ver em Value_string o número de valores únicos(298)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "      <th>Value_string</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>432.000000</td>\n",
       "      <td>432</td>\n",
       "      <td>432.0</td>\n",
       "      <td>432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>432</td>\n",
       "      <td>NaN</td>\n",
       "      <td>298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-01 20:40:00+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.090000152587891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.456759</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.637152</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.870000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.687500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.490000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.270000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>11.180000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Value                  Timestamp  Quality       Value_string\n",
       "count   432.000000                        432    432.0                432\n",
       "unique         NaN                        432      NaN                298\n",
       "top            NaN  2016-01-01 20:40:00+00:00      NaN  6.090000152587891\n",
       "freq           NaN                          1      NaN                  5\n",
       "mean      6.456759                        NaN      0.0                NaN\n",
       "std       1.637152                        NaN      0.0                NaN\n",
       "min       0.870000                        NaN      0.0                NaN\n",
       "25%       5.687500                        NaN      0.0                NaN\n",
       "50%       6.490000                        NaN      0.0                NaN\n",
       "75%       7.270000                        NaN      0.0                NaN\n",
       "max      11.180000                        NaN      0.0                NaN"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna.describe(include='all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###                              MÉTODOS STRINGS\n",
    "####              Os métodos strings no Pandas são acessados via .str\n",
    "\n",
    "#### fazendo um filtro por string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "      <th>Value_string</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>5.66</td>\n",
       "      <td>2016-01-01 04:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.659999847412109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>5.16</td>\n",
       "      <td>2016-01-01 05:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.159999847412109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>2.21</td>\n",
       "      <td>2016-01-01 08:00:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>2.2100000381469727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>5.41</td>\n",
       "      <td>2016-01-01 11:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.409999847412109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>5.91</td>\n",
       "      <td>2016-01-01 12:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.909999847412109</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Value                 Timestamp  Quality        Value_string\n",
       "26   5.66 2016-01-01 04:20:00+00:00        0   5.659999847412109\n",
       "33   5.16 2016-01-01 05:30:00+00:00        0   5.159999847412109\n",
       "48   2.21 2016-01-01 08:00:00+00:00        0  2.2100000381469727\n",
       "71   5.41 2016-01-01 11:50:00+00:00        0   5.409999847412109\n",
       "74   5.91 2016-01-01 12:20:00+00:00        0   5.909999847412109"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna[df_nova_coluna.Value_string.str.contains('21.')].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###                              Aprofundando em DataTypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value                       float32\n",
       "Timestamp       datetime64[ns, UTC]\n",
       "Quality                       int64\n",
       "Value_string                 object\n",
       "dtype: object"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2 = df_nova_coluna.copy()\n",
    "df_nova_coluna2.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Alterando o tipo da coluna Quality para object(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value                       float32\n",
       "Timestamp       datetime64[ns, UTC]\n",
       "Quality                      object\n",
       "Value_string                 object\n",
       "dtype: object"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2['Quality'] = df_nova_coluna2.Quality.astype(object)\n",
    "df_nova_coluna2.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Manipulando string para pegar somente a parte inteira de Value_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "      <th>Value_string</th>\n",
       "      <th>Value_int</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.14</td>\n",
       "      <td>2016-01-01 00:00:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>6.139999866485596</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.75</td>\n",
       "      <td>2016-01-01 00:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.75</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.71</td>\n",
       "      <td>2016-01-01 00:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.710000038146973</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.07</td>\n",
       "      <td>2016-01-01 00:30:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>6.070000171661377</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.75</td>\n",
       "      <td>2016-01-01 00:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.75</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Value                 Timestamp Quality       Value_string Value_int\n",
       "0   6.14 2016-01-01 00:00:00+00:00       0  6.139999866485596         6\n",
       "1   5.75 2016-01-01 00:10:00+00:00       0               5.75         5\n",
       "2   5.71 2016-01-01 00:20:00+00:00       0  5.710000038146973         5\n",
       "3   6.07 2016-01-01 00:30:00+00:00       0  6.070000171661377         6\n",
       "4   5.75 2016-01-01 00:40:00+00:00       0               5.75         5"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2['Value_int'] = df_nova_coluna2.Value_string.str.split('.').str[0]\n",
    "df_nova_coluna2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "      <th>Value_string</th>\n",
       "      <th>Value_int</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>432.000000</td>\n",
       "      <td>432</td>\n",
       "      <td>432.0</td>\n",
       "      <td>432</td>\n",
       "      <td>432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>432</td>\n",
       "      <td>1.0</td>\n",
       "      <td>298</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-01 20:40:00+00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.090000152587891</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>432.0</td>\n",
       "      <td>5</td>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.456759</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.637152</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.870000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.687500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.490000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.270000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>11.180000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Value                  Timestamp  Quality       Value_string  \\\n",
       "count   432.000000                        432    432.0                432   \n",
       "unique         NaN                        432      1.0                298   \n",
       "top            NaN  2016-01-01 20:40:00+00:00      0.0  6.090000152587891   \n",
       "freq           NaN                          1    432.0                  5   \n",
       "mean      6.456759                        NaN      NaN                NaN   \n",
       "std       1.637152                        NaN      NaN                NaN   \n",
       "min       0.870000                        NaN      NaN                NaN   \n",
       "25%       5.687500                        NaN      NaN                NaN   \n",
       "50%       6.490000                        NaN      NaN                NaN   \n",
       "75%       7.270000                        NaN      NaN                NaN   \n",
       "max      11.180000                        NaN      NaN                NaN   \n",
       "\n",
       "       Value_int  \n",
       "count        432  \n",
       "unique        12  \n",
       "top            6  \n",
       "freq         152  \n",
       "mean         NaN  \n",
       "std          NaN  \n",
       "min          NaN  \n",
       "25%          NaN  \n",
       "50%          NaN  \n",
       "75%          NaN  \n",
       "max          NaN  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2.describe(include='all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### expand = True são retornadas duas colunas, uma com a parte interia e outra com a parte fracionária"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>139999866485596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>710000038146973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>070000171661377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0                1\n",
       "0  6  139999866485596\n",
       "1  5               75\n",
       "2  5  710000038146973\n",
       "3  6  070000171661377\n",
       "4  5               75"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2.Value_string.str.split('.', expand=True).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Converter uma serie boolean em integer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "todas_ocorrencias = df_nova_coluna2.Value_string.str.contains('21.').astype(int)\n",
    "print(type(todas_ocorrencias))\n",
    "\n",
    "#numero de ocorrencias de valores que contenham 21.\n",
    "todas_ocorrencias.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "###                              TRATANDO VALORES NULOS (NaN)\n",
    "#### Como este dataframe não possui valores NaN, adicionei alguns quando o valor da coluna Value_int for igual a 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_nova_coluna2.loc[df_nova_coluna2.Value_int == '5', 'Value_int'] = np.nan"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saber quantidade de nulos em uma coluna\n",
    "\n",
    "#### Agora a coluna Value_int possui 99 NaN, ou seja, anteriormente possuia 99 itens com valor 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value            0\n",
       "Timestamp        0\n",
       "Quality          0\n",
       "Value_string     0\n",
       "Value_int       99\n",
       "dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filtrar para verificar quais linhas tem nulos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "      <th>Value_string</th>\n",
       "      <th>Value_int</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>359</th>\n",
       "      <td>5.65</td>\n",
       "      <td>2016-01-03 11:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.650000095367432</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>5.58</td>\n",
       "      <td>2016-01-03 12:00:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.579999923706055</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>5.87</td>\n",
       "      <td>2016-01-03 12:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.869999885559082</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>5.45</td>\n",
       "      <td>2016-01-03 12:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.449999809265137</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>5.88</td>\n",
       "      <td>2016-01-03 12:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.880000114440918</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Value                 Timestamp Quality       Value_string Value_int\n",
       "359   5.65 2016-01-03 11:50:00+00:00       0  5.650000095367432       NaN\n",
       "360   5.58 2016-01-03 12:00:00+00:00       0  5.579999923706055       NaN\n",
       "361   5.87 2016-01-03 12:10:00+00:00       0  5.869999885559082       NaN\n",
       "362   5.45 2016-01-03 12:20:00+00:00       0  5.449999809265137       NaN\n",
       "364   5.88 2016-01-03 12:40:00+00:00       0  5.880000114440918       NaN"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2[df_nova_coluna2.Value_int.isnull()].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "432"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remover linhas que contenham valores NaN(nulo)\n",
    "\n",
    "#### Com dropna inplace = True, altera o dataframe original\n",
    "#### how = 'any' -> se existir um NaN na linha então deleta toda a linha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "333"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_linhas_removidas = df_nova_coluna2.dropna(how='any')\n",
    "df_linhas_removidas.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### how = 'all' -> se todos os valores da linha for NaN então deleta a linha\n",
    "não foi deletado nada porque não existe uma linha com todos valores NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(432, 5)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = df_nova_coluna2.dropna(how='all')\n",
    "A.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Utilizando parametro subset, que recebe uma lista\n",
    "não foi deletado nada porque não existe uma linha Value_int E Timestamp NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(432, 5)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = df_nova_coluna2.dropna(subset=['Value_int', 'Timestamp'],how='all')\n",
    "A.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Foi deletado porque existe uma linha Value_int OU Timestamp NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(333, 5)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B = df_nova_coluna2.dropna(subset=['Value_int', 'Timestamp'],how='any')\n",
    "B.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### value_counts SEM contar os valores NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6     152\n",
       "7      83\n",
       "8      28\n",
       "2      15\n",
       "4      13\n",
       "9      13\n",
       "10     11\n",
       "3      10\n",
       "1       5\n",
       "11      2\n",
       "Name: Value_int, dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2.Value_int.value_counts().head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### value_counts INCLUINDO os valores NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6      152\n",
       "NaN     99\n",
       "7       83\n",
       "8       28\n",
       "2       15\n",
       "4       13\n",
       "9       13\n",
       "10      11\n",
       "3       10\n",
       "1        5\n",
       "Name: Value_int, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nova_coluna2.Value_int.value_counts(dropna=False).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preenchendo os valores que são NaN com -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Quality</th>\n",
       "      <th>Value_string</th>\n",
       "      <th>Value_int</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>359</th>\n",
       "      <td>5.65</td>\n",
       "      <td>2016-01-03 11:50:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.650000095367432</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>5.58</td>\n",
       "      <td>2016-01-03 12:00:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.579999923706055</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>5.87</td>\n",
       "      <td>2016-01-03 12:10:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.869999885559082</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>5.45</td>\n",
       "      <td>2016-01-03 12:20:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.449999809265137</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>5.88</td>\n",
       "      <td>2016-01-03 12:40:00+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>5.880000114440918</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Value                 Timestamp  Quality       Value_string Value_int\n",
       "359   5.65 2016-01-03 11:50:00+00:00        0  5.650000095367432        -1\n",
       "360   5.58 2016-01-03 12:00:00+00:00        0  5.579999923706055        -1\n",
       "361   5.87 2016-01-03 12:10:00+00:00        0  5.869999885559082        -1\n",
       "362   5.45 2016-01-03 12:20:00+00:00        0  5.449999809265137        -1\n",
       "364   5.88 2016-01-03 12:40:00+00:00        0  5.880000114440918        -1"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C = df_nova_coluna2.fillna(value=-1)\n",
    "C[C.Value_int == -1].tail()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}