PySpark_RDD e DataFrame.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Configuração de biblioteca do PySpark no Google"
      ],
      "metadata": {
        "id": "kT5SPWFMVURY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zdd7ESWeS3U1",
        "outputId": "dba2e0d4-cfab-4d0f-dc73-26d1d212aefb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pyspark in /usr/local/lib/python3.10/dist-packages (3.4.1)\n",
            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n"
          ]
        }
      ],
      "source": [
        "!pip install pyspark"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Criando a sessão do SparkContext e SparkSession"
      ],
      "metadata": {
        "id": "iXZzK_F0VW5R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark import SparkContext\n",
        "from pyspark.sql import SparkSession"
      ],
      "metadata": {
        "id": "SVC51xpOTF4h"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sc = SparkContext.getOrCreate()"
      ],
      "metadata": {
        "id": "uNjNxORcTKEb"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark = SparkSession.builder.appName('PySpark DataFrame From RDD').getOrCreate()"
      ],
      "metadata": {
        "id": "MgwbLZPBTMJF"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create PySpark DataFrame From an Existing RDD"
      ],
      "metadata": {
        "id": "NqmeysoNTsw9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "rdd = sc.parallelize([('C',85,76,87,91), ('B',85,76,87,91), (\"A\", 85,78,96,92), (\"A\", 92,76,89,96)], 4)"
      ],
      "metadata": {
        "id": "LbGDD240TOz3"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(rdd))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5xUhfDpLTSFX",
        "outputId": "d4b024c7-98cc-403a-a21c-c98bc79086fc"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pyspark.rdd.RDD'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sub = ['Division','English','Mathematics','Physics','Chemistry']\n",
        "marks_df = spark.createDataFrame(rdd, schema=sub)"
      ],
      "metadata": {
        "id": "NGzl22XDTUxf"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(marks_df))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w9CrNRlPTW0b",
        "outputId": "936c9592-4de1-4d71-9216-7a70d1ffe7a5"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pyspark.sql.dataframe.DataFrame'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "marks_df.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H94oSH23TYp9",
        "outputId": "b62ae417-1966-4d50-fef4-cf65799f53d8"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Division: string (nullable = true)\n",
            " |-- English: long (nullable = true)\n",
            " |-- Mathematics: long (nullable = true)\n",
            " |-- Physics: long (nullable = true)\n",
            " |-- Chemistry: long (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "marks_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sf4pHHZfTbD_",
        "outputId": "863fc2a4-81f0-42d4-a68d-421278299e2d"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+-------+-----------+-------+---------+\n",
            "|Division|English|Mathematics|Physics|Chemistry|\n",
            "+--------+-------+-----------+-------+---------+\n",
            "|       C|     85|         76|     87|       91|\n",
            "|       B|     85|         76|     87|       91|\n",
            "|       A|     85|         78|     96|       92|\n",
            "|       A|     92|         76|     89|       96|\n",
            "+--------+-------+-----------+-------+---------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create PySpark DataFrame From an External File"
      ],
      "metadata": {
        "id": "G7tzGYFoTvvu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession"
      ],
      "metadata": {
        "id": "M_rIOv_-TxtX"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark = SparkSession.builder.appName('PySpark DataFrame From External Files').getOrCreate()"
      ],
      "metadata": {
        "id": "56SR03IGTzdx"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "csv_file = spark.read.csv('Fish.csv', sep = ',', inferSchema = True, header = True)"
      ],
      "metadata": {
        "id": "DgUKPm9bT1IQ"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "txt_file = spark.read.text(\"example.txt\")"
      ],
      "metadata": {
        "id": "B3KPNVERUEsl"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "json_file = spark.read.json(\"sample.json\", multiLine=True)"
      ],
      "metadata": {
        "id": "lnEuj94PUUNO"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(csv_file))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B0E4RZcsUXDg",
        "outputId": "57d37c63-eea9-441e-f897-1823b37b400d"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pyspark.sql.dataframe.DataFrame'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(txt_file))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kmAUj9z_UYW_",
        "outputId": "f86d43ee-3a4d-4186-fdbf-cad562cc0c81"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pyspark.sql.dataframe.DataFrame'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(json_file))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Miuoeo7VUaIN",
        "outputId": "5499c487-e973-4d22-f7f8-86d84d5306e3"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pyspark.sql.dataframe.DataFrame'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "csv_file.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iSoTE3sBUcXe",
        "outputId": "5aa960ab-18d5-4178-cea6-e0bfa67c7f6b"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Species: string (nullable = true)\n",
            " |-- Weight: double (nullable = true)\n",
            " |-- Length1: double (nullable = true)\n",
            " |-- Length2: double (nullable = true)\n",
            " |-- Length3: double (nullable = true)\n",
            " |-- Height: double (nullable = true)\n",
            " |-- Width: double (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "txt_file.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9tPb1wJqUdmk",
        "outputId": "0b2869b6-05dc-4c2c-d063-8905e8f3bf17"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- value: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "json_file.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7WKa1jWrUfMw",
        "outputId": "fffafedb-8551-4d4e-af6b-7a8dd4ae5de2"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- employees: array (nullable = true)\n",
            " |    |-- element: struct (containsNull = true)\n",
            " |    |    |-- firstName: string (nullable = true)\n",
            " |    |    |-- lastName: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# PySpark DataFrame to Pandas DataFrame"
      ],
      "metadata": {
        "id": "TTYZQ1BuUm91"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = csv_file.toPandas()"
      ],
      "metadata": {
        "id": "-YkSgpyyUpLK"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "type(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_oaFwM6tUrI4",
        "outputId": "823a8a51-c5e0-49a5-d290-e6e60811fcab"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pandas.core.frame.DataFrame"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    }
  ]
}