pyspark-on-google-colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pyspark_on_Google_Colab.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "RgbtUdQNv-TK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q http://mirror.its.dal.ca/apache/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz\n",
        "!tar xvf spark-2.4.0-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AJc_O-34weud",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dCItH-BszUxV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-2.4.0-bin-hadoop2.7\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3KmmLD_mx3oN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "7fbdae40-5e63-42af-b1c7-77031b947e37"
      },
      "cell_type": "code",
      "source": [
        "print (os.listdir('./sample_data'))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['README.md', 'anscombe.json', 'california_housing_train.csv', 'california_housing_test.csv', 'mnist_train_small.csv', 'mnist_test.csv']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "dT_P2w10w6WW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "file_loc = './sample_data/california_housing_train.csv'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4dDKF5ttxGP4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import findspark\n",
        "findspark.init()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "k25gV_d0xI2D",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZmQNVyl21dbZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = spark.read.csv(file_loc, inferSchema=True, header =True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "T9m9qHUP2Osh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d35df8cb-3d66-43a9-befe-dee9ff1112df"
      },
      "cell_type": "code",
      "source": [
        "print (type(df))"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pyspark.sql.dataframe.DataFrame'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "dGT2lz_m1lnP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        },
        "outputId": "1f140353-af1e-45d3-a974-e6b005bf6e0e"
      },
      "cell_type": "code",
      "source": [
        "df.printSchema()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- longitude: double (nullable = true)\n",
            " |-- latitude: double (nullable = true)\n",
            " |-- housing_median_age: double (nullable = true)\n",
            " |-- total_rooms: double (nullable = true)\n",
            " |-- total_bedrooms: double (nullable = true)\n",
            " |-- population: double (nullable = true)\n",
            " |-- households: double (nullable = true)\n",
            " |-- median_income: double (nullable = true)\n",
            " |-- median_house_value: double (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "l-T1bgDt1qbZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 468
        },
        "outputId": "1aa5f048-85cb-4777-e7e4-e6b663eea212"
      },
      "cell_type": "code",
      "source": [
        "df.show()"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---------+--------+------------------+-----------+--------------+----------+----------+-------------+------------------+\n",
            "|longitude|latitude|housing_median_age|total_rooms|total_bedrooms|population|households|median_income|median_house_value|\n",
            "+---------+--------+------------------+-----------+--------------+----------+----------+-------------+------------------+\n",
            "|  -114.31|   34.19|              15.0|     5612.0|        1283.0|    1015.0|     472.0|       1.4936|           66900.0|\n",
            "|  -114.47|    34.4|              19.0|     7650.0|        1901.0|    1129.0|     463.0|         1.82|           80100.0|\n",
            "|  -114.56|   33.69|              17.0|      720.0|         174.0|     333.0|     117.0|       1.6509|           85700.0|\n",
            "|  -114.57|   33.64|              14.0|     1501.0|         337.0|     515.0|     226.0|       3.1917|           73400.0|\n",
            "|  -114.57|   33.57|              20.0|     1454.0|         326.0|     624.0|     262.0|        1.925|           65500.0|\n",
            "|  -114.58|   33.63|              29.0|     1387.0|         236.0|     671.0|     239.0|       3.3438|           74000.0|\n",
            "|  -114.58|   33.61|              25.0|     2907.0|         680.0|    1841.0|     633.0|       2.6768|           82400.0|\n",
            "|  -114.59|   34.83|              41.0|      812.0|         168.0|     375.0|     158.0|       1.7083|           48500.0|\n",
            "|  -114.59|   33.61|              34.0|     4789.0|        1175.0|    3134.0|    1056.0|       2.1782|           58400.0|\n",
            "|   -114.6|   34.83|              46.0|     1497.0|         309.0|     787.0|     271.0|       2.1908|           48100.0|\n",
            "|   -114.6|   33.62|              16.0|     3741.0|         801.0|    2434.0|     824.0|       2.6797|           86500.0|\n",
            "|   -114.6|    33.6|              21.0|     1988.0|         483.0|    1182.0|     437.0|        1.625|           62000.0|\n",
            "|  -114.61|   34.84|              48.0|     1291.0|         248.0|     580.0|     211.0|       2.1571|           48600.0|\n",
            "|  -114.61|   34.83|              31.0|     2478.0|         464.0|    1346.0|     479.0|        3.212|           70400.0|\n",
            "|  -114.63|   32.76|              15.0|     1448.0|         378.0|     949.0|     300.0|       0.8585|           45000.0|\n",
            "|  -114.65|   34.89|              17.0|     2556.0|         587.0|    1005.0|     401.0|       1.6991|           69100.0|\n",
            "|  -114.65|    33.6|              28.0|     1678.0|         322.0|     666.0|     256.0|       2.9653|           94900.0|\n",
            "|  -114.65|   32.79|              21.0|       44.0|          33.0|      64.0|      27.0|       0.8571|           25000.0|\n",
            "|  -114.66|   32.74|              17.0|     1388.0|         386.0|     775.0|     320.0|       1.2049|           44000.0|\n",
            "|  -114.67|   33.92|              17.0|       97.0|          24.0|      29.0|      15.0|       1.2656|           27500.0|\n",
            "+---------+--------+------------------+-----------+--------------+----------+----------+-------------+------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "-GRBFoUT1r8a",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}