{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.1.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>22</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>35</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bonnell, Miss. Elizabeth</td>\n",
       "      <td>58</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Name  Age     Sex\n",
       "0   Braund, Mr. Owen Harris   22    male\n",
       "1  Allen, Mr. William Henry   35    male\n",
       "2  Bonnell, Miss. Elizabeth   58  female"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A DataFrame is a 2d data structure that can store data of different types\n",
    "# in columns. It is similar to a spreadsheet, a SQL table or data.frame in R\n",
    "df = pd.DataFrame({\n",
    "    \"Name\": [\"Braund, Mr. Owen Harris\",\n",
    "             \"Allen, Mr. William Henry\",\n",
    "             \"Bonnell, Miss. Elizabeth\"],\n",
    "    \"Age\": [22, 35, 58],\n",
    "    \"Sex\": [\"male\", \"male\", \"female\"],\n",
    "})\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ages from DataFrame\n",
      "0    22\n",
      "1    35\n",
      "2    58\n",
      "Name: Age, dtype: int64\n",
      "\n",
      "ages from scratch\n",
      "0    22\n",
      "1    35\n",
      "2    58\n",
      "Name: Age, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Each column in a DataFrame is a Series\n",
    "ages = df[\"Age\"]\n",
    "print('\\nages from DataFrame')\n",
    "print(ages)\n",
    "\n",
    "# You can create a Series from scratch as well:\n",
    "ages = pd.Series([22, 35, 58], name=\"Age\")\n",
    "print('\\nages from scratch')\n",
    "print(ages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "58"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Series operations\n",
    "ages.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>38.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>18.230012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>22.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>28.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>35.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>46.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>58.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Age\n",
       "count   3.000000\n",
       "mean   38.333333\n",
       "std    18.230012\n",
       "min    22.000000\n",
       "25%    28.500000\n",
       "50%    35.000000\n",
       "75%    46.500000\n",
       "max    58.000000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# DataFame statistic: textual columns will be omitted\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Working with csv\n",
    "# https://github.com/pandas-dev/pandas/tree/master/doc/data/titanic.csv\n",
    "titanic = pd.read_csv(\"data/titanic.csv\")\n",
    "titanic.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 12 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   PassengerId  891 non-null    int64  \n",
      " 1   Survived     891 non-null    int64  \n",
      " 2   Pclass       891 non-null    int64  \n",
      " 3   Name         891 non-null    object \n",
      " 4   Sex          891 non-null    object \n",
      " 5   Age          714 non-null    float64\n",
      " 6   SibSp        891 non-null    int64  \n",
      " 7   Parch        891 non-null    int64  \n",
      " 8   Ticket       891 non-null    object \n",
      " 9   Fare         891 non-null    float64\n",
      " 10  Cabin        204 non-null    object \n",
      " 11  Embarked     889 non-null    object \n",
      "dtypes: float64(2), int64(5), object(5)\n",
      "memory usage: 83.7+ KB\n"
     ]
    }
   ],
   "source": [
    "# Get technical summary of DataFrame\n",
    "titanic.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Multiple-Columns subset, result is a new DataFrame\n",
    "age_sex = titanic[['Age', 'Sex']]\n",
    "type(age_sex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(891, 2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_sex.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>McCarthy, Mr. Timothy J</td>\n",
       "      <td>male</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17463</td>\n",
       "      <td>51.8625</td>\n",
       "      <td>E46</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Bonnell, Miss. Elizabeth</td>\n",
       "      <td>female</td>\n",
       "      <td>58.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>113783</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>C103</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Andersson, Mr. Anders Johan</td>\n",
       "      <td>male</td>\n",
       "      <td>39.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>347082</td>\n",
       "      <td>31.2750</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Hewlett, Mrs. (Mary D Kingcome)</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>248706</td>\n",
       "      <td>16.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    PassengerId  Survived  Pclass  \\\n",
       "1             2         1       1   \n",
       "6             7         0       1   \n",
       "11           12         1       1   \n",
       "13           14         0       3   \n",
       "15           16         1       2   \n",
       "\n",
       "                                                 Name     Sex   Age  SibSp  \\\n",
       "1   Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "6                             McCarthy, Mr. Timothy J    male  54.0      0   \n",
       "11                           Bonnell, Miss. Elizabeth  female  58.0      0   \n",
       "13                        Andersson, Mr. Anders Johan    male  39.0      1   \n",
       "15                   Hewlett, Mrs. (Mary D Kingcome)   female  55.0      0   \n",
       "\n",
       "    Parch    Ticket     Fare Cabin Embarked  \n",
       "1       0  PC 17599  71.2833   C85        C  \n",
       "6       0     17463  51.8625   E46        S  \n",
       "11      0    113783  26.5500  C103        S  \n",
       "13      5    347082  31.2750   NaN        S  \n",
       "15      0    248706  16.0000   NaN        S  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filtering rows\n",
    "\n",
    "age_above_35 = titanic[titanic['Age'] > 35]\n",
    "age_above_35.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0      False\n",
       "1       True\n",
       "2      False\n",
       "3      False\n",
       "4      False\n",
       "       ...  \n",
       "886    False\n",
       "887    False\n",
       "888    False\n",
       "889    False\n",
       "890    False\n",
       "Name: Age, Length: 891, dtype: bool"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How filtering expression works\n",
    "exp = titanic['Age'] > 35 # output a mask Series\n",
    "print(type(exp))\n",
    "exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2</td>\n",
       "      <td>Nasser, Mrs. Nicholas (Adele Achem)</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3</td>\n",
       "      <td>Sandstrom, Miss. Marguerite Rut</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>Bonnell, Miss. Elizabeth</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>3</td>\n",
       "      <td>Saundercock, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>3</td>\n",
       "      <td>Andersson, Mr. Anders Johan</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>3</td>\n",
       "      <td>Vestrom, Miss. Hulda Amanda Adolfina</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2</td>\n",
       "      <td>Hewlett, Mrs. (Mary D Kingcome)</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3</td>\n",
       "      <td>Rice, Master. Eugene</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2</td>\n",
       "      <td>Williams, Mr. Charles Eugene</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>3</td>\n",
       "      <td>Vander Planke, Mrs. Julius (Emelia Maria Vande...</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>3</td>\n",
       "      <td>Masselmani, Mrs. Fatima</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2</td>\n",
       "      <td>Fynney, Mr. Joseph J</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>Beesley, Mr. Lawrence</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3</td>\n",
       "      <td>McGowan, Miss. Anna \"Annie\"</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>Sloper, Mr. William Thompson</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>3</td>\n",
       "      <td>Palsson, Miss. Torborg Danira</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Pclass                                               Name     Sex\n",
       "9        2                Nasser, Mrs. Nicholas (Adele Achem)  female\n",
       "10       3                    Sandstrom, Miss. Marguerite Rut  female\n",
       "11       1                           Bonnell, Miss. Elizabeth  female\n",
       "12       3                     Saundercock, Mr. William Henry    male\n",
       "13       3                        Andersson, Mr. Anders Johan    male\n",
       "14       3               Vestrom, Miss. Hulda Amanda Adolfina  female\n",
       "15       2                   Hewlett, Mrs. (Mary D Kingcome)   female\n",
       "16       3                               Rice, Master. Eugene    male\n",
       "17       2                       Williams, Mr. Charles Eugene    male\n",
       "18       3  Vander Planke, Mrs. Julius (Emelia Maria Vande...  female\n",
       "19       3                            Masselmani, Mrs. Fatima  female\n",
       "20       2                               Fynney, Mr. Joseph J    male\n",
       "21       2                              Beesley, Mr. Lawrence    male\n",
       "22       3                        McGowan, Miss. Anna \"Annie\"  female\n",
       "23       1                       Sloper, Mr. William Thompson    male\n",
       "24       3                      Palsson, Miss. Torborg Danira  female"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filtering by isin, can not use or/and operator in this case\n",
    "class_23 = titanic[(titanic[\"Pclass\"] == 2) | (titanic[\"Pclass\"] == 3)]\n",
    "# Filtering by notna\n",
    "age_no_na = titanic[titanic['Age'].notna()]\n",
    "# Filtering and Subset: df.loc[rows_filtering, cols_filtering]\n",
    "adult_names = titanic.loc[titanic[\"Age\"] > 35, \"Name\"]\n",
    "# Select by region: df.iloc[row_start:row_end, col_start:col_end]\n",
    "titanic.iloc[9:25, 2:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                           Name     Sex   Age  SibSp  Parch  \\\n",
       "0                                     anonymous    male  22.0      1      0   \n",
       "1                                     anonymous  female  38.0      1      0   \n",
       "2                                     anonymous  female  26.0      0      0   \n",
       "3  Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1      0   \n",
       "4                      Allen, Mr. William Henry    male  35.0      0      0   \n",
       "\n",
       "             Ticket     Fare Cabin Embarked  \n",
       "0         A/5 21171   7.2500   NaN        S  \n",
       "1          PC 17599  71.2833   C85        C  \n",
       "2  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3            113803  53.1000  C123        S  \n",
       "4            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assign new values to selection base on loc/iloc\n",
    "titanic.iloc[0:3, 3] = \"anonymous\"\n",
    "titanic.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1035 entries, 0 to 1034\n",
      "Data columns (total 4 columns):\n",
      " #   Column           Non-Null Count  Dtype  \n",
      "---  ------           --------------  -----  \n",
      " 0   datetime         1035 non-null   object \n",
      " 1   station_antwerp  95 non-null     float64\n",
      " 2   station_paris    1004 non-null   float64\n",
      " 3   station_london   969 non-null    float64\n",
      "dtypes: float64(3), object(1)\n",
      "memory usage: 32.5+ KB\n",
      "None\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 1035 entries, 2019-05-07 02:00:00 to 2019-06-21 02:00:00\n",
      "Data columns (total 3 columns):\n",
      " #   Column           Non-Null Count  Dtype  \n",
      "---  ------           --------------  -----  \n",
      " 0   station_antwerp  95 non-null     float64\n",
      " 1   station_paris    1004 non-null   float64\n",
      " 2   station_london   969 non-null    float64\n",
      "dtypes: float64(3)\n",
      "memory usage: 32.3 KB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station_antwerp</th>\n",
       "      <th>station_paris</th>\n",
       "      <th>station_london</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-05-07 02:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 03:00:00</th>\n",
       "      <td>50.5</td>\n",
       "      <td>25.0</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 04:00:00</th>\n",
       "      <td>45.0</td>\n",
       "      <td>27.7</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 05:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>50.4</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 06:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>61.9</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     station_antwerp  station_paris  station_london\n",
       "datetime                                                           \n",
       "2019-05-07 02:00:00              NaN            NaN            23.0\n",
       "2019-05-07 03:00:00             50.5           25.0            19.0\n",
       "2019-05-07 04:00:00             45.0           27.7            19.0\n",
       "2019-05-07 05:00:00              NaN           50.4            16.0\n",
       "2019-05-07 06:00:00              NaN           61.9             NaN"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Plotting\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# https://raw.githubusercontent.com/pandas-dev/pandas/master/doc/data/air_quality_no2.csv\n",
    "air_quality = pd.read_csv(\"data/air_quality_no2.csv\")\n",
    "print(air_quality.info())\n",
    "air_quality = pd.read_csv(\"data/air_quality_no2.csv\", index_col=0, parse_dates=[\"datetime\"])\n",
    "print(air_quality.info())\n",
    "air_quality.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 137712 characters]\n",
      "text/plain": [
       "<Figure size 921.6x691.2 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 107168 characters]\n",
      "text/plain": [
       "<Figure size 921.6x691.2 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 921.6x691.2 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 14552 characters]\n",
      "text/plain": [
       "<Figure size 921.6x691.2 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# quick visual check\n",
    "plt.figure()\n",
    "air_quality.plot(figsize=(12.8, 9.6))\n",
    "\n",
    "\n",
    "# plot only paris\n",
    "plt.figure()\n",
    "air_quality[\"station_paris\"].plot(figsize=(12.8, 9.6))\n",
    "\n",
    "# box plot\n",
    "plt.figure(figsize=(12.8, 9.6))\n",
    "air_quality.plot.box(figsize=(12.8, 9.6))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([<AxesSubplot:xlabel='datetime'>, <AxesSubplot:xlabel='datetime'>,\n",
       "       <AxesSubplot:xlabel='datetime'>], dtype=object)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 49392 characters]\n",
      "text/plain": [
       "<Figure size 864x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# area plot\n",
    "plt.figure()\n",
    "air_quality.plot.area(figsize=(12, 4), subplots=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station_antwerp</th>\n",
       "      <th>station_paris</th>\n",
       "      <th>station_london</th>\n",
       "      <th>london_mg_per_cubic</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-05-07 02:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23.0</td>\n",
       "      <td>43.286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 03:00:00</th>\n",
       "      <td>50.5</td>\n",
       "      <td>25.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>35.758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 04:00:00</th>\n",
       "      <td>45.0</td>\n",
       "      <td>27.7</td>\n",
       "      <td>19.0</td>\n",
       "      <td>35.758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 05:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>50.4</td>\n",
       "      <td>16.0</td>\n",
       "      <td>30.112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 06:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>61.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     station_antwerp  station_paris  station_london  \\\n",
       "datetime                                                              \n",
       "2019-05-07 02:00:00              NaN            NaN            23.0   \n",
       "2019-05-07 03:00:00             50.5           25.0            19.0   \n",
       "2019-05-07 04:00:00             45.0           27.7            19.0   \n",
       "2019-05-07 05:00:00              NaN           50.4            16.0   \n",
       "2019-05-07 06:00:00              NaN           61.9             NaN   \n",
       "\n",
       "                     london_mg_per_cubic  \n",
       "datetime                                  \n",
       "2019-05-07 02:00:00               43.286  \n",
       "2019-05-07 03:00:00               35.758  \n",
       "2019-05-07 04:00:00               35.758  \n",
       "2019-05-07 05:00:00               30.112  \n",
       "2019-05-07 06:00:00                  NaN  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create new columns derived from existing columns\n",
    "air_quality[\"london_mg_per_cubic\"] = air_quality[\"station_london\"] * 1.882\n",
    "air_quality.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station_antwerp</th>\n",
       "      <th>station_paris</th>\n",
       "      <th>station_london</th>\n",
       "      <th>london_mpc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-05-07 02:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23.0</td>\n",
       "      <td>43.286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 03:00:00</th>\n",
       "      <td>50.5</td>\n",
       "      <td>25.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>35.758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 04:00:00</th>\n",
       "      <td>45.0</td>\n",
       "      <td>27.7</td>\n",
       "      <td>19.0</td>\n",
       "      <td>35.758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 05:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>50.4</td>\n",
       "      <td>16.0</td>\n",
       "      <td>30.112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 06:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>61.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     station_antwerp  station_paris  station_london  \\\n",
       "datetime                                                              \n",
       "2019-05-07 02:00:00              NaN            NaN            23.0   \n",
       "2019-05-07 03:00:00             50.5           25.0            19.0   \n",
       "2019-05-07 04:00:00             45.0           27.7            19.0   \n",
       "2019-05-07 05:00:00              NaN           50.4            16.0   \n",
       "2019-05-07 06:00:00              NaN           61.9             NaN   \n",
       "\n",
       "                     london_mpc  \n",
       "datetime                         \n",
       "2019-05-07 02:00:00      43.286  \n",
       "2019-05-07 03:00:00      35.758  \n",
       "2019-05-07 04:00:00      35.758  \n",
       "2019-05-07 05:00:00      30.112  \n",
       "2019-05-07 06:00:00         NaN  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rename column\n",
    "air_quality_renamed = air_quality.rename(columns={'london_mg_per_cubic': 'london_mpc'}) # columns=str.lower\n",
    "air_quality_renamed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station_antwerp</th>\n",
       "      <th>station_paris</th>\n",
       "      <th>station_london</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-05-07 02:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 03:00:00</th>\n",
       "      <td>50.5</td>\n",
       "      <td>25.0</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 04:00:00</th>\n",
       "      <td>45.0</td>\n",
       "      <td>27.7</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 05:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>50.4</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-07 06:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>61.9</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     station_antwerp  station_paris  station_london\n",
       "datetime                                                           \n",
       "2019-05-07 02:00:00              NaN            NaN            23.0\n",
       "2019-05-07 03:00:00             50.5           25.0            19.0\n",
       "2019-05-07 04:00:00             45.0           27.7            19.0\n",
       "2019-05-07 05:00:00              NaN           50.4            16.0\n",
       "2019-05-07 06:00:00              NaN           61.9             NaN"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# delete column\n",
    "del air_quality['london_mg_per_cubic']\n",
    "air_quality.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "age mean: \n",
      "29.69911764705882\n",
      "\n",
      "age/fare medians: \n",
      "Age     28.0000\n",
      "Fare    14.4542\n",
      "dtype: float64\n",
      "\n",
      "age/fare describe\n",
      "              Age        Fare\n",
      "count  714.000000  891.000000\n",
      "mean    29.699118   32.204208\n",
      "std     14.526497   49.693429\n",
      "min      0.420000    0.000000\n",
      "25%     20.125000    7.910400\n",
      "50%     28.000000   14.454200\n",
      "75%     38.000000   31.000000\n",
      "max     80.000000  512.329200\n",
      "\n",
      "customize aggregation\n",
      "              Age        Fare\n",
      "max     80.000000  512.329200\n",
      "mean          NaN   32.204208\n",
      "median  28.000000   14.454200\n",
      "min      0.420000    0.000000\n",
      "skew     0.389108         NaN\n"
     ]
    }
   ],
   "source": [
    "# aggregating statistics\n",
    "print('age mean: ')\n",
    "print(titanic[\"Age\"].mean())\n",
    "print()\n",
    "print('age/fare medians: ')\n",
    "print(titanic[[\"Age\", \"Fare\"]].median())\n",
    "print()\n",
    "print('age/fare describe')\n",
    "print(titanic[[\"Age\", \"Fare\"]].describe())\n",
    "print()\n",
    "print('customize aggregation')\n",
    "# skew: negative skewness means left tailed, aka mass of distribution is concentrated on te right\n",
    "print(titanic.agg({'Age': ['min', 'max', 'median', 'skew'],\n",
    "                   'Fare': ['min', 'max', 'median', 'mean']}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "titanic[[\"Sex\", \"Age\"]].groupby(\"Sex\").mean()\n",
      "              Age\n",
      "Sex              \n",
      "female  27.915709\n",
      "male    30.726645\n",
      "\n",
      "titanic.groupby([\"Sex\", \"Pclass\"])[\"Fare\"].mean()\n",
      "Sex     Pclass\n",
      "female  1         106.125798\n",
      "        2          21.970121\n",
      "        3          16.118810\n",
      "male    1          67.226127\n",
      "        2          19.741782\n",
      "        3          12.661633\n",
      "Name: Fare, dtype: float64\n",
      "\n",
      "titanic[\"Pclass\"].value_counts()  shortcut to  titanic.groupby(\"Pclass\")[\"Pclass\"].count()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3    491\n",
       "1    216\n",
       "2    184\n",
       "Name: Pclass, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculat average age for male vs female\n",
    "print('titanic[[\"Sex\", \"Age\"]].groupby(\"Sex\").mean()')\n",
    "print(titanic[[\"Sex\", \"Age\"]].groupby(\"Sex\").mean())\n",
    "print()\n",
    "\n",
    "print('titanic.groupby([\"Sex\", \"Pclass\"])[\"Fare\"].mean()')\n",
    "print(titanic.groupby([\"Sex\", \"Pclass\"])[\"Fare\"].mean())\n",
    "print()\n",
    "\n",
    "# `size` includes NaN values while `count` excludes them\n",
    "print('titanic[\"Pclass\"].value_counts()  shortcut to  titanic.groupby(\"Pclass\")[\"Pclass\"].count()')\n",
    "titanic[\"Pclass\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                           Name     Sex   Age  SibSp  Parch  \\\n",
       "0                                     anonymous    male  22.0      1      0   \n",
       "1                                     anonymous  female  38.0      1      0   \n",
       "2                                     anonymous  female  26.0      0      0   \n",
       "3  Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1      0   \n",
       "4                      Allen, Mr. William Henry    male  35.0      0      0   \n",
       "\n",
       "             Ticket     Fare Cabin Embarked  \n",
       "0         A/5 21171   7.2500   NaN        S  \n",
       "1          PC 17599  71.2833   C85        C  \n",
       "2  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3            113803  53.1000  C123        S  \n",
       "4            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titanic.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>851</th>\n",
       "      <td>852</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Svensson, Mr. Johan</td>\n",
       "      <td>male</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>347060</td>\n",
       "      <td>7.7750</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>117</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Connors, Mr. Patrick</td>\n",
       "      <td>male</td>\n",
       "      <td>70.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>370369</td>\n",
       "      <td>7.7500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>281</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Duane, Mr. Frank</td>\n",
       "      <td>male</td>\n",
       "      <td>65.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>336439</td>\n",
       "      <td>7.7500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483</th>\n",
       "      <td>484</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Turkula, Mrs. (Hedwig)</td>\n",
       "      <td>female</td>\n",
       "      <td>63.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4134</td>\n",
       "      <td>9.5875</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>327</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Nysveen, Mr. Johan Hansen</td>\n",
       "      <td>male</td>\n",
       "      <td>61.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>345364</td>\n",
       "      <td>6.2375</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     PassengerId  Survived  Pclass                       Name     Sex   Age  \\\n",
       "851          852         0       3        Svensson, Mr. Johan    male  74.0   \n",
       "116          117         0       3       Connors, Mr. Patrick    male  70.5   \n",
       "280          281         0       3           Duane, Mr. Frank    male  65.0   \n",
       "483          484         1       3     Turkula, Mrs. (Hedwig)  female  63.0   \n",
       "326          327         0       3  Nysveen, Mr. Johan Hansen    male  61.0   \n",
       "\n",
       "     SibSp  Parch  Ticket    Fare Cabin Embarked  \n",
       "851      0      0  347060  7.7750   NaN        S  \n",
       "116      0      0  370369  7.7500   NaN        Q  \n",
       "280      0      0  336439  7.7500   NaN        Q  \n",
       "483      0      0    4134  9.5875   NaN        S  \n",
       "326      0      0  345364  6.2375   NaN        S  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sorting\n",
    "titanic.sort_values(by=['Pclass', 'Age'], ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>country</th>\n",
       "      <th>location</th>\n",
       "      <th>parameter</th>\n",
       "      <th>value</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date.utc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-06-18 06:00:00+00:00</th>\n",
       "      <td>Antwerpen</td>\n",
       "      <td>BE</td>\n",
       "      <td>BETR801</td>\n",
       "      <td>pm25</td>\n",
       "      <td>18.0</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-17 08:00:00+00:00</th>\n",
       "      <td>Antwerpen</td>\n",
       "      <td>BE</td>\n",
       "      <td>BETR801</td>\n",
       "      <td>pm25</td>\n",
       "      <td>6.5</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-17 07:00:00+00:00</th>\n",
       "      <td>Antwerpen</td>\n",
       "      <td>BE</td>\n",
       "      <td>BETR801</td>\n",
       "      <td>pm25</td>\n",
       "      <td>18.5</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-17 06:00:00+00:00</th>\n",
       "      <td>Antwerpen</td>\n",
       "      <td>BE</td>\n",
       "      <td>BETR801</td>\n",
       "      <td>pm25</td>\n",
       "      <td>16.0</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-17 05:00:00+00:00</th>\n",
       "      <td>Antwerpen</td>\n",
       "      <td>BE</td>\n",
       "      <td>BETR801</td>\n",
       "      <td>pm25</td>\n",
       "      <td>7.5</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                city country location parameter  value   unit\n",
       "date.utc                                                                     \n",
       "2019-06-18 06:00:00+00:00  Antwerpen      BE  BETR801      pm25   18.0  µg/m³\n",
       "2019-06-17 08:00:00+00:00  Antwerpen      BE  BETR801      pm25    6.5  µg/m³\n",
       "2019-06-17 07:00:00+00:00  Antwerpen      BE  BETR801      pm25   18.5  µg/m³\n",
       "2019-06-17 06:00:00+00:00  Antwerpen      BE  BETR801      pm25   16.0  µg/m³\n",
       "2019-06-17 05:00:00+00:00  Antwerpen      BE  BETR801      pm25    7.5  µg/m³"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "air_quality_long = pd.read_csv(\"data/air_quality_long.csv\", index_col='date.utc', parse_dates=True)\n",
    "air_quality_long.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FR04014               1676\n",
      "London Westminster    1608\n",
      "BETR801                163\n",
      "Name: location, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>country</th>\n",
       "      <th>location</th>\n",
       "      <th>parameter</th>\n",
       "      <th>value</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date.utc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-06-21 00:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>20.0</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-20 23:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>21.8</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-20 22:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>26.5</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-20 21:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>24.9</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-20 20:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>21.4</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            city country location parameter  value   unit\n",
       "date.utc                                                                 \n",
       "2019-06-21 00:00:00+00:00  Paris      FR  FR04014       no2   20.0  µg/m³\n",
       "2019-06-20 23:00:00+00:00  Paris      FR  FR04014       no2   21.8  µg/m³\n",
       "2019-06-20 22:00:00+00:00  Paris      FR  FR04014       no2   26.5  µg/m³\n",
       "2019-06-20 21:00:00+00:00  Paris      FR  FR04014       no2   24.9  µg/m³\n",
       "2019-06-20 20:00:00+00:00  Paris      FR  FR04014       no2   21.4  µg/m³"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pivot table\n",
    "no2 = air_quality_long[air_quality_long[\"parameter\"] == 'no2']\n",
    "print(no2[\"location\"].value_counts())\n",
    "no2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>country</th>\n",
       "      <th>location</th>\n",
       "      <th>parameter</th>\n",
       "      <th>value</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date.utc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-04-09 01:00:00+00:00</th>\n",
       "      <td>Antwerpen</td>\n",
       "      <td>BE</td>\n",
       "      <td>BETR801</td>\n",
       "      <td>no2</td>\n",
       "      <td>22.5</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-09 01:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>24.4</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-09 02:00:00+00:00</th>\n",
       "      <td>London</td>\n",
       "      <td>GB</td>\n",
       "      <td>London Westminster</td>\n",
       "      <td>no2</td>\n",
       "      <td>67.0</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-09 02:00:00+00:00</th>\n",
       "      <td>Antwerpen</td>\n",
       "      <td>BE</td>\n",
       "      <td>BETR801</td>\n",
       "      <td>no2</td>\n",
       "      <td>53.5</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-09 02:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>27.4</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-09 03:00:00+00:00</th>\n",
       "      <td>London</td>\n",
       "      <td>GB</td>\n",
       "      <td>London Westminster</td>\n",
       "      <td>no2</td>\n",
       "      <td>67.0</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                city country            location parameter  \\\n",
       "date.utc                                                                     \n",
       "2019-04-09 01:00:00+00:00  Antwerpen      BE             BETR801       no2   \n",
       "2019-04-09 01:00:00+00:00      Paris      FR             FR04014       no2   \n",
       "2019-04-09 02:00:00+00:00     London      GB  London Westminster       no2   \n",
       "2019-04-09 02:00:00+00:00  Antwerpen      BE             BETR801       no2   \n",
       "2019-04-09 02:00:00+00:00      Paris      FR             FR04014       no2   \n",
       "2019-04-09 03:00:00+00:00     London      GB  London Westminster       no2   \n",
       "\n",
       "                           value   unit  \n",
       "date.utc                                 \n",
       "2019-04-09 01:00:00+00:00   22.5  µg/m³  \n",
       "2019-04-09 01:00:00+00:00   24.4  µg/m³  \n",
       "2019-04-09 02:00:00+00:00   67.0  µg/m³  \n",
       "2019-04-09 02:00:00+00:00   53.5  µg/m³  \n",
       "2019-04-09 02:00:00+00:00   27.4  µg/m³  \n",
       "2019-04-09 03:00:00+00:00   67.0  µg/m³  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use 2 measurements (head) for each location (groupby)\n",
    "no2_subset = no2.sort_index().groupby('location').head(2)\n",
    "no2_subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>location</th>\n",
       "      <th>BETR801</th>\n",
       "      <th>FR04014</th>\n",
       "      <th>London Westminster</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date.utc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-04-09 01:00:00+00:00</th>\n",
       "      <td>22.5</td>\n",
       "      <td>24.4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-09 02:00:00+00:00</th>\n",
       "      <td>53.5</td>\n",
       "      <td>27.4</td>\n",
       "      <td>67.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-09 03:00:00+00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>67.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "location                   BETR801  FR04014  London Westminster\n",
       "date.utc                                                       \n",
       "2019-04-09 01:00:00+00:00     22.5     24.4                 NaN\n",
       "2019-04-09 02:00:00+00:00     53.5     27.4                67.0\n",
       "2019-04-09 03:00:00+00:00      NaN      NaN                67.0"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pivot table\n",
    "no2_subset.pivot(columns=\"location\", values=\"value\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='date.utc'>"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 85056 characters]\n",
      "text/plain": [
       "<Figure size 864x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "no2.pivot(columns=\"location\", values=\"value\").plot(figsize=(12,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>parameter</th>\n",
       "      <th>no2</th>\n",
       "      <th>pm25</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BETR801</th>\n",
       "      <td>26.950920</td>\n",
       "      <td>23.169492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FR04014</th>\n",
       "      <td>29.374284</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>London Westminster</th>\n",
       "      <td>29.740050</td>\n",
       "      <td>13.443568</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "parameter                 no2       pm25\n",
       "location                                \n",
       "BETR801             26.950920  23.169492\n",
       "FR04014             29.374284        NaN\n",
       "London Westminster  29.740050  13.443568"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pivot table\n",
    "air_quality_long.pivot_table(\n",
    "    values=\"value\",\n",
    "    index=\"location\",\n",
    "    columns=[\"parameter\"],\n",
    "    aggfunc=\"mean\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    city country                  date.utc location parameter  value   unit\n",
      "0  Paris      FR 2019-06-21 00:00:00+00:00  FR04014       no2   20.0  µg/m³\n",
      "1  Paris      FR 2019-06-20 23:00:00+00:00  FR04014       no2   21.8  µg/m³\n",
      "2  Paris      FR 2019-06-20 22:00:00+00:00  FR04014       no2   26.5  µg/m³\n",
      "3  Paris      FR 2019-06-20 21:00:00+00:00  FR04014       no2   24.9  µg/m³\n",
      "4  Paris      FR 2019-06-20 20:00:00+00:00  FR04014       no2   21.4  µg/m³\n",
      "        city country                  date.utc location parameter  value  \\\n",
      "0  Antwerpen      BE 2019-06-18 06:00:00+00:00  BETR801      pm25   18.0   \n",
      "1  Antwerpen      BE 2019-06-17 08:00:00+00:00  BETR801      pm25    6.5   \n",
      "2  Antwerpen      BE 2019-06-17 07:00:00+00:00  BETR801      pm25   18.5   \n",
      "3  Antwerpen      BE 2019-06-17 06:00:00+00:00  BETR801      pm25   16.0   \n",
      "4  Antwerpen      BE 2019-06-17 05:00:00+00:00  BETR801      pm25    7.5   \n",
      "\n",
      "    unit  \n",
      "0  µg/m³  \n",
      "1  µg/m³  \n",
      "2  µg/m³  \n",
      "3  µg/m³  \n",
      "4  µg/m³  \n"
     ]
    }
   ],
   "source": [
    "# combine data from multiple dataframe, concatenation\n",
    "\n",
    "air_quality_no2_long = pd.read_csv('data/air_quality_no2_long.csv', parse_dates=[\"date.utc\"])\n",
    "air_quality_pm25_long = pd.read_csv('data/air_quality_pm25_long.csv', parse_dates=[\"date.utc\"])\n",
    "\n",
    "print(air_quality_no2_long.head())\n",
    "print(air_quality_pm25_long.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    city country                   date.utc location parameter  value   unit\n",
      "0  Paris      FR  2019-06-21 00:00:00+00:00  FR04014       no2   20.0  µg/m³\n",
      "1  Paris      FR  2019-06-20 23:00:00+00:00  FR04014       no2   21.8  µg/m³\n",
      "2  Paris      FR  2019-06-20 22:00:00+00:00  FR04014       no2   26.5  µg/m³\n",
      "3  Paris      FR  2019-06-20 21:00:00+00:00  FR04014       no2   24.9  µg/m³\n",
      "4  Paris      FR  2019-06-20 20:00:00+00:00  FR04014       no2   21.4  µg/m³\n",
      "        city country                   date.utc            location parameter  \\\n",
      "1105  London      GB  2019-05-07 06:00:00+00:00  London Westminster      pm25   \n",
      "1106  London      GB  2019-05-07 04:00:00+00:00  London Westminster      pm25   \n",
      "1107  London      GB  2019-05-07 03:00:00+00:00  London Westminster      pm25   \n",
      "1108  London      GB  2019-05-07 02:00:00+00:00  London Westminster      pm25   \n",
      "1109  London      GB  2019-05-07 01:00:00+00:00  London Westminster      pm25   \n",
      "\n",
      "      value   unit  \n",
      "1105    9.0  µg/m³  \n",
      "1106    8.0  µg/m³  \n",
      "1107    8.0  µg/m³  \n",
      "1108    8.0  µg/m³  \n",
      "1109    8.0  µg/m³  \n"
     ]
    }
   ],
   "source": [
    "# combine by rows\n",
    "air_quality_all = pd.concat([air_quality_no2_long, air_quality_pm25_long], axis=0)\n",
    "print(air_quality_all.head())\n",
    "print(air_quality_all.tail())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# more combination methods\n",
    "https://pandas.pydata.org/docs/getting_started/intro_tutorasdfials/08_combine_dataframes.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Paris', 'Antwerpen', 'London'], dtype=object)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "air_quality_all.city.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Paris', 'Antwerpen', 'London'], dtype=object)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "air_quality_all['city'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "air_quality_all = air_quality_all.rename(columns={'date.utc': 'datetime'})\n",
    "air_quality_all.datetime = pd.to_datetime(air_quality_all.datetime) # convert to pd.Timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      2019-06-21 00:00:00+00:00\n",
       "1      2019-06-20 23:00:00+00:00\n",
       "2      2019-06-20 22:00:00+00:00\n",
       "3      2019-06-20 21:00:00+00:00\n",
       "4      2019-06-20 20:00:00+00:00\n",
       "                  ...           \n",
       "1105   2019-05-07 06:00:00+00:00\n",
       "1106   2019-05-07 04:00:00+00:00\n",
       "1107   2019-05-07 03:00:00+00:00\n",
       "1108   2019-05-07 02:00:00+00:00\n",
       "1109   2019-05-07 01:00:00+00:00\n",
       "Name: datetime, Length: 3178, dtype: datetime64[ns, UTC]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "air_quality_all.datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      2019-06-21 00:00:00+00:00\n",
       "1      2019-06-20 23:00:00+00:00\n",
       "2      2019-06-20 22:00:00+00:00\n",
       "3      2019-06-20 21:00:00+00:00\n",
       "4      2019-06-20 20:00:00+00:00\n",
       "                  ...           \n",
       "2063   2019-05-07 06:00:00+00:00\n",
       "2064   2019-05-07 04:00:00+00:00\n",
       "2065   2019-05-07 03:00:00+00:00\n",
       "2066   2019-05-07 02:00:00+00:00\n",
       "2067   2019-05-07 01:00:00+00:00\n",
       "Name: date.utc, Length: 2068, dtype: datetime64[ns, UTC]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "air_quality_no2_long['date.utc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Timedelta('44 days 23:00:00')"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculates timestamp delta\n",
    "air_quality_all.datetime.max() - air_quality_all.datetime.min() # returns pd.Timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date.utc  location          \n",
       "0         BETR801               27.875000\n",
       "          FR04014               24.856250\n",
       "          London Westminster    23.969697\n",
       "1         BETR801               22.214286\n",
       "          FR04014               30.999359\n",
       "          London Westminster    24.885714\n",
       "2         BETR801               21.125000\n",
       "          FR04014               29.165753\n",
       "          London Westminster    23.460432\n",
       "3         BETR801               27.500000\n",
       "          FR04014               28.600690\n",
       "          London Westminster    24.780142\n",
       "4         BETR801               28.400000\n",
       "          FR04014               31.617986\n",
       "          London Westminster    26.446809\n",
       "5         BETR801               33.500000\n",
       "          FR04014               25.266154\n",
       "          London Westminster    24.977612\n",
       "6         BETR801               21.896552\n",
       "          FR04014               23.274306\n",
       "          London Westminster    24.859155\n",
       "Name: value, dtype: float64"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculates average NO2 concentrationfor each day of week \n",
    "# for each of the measurement locations\n",
    "\n",
    "air_quality_no2_long.groupby([\n",
    "    air_quality_no2_long[\"date.utc\"].dt.weekday,\n",
    "    \"location\"\n",
    "])[\"value\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9640 characters]\n",
      "text/plain": [
       "<Figure size 864x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the typical NO2 pattern during the day of our time series\n",
    "# of all stations together. that is the average value for each hour of day\n",
    "fig, axs = plt.subplots(figsize=(12, 4))\n",
    "g = air_quality_no2_long.groupby([air_quality_no2_long[\"date.utc\"].dt.hour])\n",
    "g[\"value\"].mean().plot(kind=\"bar\", rot=0, ax=axs)\n",
    "plt.xlabel(\"Hour of the day\")\n",
    "plt.ylabel(\"$NO_2 (\\mu g/m^3)$\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2068 entries, 0 to 2067\n",
      "Data columns (total 7 columns):\n",
      " #   Column     Non-Null Count  Dtype              \n",
      "---  ------     --------------  -----              \n",
      " 0   city       2068 non-null   object             \n",
      " 1   country    2068 non-null   object             \n",
      " 2   date.utc   2068 non-null   datetime64[ns, UTC]\n",
      " 3   location   2068 non-null   object             \n",
      " 4   parameter  2068 non-null   object             \n",
      " 5   value      2068 non-null   float64            \n",
      " 6   unit       2068 non-null   object             \n",
      "dtypes: datetime64[ns, UTC](1), float64(1), object(5)\n",
      "memory usage: 113.2+ KB\n"
     ]
    }
   ],
   "source": [
    "air_quality_no2_long.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 2068 entries, 2019-06-21 00:00:00+00:00 to 2019-05-07 01:00:00+00:00\n",
      "Data columns (total 6 columns):\n",
      " #   Column     Non-Null Count  Dtype  \n",
      "---  ------     --------------  -----  \n",
      " 0   city       2068 non-null   object \n",
      " 1   country    2068 non-null   object \n",
      " 2   location   2068 non-null   object \n",
      " 3   parameter  2068 non-null   object \n",
      " 4   value      2068 non-null   float64\n",
      " 5   unit       2068 non-null   object \n",
      "dtypes: float64(1), object(5)\n",
      "memory usage: 113.1+ KB\n",
      "None\n",
      "Int64Index([2019, 2019, 2019, 2019, 2019, 2019, 2019, 2019, 2019, 2019,\n",
      "            ...\n",
      "            2019, 2019, 2019, 2019, 2019, 2019, 2019, 2019, 2019, 2019],\n",
      "           dtype='int64', name='date.utc', length=2068)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>country</th>\n",
       "      <th>location</th>\n",
       "      <th>parameter</th>\n",
       "      <th>value</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date.utc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-06-21 00:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>20.0</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-20 23:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>21.8</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-20 22:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>26.5</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-20 21:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>24.9</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-20 20:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>FR</td>\n",
       "      <td>FR04014</td>\n",
       "      <td>no2</td>\n",
       "      <td>21.4</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            city country location parameter  value   unit\n",
       "date.utc                                                                 \n",
       "2019-06-21 00:00:00+00:00  Paris      FR  FR04014       no2   20.0  µg/m³\n",
       "2019-06-20 23:00:00+00:00  Paris      FR  FR04014       no2   21.8  µg/m³\n",
       "2019-06-20 22:00:00+00:00  Paris      FR  FR04014       no2   26.5  µg/m³\n",
       "2019-06-20 21:00:00+00:00  Paris      FR  FR04014       no2   24.9  µg/m³\n",
       "2019-06-20 20:00:00+00:00  Paris      FR  FR04014       no2   21.4  µg/m³"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use date.utc as index by using DataFrame.set_index method\n",
    "no2_long = air_quality_no2_long.set_index('date.utc')\n",
    "print(no2_long.info())\n",
    "print(no2_long.index.year)\n",
    "no2_long.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>country</th>\n",
       "      <th>location</th>\n",
       "      <th>parameter</th>\n",
       "      <th>value</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date.utc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-05-31 00:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>GB</td>\n",
       "      <td>London Westminster</td>\n",
       "      <td>no2</td>\n",
       "      <td>97.0</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-30 00:00:00+00:00</th>\n",
       "      <td>Paris</td>\n",
       "      <td>GB</td>\n",
       "      <td>London Westminster</td>\n",
       "      <td>no2</td>\n",
       "      <td>84.7</td>\n",
       "      <td>µg/m³</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            city country            location parameter  value  \\\n",
       "date.utc                                                                        \n",
       "2019-05-31 00:00:00+00:00  Paris      GB  London Westminster       no2   97.0   \n",
       "2019-06-30 00:00:00+00:00  Paris      GB  London Westminster       no2   84.7   \n",
       "\n",
       "                            unit  \n",
       "date.utc                          \n",
       "2019-05-31 00:00:00+00:00  µg/m³  \n",
       "2019-06-30 00:00:00+00:00  µg/m³  "
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# aggregate the current hourly time series values to the monthly maximum value \n",
    "# resample provides a time-based grouping: M,5H..., requires an aggregation funciton\n",
    "no2_long.resample('M').max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                         anonymous\n",
       "1                                         anonymous\n",
       "2                                         anonymous\n",
       "3      futrelle, mrs. jacques heath (lily may peel)\n",
       "4                          allen, mr. william henry\n",
       "                           ...                     \n",
       "886                           montvila, rev. juozas\n",
       "887                    graham, miss. margaret edith\n",
       "888        johnston, miss. catherine helen \"carrie\"\n",
       "889                           behr, mr. karl howell\n",
       "890                             dooley, mr. patrick\n",
       "Name: Name, Length: 891, dtype: object"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# manipulate textual data\n",
    "\n",
    "titanic[\"Name\"].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Surname</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>anonymous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "      <td>anonymous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>anonymous</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>anonymous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "      <td>Futrelle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>Allen</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                           Name     Sex   Age  SibSp  Parch  \\\n",
       "0                                     anonymous    male  22.0      1      0   \n",
       "1                                     anonymous  female  38.0      1      0   \n",
       "2                                     anonymous  female  26.0      0      0   \n",
       "3  Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1      0   \n",
       "4                      Allen, Mr. William Henry    male  35.0      0      0   \n",
       "\n",
       "             Ticket     Fare Cabin Embarked    Surname  \n",
       "0         A/5 21171   7.2500   NaN        S  anonymous  \n",
       "1          PC 17599  71.2833   C85        C  anonymous  \n",
       "2  STON/O2. 3101282   7.9250   NaN        S  anonymous  \n",
       "3            113803  53.1000  C123        S   Futrelle  \n",
       "4            373450   8.0500   NaN        S      Allen  "
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a Surname from Name column\n",
    "titanic[\"Surname\"] = titanic[\"Name\"].str.split(\",\").str.get(0)\n",
    "titanic.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Surname</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>759</th>\n",
       "      <td>760</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Rothes, the Countess. of (Lucy Noel Martha Dye...</td>\n",
       "      <td>female</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110152</td>\n",
       "      <td>86.5</td>\n",
       "      <td>B77</td>\n",
       "      <td>S</td>\n",
       "      <td>Rothes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     PassengerId  Survived  Pclass  \\\n",
       "759          760         1       1   \n",
       "\n",
       "                                                  Name     Sex   Age  SibSp  \\\n",
       "759  Rothes, the Countess. of (Lucy Noel Martha Dye...  female  33.0      0   \n",
       "\n",
       "     Parch  Ticket  Fare Cabin Embarked Surname  \n",
       "759      0  110152  86.5   B77        S  Rothes  "
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the passenger data about the Countesses on board of the Titanic\n",
    "titanic[titanic[\"Name\"].str.contains(\"Countess\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Penasco y Castellana, Mrs. Victor de Satode (Maria Josefa Perez de Soto y Vallejo)'"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get index of longest name\n",
    "titanic[\"Name\"].str.len().idxmax()\n",
    "# get longest name of table\n",
    "titanic.loc[titanic[\"Name\"].str.len().idxmax(), \"Name\"]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
