DataSearch-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New Jersey , Guttenberg :  40.7920454,-74.0037505,5mi\n",
      "2018-07-02, New Jersey , Guttenberg: attempt 1, 25 tweets gathered.\n",
      "2018-07-01, New Jersey , Guttenberg: attempt 1, 50 tweets gathered.\n",
      "Sample Tweet: @Gerritzen @SpaceX @FortniteGame Haha thanks.\n",
      "\n",
      "\n",
      "New Jersey , Union City :  40.7795455,-74.02375119999999,5mi\n",
      "2018-07-02, New Jersey , Union City: attempt 1, 25 tweets gathered.\n",
      "2018-07-01, New Jersey , Union City: attempt 1, 50 tweets gathered.\n",
      "Sample Tweet: @Gerritzen @SpaceX @FortniteGame Haha thanks.\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>followers</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>statuses_count</th>\n",
       "      <th>vader</th>\n",
       "      <th>average times</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <th>day</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">New Jersey , Guttenberg</th>\n",
       "      <th>2018-06-30</th>\n",
       "      <td>777.68</td>\n",
       "      <td>1022.0</td>\n",
       "      <td>27576.80</td>\n",
       "      <td>0.024932</td>\n",
       "      <td>117.356199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-07-01</th>\n",
       "      <td>16884.20</td>\n",
       "      <td>1257.2</td>\n",
       "      <td>23352.64</td>\n",
       "      <td>0.113980</td>\n",
       "      <td>231.466081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">New Jersey , Union City</th>\n",
       "      <th>2018-06-30</th>\n",
       "      <td>777.68</td>\n",
       "      <td>1022.0</td>\n",
       "      <td>27576.84</td>\n",
       "      <td>0.024932</td>\n",
       "      <td>129.374710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-07-01</th>\n",
       "      <td>16884.20</td>\n",
       "      <td>1257.2</td>\n",
       "      <td>23352.64</td>\n",
       "      <td>0.113980</td>\n",
       "      <td>255.170646</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    followers  friends_count  statuses_count  \\\n",
       "location                day                                                    \n",
       "New Jersey , Guttenberg 2018-06-30     777.68         1022.0        27576.80   \n",
       "                        2018-07-01   16884.20         1257.2        23352.64   \n",
       "New Jersey , Union City 2018-06-30     777.68         1022.0        27576.84   \n",
       "                        2018-07-01   16884.20         1257.2        23352.64   \n",
       "\n",
       "                                       vader  average times  \n",
       "location                day                                  \n",
       "New Jersey , Guttenberg 2018-06-30  0.024932     117.356199  \n",
       "                        2018-07-01  0.113980     231.466081  \n",
       "New Jersey , Union City 2018-06-30  0.024932     129.374710  \n",
       "                        2018-07-01  0.113980     255.170646  "
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import wikipedia as wp\n",
    "from pygeocoder import Geocoder\n",
    "import time\n",
    "import numpy as np\n",
    "from googletrans import Translator\n",
    "# Import and Initialize Sentiment Analyzer\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "analyzer = SentimentIntensityAnalyzer()\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "from datetime import datetime\n",
    "from datetime import date, timedelta\n",
    "\n",
    "import tweepy; import json\n",
    "from apikeys import twitterAccessToken as access_token\n",
    "from apikeys import twitterAccessTokenSecret as access_token_secret\n",
    "from apikeys import twitterConsumerKey as consumer_key\n",
    "from apikeys import twitterConsumerSecretKey as consumer_secret\n",
    "\n",
    "def worldTweets(search_term,numTweets, numDays):\n",
    "    #-- Get the most populated cities from wikipedia (Thank you wikipedia library!) --\n",
    "    html = wp.page(\"List_of_cities_by_population_density\").html().encode(\"UTF-8\")\n",
    "    worldCities = pd.read_html(html)[1]\n",
    "    worldCities = worldCities.drop([2,3,4],axis=1)\n",
    "    worldCities = worldCities.rename(columns={0:'city',1:'population',5:'density',6:'country'})\n",
    "    worldCities = worldCities.iloc[1:]\n",
    "    worldCities['population'] = [int(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in worldCities['population']]\n",
    "    worldCities['density'] = [int(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in worldCities['density']]\n",
    "    #--- ---- ----- ----- ---- ---- ----- ---- ---- ----- --- ---- ---- ----- ---- ---\n",
    "    \n",
    "    #--- Get tweets by Worlds most densily populated cities ---- --- ---- ---- --- ---\n",
    "    translator = Translator()\n",
    "    comparisons=pd.DataFrame(columns=['time density','sentiment'])\n",
    "    cityCount = 3\n",
    "    for index,row in worldCities.iterrows():\n",
    "        #-- location --- ----- --- ----\n",
    "        city,pop,density,country = row\n",
    "        cityCountry = city+' , '+country\n",
    "\n",
    "        #-- language conversion --- ---- --\n",
    "        languagesDf = getCountryLanguages()\n",
    "        if translateToLocalLanguage:\n",
    "            try:\n",
    "                translatedSearch = translator.translate(search_term, src='en', dest=languagesDf.loc[country,'language']).text\n",
    "            except ValueError:\n",
    "                print(\"could not translate \", languagesDf.loc[country,'language'])\n",
    "                translatedSearch=search_term\n",
    "                print('translated word: ',translatedSearch)\n",
    "        else:\n",
    "            translatedSearch=search_term\n",
    "        #--- --- --- ---- ---- --- --- ---\n",
    "        \n",
    "        try:\n",
    "            tweetsWorld = pd.concat([tweetsWorld, SearchForData(translatedSearch, numTweets, cityCountry, 100)], axis=0)\n",
    "        except:\n",
    "            tweetsWorld = SearchForData(translatedSearch, numTweets, cityCountry, 100)\n",
    "        print('\\n')\n",
    "        time.sleep(4)\n",
    "        #if cityCount==0:\n",
    "        #    break\n",
    "        #else:\n",
    "        #    cityCount-=1\n",
    "    return tweetsWorld\n",
    "\n",
    "def usTweets(search_term, numTweets, numDays):\n",
    "    #--- population per cities in United States --- ---- ---- --- ---- --- --- --- ---\n",
    "    html = wp.page(\"List_of_United_States_cities_by_population_density\").html().encode(\"UTF-8\")\n",
    "    UScities = pd.read_html(html)[1]\n",
    "    UScities = UScities.drop([0,2,4,6,8],axis=1)\n",
    "    UScities = UScities.rename(columns={1:'city',3:'state',5: 'land area (mi^2)',7:'density'})\n",
    "    UScities = UScities.iloc[1:]\n",
    "    #df['population']=[int(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in df['population']]\n",
    "    UScities['density'] = [float(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in UScities['density']]\n",
    "    UScities['land area (mi^2)']=[float(area.split('\\xa0')[-1]) for area in UScities['land area (mi^2)']]\n",
    "    #--- ---- ----- ----- ---- ---- ----- ---- ---- ----- --- ---- ---- ----- ---- ---\n",
    "    \n",
    "    \n",
    "    \n",
    "    #--- Add US Cities --- ---- ---- ---- ---- ---\n",
    "    cityCount = 1\n",
    "    for index,row in UScities.iterrows():\n",
    "        #-- location --- ----- --- ----\n",
    "        city,state,area,density = row\n",
    "        cityState = state+' , '+city\n",
    "\n",
    "        try:\n",
    "            tweetsUS = pd.concat([tweetsUS, SearchForData(search_term=search_term, nTweets=numTweets, cityCountry=cityState,radius=max(area,5), numDays=numDays )], axis=0)\n",
    "        except:\n",
    "            tweetsUS = SearchForData(search_term=search_term, nTweets=numTweets, cityCountry=cityState,radius=max(area,5), numDays=numDays)\n",
    "        \n",
    "        print('\\n')\n",
    "        time.sleep(4)\n",
    "        if cityCount==0:\n",
    "            break\n",
    "        else:\n",
    "            cityCount-=1\n",
    "    return tweetsUS\n",
    "\n",
    "def parse_url( url):\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.text, 'lxml')\n",
    "    listylist=[]\n",
    "    for table in soup.find_all('table'):\n",
    "        listylist.append(parse_html_table(table))\n",
    "    return listylist\n",
    "\n",
    "def parse_html_table( table):\n",
    "    n_columns = 0; n_rows=0; column_names = []\n",
    "\n",
    "    # Find number of rows and columns\n",
    "    # we also find the column titles if we can\n",
    "    for row in table.find_all('tr'):\n",
    "\n",
    "        # Determine the number of rows in the table\n",
    "        td_tags = row.find_all('td')\n",
    "        if len(td_tags) > 0:\n",
    "            n_rows+=1\n",
    "            if n_columns == 0:\n",
    "                # Set the number of columns for our table\n",
    "                n_columns = len(td_tags)\n",
    "\n",
    "        # Handle column names if we find them\n",
    "        th_tags = row.find_all('th') \n",
    "        if len(th_tags) > 0 and len(column_names) == 0:\n",
    "            for th in th_tags:\n",
    "                column_names.append(th.get_text())\n",
    "\n",
    "    # Safeguard on Column Titles\n",
    "    if len(column_names) > 0 and len(column_names) != n_columns:\n",
    "        raise Exception(\"Column titles do not match the number of columns\")\n",
    "\n",
    "    columns = column_names if len(column_names) > 0 else range(0,n_columns)\n",
    "    df = pd.DataFrame(columns = columns,\n",
    "                      index= range(0,n_rows))\n",
    "    row_marker = 0\n",
    "    for row in table.find_all('tr'):\n",
    "        column_marker = 0\n",
    "        columns = row.find_all('td')\n",
    "        for column in columns:\n",
    "            df.iat[row_marker,column_marker] = column.get_text()\n",
    "            column_marker += 1\n",
    "        if len(columns) > 0:\n",
    "            row_marker += 1\n",
    "\n",
    "    # Convert to float if possible\n",
    "    for col in df:\n",
    "        try:\n",
    "            df[col] = df[col].astype(float)\n",
    "        except ValueError:\n",
    "            pass\n",
    "\n",
    "    return df\n",
    "\n",
    "def getCountryLanguages():\n",
    "    #TODO: Use the .apply to just change the table to one dialect. Imrpove language scope later.\n",
    "    df = parse_url('https://www.infoplease.com/world/countries-world/languages-spoken-each-country-world')\n",
    "    countryLanguages = df[0].rename(columns={0:'country',1:'language'}).set_index('country')\n",
    "    countryLanguages['language'] = [re.sub('\\d+|%|\\(.*\\)|\\s','',i).split(',')[0].split(';')[0] for i in countryLanguages['language']]\n",
    "    return countryLanguages\n",
    "\n",
    "\n",
    "#returns hashtag, followers, following, text, geo, date\n",
    "#cityCountry example: 'paris,france'\n",
    "def SearchForData(search_term, nTweets, cityCountry='',radius=100, numDays=1):\n",
    "    \n",
    "    # Setup Tweepy API Authentication\n",
    "    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "    auth.set_access_token(access_token, access_token_secret)\n",
    "    api = tweepy.API(auth, parser=tweepy.parsers.JSONParser())\n",
    "    \n",
    "    #--- Calculate geocordinates from cityCountry --- --- ---- ---- --- --- --- --\n",
    "    geoConvertTries = 0\n",
    "    while True:\n",
    "        try:\n",
    "            result = Geocoder.geocode(cityCountry)\n",
    "        except Exception as error:\n",
    "            #print('errrooooorrrrr: ',error.message)\n",
    "            if 'OVER_QUERY_LIMIT' in str(error):\n",
    "                print('Encountered an error:{0}\\nWaiting 30 seconds and trying again.'.format(error))\n",
    "                time.sleep(30)\n",
    "                if geoConvertTries>10:\n",
    "                    print(\"Could not convert geo. returning empty list\")\n",
    "                    return []\n",
    "            elif not (re.search('^\\w+,\\w+$',cityCountry)):\n",
    "                print(\"cityCountry input format is incorrect. It should be \\'city,Country\\' like \\'paris,france\\'\")\n",
    "                return []\n",
    "            else:\n",
    "                print(\"Could not convert geo. returning empty list\")\n",
    "                return []\n",
    "        else:\n",
    "            break\n",
    "        geoConvertTries+=1\n",
    "    # 34.0934,56.134,50mi\n",
    "    coords = str(result[0].coordinates).replace('(','').replace(')','') + f',{radius}mi'\n",
    "    coords=coords.replace(' ','')\n",
    "    print(cityCountry, \": \", coords)\n",
    "    #--- ---- ----- ---- ---- ---- ---- ---- --- ---- ---- --- ---- --- ---- --- --\n",
    "\n",
    "    #--- grab tweets --- ---- ---- ---- ---- ---- ---- ---- --- --- ---- ---- ----\n",
    "    maxTweets = 10000; oldest_tweet = None; unique_ids = []; desiredTweets = [];nTweetsPerDay=nTweets/numDays\n",
    "    for day,num in zip([str(date.today() - timedelta(i)).split()[0] for i in range(numDays)], range(1,numDays+1)):\n",
    "        tweetsPerDay=[];tries=1\n",
    "        while len(desiredTweets) < min(nTweetsPerDay*num , maxTweets/numDays):\n",
    "            \n",
    "            #--- determine whether to grab tweets by geo or not --- ---- --- ----- --\n",
    "            while True:\n",
    "                try:\n",
    "                    if cityCountry:\n",
    "                        tweetsPerDay = api.search(search_term, count=nTweetsPerDay, result_type=\"recent\", max_id=oldest_tweet, geocode=coords, until=day)\n",
    "                    else:\n",
    "                        tweetsPerDay = api.search(search_term, count=nTweetsPerDay, result_type=\"recent\", max_id=oldest_tweet, until=day)\n",
    "                    time.sleep(2)\n",
    "                except Exception as error:\n",
    "                    print(error[0]['message'],'Trying again after 1 minute.')\n",
    "                    time.sleep(60)\n",
    "                else:\n",
    "                    break\n",
    "            \n",
    "            #---- ----- ----- ---- ----- ---- ----- ---- ----- ---- ---- ---- ---- --\n",
    "\n",
    "            #--- Dont go through an infinite loop trying to fill tweets that don't exist -----\n",
    "            if len(tweetsPerDay['statuses'])==0:\n",
    "                print(f'No tweets returned while searching for \\'{search_term}\\'',len(desiredTweets)\\\n",
    "                     ,' ',day)\n",
    "                if desiredTweets==[]:\n",
    "                    return [{'text':None, 'vader':None,\n",
    "                                          'location':cityCountry, 'geo':coords,\n",
    "                                          'hashtags':None, 'followers':None,\n",
    "                                         'friends_count':None,'statuses_count':None,\n",
    "                                          'created_at':None}]\n",
    "                return pd.DataFrame(desiredTweets)\n",
    "\n",
    "            #--- Append relevent tweets to output listy list ---- --- ---- ---- ---- --- ---\n",
    "            for tweet in tweetsPerDay['statuses']:\n",
    "                # Append tweet_id to ids list if it doesn't already exist. This allows checking for duplicate tweets\n",
    "                if tweet[\"id\"] not in unique_ids :\n",
    "                    unique_ids.append(tweet['id'])\n",
    "                    if coords:\n",
    "                        desiredTweets.append({'text':tweet['text'], 'vader':analyzer.polarity_scores(tweet['text'])['compound'],\n",
    "                                          'location':cityCountry, 'geo':coords,\n",
    "                                          'hashtags':tweet['entities']['hashtags'], 'followers':tweet['user']['followers_count'],\n",
    "                                         'friends_count':tweet['user']['friends_count'],'statuses_count':tweet['user']['statuses_count'],\n",
    "                                          'created_at':datetime.strptime(tweet['created_at'],'%a %b %d %H:%M:%S %z %Y')})\n",
    "                    else:\n",
    "                        desiredTweets.append({'text':tweet['text'], 'vader':analyzer.polarity_scores(tweet['text'])['compound'],\n",
    "                                          'hashtags':tweet['entities']['hashtags'], 'followers':tweet['user']['followers_count'],\n",
    "                                         'friends_count':tweet['user']['friends_count'],'statuses_count':tweet['user']['statuses_count'],\n",
    "                                          'created_at':datetime.strptime(tweet['created_at'],'%a %b %d %H:%M:%S %z %Y')})\n",
    "                \n",
    "                # Reassign the the oldest tweet (i.e. the max_id) subtract 1 so the previous oldest isn't included\n",
    "                oldest_tweet = tweet[\"id\"] - 1\n",
    "            print(f'{day}, {cityCountry}: attempt {tries}, {len(desiredTweets)} tweets gathered.');tries+=1\n",
    "\n",
    "    #--- Print sample tweet --- --- ---- ---- --- ---- ---- --- ---- ---- ---\n",
    "    translator = Translator()\n",
    "    try:\n",
    "        print ('Sample Tweet:',translator.translate(desiredTweets[0]['text'], dest='en').text)\n",
    "    except:\n",
    "        print('there was an error translating sample tweet: ',desiredTweets[0]['text'])\n",
    "    return pd.DataFrame(desiredTweets)\n",
    "\n",
    "\n",
    "def GetTweetsByPopularCities(search_term, numTweets, translateToLocalLanguage = True, numDays = 1):\n",
    "    #tweetsWorld = worldTweets()\n",
    "    tweetsUS = usTweets(search_term, numTweets, numDays)\n",
    "    tweetsUS['day']=[time.date() for time in tweetsUS['created_at']]\n",
    "    \n",
    "    #Calculate time density of tweets per city\n",
    "    cityData = tweetsUS.groupby(['location','day']).mean()\n",
    "    aveTimes={}\n",
    "    for date in tweetsUS.groupby(['location','day'])['created_at']:\n",
    "        #print (date[0])\n",
    "        #return\n",
    "        cityTime, timeseries = date\n",
    "        (cityState,timeobj)=cityTime\n",
    "        #print(cit)\n",
    "        [state,city]=cityState.split(',')\n",
    "        city=city.lstrip().rstrip(); state=state.rstrip().lstrip()\n",
    "        cityDensity=UScities.loc[UScities['city']==city].loc[UScities['state']==state].loc[:,'density'].values[0]\n",
    "        #return timeseries\n",
    "        try:\n",
    "            aveTimes[cityTime]=np.mean([(b-a).seconds*10000/cityDensity for a,b in zip(timeseries.tolist()[1:],timeseries.tolist()[0:-1])])\n",
    "        except:\n",
    "            if len(timeseries.tolist())<2:\n",
    "                aveTimes[cityTime]=0\n",
    "        #print(f'{city}, {cityDensity}, {aveTimes[cityState]}')\n",
    "    #KeyError: ('New Jersey , Guttenberg', datetime.date(2018, 6, 30))\n",
    "    cityData['time between tweets']=[aveTimes[citytime] for citytime in cityData.index]\n",
    "    cityData\n",
    "    return cityData\n",
    "\n",
    "\n",
    "\n",
    "cityDataMain = GetTweetsByPopularCities(search_term='spacex', numTweets=50,translateToLocalLanguage= False, numDays = 2)\n",
    "cityDataMain\n",
    "#print(tweetsUS.groupby('location').mean())\n",
    "#print(tweetsWorld.groupby('location').mean())\n",
    "\n",
    "\n",
    "                                                               #locations are not required inputs\n",
    "#tweets = SearchForData(search_term='baguettes', nTweets=100, cityCountry='paris,france',radius=100)\n",
    "#tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAegAAAEWCAYAAACtyARlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XucVXW9//HXe4aLN0QkRQ+oeJRUvKEiYkV5CyEhNDExUjOT1EOdU4eOdipC6tjR7Fgd/Fl41yJvHGtKFC28lLfAxAuigaIyYqKCCF6B+fz+WN/BzWYPs0f3zKw9vJ8+1oO1vuv7/a7v2s5jf/b3u75rLUUEZmZmli817d0AMzMz25ADtJmZWQ45QJuZmeWQA7SZmVkOOUCbmZnlkAO0mZlZDjlAm1m7k3ScpMWSVkk6QNIekh6RtFLS19u7fWbtQb4P2swqSdIkYPeI+GILyjwDfDMifpe2rwDeiIhvtE4rzfLPPWgzy4NdgHkb2Tbb5LgHbWYfmKRzgK8DWwNLgG8CtwAC3gWeiYj9JT0HfCUi/pjKTQJ2B04HXgO2BN4C/gE8D3wKWA2sAQ6MiL+33VmZ5UOn9m6AmVUnSXsA44GDI2KJpL5ALXA+ZQ5xR8S7wFaSAtg/Ihamuu8GfhURl7dS881yzwHazD6otUBXoL+kVyLiOQBJ7doos47C16DN7ANJvd1/AyYBSyVdL+mf2rdVZh2HA7SZfWARMS0iPkE2qSuAC9K/xd4EtijY3qENmmdW1RygzewDSfcqHyGpK/AO8DbZsPfLQF9Jhd8vc4ExkjpLGgiMbvsWm1UXB2gz+6C6Av8NvEo2+3p74D+Bm9L+1yT9La1/D9gNWA6cB0xr26aaVR/fZmVmZpZD7kGbmZnlkAO0mZlZDjlAm5mZ5ZADtJmZWQ75SWLWKqZpD88+tA2MPbNHezfBciguffBDP35OZw0u+zunEsdrC+5Bm1mbcHA2axn3oM3MrOqppio6xS3iAG1mZlWvplPHGxB2gDYzs6rnHrSZmVkOdcTXnDpAm5lZ1XMP2szMLIccoM3MzHLIAdrMzCyHPIvbzMwsh9yDNjMzyyEHaDMzsxzybVZmZmY55B60mZlZDnmSmJmZWQ65B21mZpZDDtBmZmY55ABtZmaWQx0xQHe8q+pmZrbJUY3KXpqtSxom6WlJCyWdW2L/JyX9TdIaSaOL9p0qaUFaTi1IvzvVOTct2zfXDvegzcys6tXUVqa/KakWuAT4NFAPzJZUFxFPFmR7AfgSMKGo7LbA94GBQAAPp7LLU5axETGn3La4B21mZlWvgj3oQcDCiHg2It4DrgdGFWaIiOci4jGgoajs0cCdEbEsBeU7gWEf9JwcoM3MrOq1JEBLGidpTsEyrqCq3sDigu36lFaO5spelYa3v6cyHn3mIW4zM6t6NS3obkbEVGBqE7tLBc4os+qNlR0bES9K6gZMB04Grt1YZe5Bm5lZ1auVyl6aUQ/sVLDdB1hSZjOaLBsRL6Z/VwLTyIbSN8oB2szMql6X2pqyl2bMBvpJ2lVSF2AMUFdmM2YCQyX1kNQDGArMlNRJ0kcAJHUGRgBPNFeZh7jNzKzqVWgSNxGxRtJ4smBbC1wZEfMkTQbmRESdpIOBW4AewEhJ50XE3hGxTNIPyII8wOSUtiVZoO6c6vwjcFlzbXGANjOzqlfG0HXZImIGMKMobWLB+myy4etSZa8ErixKexM4qKXtcIA2M7OqV8kAnRcO0GZmVvVqO+CjPh2gzcys6tV2vPjsAG1mZtWvjNnZVccB2szMqp6HuM3MzHLIQ9xmZmY55B60mZlZDvk2KzMzsxxygDYzM8uhLh3wIrQDtJmZVT1fgzYzM8shD3GbmZnlUAd8TokDtJmZVT/3oM3MzHLIj/o0MzPLoQ4Ynx2gzcys+nmI28zMLIc64G3QDtBmZlb9atyDNjMzyx/3oM3MzHKosyeJmZmZ5U+NH/VpZmaWPx1xiLsDDgqYtZ4djx7CiKduZ+SCO+h/zhkb7K/p0pmPX38xIxfcwdAHb2TLXXoDsMNRH2PYnOl85rE6hs2ZTq/DB68rc+Rd1zLiqdsZ/shvGf7Ib+m63bZtdj5WeUf3H8xTk25gwXk3cc7QkzfYP2T3ATz87WtYPeUvHH/A4evtu238xSz/yZ38/uyL2qq5HUaNyl+qRasFaEkh6ScF2xMkTapg/X0lPVGp+lqDpGGS/irpKUlzJd0gaedmygyQ9JmC7cMkfexDtCH3n1O1UE0NAy+ZyF3Dv8Kt/Y9hl5NGsPVeu62XZ7fTT+C95W/w+35DefriqxlwwQQA3n11OfeMPIsZ+32WB049l0Ovu3C9cvePncBtBxzLbQccy7uvLGuzc7LKqlENl4yZwPAp36D/5JM46eCh7LVD3/XyvLDsZb507Q+YNvuODcr/+M5fc/LV57VRazuWWpW/VIvW7EG/C3xO0kda8Rhlk1TbxsfbB/hf4NSI2DMiBgC/Bvo2U3QA8JmC7cOADxygPyxJvgyS9By0H6sWPs+bi+ppWL2a56+/lT6jjlwvT59RR7DomlsAeOHmmfQ68lAAls+dz9svLQVgxbwF1G7WhZoundv2BKzVDerbn4Wv1LPo1SWsXruG6+fcyaj9P7lenueXvcTjLy6kIWKD8rOensPKd95qq+Z2KDVS2Uu1aM0AvQaYCnyjeIek7SRNlzQ7LR9P6Y9L2kaZ1ySdktKvk3RUUweSVCvpx6muxyR9NaUfJukuSdOAxyVtKelWSY9KekLSiSnfQZLukfSwpJmSdpS0m6S/FRyjn6SHW3D+5wDnR8T8xoSIqIuIe1N9d0samNY/Iuk5SV2AycCJqcd9DnAm8I20PWQjn90kSVemep+V9PWCtnSSdE36bG6WtEVT513QtvMl3QP8a/osHkzHmyxpVQs+hw5j8969eHPxP9Ztv1X/Mlv07lUiz0sAxNq1rF6xkq49e6yXZ6fjj2b5I/NpeG/1urTBV53P8Ed+yz7fPbsVz8BaW+9ttmPx8qXrtuuXL6X3Ntu1Y4s2HZ1ryl+qRWv3ji4BHpN0YVH6z4CLI+Ivach3JrAXcB/wceB54FlgCHAtMBg4ayPHOR1YEREHS+oK3CepcfxoELBPRCySdDywJCKOAZDUXVJnsp7uqIh4JQXt/4qIL0taIWlARMwFTgOubsG57w206EJSRLwnaSIwMCLGpzZuDqyKiIvS9jRKf3YAewKHA92ApyVdmtL3AE6PiPskXQmcLelnpc4b+HIqs01EfCod8w/AzyLiN5LObKr9ksYB4wBOZ3uOYJuWnH7+lfjlHcW9oGbydO+/OwMumMBdQ7+8Lu3+sRN4e8lSOm21JUOm/5xdTx7Fout+V7l2W5tRyf//7dCQTVBHfNRnq/6WiIg3yALs14t2HQVMkTQXqAO2ltQN+DPwybRcCuwrqTewLCI21msbCpyS6nsI6An0S/v+GhGL0vrjwFGSLpA0JCJWkAWvfYA7U/nvAn1S/suB09Lw+InAtA/yOUjqmXrAf5c04YPUUaCpzw7g1oh4NyJeBZYCjd27xRFxX1r/FfAJNn7eADcUrB8K3JTWm/wMImJqRAyMiIEdLjgDb9f/gy132mHd9hZ9evH2kqUl8uwIgGpr6dy9G+8tex3IetdDbpnCA6ecw6pnF79fJtWxZtWbPDftD/QctF9rn4q1kvrlS9mpx/brtvv02J4lK15pxxZtOio5SUzZ/KGnJS2UdG6J/V2VzSlaKOkhSX1TehdJV6XR4EclHVZQ5qCUvlDSz1Xq11zxObXg/D+on5L1cLcsOu6hETEgLb0jYiVwL1mveQhwN/AKMJoscG+MgK8V1LdrRDT2oN9szBQRfwcOIgvUP0q9VQHzCsruGxFDU5HpwHBgBPBwRLzWgvOeBxyYjvtaugY9Fdgq7V/D+5//Zi2ot6nPDrLr/o3W8v4ISfFv+GDj5w0Fn5tlXpv9ON369WXLvn2o6dyZXcYcw4t1s9bLU183i11PPQ6AnUcfzcuzHgSgc/duHHbrVB799v/w6v3rrpyg2tp1Q+Dq1IneIw7j9ScWtNEZWaXNfn4+/bbfib49d6RzbSfGDPw0dY819/VllVCpSWKpQ3YJ2Xd/f+AkSf2Lsp0OLI+I3YGLgQtS+hkAEbEv8GngJ5Iav+cvJRth7JeWYc2dU6sH6IhYBtxIdkKN7gDGN25IGpDyLgY+AvSLiGeBvwATaD5AzwTOSsPVSPqopC2LM0n6J+CtiPgV2fDzgcDTwHaSDk15OkvaO7XnnVT3pcBVpQ4sabyk8SV2XQh8R9JeBWlbFKw/R/ZjAbIfIY1Wkg1RN7Vd8rNrxs6N5wecRPa5NnneJTwIHJ/Wx5RxvA4p1q5lzvjJHD7zco6ZP4MXbryNFU8uZN/zvk7vkUcA8MwVN9O15zaMXHAHe37zNOaem13l+Oj4L9Jt953Z53tnr3c7VU3XLhw+83KGP1rH8Lm/5e0Xl/LMZTe252nah7C2YS3jr7+ImV/7GfO/fz03PvwnnnxpEeeNOIOR+w0BYOAue7H4/DpOOPAIfvmFc3nie+8PSt3777/gpjP+iyP3GMji8+sYutch7XUqVaempvylGYOAhRHxbES8B1wPjCrKMwq4Jq3fDByZesT9gT8BRMRS4HVgYJrfs3VEPBDZNa9rgWOba4g2uIZWIZJWRcRWab0XsAi4MCImKZvZfQnZtdNOwL0RcWbKex1QGxFfUHZ70V+A7Yp7r5J2B26IiIPSL5QfAiPJeoavkJ38AcCEiBiRyhwN/BhoAFYDZ0XEnBTkfg50T+35aURclsoMJutJ7xwRa0uc5xTgvoj4TYl9xwCTyALsa8ALwPcj4u+S9iT74bIKmAV8MSL6StqW7EdBZ+BHwCNkfwANwNeA+aU+O2W3sBVeq36CrOcPMINsdOJjwALg5Ih4q6nzlnR3+tzmpLr6kQ2NC7gVGBcRvYvPt9A07eErb7aesWf2aD6TbZLi0gc/9AXkKY+dVfZ3ztf2/8VXSfNlkqkRMRVA0mhgWER8JW2fDBzSOC8opT2R8tSn7WeAQ4DPkfWcTwJ2Ivv+Pp1sXtV/R8RRKf8Q4JzG2NSUVpsk1hic0/rLFPQe0zXSE5sod3LB+v003cvfG3gm5WsA/jMthe5OS2N9M8mCX/Ex55Jd9y7lE8CVpYJz0hf4ZqkdEXErWUArte8poPBi43dT+jLg4KLsxRclN/jsImJS0fY+BZvFwzONeUqed0QcVpT0IjA4IkLSGGBOqfrMzNpLS+5vTsF4ahO7S9VUHPybynMlWedpDllQvp/scmY5dW6gKu9xlTSZbIjhS618nFuA3YAjmsrT3C+gDuIgsolpIhuy+XIz+c3M2lQFnxBWT9b7bdQHWNJEnnplz4roTjaZOSi4tVjS/WSjlstZfxJuqTo3UJUBOiImAhPb4DjHtfYxqkFE/BnYv73bYWbWlAreZjUb6CdpV7LRwzHAF4ry1AGnAg+QzSGalUYYtyC7dPympE8DayLiSQBJK9Ml04eAU8huc92oqgzQZmZmhSrVg46INWni70ygluwS57w0cjsnIuqAK4DrJC0ElvH+5NntgZmSGsiCe+HD2M8ie5bG5sBtadkoB2gzM6t6lXzGdkTMIJtcW5g2sWD9HeCEEuWeI3vGRKk655A9e6JsDtBmZlb1OlfTa6rK5ABtZmZVr5peglEuB2gzM6t6DtBmZmY55ABtZmaWQzVqi1dLtC0HaDMzq3ruQZuZmeVQpzLeglFtHKDNzKzq1bTJ25PblgO0mZlVPQ9xm5mZ5ZADtJmZWQ55FreZmVkOeZKYmZlZDnmI28zMLIc8xG1mZpZDNbgHbWZmljse4jYzM8shD3GbmZnlkGdxm5mZ5ZB70GZmZjnka9BmZmY55FncZmZmOeQetJmZWQ51qqlt7yZUnAO0mZlVPXmSmJmZWf7U4ABtZmaWO+5Bm5mZ5VBHvA+6452RmZltckRN2UuzdUnDJD0taaGkc0vs7yrphrT/IUl9U/pYSXMLlgZJA9K+u1Odjfu2b64dZfWgJX0U+BawS2GZiDiinPJmZmatqVaVGRCWVAtcAnwaqAdmS6qLiCcLsp0OLI+I3SWNAS4AToyIXwO/TvXsC/wuIuYWlBsbEXPKbUu5Z3QT8AvgMmBtuZWbmZm1hQoOcQ8CFkbEswCSrgdGAYUBehQwKa3fDEyRpIiIgjwnAb/5MA0pN0CviYhLP8yBzMzMWksFJ4n1BhYXbNcDhzSVJyLWSFoB9AReLchzIlkgL3SVpLXAdOCHRQF9A+We0e8lnS1pR0nbNi5lljUzM2tVNS34T9I4SXMKlnEFVZV6JFlxIN1oHkmHAG9FxBMF+8dGxL7AkLSc3Nw5lduDPjX9+62ixvxzmeXNzMxaTUt60BExFZjaxO56YKeC7T7Akiby1EvqBHQHlhXsH0PR8HZEvJj+XSlpGtlQ+rUba2dZAToidi0nn5mZWXuo4DXo2UA/SbsCL5IF2y8U5akj67g+AIwGZjUOVyv7pXAC8MnGzCmIbxMRr0rqDIwA/thcQ8qdxd0ZOKvggHcDv4yI1eWUNzMza0216lyRetI15fHATKAWuDIi5kmaDMyJiDrgCuA6SQvJes5jCqr4JFDfOMks6QrMTLG0liw4X9ZcW8od4r4U6Az8v7R9ckr7SpnlzczMWk059zeXKyJmADOK0iYWrL9D1ksuVfZuYHBR2pvAQS1tR7kB+uCI2L9ge5akR1t6MDMzs9bQEZ8kVm6AXitpt4h4BkDSP+P7oc3MLCc25Wdxfwu4S9KzZNPLdwFOa7VWmZmZtcAm+zariPiTpH7AHmQB+qmIeLdVW2ZmZlam2pqO9+6njZ6RpCMiYpakzxXt2k0SEfF/rdg2MzOzslRyklheNPeT41PALGBkiX0BOECbmVm72+QmiUXE99Pq5IhYVLgv3cRtZmbW7jpiD7rcM5peIu3mSjbEzMzsg6pRTdlLtWjuGvSewN5A96Lr0FsDm7Vmw8ysY/n1L5Yz9swe7d0M66A2xdus9iB7Zug2rH8deiVwRms1ysw6Hgdna01qaCg/c5XE8uauQf8O+J2kQyPigTZqk5mZWctECwJ0lSj3xrFHJP0L2XD3uqHtiPhyq7TKzMysJTpggC63o38dsANwNHAP2fsxV7ZWo8zMzFokGspfqkS5AXr3iPge8GZEXAMcA+zbes0yMzNrgYaG8pcqUe4Qd+N7n1+XtA/wD6Bvq7TIzMyspaqoZ1yucgP0VEk9gO8CdcBWwMSNFzEzM2sjDWvauwUVV+7LMi5Pq/cC/9x6zTEzM/sAqmjoulxlXYOWdL6kbQq2e0j6Yes1y8zMrAU24UliwyPi9caNiFgOfKZ1mmRmZtZCHTBAl3sNulZS18Z3QEvaHOjaes0yMzNrgSoKvOUqN0D/CviTpKvIXjP5ZeCaVmuVmZlZC8Ta1c1nStSK7aikcieJXSjpMeColPSDiJjZes0yMzNrgU24Bw3wCNCZrAf9SOs0x8zM7APogAG63Fncnwf+CowGPg88JGl0azbMzMysbJvwJLHvAAdHxFIASdsBfwRubq2GmZmZla2KAm+5yg3QNY3BOXmNqnmjppmZdXgd8EEl5Qbo2yXNBH6Ttk8EZrROk8zMzFpoE37U57ckHQ98nGyG+tSIuKVVW2ZmZlauDjjEXfYwdURMj4hvRsQ3HJzNzCxXKvi6SUnDJD0taaGkc0vs7yrphrT/IUl9C/btJ+kBSfMkPS5ps5R+UNpeKOnnkpq9HXujAVrSSklvlFhWSnqj2bM0MzNrCxWaxS2pFrgEGA70B06S1L8o2+nA8ojYHbgYuCCV7UT2YK8zI2Jv4DDef13zpcA4oF9ahjV3ShsN0BHRLSK2LrF0i4itm6vczMysTVTuNqtBwMKIeDYi3gOuB0YV5RnF+0/TvBk4MvWIhwKPRcSjABHxWkSslbQjsHVEPBARAVwLHNtcQzwT28zMql8LhrgljZM0p2AZV1BTb2BxwXZ9SqNUnohYA6wAegIfBULSTEl/k/QfBfnrm6lzAy15kpiZmVk+rVlbdtaImApMbWJ3qWvDUWaeTsAngIOBt8jeYfEwUOqScHGdG3AP2szMql/lJonVAzsVbPcBljSVJ1137g4sS+n3RMSrEfEW2e3IB6b0Ps3UuQEHaDMzq34NUf6ycbOBfpJ2ldQFGAPUFeWpA05N66OBWena8kxgP0lbpMD9KeDJiHgJWClpcLpWfQrwu+Ya4iFuMzOrfhV6klhErJE0nizY1gJXRsQ8SZOBORFRB1wBXCdpIVnPeUwqu1zS/5AF+QBmRMStqeqzgKuBzYHb0rJRDtBmZlb9Kvioz4iYQdHTMiNiYsH6O8AJTZT9FdmtVsXpc4B9WtIOB2gzM6t+LZgkVi0coM3MrPo1f2256jhAm5lZ9duE32ZlZmaWXw7QZmZm+ZPd5VSeZt9SkRMO0GZmVv3cgzYzM8shz+I2MzPLIfegzczMcsgB2szMLId8H7SZmVkOuQdtZmaWQ54kZmZmlkPuQZuZmeWQA7SZmVkOeZKYmZlZDrkHbWZmlj+x1j1oMzOz/FntHrSZmVnuhK9Bm5mZ5ZCHuM3MzHJobccb4q5p7waYVZMdjx7CiKduZ+SCO+h/zhkb7K/p0pmPX38xIxfcwdAHb2TLXXoDsMNRH2PYnOl85rE6hs2ZTq/DB68rc+Rd1zLiqdsZ/shvGf7Ib+m63bZtdj5WeUf3H8xTk25gwXk3cc7QkzfYP2T3ATz87WtYPeUvHH/A4evtu238xSz/yZ38/uyL2qq5HUY0RNlLtWjzAC0pJP2kYHuCpEkVrL+vpCcqVV+lSZokaUJR2nOSPtJMucmSjqpQGwZJulfS05KeknS5pC0kfVbSuSnPsZL6V+J4HYVqahh4yUTuGv4Vbu1/DLucNIKt99ptvTy7nX4C7y1/g9/3G8rTF1/NgAuy/9Xvvrqce0aexYz9PssDp57LoddduF65+8dO4LYDjuW2A47l3VeWtdk5WWXVqIZLxkxg+JRv0H/ySZx08FD22qHvenleWPYyX7r2B0ybfccG5X985685+erz2qi1HczaKH+pEu3Rg34X+FxzAamtSKpt7zaUIyImRsQfP2w9knoBNwHnRMQewF7A7UC3iKiLiP9OWY8FHKAL9By0H6sWPs+bi+ppWL2a56+/lT6jjlwvT59RR7DomlsAeOHmmfQ68lAAls+dz9svLQVgxbwF1G7WhZoundv2BKzVDerbn4Wv1LPo1SWsXruG6+fcyaj9P7lenueXvcTjLy6kITYMFLOensPKd95qq+Z2KLG6oeylWrRHgF4DTAW+UbxD0naSpkuanZaPp/THJW2jzGuSTknp122sVympVtKPU12PSfpqSj9M0l2SpgGPS9pS0q2SHpX0hKQTU76DJN0j6WFJMyXtKGk3SX8rOEY/SQ9X4oNJvf/5ki6TNE/SHZI2T/uuljQ6rR8p6ZH0uVwpqWtKf07SeZL+lvbtWeIw/wJcExEPAETm5oh4WdKXJE2R9DHgs8CPJc1tzXOuJpv37sWbi/+xbvut+pfZonevEnleAiDWrmX1ipV07dljvTw7HX80yx+ZT8N7q9elDb7qfIY/8lv2+e7ZrXgG1tp6b7Mdi5cvXbddv3wpvbfZrh1btAlpaCh/qRLtdQ36EmCspO5F6T8DLo6Ig4HjgctT+n3Ax4G9gWeBISl9MPDgRo5zOrAi1XcwcIakXdO+QcB3IqI/MAxYEhH7R8Q+wO2SOgP/C4yOiIOAK4H/iohngBWSBqR6TgOubvEn0LR+wCURsTfwOtnnsI6kzdLxToyIfckm+p1VkOXViDgQuBRYbyg92QfYaHCNiPuBOuBbETGg3HOWNE7SHElzZvF6sydadaQNkqK4F9RMnu79d2fABRP461cnrku7f+wEZuz3We4cMpbthhzEriePqlybrU2p5P//dmjIpshD3JUREW8A1wJfL9p1FDBF0lyyALG1pG7An4FPpuVSYF9JvYFlEbFqI4caCpyS6nsI6EkWAAH+GhGL0vrjwFGSLpA0JCJWAHuQBbM7U/nvAn1S/suB09Lw+InAtJacfjPpiyJiblp/GOhblG+PlOfvafsass+l0f9tpOyH0ew5R8TUiBgYEQOPYJsKHjof3q7/B1vutMO67S369OLtJUtL5NkRANXW0rl7N95blv1Y2bx3L4bcMoUHTjmHVc8ufr9MqmPNqjd5btof6Dlov9Y+FWsl9cuXslOP7ddt9+mxPUtWvNKOLdp0eJJYZf2UrIe7ZUFaDXBo6rUNiIjeEbESuJes1zwEuBt4BRhNFrg3RsDXCurbNSIaZ2a82ZgpBbuDyAL1jyRNTGXnFZTdNyKGpiLTgeHACODhiHitBef9GtCjKK0brOtyvluQvpYNb4Xb8Cf6+hrLlyoLMI/sXFvqw5xzh/Da7Mfp1q8vW/btQ03nzuwy5hherJu1Xp76ulnseupxAOw8+mhenpUN8HTu3o3Dbp3Ko9/+H169f93VAlRbu24IXJ060XvEYbz+xII2OiOrtNnPz6ff9jvRt+eOdK7txJiBn6busea+pqwi3IOunIhYBtxIFqQb3QGMb9xoHFKNiMXAR4B+EfEs8Bey4dvm/vJnAmel4WokfVTSlsWZJP0T8FZE/Aq4CDgQeBrYTtKhKU9nSXun9ryT6r4UuKrUgSWNlzS+xK57gc+mkQEkfQ54NCLKfdv4U0BfSbun7ZOBe8osCzAFOFXSIQVt/aKkHYryrST74QCUd84dXaxdy5zxkzl85uUcM38GL9x4GyueXMi+532d3iOPAOCZK26ma89tGLngDvb85mnMPTe7Xeaj479It913Zp/vnb3e7VQ1Xbtw+MzLGf5oHcPn/pa3X1zKM5fd2J6naR/C2oa1jL/+ImZ+7WfM//713Pjwn3jypUWcN+IMRu6XXZkbuMteLD6/jhMOPIJffuFcnvje+4NR9/77L7jpjP/iyD0Gsvj8OobudUhTh7IilZwkJmlYustlYeOdLUX7u0q6Ie1/SFLfov07S1qlgjt20hyhx9O8njnlnJM2uIbWyiStioit0novYBFwYURMSjO7LyGbWdxcXTUIAAAK+klEQVQJuDcizkx5rwNqI+ILaRLTX4DtintyKXDdEBEHSaoBfgiMJOt5vkI2O/kAYEJEjEhljgZ+DDQAq4GzImJO+oHwc6B7as9PI+KyVGYwWa9y51LBVdIU4L6I+E2JfV8FziYb1l4KnBkRz6b/yX9I18FJ/3O3Sp/N1WnfzZKOJPsh0QmYndr7rqTngIER8aqkgcBFEXFYieMfClwIbJ/O+V6ySXufT+XHK5ugdxlZj3x0RDzT3DkXmqY9qudnqrWJsWcWDxyZZeLSB5sbGWzWO5NHlv2ds9nE3zd5vHQZ7+/Ap4F6su/YkyLiyYI8ZwP7RcSZksYAx0XEiQX7p5N9tz4UEReltOdI38/ltrPNA3RrkzQKGBsRn2/l40wAukfE95rY/wfgcxHxXmu2oy01d86FHKCtmAO0NaUSAfrtSSPK/s7ZfNIfNhagDwUmRcTRafvbABHxo4I8M1OeByR1Av5B1mEMSceSTWp+E1j1YQJ0h3rUp6TJwCjgS618nFuA3YAjmsrT2DvvKMo5ZzOzdtOCa8uSxgHjCpKmRsTUtN4bWFywrx4ovtawLk9ErJG0Augp6W3gHLLed/FdNAHcISmAXxYcr0kdKkBHxERgYrMZP/xxjmvtY+TNpnjOZlZFWjA7OwXHpgJkqd51ceVN5TmP7FbhVSVuuft4RCyRtD3Z3UFPRcS9G2tnhwrQZma2aYrKzc6uB3Yq2O4DLGkiT30a4u4OLCPraY+WdCGwDdAg6Z2ImBIRSwAiYmkakRxENv+nSQ7QZmZW/VaXeyNMs2YD/dJDrV4ExgBfKMpTB5wKPEB2y++syCZ0NT5EC2XvmFgVEVPS3UM1EbEyrQ8FJjfXEAdoMzOrepV6AEm6pjye7LbSWuDKiJiX5jjNiYg64ArgOkkLyXrOY5qpthdwSxr27gRMi4jbm2uLA7SZmVW/Cj6AJCJmADOK0iYWrL8DnNBMHZMK1p8F9m9pOxygzcys6lXTIzzL5QBtZmZVr4KTxHLDAdrMzKqee9BmZmY5tLaMZ2xXGwdoMzOretHgAG1mZpY7vgZtZmaWQ74GbWZmlkPuQZuZmeXQ2jW+Bm1mZpY77kGbmZnlkK9Bm5mZ5ZADtJmZWQ55iNvMzCyH/KASMzOzHGrwoz7NzMzyp8HXoM3MzPLH16DNzMxyyLO4zczMcsg9aDMzsxxyD9rMzCyHPIvbzMwshzyL28zMLIc64HNKHKDNzKz6OUCbmZnlkAO0mZlZDq1Z294tqDwHaDMzq3odsQdd094NMDMz+7AaGspfmiNpmKSnJS2UdG6J/V0l3ZD2PySpb0ofJGluWh6VdFy5dZbiAG1mZlWvUgFaUi1wCTAc6A+cJKl/UbbTgeURsTtwMXBBSn8CGBgRA4BhwC8ldSqzzg04QJuZWdWrYA96ELAwIp6NiPeA64FRRXlGAdek9ZuBIyUpIt6KiDUpfTOg8ebscurcgAO0mZlVvZYEaEnjJM0pWMYVVNUbWFywXZ/SKJUnBeQVQE8ASYdImgc8DpyZ9pdT5wY8SczMzKremjXN52kUEVOBqU3sVqki5eaJiIeAvSXtBVwj6bYy69yAe9BmZlb1IqLspRn1wE4F232AJU3lkdQJ6A4sK2rPfOBNYJ8y69yAA7SZmVW9Cl6Dng30k7SrpC7AGKCuKE8dcGpaHw3MiohIZToBSNoF2AN4rsw6N+AhbjMzq3qVug86ItZIGg/MBGqBKyNinqTJwJyIqAOuAK6TtJCs5zwmFf8EcK6k1UADcHZEvApQqs7m2uIAbWZmVa+SDyqJiBnAjKK0iQXr7wAnlCh3HXBduXU2xwHazMyqXkd8kpgDtJmZVb2WzOKuFg7QZmZW9dyDNjMzy6GGZu+eqj4O0GZmVvXcgzYzM8uhjhig/aASM2sTv/7F8vZugnVga9aUv1QLlfHYMzP7ECSNS8/+NVvHfxfWHPegzVrfuOaz2CbIfxe2UQ7QZmZmOeQAbWZmlkMO0Gatz9cZrRT/XdhGeZKYmZlZDrkHbWZmlkMO0GZmZjnkAG2bHEk7SbpL0nxJ8yT9a0rfVtKdkhakf3uk9D0lPSDpXUkTiur6RqrjCUm/kbRZE8c8NdW7QNKpBeldJE2V9HdJT0k6vonyt0t6NB3rF5JqN9Zma7m8/F1I6iZpbsHyqqSfNlH+IEmPS1oo6eeSlNJPSMdvkDSwkp+TtR0HaNsUrQH+PSL2AgYD/yKpP3Au8KeI6Af8KW0DLAO+DlxUWImk3il9YETsA9QCY4oPJmlb4PvAIcAg4PsFgfQ7wNKI+CjQH7iniTZ/PiL2B/YBtuP9l8U31WZruVz8XUTEyogY0LgAzwP/10SbLyW7n7pfWoal9CeAzwH3tvxjsLxwgLZNTkS8FBF/S+srgflAb2AUcE3Kdg1wbMqzNCJmA6tLVNcJ2FxSJ2ALYEmJPEcDd0bEsohYDtzJ+1+kXwZ+lI7TEBGvNtHmNwqO1wVonN1Zss3Wcjn7uwBAUj9ge+DPxYUl7QhsHREPRDbb99qCts2PiKdbcv6WPw7QtkmT1Bc4AHgI6BURL0H2ZU32xdikiHiRrPf0AvASsCIi7iiRtTewuGC7HugtaZu0/QNJf5N0k6ReG2nrTGApsBK4OSW3qM1Wnvb8uyjKcxJwQ5S+3aZ3KrOx8lbFHKBtkyVpK2A68G8FPdSWlO9B1rvaFfgnYEtJXyyVtURakPWy+gD3RcSBwAMUDZeuVyDiaGBHoCtwREvba+XJwd9FoTHAb5o6VBnlrYo5QNsmSVJnsi/hX0dE4/W9l9OwYePw4dJmqjkKWBQRr0TEarLrhB+TdEjBBJ/PkvVsdioo14dsyPM14C3glpR+E3CgpNqC8pMLDxgR7wB1ZAHgg7TZNiInfxeNbdkf6BQRD6ft4r+L+lSmZHmrfn4ftG1y0kzXK4D5EfE/BbvqgFOB/07//q6Zql4ABkvaAngbOBKYExEPAQMKjrctcH7BxLChwLcjIiT9HjgMmJXKPxkRa4vKbwV0i4iX0jXNz/D+NcmWttmakJe/i4J6TqKg91z8d5HqWClpMNlQ/CnA/5Z3tlYVIsKLl01qAT5BNhT4GDA3LZ8BepLN0l2Q/t025d+BrLfyBvB6Wt867TsPeIps1ux1QNcmjvllYGFaTitI34Vspu1j6Zg7lyjbC5id8swj+xLulPaVbLOX6v67SPueBfZsps0D0zGeAabw/tMhj0vteRd4GZjZ3p+vl5YvftSnmZlZDvkatJmZWQ45QJuZmeWQA7SZmVkOOUCbmZnlkAO0mZlZDjlAm1mrkDSp+C1PZlY+B2gzM7MccoA2s4qR9B1JT0v6I7BHSjtD0uz0PuvpkrZI7zxelB6tiaStJT3XuG1mDtBmViGSDiJ7ucMBZO8iPjjt+r+IODiy91nPB06P7HWOdwPHpDxjgOmRPbvazHCANrPKGQLcEhFvRfYWqLqUvo+kP0t6HBgL7J3SLwdOS+unAVe1aWvNcs4B2swqqdSzg68GxkfEvmTPqN4MICLuA/pK+hRQGxFPtFkrzaqAA7SZVcq9wHGSNpfUDRiZ0rsBL6Xry2OLylxL9sYm957NivhlGWZWMZK+Q/baw+fJ3qb0JPAm8B8p7XGyV2d+KeXfAVgE7BgRr7dHm83yygHazNqNpNHAqIg4ub3bYpY3ndq7AWa2aZL0v8Bwsncum1kR96DNzMxyyJPEzMzMcsgB2szMLIccoM3MzHLIAdrMzCyHHKDNzMxy6P8D7KonPMDkJdcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11276ec50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from seaborn import heatmap\n",
    "#c_matrix = pd.pivot_table(news_data_new, \"score\", index=[\"name\"], columns=[\"year_month\"], aggfunc=np.mean)                          \n",
    "matrix=pd.pivot_table(cityDataMain,'vader',index=['location'],columns=['day'],aggfunc=np.mean)\n",
    "hm = heatmap(matrix, cmap='RdYlGn', annot=True)\n",
    "hm.set_title('stuff')\n",
    "plt.savefig('fig.png',dpi=100)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>followers</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>statuses_count</th>\n",
       "      <th>vader</th>\n",
       "      <th>average times</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>California , Albany</th>\n",
       "      <td>12664.475000</td>\n",
       "      <td>2494.085000</td>\n",
       "      <td>41845.640000</td>\n",
       "      <td>-0.258858</td>\n",
       "      <td>2931.362055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Alhambra</th>\n",
       "      <td>1807.570000</td>\n",
       "      <td>2007.625000</td>\n",
       "      <td>36424.005000</td>\n",
       "      <td>0.005641</td>\n",
       "      <td>2699.818191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Artesia</th>\n",
       "      <td>1704.073892</td>\n",
       "      <td>1966.793103</td>\n",
       "      <td>24957.950739</td>\n",
       "      <td>-0.008311</td>\n",
       "      <td>2985.501343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Baldwin Park</th>\n",
       "      <td>1406.316038</td>\n",
       "      <td>1314.174528</td>\n",
       "      <td>28348.037736</td>\n",
       "      <td>-0.005242</td>\n",
       "      <td>2573.503197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Bell</th>\n",
       "      <td>12797.990521</td>\n",
       "      <td>13001.018957</td>\n",
       "      <td>70891.734597</td>\n",
       "      <td>-0.126618</td>\n",
       "      <td>2017.529214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Bell Gardens</th>\n",
       "      <td>3089.039409</td>\n",
       "      <td>1426.871921</td>\n",
       "      <td>26007.197044</td>\n",
       "      <td>-0.085866</td>\n",
       "      <td>1843.151112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Bellflower</th>\n",
       "      <td>1451.950355</td>\n",
       "      <td>1787.907801</td>\n",
       "      <td>19833.773050</td>\n",
       "      <td>-0.000277</td>\n",
       "      <td>2529.536833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Berkeley</th>\n",
       "      <td>12640.465000</td>\n",
       "      <td>2150.355000</td>\n",
       "      <td>41316.020000</td>\n",
       "      <td>-0.067293</td>\n",
       "      <td>2826.648354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Cudahy</th>\n",
       "      <td>1064.834906</td>\n",
       "      <td>1354.113208</td>\n",
       "      <td>21860.179245</td>\n",
       "      <td>-0.039462</td>\n",
       "      <td>1423.611557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Daly City</th>\n",
       "      <td>2115.755000</td>\n",
       "      <td>1776.705000</td>\n",
       "      <td>48826.885000</td>\n",
       "      <td>0.061896</td>\n",
       "      <td>2218.263782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , East Palo Alto</th>\n",
       "      <td>6297.530000</td>\n",
       "      <td>2247.155000</td>\n",
       "      <td>79389.500000</td>\n",
       "      <td>-0.098829</td>\n",
       "      <td>2629.478149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , El Monte</th>\n",
       "      <td>1811.465000</td>\n",
       "      <td>2010.430000</td>\n",
       "      <td>36439.955000</td>\n",
       "      <td>0.010876</td>\n",
       "      <td>2503.617391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Hawaiian Gardens</th>\n",
       "      <td>1397.970443</td>\n",
       "      <td>1613.059113</td>\n",
       "      <td>24971.073892</td>\n",
       "      <td>-0.016217</td>\n",
       "      <td>1973.770853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Hawthorne</th>\n",
       "      <td>4428.975728</td>\n",
       "      <td>2427.165049</td>\n",
       "      <td>51210.941748</td>\n",
       "      <td>-0.063301</td>\n",
       "      <td>2125.824656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Hermosa Beach</th>\n",
       "      <td>4730.694175</td>\n",
       "      <td>2581.582524</td>\n",
       "      <td>53511.000000</td>\n",
       "      <td>-0.085617</td>\n",
       "      <td>2272.705411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Huntington Park</th>\n",
       "      <td>1811.325000</td>\n",
       "      <td>2010.325000</td>\n",
       "      <td>36437.375000</td>\n",
       "      <td>0.010876</td>\n",
       "      <td>1584.643151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Inglewood</th>\n",
       "      <td>1814.460000</td>\n",
       "      <td>1990.245000</td>\n",
       "      <td>36245.290000</td>\n",
       "      <td>0.007341</td>\n",
       "      <td>2466.212232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , La Puente</th>\n",
       "      <td>3491.522167</td>\n",
       "      <td>1928.679803</td>\n",
       "      <td>45418.852217</td>\n",
       "      <td>0.039831</td>\n",
       "      <td>2654.201018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Lawndale</th>\n",
       "      <td>4495.888350</td>\n",
       "      <td>2469.733010</td>\n",
       "      <td>51411.271845</td>\n",
       "      <td>-0.092004</td>\n",
       "      <td>1839.853008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Lomita</th>\n",
       "      <td>3796.769697</td>\n",
       "      <td>3548.636364</td>\n",
       "      <td>34504.745455</td>\n",
       "      <td>0.007445</td>\n",
       "      <td>2992.532309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Lynwood</th>\n",
       "      <td>1674.735043</td>\n",
       "      <td>2267.948718</td>\n",
       "      <td>26078.538462</td>\n",
       "      <td>-0.032242</td>\n",
       "      <td>2141.015484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Maywood</th>\n",
       "      <td>13274.779412</td>\n",
       "      <td>13488.235294</td>\n",
       "      <td>72953.416667</td>\n",
       "      <td>-0.113848</td>\n",
       "      <td>1330.726768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Norwalk</th>\n",
       "      <td>1203.611940</td>\n",
       "      <td>1526.373134</td>\n",
       "      <td>32477.417910</td>\n",
       "      <td>0.108944</td>\n",
       "      <td>2839.481233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Paramount</th>\n",
       "      <td>1094.943396</td>\n",
       "      <td>1326.801887</td>\n",
       "      <td>21402.113208</td>\n",
       "      <td>-0.024866</td>\n",
       "      <td>2836.494059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Redondo Beach</th>\n",
       "      <td>4699.174757</td>\n",
       "      <td>2556.689320</td>\n",
       "      <td>53883.388350</td>\n",
       "      <td>-0.090458</td>\n",
       "      <td>2931.346070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Rosemead</th>\n",
       "      <td>1088.295567</td>\n",
       "      <td>996.596059</td>\n",
       "      <td>29237.576355</td>\n",
       "      <td>0.001448</td>\n",
       "      <td>2972.901225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , San Francisco</th>\n",
       "      <td>15772.785000</td>\n",
       "      <td>4320.095000</td>\n",
       "      <td>41796.680000</td>\n",
       "      <td>-0.057079</td>\n",
       "      <td>1627.162527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , San Pablo</th>\n",
       "      <td>1098.335000</td>\n",
       "      <td>1197.360000</td>\n",
       "      <td>15893.565000</td>\n",
       "      <td>-0.046086</td>\n",
       "      <td>2988.199060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Santa Ana</th>\n",
       "      <td>1839.510000</td>\n",
       "      <td>2036.550000</td>\n",
       "      <td>36385.895000</td>\n",
       "      <td>0.022561</td>\n",
       "      <td>2440.801248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California , Santa Monica</th>\n",
       "      <td>5765.440000</td>\n",
       "      <td>3442.655000</td>\n",
       "      <td>37148.630000</td>\n",
       "      <td>-0.089412</td>\n",
       "      <td>2987.848356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Manorhaven</th>\n",
       "      <td>2652.230000</td>\n",
       "      <td>2801.655000</td>\n",
       "      <td>39507.910000</td>\n",
       "      <td>0.032755</td>\n",
       "      <td>2394.267055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Mineola</th>\n",
       "      <td>2264.443396</td>\n",
       "      <td>1940.485849</td>\n",
       "      <td>37675.655660</td>\n",
       "      <td>0.015373</td>\n",
       "      <td>2914.701342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Mount Vernon</th>\n",
       "      <td>2225.096774</td>\n",
       "      <td>2204.571429</td>\n",
       "      <td>61426.087558</td>\n",
       "      <td>-0.104092</td>\n",
       "      <td>1786.951522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , New Hyde Park</th>\n",
       "      <td>1286.940092</td>\n",
       "      <td>1214.041475</td>\n",
       "      <td>24458.700461</td>\n",
       "      <td>0.014142</td>\n",
       "      <td>2594.185620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , New Square</th>\n",
       "      <td>2742.664384</td>\n",
       "      <td>2221.164384</td>\n",
       "      <td>21160.965753</td>\n",
       "      <td>0.034963</td>\n",
       "      <td>4166.821427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , New York City</th>\n",
       "      <td>2585.385000</td>\n",
       "      <td>2283.145000</td>\n",
       "      <td>51117.290000</td>\n",
       "      <td>0.002459</td>\n",
       "      <td>1124.952725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Port Chester</th>\n",
       "      <td>2279.903382</td>\n",
       "      <td>1908.743961</td>\n",
       "      <td>30056.028986</td>\n",
       "      <td>-0.021083</td>\n",
       "      <td>2603.272186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , South Floral Park</th>\n",
       "      <td>1263.658879</td>\n",
       "      <td>1243.140187</td>\n",
       "      <td>25672.570093</td>\n",
       "      <td>0.053686</td>\n",
       "      <td>1899.280251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Spring Valley</th>\n",
       "      <td>1251.617021</td>\n",
       "      <td>758.333333</td>\n",
       "      <td>10494.382979</td>\n",
       "      <td>0.016277</td>\n",
       "      <td>4560.953536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Tuckahoe</th>\n",
       "      <td>1359.942708</td>\n",
       "      <td>1539.776042</td>\n",
       "      <td>41152.619792</td>\n",
       "      <td>-0.026152</td>\n",
       "      <td>3173.420892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Valley Stream</th>\n",
       "      <td>689.995536</td>\n",
       "      <td>935.986607</td>\n",
       "      <td>13672.504464</td>\n",
       "      <td>0.024497</td>\n",
       "      <td>2610.579963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Williston Park</th>\n",
       "      <td>2646.729858</td>\n",
       "      <td>2217.720379</td>\n",
       "      <td>41157.066351</td>\n",
       "      <td>0.029821</td>\n",
       "      <td>2632.755060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York , Yonkers</th>\n",
       "      <td>1958.825000</td>\n",
       "      <td>2035.710000</td>\n",
       "      <td>50962.235000</td>\n",
       "      <td>-0.052142</td>\n",
       "      <td>2801.821970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ohio , Lakewood</th>\n",
       "      <td>5662.483254</td>\n",
       "      <td>3746.578947</td>\n",
       "      <td>48987.440191</td>\n",
       "      <td>0.029762</td>\n",
       "      <td>2865.414756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oregon , Johnson City</th>\n",
       "      <td>1427.680556</td>\n",
       "      <td>2018.416667</td>\n",
       "      <td>22262.467593</td>\n",
       "      <td>-0.030084</td>\n",
       "      <td>3236.185849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Clifton Heights</th>\n",
       "      <td>2299.608911</td>\n",
       "      <td>1325.628713</td>\n",
       "      <td>20412.386139</td>\n",
       "      <td>-0.037181</td>\n",
       "      <td>2796.794799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Collingdale</th>\n",
       "      <td>2333.838095</td>\n",
       "      <td>1373.247619</td>\n",
       "      <td>20202.452381</td>\n",
       "      <td>-0.022391</td>\n",
       "      <td>2960.272453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Conshohocken</th>\n",
       "      <td>1437.810000</td>\n",
       "      <td>1331.070000</td>\n",
       "      <td>17869.415000</td>\n",
       "      <td>-0.063159</td>\n",
       "      <td>2382.983574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Darby</th>\n",
       "      <td>2304.825871</td>\n",
       "      <td>1346.845771</td>\n",
       "      <td>19152.810945</td>\n",
       "      <td>-0.007141</td>\n",
       "      <td>2430.219019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Dormont</th>\n",
       "      <td>3147.295000</td>\n",
       "      <td>2511.865000</td>\n",
       "      <td>47464.645000</td>\n",
       "      <td>-0.074023</td>\n",
       "      <td>2422.238308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , East Lansdowne</th>\n",
       "      <td>3960.665000</td>\n",
       "      <td>2615.005000</td>\n",
       "      <td>40236.130000</td>\n",
       "      <td>-0.067131</td>\n",
       "      <td>2429.351333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Millbourne</th>\n",
       "      <td>3966.510000</td>\n",
       "      <td>2622.845000</td>\n",
       "      <td>40909.590000</td>\n",
       "      <td>-0.067074</td>\n",
       "      <td>1836.653052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Mount Oliver</th>\n",
       "      <td>3145.570000</td>\n",
       "      <td>2512.100000</td>\n",
       "      <td>47397.145000</td>\n",
       "      <td>-0.074023</td>\n",
       "      <td>2596.527861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Parkside</th>\n",
       "      <td>750.563725</td>\n",
       "      <td>1074.524510</td>\n",
       "      <td>21153.622549</td>\n",
       "      <td>0.027853</td>\n",
       "      <td>3097.330532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Pennsbury Village</th>\n",
       "      <td>5045.905660</td>\n",
       "      <td>1943.641509</td>\n",
       "      <td>39840.905660</td>\n",
       "      <td>0.033181</td>\n",
       "      <td>6664.431451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Philadelphia</th>\n",
       "      <td>2840.635000</td>\n",
       "      <td>2362.830000</td>\n",
       "      <td>56576.285000</td>\n",
       "      <td>0.012491</td>\n",
       "      <td>2705.469378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , Upper Darby[c]</th>\n",
       "      <td>3945.575000</td>\n",
       "      <td>2605.265000</td>\n",
       "      <td>40082.720000</td>\n",
       "      <td>-0.068693</td>\n",
       "      <td>2924.651436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania , West Chester</th>\n",
       "      <td>1753.645320</td>\n",
       "      <td>2067.162562</td>\n",
       "      <td>26178.615764</td>\n",
       "      <td>-0.040441</td>\n",
       "      <td>3443.109105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rhode Island , Central Falls</th>\n",
       "      <td>3699.710000</td>\n",
       "      <td>1465.180000</td>\n",
       "      <td>39257.265000</td>\n",
       "      <td>0.048007</td>\n",
       "      <td>1920.198825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Texas , Mobile City</th>\n",
       "      <td>4325.969880</td>\n",
       "      <td>4261.463855</td>\n",
       "      <td>40372.487952</td>\n",
       "      <td>-0.053428</td>\n",
       "      <td>3908.226693</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>133 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     followers  friends_count  statuses_count  \\\n",
       "location                                                                        \n",
       "California , Albany               12664.475000    2494.085000    41845.640000   \n",
       "California , Alhambra              1807.570000    2007.625000    36424.005000   \n",
       "California , Artesia               1704.073892    1966.793103    24957.950739   \n",
       "California , Baldwin Park          1406.316038    1314.174528    28348.037736   \n",
       "California , Bell                 12797.990521   13001.018957    70891.734597   \n",
       "California , Bell Gardens          3089.039409    1426.871921    26007.197044   \n",
       "California , Bellflower            1451.950355    1787.907801    19833.773050   \n",
       "California , Berkeley             12640.465000    2150.355000    41316.020000   \n",
       "California , Cudahy                1064.834906    1354.113208    21860.179245   \n",
       "California , Daly City             2115.755000    1776.705000    48826.885000   \n",
       "California , East Palo Alto        6297.530000    2247.155000    79389.500000   \n",
       "California , El Monte              1811.465000    2010.430000    36439.955000   \n",
       "California , Hawaiian Gardens      1397.970443    1613.059113    24971.073892   \n",
       "California , Hawthorne             4428.975728    2427.165049    51210.941748   \n",
       "California , Hermosa Beach         4730.694175    2581.582524    53511.000000   \n",
       "California , Huntington Park       1811.325000    2010.325000    36437.375000   \n",
       "California , Inglewood             1814.460000    1990.245000    36245.290000   \n",
       "California , La Puente             3491.522167    1928.679803    45418.852217   \n",
       "California , Lawndale              4495.888350    2469.733010    51411.271845   \n",
       "California , Lomita                3796.769697    3548.636364    34504.745455   \n",
       "California , Lynwood               1674.735043    2267.948718    26078.538462   \n",
       "California , Maywood              13274.779412   13488.235294    72953.416667   \n",
       "California , Norwalk               1203.611940    1526.373134    32477.417910   \n",
       "California , Paramount             1094.943396    1326.801887    21402.113208   \n",
       "California , Redondo Beach         4699.174757    2556.689320    53883.388350   \n",
       "California , Rosemead              1088.295567     996.596059    29237.576355   \n",
       "California , San Francisco        15772.785000    4320.095000    41796.680000   \n",
       "California , San Pablo             1098.335000    1197.360000    15893.565000   \n",
       "California , Santa Ana             1839.510000    2036.550000    36385.895000   \n",
       "California , Santa Monica          5765.440000    3442.655000    37148.630000   \n",
       "...                                        ...            ...             ...   \n",
       "New York , Manorhaven              2652.230000    2801.655000    39507.910000   \n",
       "New York , Mineola                 2264.443396    1940.485849    37675.655660   \n",
       "New York , Mount Vernon            2225.096774    2204.571429    61426.087558   \n",
       "New York , New Hyde Park           1286.940092    1214.041475    24458.700461   \n",
       "New York , New Square              2742.664384    2221.164384    21160.965753   \n",
       "New York , New York City           2585.385000    2283.145000    51117.290000   \n",
       "New York , Port Chester            2279.903382    1908.743961    30056.028986   \n",
       "New York , South Floral Park       1263.658879    1243.140187    25672.570093   \n",
       "New York , Spring Valley           1251.617021     758.333333    10494.382979   \n",
       "New York , Tuckahoe                1359.942708    1539.776042    41152.619792   \n",
       "New York , Valley Stream            689.995536     935.986607    13672.504464   \n",
       "New York , Williston Park          2646.729858    2217.720379    41157.066351   \n",
       "New York , Yonkers                 1958.825000    2035.710000    50962.235000   \n",
       "Ohio , Lakewood                    5662.483254    3746.578947    48987.440191   \n",
       "Oregon , Johnson City              1427.680556    2018.416667    22262.467593   \n",
       "Pennsylvania , Clifton Heights     2299.608911    1325.628713    20412.386139   \n",
       "Pennsylvania , Collingdale         2333.838095    1373.247619    20202.452381   \n",
       "Pennsylvania , Conshohocken        1437.810000    1331.070000    17869.415000   \n",
       "Pennsylvania , Darby               2304.825871    1346.845771    19152.810945   \n",
       "Pennsylvania , Dormont             3147.295000    2511.865000    47464.645000   \n",
       "Pennsylvania , East Lansdowne      3960.665000    2615.005000    40236.130000   \n",
       "Pennsylvania , Millbourne          3966.510000    2622.845000    40909.590000   \n",
       "Pennsylvania , Mount Oliver        3145.570000    2512.100000    47397.145000   \n",
       "Pennsylvania , Parkside             750.563725    1074.524510    21153.622549   \n",
       "Pennsylvania , Pennsbury Village   5045.905660    1943.641509    39840.905660   \n",
       "Pennsylvania , Philadelphia        2840.635000    2362.830000    56576.285000   \n",
       "Pennsylvania , Upper Darby[c]      3945.575000    2605.265000    40082.720000   \n",
       "Pennsylvania , West Chester        1753.645320    2067.162562    26178.615764   \n",
       "Rhode Island , Central Falls       3699.710000    1465.180000    39257.265000   \n",
       "Texas , Mobile City                4325.969880    4261.463855    40372.487952   \n",
       "\n",
       "                                     vader  average times  \n",
       "location                                                   \n",
       "California , Albany              -0.258858    2931.362055  \n",
       "California , Alhambra             0.005641    2699.818191  \n",
       "California , Artesia             -0.008311    2985.501343  \n",
       "California , Baldwin Park        -0.005242    2573.503197  \n",
       "California , Bell                -0.126618    2017.529214  \n",
       "California , Bell Gardens        -0.085866    1843.151112  \n",
       "California , Bellflower          -0.000277    2529.536833  \n",
       "California , Berkeley            -0.067293    2826.648354  \n",
       "California , Cudahy              -0.039462    1423.611557  \n",
       "California , Daly City            0.061896    2218.263782  \n",
       "California , East Palo Alto      -0.098829    2629.478149  \n",
       "California , El Monte             0.010876    2503.617391  \n",
       "California , Hawaiian Gardens    -0.016217    1973.770853  \n",
       "California , Hawthorne           -0.063301    2125.824656  \n",
       "California , Hermosa Beach       -0.085617    2272.705411  \n",
       "California , Huntington Park      0.010876    1584.643151  \n",
       "California , Inglewood            0.007341    2466.212232  \n",
       "California , La Puente            0.039831    2654.201018  \n",
       "California , Lawndale            -0.092004    1839.853008  \n",
       "California , Lomita               0.007445    2992.532309  \n",
       "California , Lynwood             -0.032242    2141.015484  \n",
       "California , Maywood             -0.113848    1330.726768  \n",
       "California , Norwalk              0.108944    2839.481233  \n",
       "California , Paramount           -0.024866    2836.494059  \n",
       "California , Redondo Beach       -0.090458    2931.346070  \n",
       "California , Rosemead             0.001448    2972.901225  \n",
       "California , San Francisco       -0.057079    1627.162527  \n",
       "California , San Pablo           -0.046086    2988.199060  \n",
       "California , Santa Ana            0.022561    2440.801248  \n",
       "California , Santa Monica        -0.089412    2987.848356  \n",
       "...                                    ...            ...  \n",
       "New York , Manorhaven             0.032755    2394.267055  \n",
       "New York , Mineola                0.015373    2914.701342  \n",
       "New York , Mount Vernon          -0.104092    1786.951522  \n",
       "New York , New Hyde Park          0.014142    2594.185620  \n",
       "New York , New Square             0.034963    4166.821427  \n",
       "New York , New York City          0.002459    1124.952725  \n",
       "New York , Port Chester          -0.021083    2603.272186  \n",
       "New York , South Floral Park      0.053686    1899.280251  \n",
       "New York , Spring Valley          0.016277    4560.953536  \n",
       "New York , Tuckahoe              -0.026152    3173.420892  \n",
       "New York , Valley Stream          0.024497    2610.579963  \n",
       "New York , Williston Park         0.029821    2632.755060  \n",
       "New York , Yonkers               -0.052142    2801.821970  \n",
       "Ohio , Lakewood                   0.029762    2865.414756  \n",
       "Oregon , Johnson City            -0.030084    3236.185849  \n",
       "Pennsylvania , Clifton Heights   -0.037181    2796.794799  \n",
       "Pennsylvania , Collingdale       -0.022391    2960.272453  \n",
       "Pennsylvania , Conshohocken      -0.063159    2382.983574  \n",
       "Pennsylvania , Darby             -0.007141    2430.219019  \n",
       "Pennsylvania , Dormont           -0.074023    2422.238308  \n",
       "Pennsylvania , East Lansdowne    -0.067131    2429.351333  \n",
       "Pennsylvania , Millbourne        -0.067074    1836.653052  \n",
       "Pennsylvania , Mount Oliver      -0.074023    2596.527861  \n",
       "Pennsylvania , Parkside           0.027853    3097.330532  \n",
       "Pennsylvania , Pennsbury Village  0.033181    6664.431451  \n",
       "Pennsylvania , Philadelphia       0.012491    2705.469378  \n",
       "Pennsylvania , Upper Darby[c]    -0.068693    2924.651436  \n",
       "Pennsylvania , West Chester      -0.040441    3443.109105  \n",
       "Rhode Island , Central Falls      0.048007    1920.198825  \n",
       "Texas , Mobile City              -0.053428    3908.226693  \n",
       "\n",
       "[133 rows x 5 columns]"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#--- population per cities in United States --- ---- ---- --- ---- --- --- --- ---\n",
    "html = wp.page(\"List_of_United_States_cities_by_population_density\").html().encode(\"UTF-8\")\n",
    "UScities = pd.read_html(html)[1]\n",
    "UScities = UScities.drop([0,2,4,6,8],axis=1)\n",
    "UScities = UScities.rename(columns={1:'city',3:'state',5: 'land area (mi^2)',7:'density'})\n",
    "UScities = UScities.iloc[1:]\n",
    "#df['population']=[int(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in df['population']]\n",
    "UScities['density'] = [float(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in UScities['density']]\n",
    "UScities['land area (mi^2)']=[float(area.split('\\xa0')[-1]) for area in UScities['land area (mi^2)']]\n",
    "#--- ---- ----- ----- ---- ---- ----- ---- ---- ----- --- ---- ---- ----- ---- ---\n",
    "city=UScities.loc[UScities['city']=='Guttenberg'].loc[UScities['state']=='New Jersey'].loc[:,'density'].values[0]\n",
    "\n",
    "import numpy as np\n",
    "cityData = tweetsUS.groupby('location').mean()\n",
    "#print(cityData)\n",
    "aveTimes={}\n",
    "for date in tweetsUS.groupby('location')['created_at']:\n",
    "    cityState, timeseries = date\n",
    "    re.sub('\\W','', cityState)\n",
    "    [state,city]=cityState.split(',')\n",
    "    city=city.lstrip().rstrip(); state=state.rstrip().lstrip()\n",
    "    #print(city, state)\n",
    "    #print('this line: ',UScities.loc[UScities['city']==city])\n",
    "    cityDensity=UScities.loc[UScities['city']==city].loc[UScities['state']==state].loc[:,'density'].values[0]\n",
    "    aveTimes[cityState]=np.mean([(b-a).seconds*10000/cityDensity for a,b in zip(timeseries.tolist()[1:],timeseries.tolist()[0:-1])])\n",
    "    #print(f'{city}, {cityDensity}, {aveTimes[cityState]}')\n",
    "cityData['average times']=[aveTimes[city] for city in cityData.index]\n",
    "cityData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "Inconsistent shape between the condition and the input (got (133, 1) and (133,))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-189-a024cc43b640>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mheatmap\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mhm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mheatmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcityData\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'vader'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmap\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'RdYlGn'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mannot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mhm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_title\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'stuff'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavefig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'fig.png'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdpi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/seaborn/matrix.py\u001b[0m in \u001b[0;36mheatmap\u001b[0;34m(data, vmin, vmax, cmap, center, robust, annot, fmt, annot_kws, linewidths, linecolor, cbar, cbar_kws, cbar_ax, square, xticklabels, yticklabels, mask, ax, **kwargs)\u001b[0m\n\u001b[1;32m    515\u001b[0m     plotter = _HeatMapper(data, vmin, vmax, cmap, center, robust, annot, fmt,\n\u001b[1;32m    516\u001b[0m                           \u001b[0mannot_kws\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcbar\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcbar_kws\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxticklabels\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 517\u001b[0;31m                           yticklabels, mask)\n\u001b[0m\u001b[1;32m    518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m     \u001b[0;31m# Add the pcolormesh kwargs here\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/seaborn/matrix.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, vmin, vmax, cmap, center, robust, annot, fmt, annot_kws, cbar, cbar_kws, xticklabels, yticklabels, mask)\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_matrix_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m         \u001b[0mplot_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasked_where\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplot_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0;31m# Get good names for the rows and columns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/numpy/ma/core.py\u001b[0m in \u001b[0;36mmasked_where\u001b[0;34m(condition, a, copy)\u001b[0m\n\u001b[1;32m   1928\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcshape\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mcshape\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mashape\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1929\u001b[0m         raise IndexError(\"Inconsistent shape between the condition and the input\"\n\u001b[0;32m-> 1930\u001b[0;31m                          \" (got %s and %s)\" % (cshape, ashape))\n\u001b[0m\u001b[1;32m   1931\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_mask'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1932\u001b[0m         \u001b[0mcond\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmask_or\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcond\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: Inconsistent shape between the condition and the input (got (133, 1) and (133,))"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-07-02 1\n"
     ]
    }
   ],
   "source": [
    "numDays=1\n",
    "for a,b in zip([str(date.today() - timedelta(i)).split()[0] for i in range(numDays)], range(1,numDays+1)):\n",
    "    print(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "timeBetween=[]\n",
    "prevCreatedAt = datetime.strptime(tweets[0]['created_at'],'%a %b %d %H:%M:%S %z %Y')\n",
    "for tweet in tweets[1:]:\n",
    "    #print(tweet['created_at'])\n",
    "    timeBetween.append((prevCreatedAt - datetime.strptime(tweet['created_at'],'%a %b %d %H:%M:%S %z %Y')).total_seconds()/60)\n",
    "    prevCreatedAt = datetime.strptime(tweet['created_at'],'%a %b %d %H:%M:%S %z %Y')\n",
    "plt.plot(range(999),timeBetween)\n",
    "plt.show()\n",
    "plt.plot([datetime.strptime(tweet['created_at'],'%a %b %d %H:%M:%S %z %Y') for tweet in tweets][::-1],range(1000))\n",
    "#plt.x('date',rotation='vertical')\n",
    "plt.show()\n",
    "print((datetime.strptime(tweets[-1]['created_at'],'%a %b %d %H:%M:%S %z %Y')-datetime.strptime(tweets[0]['created_at'],'%a %b %d %H:%M:%S %z %Y')).total_seconds()//60//60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This code came from the following link:\n",
    "#https://srome.github.io/Parsing-HTML-Tables-in-Python-with-BeautifulSoup-and-pandas/\n",
    "import requests\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "class HTMLTableParser():\n",
    "\n",
    "    def parse_url(self, url):\n",
    "        response = requests.get(url)\n",
    "        soup = BeautifulSoup(response.text, 'lxml')\n",
    "        listylist=[]\n",
    "        for table in soup.find_all('table'):\n",
    "            listylist.append(self.parse_html_table(table))\n",
    "        return listylist\n",
    "\n",
    "    def parse_html_table(self, table):\n",
    "        n_columns = 0; n_rows=0; column_names = []\n",
    "\n",
    "        # Find number of rows and columns\n",
    "        # we also find the column titles if we can\n",
    "        for row in table.find_all('tr'):\n",
    "\n",
    "            # Determine the number of rows in the table\n",
    "            td_tags = row.find_all('td')\n",
    "            if len(td_tags) > 0:\n",
    "                n_rows+=1\n",
    "                if n_columns == 0:\n",
    "                    # Set the number of columns for our table\n",
    "                    n_columns = len(td_tags)\n",
    "\n",
    "            # Handle column names if we find them\n",
    "            th_tags = row.find_all('th') \n",
    "            if len(th_tags) > 0 and len(column_names) == 0:\n",
    "                for th in th_tags:\n",
    "                    column_names.append(th.get_text())\n",
    "\n",
    "        # Safeguard on Column Titles\n",
    "        if len(column_names) > 0 and len(column_names) != n_columns:\n",
    "            raise Exception(\"Column titles do not match the number of columns\")\n",
    "\n",
    "        columns = column_names if len(column_names) > 0 else range(0,n_columns)\n",
    "        df = pd.DataFrame(columns = columns,\n",
    "                          index= range(0,n_rows))\n",
    "        row_marker = 0\n",
    "        for row in table.find_all('tr'):\n",
    "            column_marker = 0\n",
    "            columns = row.find_all('td')\n",
    "            for column in columns:\n",
    "                df.iat[row_marker,column_marker] = column.get_text()\n",
    "                column_marker += 1\n",
    "            if len(columns) > 0:\n",
    "                row_marker += 1\n",
    "\n",
    "        # Convert to float if possible\n",
    "        for col in df:\n",
    "            try:\n",
    "                df[col] = df[col].astype(float)\n",
    "            except ValueError:\n",
    "                pass\n",
    "\n",
    "        return df\n",
    "\n",
    "#TODO: Use the .apply to just change the table to one dialect. Imrpove language scope later.\n",
    "obj = HTMLTableParser()\n",
    "df = obj.parse_url('https://www.infoplease.com/world/countries-world/languages-spoken-each-country-world')\n",
    "countryLanguages = df[0].rename(columns={0:'country',1:'language'}).set_index('country')\n",
    "newDict = []\n",
    "for index, row in countryLanguages.iterrows():\n",
    "    language = re.sub('\\d+|%|\\(.*\\)|\\s','',countryLanguages.loc[index].values[0]).split(',')[0].split(';')[0]\n",
    "    newDict.append([index, language]) #print(index,\": \",language)\n",
    "newDict\n",
    "\n",
    "newDf = pd.DataFrame(newDict)\n",
    "newDf = newDf.rename(columns={0:'country',1:'language'}).set_index('country')\n",
    "newDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>language</th>\n",
       "      <th>languages</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Afghanistan</th>\n",
       "      <td>Dari Persian, Pashtu (both official), other Tu...</td>\n",
       "      <td>DariPersian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Albania</th>\n",
       "      <td>Albanian (Tosk is the official dialect), Greek</td>\n",
       "      <td>Albanian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Algeria</th>\n",
       "      <td>Arabic (official), French, Berber dialects</td>\n",
       "      <td>Arabic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Andorra</th>\n",
       "      <td>Catalán (official), French, Castilian, Portuguese</td>\n",
       "      <td>Catalán</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Angola</th>\n",
       "      <td>Portuguese (official), Bantu and other African...</td>\n",
       "      <td>Portuguese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Antigua and Barbuda</th>\n",
       "      <td>English (official), local dialects</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Argentina</th>\n",
       "      <td>Spanish (official), English, Italian, German, ...</td>\n",
       "      <td>Spanish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Armenia</th>\n",
       "      <td>Armenian 98%, Yezidi, Russian</td>\n",
       "      <td>Armenian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Australia</th>\n",
       "      <td>English 79%, native and other languages</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Austria</th>\n",
       "      <td>German (official nationwide); Slovene, Croatia...</td>\n",
       "      <td>German</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Azerbaijan</th>\n",
       "      <td>Azerbaijani Turkic 89%, Russian 3%, Armenian 2...</td>\n",
       "      <td>AzerbaijaniTurkic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bahamas</th>\n",
       "      <td>English (official), Creole (among Haitian immi...</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bahrain</th>\n",
       "      <td>Arabic, English, Farsi, Urdu</td>\n",
       "      <td>Arabic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bangladesh</th>\n",
       "      <td>Bangla (official), English</td>\n",
       "      <td>Bangla</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Barbados</th>\n",
       "      <td>English</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Belarus</th>\n",
       "      <td>Belorussian (White Russian), Russian, other</td>\n",
       "      <td>Belorussian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Belgium</th>\n",
       "      <td>Dutch (Flemish) 60%, French 40%, German less t...</td>\n",
       "      <td>Dutch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Belize</th>\n",
       "      <td>English (official), Spanish, Mayan, Garifuna (...</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Benin</th>\n",
       "      <td>French (official), Fon, Yoruba, tribal languages</td>\n",
       "      <td>French</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bhutan</th>\n",
       "      <td>Dzongkha (official), Tibetan dialects (among B...</td>\n",
       "      <td>Dzongkha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bolivia</th>\n",
       "      <td>Spanish, Quechua, Aymara (all official)</td>\n",
       "      <td>Spanish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bosnia and Herzegovina</th>\n",
       "      <td>Bosnian, Croatian, Serbian</td>\n",
       "      <td>Bosnian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Botswana</th>\n",
       "      <td>English 2% (official), Setswana 78%, Kalanga 8...</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brazil</th>\n",
       "      <td>Portuguese (official), Spanish, English, French</td>\n",
       "      <td>Portuguese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brunei</th>\n",
       "      <td>Malay (official), English, Chinese</td>\n",
       "      <td>Malay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bulgaria</th>\n",
       "      <td>Bulgarian 85%, Turkish 10%, Roma 4%</td>\n",
       "      <td>Bulgarian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Burkina Faso</th>\n",
       "      <td>French (official); native African (Sudanic) la...</td>\n",
       "      <td>Frenchlanguages</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Burundi</th>\n",
       "      <td>Kirundi and French (official), Swahili</td>\n",
       "      <td>KirundiandFrench</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cambodia</th>\n",
       "      <td>Khmer 95% (official), French, English</td>\n",
       "      <td>Khmer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cameroon</th>\n",
       "      <td>French, English (both official); 24 major Afri...</td>\n",
       "      <td>French</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Swaziland</th>\n",
       "      <td>English, siSwati (both official)</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sweden</th>\n",
       "      <td>Swedish, small Sami- and Finnish-speaking mino...</td>\n",
       "      <td>Swedish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Switzerland</th>\n",
       "      <td>German 64%, French 20%, Italian 7% (all offici...</td>\n",
       "      <td>German</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Syria</th>\n",
       "      <td>Arabic (official); Kurdish, Armenian, Aramaic,...</td>\n",
       "      <td>Arabic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Taiwan</th>\n",
       "      <td>Chinese (Mandarin, official), Taiwanese (Min),...</td>\n",
       "      <td>Chinese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tajikistan</th>\n",
       "      <td>Tajik (official), Russian widely used in gover...</td>\n",
       "      <td>Tajik</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tanzania</th>\n",
       "      <td>Swahili, English (both official); Arabic; many...</td>\n",
       "      <td>Swahili</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Thailand</th>\n",
       "      <td>Thai (Siamese), English (secondary language of...</td>\n",
       "      <td>Thai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Togo</th>\n",
       "      <td>French (official, commerce); Ewé, Mina (south)...</td>\n",
       "      <td>French</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tonga</th>\n",
       "      <td>Tongan (an Austronesian language), English</td>\n",
       "      <td>Tongan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trinidad and Tobago</th>\n",
       "      <td>English (official), Hindi, French, Spanish, Ch...</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tunisia</th>\n",
       "      <td>Arabic (official, commerce), French (commerce)</td>\n",
       "      <td>Arabic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Turkey</th>\n",
       "      <td>Turkish (official), Kurdish, Dimli, Azeri, Kab...</td>\n",
       "      <td>Turkish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Turkmenistan</th>\n",
       "      <td>Turkmen 72%; Russian 12%; Uzbek 9%, other 7%</td>\n",
       "      <td>Turkmen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tuvalu</th>\n",
       "      <td>Tuvaluan, English, Samoan, Kiribati (on the is...</td>\n",
       "      <td>Tuvaluan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Uganda</th>\n",
       "      <td>English (official), Ganda or Luganda, other Ni...</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ukraine</th>\n",
       "      <td>Ukrainian 67%, Russian 24%, Romanian, Polish, ...</td>\n",
       "      <td>Ukrainian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>United Arab Emirates</th>\n",
       "      <td>Arabic (official), Persian, English, Hindi, Urdu</td>\n",
       "      <td>Arabic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>United Kingdom</th>\n",
       "      <td>English, Welsh, Scots Gaelic</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>United States</th>\n",
       "      <td>English 82%, Spanish 11% (2000)</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Uruguay</th>\n",
       "      <td>Spanish, Portunol, or Brazilero</td>\n",
       "      <td>Spanish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Uzbekistan</th>\n",
       "      <td>Uzbek 74.3%, Russian 14.2%, Tajik 4.4%, other ...</td>\n",
       "      <td>Uzbek.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vanuatu</th>\n",
       "      <td>Bislama 23% (a Melanesian pidgin English), Eng...</td>\n",
       "      <td>Bislama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vatican City (Holy See)</th>\n",
       "      <td>Italian, Latin, French, various other languages</td>\n",
       "      <td>Italian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Venezuela</th>\n",
       "      <td>Spanish (official), numerous indigenous dialects</td>\n",
       "      <td>Spanish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vietnam</th>\n",
       "      <td>Vietnamese (official); English (increasingly f...</td>\n",
       "      <td>Vietnamese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Western Sahara (proposed state)</th>\n",
       "      <td>Hassaniya Arabic, Moroccan Arabic</td>\n",
       "      <td>HassaniyaArabic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yemen</th>\n",
       "      <td>Arabic</td>\n",
       "      <td>Arabic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zambia</th>\n",
       "      <td>English (official); major vernaculars: Bemba, ...</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zimbabwe</th>\n",
       "      <td>English (official), Shona, Ndebele (Sindebele)...</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>198 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                          language  \\\n",
       "country                                                                              \n",
       "Afghanistan                      Dari Persian, Pashtu (both official), other Tu...   \n",
       "Albania                             Albanian (Tosk is the official dialect), Greek   \n",
       "Algeria                                 Arabic (official), French, Berber dialects   \n",
       "Andorra                          Catalán (official), French, Castilian, Portuguese   \n",
       "Angola                           Portuguese (official), Bantu and other African...   \n",
       "Antigua and Barbuda                             English (official), local dialects   \n",
       "Argentina                        Spanish (official), English, Italian, German, ...   \n",
       "Armenia                                              Armenian 98%, Yezidi, Russian   \n",
       "Australia                                  English 79%, native and other languages   \n",
       "Austria                          German (official nationwide); Slovene, Croatia...   \n",
       "Azerbaijan                       Azerbaijani Turkic 89%, Russian 3%, Armenian 2...   \n",
       "Bahamas                          English (official), Creole (among Haitian immi...   \n",
       "Bahrain                                               Arabic, English, Farsi, Urdu   \n",
       "Bangladesh                                              Bangla (official), English   \n",
       "Barbados                                                                   English   \n",
       "Belarus                                Belorussian (White Russian), Russian, other   \n",
       "Belgium                          Dutch (Flemish) 60%, French 40%, German less t...   \n",
       "Belize                           English (official), Spanish, Mayan, Garifuna (...   \n",
       "Benin                             French (official), Fon, Yoruba, tribal languages   \n",
       "Bhutan                           Dzongkha (official), Tibetan dialects (among B...   \n",
       "Bolivia                                    Spanish, Quechua, Aymara (all official)   \n",
       "Bosnia and Herzegovina                                  Bosnian, Croatian, Serbian   \n",
       "Botswana                         English 2% (official), Setswana 78%, Kalanga 8...   \n",
       "Brazil                             Portuguese (official), Spanish, English, French   \n",
       "Brunei                                          Malay (official), English, Chinese   \n",
       "Bulgaria                                       Bulgarian 85%, Turkish 10%, Roma 4%   \n",
       "Burkina Faso                     French (official); native African (Sudanic) la...   \n",
       "Burundi                                     Kirundi and French (official), Swahili   \n",
       "Cambodia                                     Khmer 95% (official), French, English   \n",
       "Cameroon                         French, English (both official); 24 major Afri...   \n",
       "...                                                                            ...   \n",
       "Swaziland                                         English, siSwati (both official)   \n",
       "Sweden                           Swedish, small Sami- and Finnish-speaking mino...   \n",
       "Switzerland                      German 64%, French 20%, Italian 7% (all offici...   \n",
       "Syria                            Arabic (official); Kurdish, Armenian, Aramaic,...   \n",
       "Taiwan                           Chinese (Mandarin, official), Taiwanese (Min),...   \n",
       "Tajikistan                       Tajik (official), Russian widely used in gover...   \n",
       "Tanzania                         Swahili, English (both official); Arabic; many...   \n",
       "Thailand                         Thai (Siamese), English (secondary language of...   \n",
       "Togo                             French (official, commerce); Ewé, Mina (south)...   \n",
       "Tonga                                   Tongan (an Austronesian language), English   \n",
       "Trinidad and Tobago              English (official), Hindi, French, Spanish, Ch...   \n",
       "Tunisia                             Arabic (official, commerce), French (commerce)   \n",
       "Turkey                           Turkish (official), Kurdish, Dimli, Azeri, Kab...   \n",
       "Turkmenistan                          Turkmen 72%; Russian 12%; Uzbek 9%, other 7%   \n",
       "Tuvalu                           Tuvaluan, English, Samoan, Kiribati (on the is...   \n",
       "Uganda                           English (official), Ganda or Luganda, other Ni...   \n",
       "Ukraine                          Ukrainian 67%, Russian 24%, Romanian, Polish, ...   \n",
       "United Arab Emirates              Arabic (official), Persian, English, Hindi, Urdu   \n",
       "United Kingdom                                        English, Welsh, Scots Gaelic   \n",
       "United States                                      English 82%, Spanish 11% (2000)   \n",
       "Uruguay                                            Spanish, Portunol, or Brazilero   \n",
       "Uzbekistan                       Uzbek 74.3%, Russian 14.2%, Tajik 4.4%, other ...   \n",
       "Vanuatu                          Bislama 23% (a Melanesian pidgin English), Eng...   \n",
       "Vatican City (Holy See)            Italian, Latin, French, various other languages   \n",
       "Venezuela                         Spanish (official), numerous indigenous dialects   \n",
       "Vietnam                          Vietnamese (official); English (increasingly f...   \n",
       "Western Sahara (proposed state)                  Hassaniya Arabic, Moroccan Arabic   \n",
       "Yemen                                                                       Arabic   \n",
       "Zambia                           English (official); major vernaculars: Bemba, ...   \n",
       "Zimbabwe                         English (official), Shona, Ndebele (Sindebele)...   \n",
       "\n",
       "                                         languages  \n",
       "country                                             \n",
       "Afghanistan                            DariPersian  \n",
       "Albania                                   Albanian  \n",
       "Algeria                                     Arabic  \n",
       "Andorra                                    Catalán  \n",
       "Angola                                  Portuguese  \n",
       "Antigua and Barbuda                        English  \n",
       "Argentina                                  Spanish  \n",
       "Armenia                                   Armenian  \n",
       "Australia                                  English  \n",
       "Austria                                     German  \n",
       "Azerbaijan                       AzerbaijaniTurkic  \n",
       "Bahamas                                    English  \n",
       "Bahrain                                     Arabic  \n",
       "Bangladesh                                  Bangla  \n",
       "Barbados                                   English  \n",
       "Belarus                                Belorussian  \n",
       "Belgium                                      Dutch  \n",
       "Belize                                     English  \n",
       "Benin                                       French  \n",
       "Bhutan                                    Dzongkha  \n",
       "Bolivia                                    Spanish  \n",
       "Bosnia and Herzegovina                     Bosnian  \n",
       "Botswana                                   English  \n",
       "Brazil                                  Portuguese  \n",
       "Brunei                                       Malay  \n",
       "Bulgaria                                 Bulgarian  \n",
       "Burkina Faso                       Frenchlanguages  \n",
       "Burundi                           KirundiandFrench  \n",
       "Cambodia                                     Khmer  \n",
       "Cameroon                                    French  \n",
       "...                                            ...  \n",
       "Swaziland                                  English  \n",
       "Sweden                                     Swedish  \n",
       "Switzerland                                 German  \n",
       "Syria                                       Arabic  \n",
       "Taiwan                                     Chinese  \n",
       "Tajikistan                                   Tajik  \n",
       "Tanzania                                   Swahili  \n",
       "Thailand                                      Thai  \n",
       "Togo                                        French  \n",
       "Tonga                                       Tongan  \n",
       "Trinidad and Tobago                        English  \n",
       "Tunisia                                     Arabic  \n",
       "Turkey                                     Turkish  \n",
       "Turkmenistan                               Turkmen  \n",
       "Tuvalu                                    Tuvaluan  \n",
       "Uganda                                     English  \n",
       "Ukraine                                  Ukrainian  \n",
       "United Arab Emirates                        Arabic  \n",
       "United Kingdom                             English  \n",
       "United States                              English  \n",
       "Uruguay                                    Spanish  \n",
       "Uzbekistan                                  Uzbek.  \n",
       "Vanuatu                                    Bislama  \n",
       "Vatican City (Holy See)                    Italian  \n",
       "Venezuela                                  Spanish  \n",
       "Vietnam                                 Vietnamese  \n",
       "Western Sahara (proposed state)    HassaniyaArabic  \n",
       "Yemen                                       Arabic  \n",
       "Zambia                                     English  \n",
       "Zimbabwe                                   English  \n",
       "\n",
       "[198 rows x 2 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = parse_url('https://www.infoplease.com/world/countries-world/languages-spoken-each-country-world')\n",
    "countryLanguages = df[0].rename(columns={0:'country',1:'language'}).set_index('country')\n",
    "countryLanguages['languages'] = [re.sub('\\d+|%|\\(.*\\)|\\s','',i).split(',')[0].split(';')[0] for i in countryLanguages['language']]\n",
    "countryLanguages    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "translator=Translator()\n",
    "translator.translate('Hola me llamo Jennifer 😜😜', dest='en').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributors': None,\n",
       " 'coordinates': None,\n",
       " 'created_at': 'Sat Jun 30 19:33:55 +0000 2018',\n",
       " 'entities': {'hashtags': [{'indices': [87, 93], 'text': 'Tibet'},\n",
       "   {'indices': [94, 103], 'text': 'Tibetans'},\n",
       "   {'indices': [104, 114], 'text': 'FreeTibet'}],\n",
       "  'symbols': [],\n",
       "  'urls': [{'display_url': 'shar.es/anBniE',\n",
       "    'expanded_url': 'https://shar.es/anBniE',\n",
       "    'indices': [47, 70],\n",
       "    'url': 'https://t.co/zAKFBK5Mna'},\n",
       "   {'display_url': 'twitter.com/i/web/status/1…',\n",
       "    'expanded_url': 'https://twitter.com/i/web/status/1013143614623440896',\n",
       "    'indices': [116, 139],\n",
       "    'url': 'https://t.co/9GU9LWHwGp'}],\n",
       "  'user_mentions': [{'id': 23065876,\n",
       "    'id_str': '23065876',\n",
       "    'indices': [75, 85],\n",
       "    'name': 'Tsem Tulku Rinpoche',\n",
       "    'screen_name': 'tsemtulku'}]},\n",
       " 'favorite_count': 0,\n",
       " 'favorited': False,\n",
       " 'geo': None,\n",
       " 'id': 1013143614623440896,\n",
       " 'id_str': '1013143614623440896',\n",
       " 'in_reply_to_screen_name': None,\n",
       " 'in_reply_to_status_id': None,\n",
       " 'in_reply_to_status_id_str': None,\n",
       " 'in_reply_to_user_id': None,\n",
       " 'in_reply_to_user_id_str': None,\n",
       " 'is_quote_status': False,\n",
       " 'lang': 'en',\n",
       " 'metadata': {'iso_language_code': 'en', 'result_type': 'recent'},\n",
       " 'place': None,\n",
       " 'possibly_sensitive': False,\n",
       " 'retweet_count': 0,\n",
       " 'retweeted': False,\n",
       " 'source': '<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>',\n",
       " 'text': 'The Sixty-Four Yogini Temple of Hirapur, India https://t.co/zAKFBK5Mna via @tsemtulku\\n\\n#Tibet\\n#Tibetans\\n#FreeTibet… https://t.co/9GU9LWHwGp',\n",
       " 'truncated': True,\n",
       " 'user': {'contributors_enabled': False,\n",
       "  'created_at': 'Wed Nov 18 10:24:35 +0000 2015',\n",
       "  'default_profile': False,\n",
       "  'default_profile_image': False,\n",
       "  'description': '',\n",
       "  'entities': {'description': {'urls': []}},\n",
       "  'favourites_count': 7051,\n",
       "  'follow_request_sent': False,\n",
       "  'followers_count': 288,\n",
       "  'following': False,\n",
       "  'friends_count': 181,\n",
       "  'geo_enabled': True,\n",
       "  'has_extended_profile': True,\n",
       "  'id': 4277739734,\n",
       "  'id_str': '4277739734',\n",
       "  'is_translation_enabled': False,\n",
       "  'is_translator': False,\n",
       "  'lang': 'en',\n",
       "  'listed_count': 74,\n",
       "  'location': 'Bentung, Pahang',\n",
       "  'name': 'Chris Chong',\n",
       "  'notifications': False,\n",
       "  'profile_background_color': '000000',\n",
       "  'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_tile': False,\n",
       "  'profile_banner_url': 'https://pbs.twimg.com/profile_banners/4277739734/1524960980',\n",
       "  'profile_image_url': 'http://pbs.twimg.com/profile_images/780584831935586304/GLHvkq-Q_normal.jpg',\n",
       "  'profile_image_url_https': 'https://pbs.twimg.com/profile_images/780584831935586304/GLHvkq-Q_normal.jpg',\n",
       "  'profile_link_color': '1B95E0',\n",
       "  'profile_sidebar_border_color': '000000',\n",
       "  'profile_sidebar_fill_color': '000000',\n",
       "  'profile_text_color': '000000',\n",
       "  'profile_use_background_image': False,\n",
       "  'protected': False,\n",
       "  'screen_name': 'chrischong90',\n",
       "  'statuses_count': 9983,\n",
       "  'time_zone': None,\n",
       "  'translator_type': 'none',\n",
       "  'url': None,\n",
       "  'utc_offset': None,\n",
       "  'verified': False}}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Setup Tweepy API Authentication\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "api = tweepy.API(auth, parser=tweepy.parsers.JSONParser())\n",
    "tweets = api.search('dalailama')\n",
    "tweets['statuses'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>population</th>\n",
       "      <th>density</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Manila</td>\n",
       "      <td>1780148</td>\n",
       "      <td>107561</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ebeye</td>\n",
       "      <td>15000</td>\n",
       "      <td>107143</td>\n",
       "      <td>Marshall Islands</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pateros (Municipality)</td>\n",
       "      <td>64147</td>\n",
       "      <td>79114</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mumbai</td>\n",
       "      <td>12478447</td>\n",
       "      <td>73837</td>\n",
       "      <td>India</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Dhaka</td>\n",
       "      <td>8523137</td>\n",
       "      <td>73583</td>\n",
       "      <td>Bangladesh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Bnei Brak</td>\n",
       "      <td>200162</td>\n",
       "      <td>73159</td>\n",
       "      <td>Israel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Caloocan</td>\n",
       "      <td>1489040</td>\n",
       "      <td>72302</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Levallois-Perret</td>\n",
       "      <td>63436</td>\n",
       "      <td>68458</td>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Le PrÃ©-Saint-Gervais</td>\n",
       "      <td>18121</td>\n",
       "      <td>67047</td>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Chennai</td>\n",
       "      <td>4681087</td>\n",
       "      <td>66961</td>\n",
       "      <td>India</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Vincennes</td>\n",
       "      <td>48689</td>\n",
       "      <td>66371</td>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Saint-MandÃ©</td>\n",
       "      <td>22627</td>\n",
       "      <td>65115</td>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Bally</td>\n",
       "      <td>291972</td>\n",
       "      <td>64031</td>\n",
       "      <td>India</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Kolkata</td>\n",
       "      <td>4486679</td>\n",
       "      <td>62813</td>\n",
       "      <td>India</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Saint-Josse-ten-Noode</td>\n",
       "      <td>27548</td>\n",
       "      <td>62404</td>\n",
       "      <td>Belgium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Kathmandu</td>\n",
       "      <td>1183000</td>\n",
       "      <td>61972</td>\n",
       "      <td>Nepal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Subang Jaya</td>\n",
       "      <td>1683589</td>\n",
       "      <td>38482</td>\n",
       "      <td>Malaysia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Neapoli</td>\n",
       "      <td>27084</td>\n",
       "      <td>60186</td>\n",
       "      <td>Greece</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Montrouge</td>\n",
       "      <td>48410</td>\n",
       "      <td>59705</td>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>MalÃ©</td>\n",
       "      <td>133412</td>\n",
       "      <td>59559</td>\n",
       "      <td>Maldives</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Malabon</td>\n",
       "      <td>353337</td>\n",
       "      <td>58607</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Guttenberg, New Jersey</td>\n",
       "      <td>11481</td>\n",
       "      <td>58577</td>\n",
       "      <td>United States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Pasig</td>\n",
       "      <td>669773</td>\n",
       "      <td>55958</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Paris</td>\n",
       "      <td>2265886</td>\n",
       "      <td>55673</td>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Mislata</td>\n",
       "      <td>43756</td>\n",
       "      <td>54695</td>\n",
       "      <td>Spain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Macau</td>\n",
       "      <td>643100</td>\n",
       "      <td>54790</td>\n",
       "      <td>China</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Kallithea</td>\n",
       "      <td>100050</td>\n",
       "      <td>54733</td>\n",
       "      <td>Greece</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Nea Smyrni</td>\n",
       "      <td>73090</td>\n",
       "      <td>53717</td>\n",
       "      <td>Greece</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Howrah</td>\n",
       "      <td>1072161</td>\n",
       "      <td>53670</td>\n",
       "      <td>India</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Pasay</td>\n",
       "      <td>392869</td>\n",
       "      <td>53554</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>San Juan</td>\n",
       "      <td>121430</td>\n",
       "      <td>52946</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Colombo</td>\n",
       "      <td>323257</td>\n",
       "      <td>52871</td>\n",
       "      <td>Sri Lanka</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>L'Hospitalet de Llobregat</td>\n",
       "      <td>252171</td>\n",
       "      <td>52536</td>\n",
       "      <td>Spain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Union City, New Jersey</td>\n",
       "      <td>66455</td>\n",
       "      <td>51810</td>\n",
       "      <td>United States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Cairo</td>\n",
       "      <td>11742120</td>\n",
       "      <td>50180</td>\n",
       "      <td>Egypt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Makati</td>\n",
       "      <td>529039</td>\n",
       "      <td>50080</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>West New York, New Jersey</td>\n",
       "      <td>49708</td>\n",
       "      <td>49362</td>\n",
       "      <td>United States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Saint-Gilles</td>\n",
       "      <td>46931</td>\n",
       "      <td>48234</td>\n",
       "      <td>Belgium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Bandung</td>\n",
       "      <td>3075478</td>\n",
       "      <td>48054</td>\n",
       "      <td>Indonesia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Monaco</td>\n",
       "      <td>36950</td>\n",
       "      <td>47372</td>\n",
       "      <td>Monaco</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Boulogne-Billancourt</td>\n",
       "      <td>113085</td>\n",
       "      <td>47240</td>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Quezon City</td>\n",
       "      <td>2936116</td>\n",
       "      <td>45999</td>\n",
       "      <td>Philippines</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         city  population  density           country\n",
       "1                      Manila     1780148   107561       Philippines\n",
       "2                       Ebeye       15000   107143  Marshall Islands\n",
       "3      Pateros (Municipality)       64147    79114       Philippines\n",
       "4                      Mumbai    12478447    73837             India\n",
       "5                       Dhaka     8523137    73583        Bangladesh\n",
       "6                   Bnei Brak      200162    73159            Israel\n",
       "7                    Caloocan     1489040    72302       Philippines\n",
       "8            Levallois-Perret       63436    68458            France\n",
       "9       Le PrÃ©-Saint-Gervais       18121    67047            France\n",
       "10                    Chennai     4681087    66961             India\n",
       "11                  Vincennes       48689    66371            France\n",
       "12               Saint-MandÃ©       22627    65115            France\n",
       "13                      Bally      291972    64031             India\n",
       "14                    Kolkata     4486679    62813             India\n",
       "15      Saint-Josse-ten-Noode       27548    62404           Belgium\n",
       "16                  Kathmandu     1183000    61972             Nepal\n",
       "17                Subang Jaya     1683589    38482          Malaysia\n",
       "18                    Neapoli       27084    60186            Greece\n",
       "19                  Montrouge       48410    59705            France\n",
       "20                      MalÃ©      133412    59559          Maldives\n",
       "21                    Malabon      353337    58607       Philippines\n",
       "22     Guttenberg, New Jersey       11481    58577     United States\n",
       "23                      Pasig      669773    55958       Philippines\n",
       "24                      Paris     2265886    55673            France\n",
       "25                    Mislata       43756    54695             Spain\n",
       "26                      Macau      643100    54790             China\n",
       "27                  Kallithea      100050    54733            Greece\n",
       "28                 Nea Smyrni       73090    53717            Greece\n",
       "29                     Howrah     1072161    53670             India\n",
       "30                      Pasay      392869    53554       Philippines\n",
       "31                   San Juan      121430    52946       Philippines\n",
       "32                    Colombo      323257    52871         Sri Lanka\n",
       "33  L'Hospitalet de Llobregat      252171    52536             Spain\n",
       "34     Union City, New Jersey       66455    51810     United States\n",
       "35                      Cairo    11742120    50180             Egypt\n",
       "36                     Makati      529039    50080       Philippines\n",
       "37  West New York, New Jersey       49708    49362     United States\n",
       "38               Saint-Gilles       46931    48234           Belgium\n",
       "39                    Bandung     3075478    48054         Indonesia\n",
       "40                     Monaco       36950    47372            Monaco\n",
       "41       Boulogne-Billancourt      113085    47240            France\n",
       "42                Quezon City     2936116    45999       Philippines"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import wikipedia as wp\n",
    "from pygeocoder import Geocoder\n",
    "import time\n",
    "import re\n",
    "    \n",
    "#-- Get the most populated cities from wikipedia (Thank you wikipedia library!) --\n",
    "html = wp.page(\"List_of_cities_by_population_density\").html().encode(\"UTF-8\")\n",
    "df = pd.read_html(html)[1]\n",
    "df=df.drop([2,3,4],axis=1)\n",
    "df=df.rename(columns={0:'city',1:'population',5:'density',6:'country'})\n",
    "df=df.iloc[1:]\n",
    "df['population']=[int(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in df['population']]\n",
    "df['density']=[int(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in df['density']]\n",
    "df\n",
    "#--- ---- ----- ----- ---- ---- ----- ---- ---- ----- --- ---- ---- ----- ---- ---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1780148,\n",
       " 15000,\n",
       " 64147,\n",
       " 12478447,\n",
       " 8523137,\n",
       " 200162,\n",
       " 1489040,\n",
       " 63436,\n",
       " 18121,\n",
       " 4681087,\n",
       " 48689,\n",
       " 22627,\n",
       " 291972,\n",
       " 4486679,\n",
       " 27548,\n",
       " 1183000,\n",
       " 1683589,\n",
       " 27084,\n",
       " 48410,\n",
       " 133412,\n",
       " 353337,\n",
       " 11481,\n",
       " 669773,\n",
       " 2265886,\n",
       " 43756,\n",
       " 643100,\n",
       " 100050,\n",
       " 73090,\n",
       " 1072161,\n",
       " 392869,\n",
       " 121430,\n",
       " 323257,\n",
       " 252171,\n",
       " 66455,\n",
       " 11742120,\n",
       " 529039,\n",
       " 49708,\n",
       " 46931,\n",
       " 3075478,\n",
       " 36950,\n",
       " 113085,\n",
       " 2936116]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[int(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in df['population']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>followers</th>\n",
       "      <th>following</th>\n",
       "      <th>geo</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>statuses_count</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>154060</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>22455</td>\n",
       "      <td>@ryapee Hi Rya order ako ulit polyblender mejo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>83</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>3030</td>\n",
       "      <td>Yung dating saling pusa naging aso bigla // 🎶</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>643</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'text': 'AdoptDontShop', 'indices': [38, 52]}]</td>\n",
       "      <td>8529</td>\n",
       "      <td>sana meron din dito sa Pilipinas yung #AdoptDo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2729</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>28771</td>\n",
       "      <td>Aso nga kasi ako, bantay ako dito hahaha 😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>14</td>\n",
       "      <td>lakas mangahol ng kaklase ko dinaig pa aso nam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>1993</td>\n",
       "      <td>@LampanoElla Dun sa aso oo HAHA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>113</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>4495</td>\n",
       "      <td>Me: labas mo dila para lumabas dila ng aso \\nS...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>130</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>1745</td>\n",
       "      <td>cute kong aso https://t.co/twDprm7o5P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>54</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>727</td>\n",
       "      <td>me: pabili pong dog food\\ntindero: alin? ung p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>24</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>296</td>\n",
       "      <td>@dsgalarpez hahahaha aso ka na ba ngayon?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>841</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>29025</td>\n",
       "      <td>@DenniceRoselle Uy kawawa mga aso. Di naman si...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>260</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>8486</td>\n",
       "      <td>Nagtanggal tuloy ako nang mga tae nang aso kai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>260</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>8486</td>\n",
       "      <td>Nakakapikon ung aso 😭😭</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>176</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>4682</td>\n",
       "      <td>May mga sakit aso namin hanep</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1357</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>15736</td>\n",
       "      <td>i hate when strangers esp. men look at you str...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>422</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>21045</td>\n",
       "      <td>@dnnkthryn Ngek malas. Sa rosewood naman okay ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>422</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>21045</td>\n",
       "      <td>@dnnkthryn Yup. Iba talaga pag bahay super fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>365</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>2083</td>\n",
       "      <td>Tang ina sobrang iba pala pakiramdam pag namat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>243</td>\n",
       "      <td>aso't pusa ❤\\ngoodmorning hubby babe ! https:/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1021</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>11119</td>\n",
       "      <td>RT @akoposimarcelo: Yung buti pa yung mga aso,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>905</td>\n",
       "      <td>napaka clingy ng aso ko, nebeyen hehe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>147</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>3406</td>\n",
       "      <td>ang laki ng aso!!! panay naka tahol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>269</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>21310</td>\n",
       "      <td>Alam ata ng aso ko na birthday ko ngayon. Iba ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>548</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>3553</td>\n",
       "      <td>Nagduduet nanaman yung dalawang aso hays</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>88</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>1470</td>\n",
       "      <td>Yung aso naming maligalig na palundag lundag pa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>394</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>67411</td>\n",
       "      <td>Mukha talaga akong tanga kapag nakikipaglaro a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>417</td>\n",
       "      <td>RT @akoposimarcelo: Yung buti pa yung mga aso,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>174</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>4033</td>\n",
       "      <td>nagdisitahulan mga aso ang creepy huhu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>828</td>\n",
       "      <td>@glbysrcmny aso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>850</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>11211</td>\n",
       "      <td>Distemper virus. may ganyan plang sakit nang m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>1208</td>\n",
       "      <td>Plus two agad aso namen AAHAHAHAHAH saya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>180</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>6216</td>\n",
       "      <td>@jhnlstrpgnsn Hahaha hindi ko aso yun sa ate k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>1280</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>5483</td>\n",
       "      <td>RT @akoposimarcelo: Yung buti pa yung mga aso,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>74</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>2055</td>\n",
       "      <td>Mama: Tanga Di Mo Pa Pinapakain Yung Aso.\\n\\nT...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>72</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>456</td>\n",
       "      <td>@cescamarii di yan kusa aso yan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>365</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>2083</td>\n",
       "      <td>Nasagasaan aso ko 😢</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>301</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>4788</td>\n",
       "      <td>Ngayon ko nalang na appreciate ulit yung ganda...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>561</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>13927</td>\n",
       "      <td>pag gantong nalulungkot ako imbis na maghanap ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>180</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>6718</td>\n",
       "      <td>creepy ng aso pero mahal ko kayo HAHAHHHAHAHA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>329</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>1259</td>\n",
       "      <td>Pagod bebi ko ako inaantok na tagal ni aso HAH...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>460</td>\n",
       "      <td>So ayun diba may hamster kami si Luxus saka si...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>797</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>42407</td>\n",
       "      <td>@Anniefernando6 @aldenAllTheWay Baka kayo ang ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>293</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>3881</td>\n",
       "      <td>hindi ako to pramis. hahahahahahaha hindi ako ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>293</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>3881</td>\n",
       "      <td>HUY ANG WEIRD TALAGA KASI NAGSESAVE AKO NG PIC...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>84</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>15631</td>\n",
       "      <td>Fun fact about you — Sobrang love ko yung mga ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>293</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>3881</td>\n",
       "      <td>diko maintindihan bakit ako nagsesave ng pictu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>440</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>1375</td>\n",
       "      <td>Hayaan mo lang na husgahan ka nila.Hindi yung ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>476</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>7158</td>\n",
       "      <td>@pauiicosta lumabas na naman pagka aso mo haha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>1902</td>\n",
       "      <td>nagttampo ako kay potchi, tangina aso lang yon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>183</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>1154</td>\n",
       "      <td>Ako: pare iiyak ka pag namatay aso mo? \\nRoque...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>62</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>756</td>\n",
       "      <td>Aso ko e pero di na maghahabol🐶 https://t.co/Y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>44</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>64</td>\n",
       "      <td>hi aso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>167</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>2591</td>\n",
       "      <td>Nangapitbahay nako para sa aso. Happy pill! 😊 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>536</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>6135</td>\n",
       "      <td>NP: Banal na aso, santong kabayo\\n\\nNatatawa a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>3228</td>\n",
       "      <td>False</td>\n",
       "      <td>{'type': 'Point', 'coordinates': [14.61941886,...</td>\n",
       "      <td>[]</td>\n",
       "      <td>10223</td>\n",
       "      <td>Late post: Buti na lang talaga alert ako..kunc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>242</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>4463</td>\n",
       "      <td>tao,ahas at aso.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>473</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>5234</td>\n",
       "      <td>ALAM NIYO BANG MUNTIKAN NG GAWING PAGKAIN NG A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>220</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>10928</td>\n",
       "      <td>Dang kyot ng aso nila Sir huhu i want 😍</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>141</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>12781</td>\n",
       "      <td>Hindi sa dinidepensahan pero may umuulol na na...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>200</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>8882</td>\n",
       "      <td>RT @Itsmeearlbravo: Di naman siguro ako pinang...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    followers  following                                                geo  \\\n",
       "0      154060      False                                               None   \n",
       "1          83      False                                               None   \n",
       "2         643      False                                               None   \n",
       "3        2729      False                                               None   \n",
       "4          25      False                                               None   \n",
       "5         116      False                                               None   \n",
       "6         113      False                                               None   \n",
       "7         130      False                                               None   \n",
       "8          54      False                                               None   \n",
       "9          24      False                                               None   \n",
       "10        841      False                                               None   \n",
       "11        260      False                                               None   \n",
       "12        260      False                                               None   \n",
       "13        176      False                                               None   \n",
       "14       1357      False                                               None   \n",
       "15        422      False                                               None   \n",
       "16        422      False                                               None   \n",
       "17        365      False                                               None   \n",
       "18         75      False                                               None   \n",
       "19       1021      False                                               None   \n",
       "20         47      False                                               None   \n",
       "21        147      False                                               None   \n",
       "22        269      False                                               None   \n",
       "23        548      False                                               None   \n",
       "24         88      False                                               None   \n",
       "25        394      False                                               None   \n",
       "26        105      False                                               None   \n",
       "27        174      False                                               None   \n",
       "28         90      False                                               None   \n",
       "29        850      False                                               None   \n",
       "..        ...        ...                                                ...   \n",
       "70         80      False                                               None   \n",
       "71        180      False                                               None   \n",
       "72       1280      False                                               None   \n",
       "73         74      False                                               None   \n",
       "74         72      False                                               None   \n",
       "75        365      False                                               None   \n",
       "76        301      False                                               None   \n",
       "77        561      False                                               None   \n",
       "78        180      False                                               None   \n",
       "79        329      False                                               None   \n",
       "80          5      False                                               None   \n",
       "81        797      False                                               None   \n",
       "82        293      False                                               None   \n",
       "83        293      False                                               None   \n",
       "84         84      False                                               None   \n",
       "85        293      False                                               None   \n",
       "86        440      False                                               None   \n",
       "87        476      False                                               None   \n",
       "88         35      False                                               None   \n",
       "89        183      False                                               None   \n",
       "90         62      False                                               None   \n",
       "91         44      False                                               None   \n",
       "92        167      False                                               None   \n",
       "93        536      False                                               None   \n",
       "94       3228      False  {'type': 'Point', 'coordinates': [14.61941886,...   \n",
       "95        242      False                                               None   \n",
       "96        473      False                                               None   \n",
       "97        220      False                                               None   \n",
       "98        141      False                                               None   \n",
       "99        200      False                                               None   \n",
       "\n",
       "                                            hashtags  statuses_count  \\\n",
       "0                                                 []           22455   \n",
       "1                                                 []            3030   \n",
       "2   [{'text': 'AdoptDontShop', 'indices': [38, 52]}]            8529   \n",
       "3                                                 []           28771   \n",
       "4                                                 []              14   \n",
       "5                                                 []            1993   \n",
       "6                                                 []            4495   \n",
       "7                                                 []            1745   \n",
       "8                                                 []             727   \n",
       "9                                                 []             296   \n",
       "10                                                []           29025   \n",
       "11                                                []            8486   \n",
       "12                                                []            8486   \n",
       "13                                                []            4682   \n",
       "14                                                []           15736   \n",
       "15                                                []           21045   \n",
       "16                                                []           21045   \n",
       "17                                                []            2083   \n",
       "18                                                []             243   \n",
       "19                                                []           11119   \n",
       "20                                                []             905   \n",
       "21                                                []            3406   \n",
       "22                                                []           21310   \n",
       "23                                                []            3553   \n",
       "24                                                []            1470   \n",
       "25                                                []           67411   \n",
       "26                                                []             417   \n",
       "27                                                []            4033   \n",
       "28                                                []             828   \n",
       "29                                                []           11211   \n",
       "..                                               ...             ...   \n",
       "70                                                []            1208   \n",
       "71                                                []            6216   \n",
       "72                                                []            5483   \n",
       "73                                                []            2055   \n",
       "74                                                []             456   \n",
       "75                                                []            2083   \n",
       "76                                                []            4788   \n",
       "77                                                []           13927   \n",
       "78                                                []            6718   \n",
       "79                                                []            1259   \n",
       "80                                                []             460   \n",
       "81                                                []           42407   \n",
       "82                                                []            3881   \n",
       "83                                                []            3881   \n",
       "84                                                []           15631   \n",
       "85                                                []            3881   \n",
       "86                                                []            1375   \n",
       "87                                                []            7158   \n",
       "88                                                []            1902   \n",
       "89                                                []            1154   \n",
       "90                                                []             756   \n",
       "91                                                []              64   \n",
       "92                                                []            2591   \n",
       "93                                                []            6135   \n",
       "94                                                []           10223   \n",
       "95                                                []            4463   \n",
       "96                                                []            5234   \n",
       "97                                                []           10928   \n",
       "98                                                []           12781   \n",
       "99                                                []            8882   \n",
       "\n",
       "                                                 text  \n",
       "0   @ryapee Hi Rya order ako ulit polyblender mejo...  \n",
       "1       Yung dating saling pusa naging aso bigla // 🎶  \n",
       "2   sana meron din dito sa Pilipinas yung #AdoptDo...  \n",
       "3          Aso nga kasi ako, bantay ako dito hahaha 😂  \n",
       "4   lakas mangahol ng kaklase ko dinaig pa aso nam...  \n",
       "5                     @LampanoElla Dun sa aso oo HAHA  \n",
       "6   Me: labas mo dila para lumabas dila ng aso \\nS...  \n",
       "7               cute kong aso https://t.co/twDprm7o5P  \n",
       "8   me: pabili pong dog food\\ntindero: alin? ung p...  \n",
       "9           @dsgalarpez hahahaha aso ka na ba ngayon?  \n",
       "10  @DenniceRoselle Uy kawawa mga aso. Di naman si...  \n",
       "11  Nagtanggal tuloy ako nang mga tae nang aso kai...  \n",
       "12                             Nakakapikon ung aso 😭😭  \n",
       "13                      May mga sakit aso namin hanep  \n",
       "14  i hate when strangers esp. men look at you str...  \n",
       "15  @dnnkthryn Ngek malas. Sa rosewood naman okay ...  \n",
       "16  @dnnkthryn Yup. Iba talaga pag bahay super fre...  \n",
       "17  Tang ina sobrang iba pala pakiramdam pag namat...  \n",
       "18  aso't pusa ❤\\ngoodmorning hubby babe ! https:/...  \n",
       "19  RT @akoposimarcelo: Yung buti pa yung mga aso,...  \n",
       "20              napaka clingy ng aso ko, nebeyen hehe  \n",
       "21                ang laki ng aso!!! panay naka tahol  \n",
       "22  Alam ata ng aso ko na birthday ko ngayon. Iba ...  \n",
       "23           Nagduduet nanaman yung dalawang aso hays  \n",
       "24    Yung aso naming maligalig na palundag lundag pa  \n",
       "25  Mukha talaga akong tanga kapag nakikipaglaro a...  \n",
       "26  RT @akoposimarcelo: Yung buti pa yung mga aso,...  \n",
       "27             nagdisitahulan mga aso ang creepy huhu  \n",
       "28                                    @glbysrcmny aso  \n",
       "29  Distemper virus. may ganyan plang sakit nang m...  \n",
       "..                                                ...  \n",
       "70           Plus two agad aso namen AAHAHAHAHAH saya  \n",
       "71  @jhnlstrpgnsn Hahaha hindi ko aso yun sa ate k...  \n",
       "72  RT @akoposimarcelo: Yung buti pa yung mga aso,...  \n",
       "73  Mama: Tanga Di Mo Pa Pinapakain Yung Aso.\\n\\nT...  \n",
       "74                    @cescamarii di yan kusa aso yan  \n",
       "75                                Nasagasaan aso ko 😢  \n",
       "76  Ngayon ko nalang na appreciate ulit yung ganda...  \n",
       "77  pag gantong nalulungkot ako imbis na maghanap ...  \n",
       "78      creepy ng aso pero mahal ko kayo HAHAHHHAHAHA  \n",
       "79  Pagod bebi ko ako inaantok na tagal ni aso HAH...  \n",
       "80  So ayun diba may hamster kami si Luxus saka si...  \n",
       "81  @Anniefernando6 @aldenAllTheWay Baka kayo ang ...  \n",
       "82  hindi ako to pramis. hahahahahahaha hindi ako ...  \n",
       "83  HUY ANG WEIRD TALAGA KASI NAGSESAVE AKO NG PIC...  \n",
       "84  Fun fact about you — Sobrang love ko yung mga ...  \n",
       "85  diko maintindihan bakit ako nagsesave ng pictu...  \n",
       "86  Hayaan mo lang na husgahan ka nila.Hindi yung ...  \n",
       "87  @pauiicosta lumabas na naman pagka aso mo haha...  \n",
       "88  nagttampo ako kay potchi, tangina aso lang yon...  \n",
       "89  Ako: pare iiyak ka pag namatay aso mo? \\nRoque...  \n",
       "90  Aso ko e pero di na maghahabol🐶 https://t.co/Y...  \n",
       "91                                             hi aso  \n",
       "92  Nangapitbahay nako para sa aso. Happy pill! 😊 ...  \n",
       "93  NP: Banal na aso, santong kabayo\\n\\nNatatawa a...  \n",
       "94  Late post: Buti na lang talaga alert ako..kunc...  \n",
       "95                                   tao,ahas at aso.  \n",
       "96  ALAM NIYO BANG MUNTIKAN NG GAWING PAGKAIN NG A...  \n",
       "97            Dang kyot ng aso nila Sir huhu i want 😍  \n",
       "98  Hindi sa dinidepensahan pero may umuulol na na...  \n",
       "99  RT @Itsmeearlbravo: Di naman siguro ako pinang...  \n",
       "\n",
       "[100 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets['Manila,Philippines']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Delft, Holland'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "# Setup Tweepy API Authentication\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "api = tweepy.API(auth, parser=tweepy.parsers.JSONParser())\n",
    "tweetsa=api.search(q='trump', geocode='52.132633,5.29126,100km')\n",
    "tweetsa['statuses'][0]['user']['location']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "If using all scalar values, you must pass an index",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-114-f7373bd5071b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'a'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'b'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdf2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'a'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'b'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m+=\u001b[0m\u001b[0mdf2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[1;32m    328\u001b[0m                                  dtype=dtype, copy=copy)\n\u001b[1;32m    329\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 330\u001b[0;31m             \u001b[0mmgr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    331\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMaskedArray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmrecords\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmrecords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_init_dict\u001b[0;34m(self, data, index, columns, dtype)\u001b[0m\n\u001b[1;32m    459\u001b[0m             \u001b[0marrays\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    460\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 461\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_arrays_to_mgr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    462\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    463\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_init_ndarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_arrays_to_mgr\u001b[0;34m(arrays, arr_names, index, columns, dtype)\u001b[0m\n\u001b[1;32m   6161\u001b[0m     \u001b[0;31m# figure out the index, if necessary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6162\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6163\u001b[0;31m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextract_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   6164\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6165\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_ensure_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mextract_index\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m   6200\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6201\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mindexes\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mraw_lengths\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6202\u001b[0;31m             raise ValueError('If using all scalar values, you must pass'\n\u001b[0m\u001b[1;32m   6203\u001b[0m                              ' an index')\n\u001b[1;32m   6204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: If using all scalar values, you must pass an index"
     ]
    }
   ],
   "source": [
    "df=pd.DataFrame({'a':1,'b':2,'c':3})\n",
    "df2=pd.DataFrame({'a':1,'b':2,'c':3})\n",
    "df+=df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Jun 29 20:43:37 +0000 2018\n",
      "Thu Jun 28 14:21:15 +0000 2018\n",
      "Wed Jun 27 23:51:55 +0000 2018\n",
      "Tue Jun 26 23:12:07 +0000 2018\n",
      "Mon Jun 25 19:23:04 +0000 2018\n",
      "Sun Jun 24 23:58:25 +0000 2018\n",
      "Sat Jun 23 23:06:10 +0000 2018\n",
      "Fri Jun 22 17:51:26 +0000 2018\n",
      "Thu Jun 21 22:46:17 +0000 2018\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-82-e5c117ad83b9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mday\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoday\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtimedelta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m70\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mtweets\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mapi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cat'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mgeocode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'34.0934,56.134,200mi'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muntil\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mday\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'1012848185839058950'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtweets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'statuses'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'created_at'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "from datetime import date, timedelta\n",
    "date.today() - timedelta(0)\n",
    "for day in [str(date.today() - timedelta(i)).split()[0] for i in range(70)]:\n",
    "    tweets=api.search('cat',geocode = '34.0934,56.134,200mi', until=day, max_id='1012848185839058950')\n",
    "    print(tweets['statuses'][0]['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 2\n",
      "2 3\n"
     ]
    }
   ],
   "source": [
    "for a,b in zip([1,2],[2,3]):\n",
    "    print(a,b)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>state</th>\n",
       "      <th>city</th>\n",
       "      <th>density</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>57116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>51810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>49362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>39066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New York</td>\n",
       "      <td>27788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New York</td>\n",
       "      <td>27016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>24577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>24060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>23216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>22437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New York</td>\n",
       "      <td>21635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Miami</td>\n",
       "      <td>Florida</td>\n",
       "      <td>21484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>21254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Miami</td>\n",
       "      <td>Florida</td>\n",
       "      <td>20518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>19179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Boston</td>\n",
       "      <td>Massachusetts</td>\n",
       "      <td>18431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>18297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>18218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Louisville</td>\n",
       "      <td>Kentucky</td>\n",
       "      <td>18100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>18801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>San Francisco</td>\n",
       "      <td>California</td>\n",
       "      <td>18679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Miami</td>\n",
       "      <td>Florida</td>\n",
       "      <td>17023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>16896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>16557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>16377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Boston</td>\n",
       "      <td>Massachusetts</td>\n",
       "      <td>16354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Providence</td>\n",
       "      <td>Rhode Island</td>\n",
       "      <td>16146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>16093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Boston</td>\n",
       "      <td>Massachusetts</td>\n",
       "      <td>16036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>16036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>Detroit</td>\n",
       "      <td>Michigan</td>\n",
       "      <td>10900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>10897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>10882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New York</td>\n",
       "      <td>10847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>San Francisco</td>\n",
       "      <td>California</td>\n",
       "      <td>10752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>10667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>10572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>Miami</td>\n",
       "      <td>Florida</td>\n",
       "      <td>10474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>10398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>10397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>San Francisco</td>\n",
       "      <td>California</td>\n",
       "      <td>10368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>Boston</td>\n",
       "      <td>Massachusetts</td>\n",
       "      <td>10351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New York</td>\n",
       "      <td>10337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>10256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>Cleveland</td>\n",
       "      <td>Ohio</td>\n",
       "      <td>10208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New York</td>\n",
       "      <td>10188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New York</td>\n",
       "      <td>10187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>10178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>New York City</td>\n",
       "      <td>New Jersey</td>\n",
       "      <td>10144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>10126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>10107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>Chicago</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>10094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>10065</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>133 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             state           city  density\n",
       "1    New York City     New Jersey    57116\n",
       "2    New York City     New Jersey    51810\n",
       "3    New York City     New Jersey    49362\n",
       "4    New York City     New Jersey    39066\n",
       "5    New York City       New York    27788\n",
       "6    New York City       New York    27016\n",
       "7    New York City     New Jersey    24577\n",
       "8    New York City     New Jersey    24060\n",
       "9      Los Angeles     California    23216\n",
       "10   New York City     New Jersey    22437\n",
       "11   New York City       New York    21635\n",
       "12           Miami        Florida    21484\n",
       "13     Los Angeles     California    21254\n",
       "14           Miami        Florida    20518\n",
       "15     Los Angeles     California    19179\n",
       "16          Boston  Massachusetts    18431\n",
       "17     Los Angeles     California    18297\n",
       "18   New York City     New Jersey    18218\n",
       "19      Louisville       Kentucky    18100\n",
       "20   New York City     New Jersey    18801\n",
       "21   San Francisco     California    18679\n",
       "22           Miami        Florida    17023\n",
       "23     Los Angeles     California    16896\n",
       "24    Philadelphia   Pennsylvania    16557\n",
       "25   New York City     New Jersey    16377\n",
       "26          Boston  Massachusetts    16354\n",
       "27      Providence   Rhode Island    16146\n",
       "28   New York City     New Jersey    16093\n",
       "29          Boston  Massachusetts    16036\n",
       "30     Los Angeles     California    16036\n",
       "..             ...            ...      ...\n",
       "104        Detroit       Michigan    10900\n",
       "105   Philadelphia   Pennsylvania    10897\n",
       "106   Philadelphia   Pennsylvania    10882\n",
       "107  New York City     New Jersey    10855\n",
       "108  New York City       New York    10847\n",
       "109  New York City     New Jersey    10841\n",
       "110  San Francisco     California    10752\n",
       "111  New York City     New Jersey    10744\n",
       "112    Los Angeles     California    10667\n",
       "113  New York City     New Jersey    10590\n",
       "114  New York City     New Jersey    10582\n",
       "115    Los Angeles     California    10572\n",
       "116  New York City     New Jersey    10556\n",
       "117          Miami        Florida    10474\n",
       "118    Los Angeles     California    10398\n",
       "119   Philadelphia   Pennsylvania    10397\n",
       "120  San Francisco     California    10368\n",
       "121  New York City     New Jersey    10358\n",
       "122         Boston  Massachusetts    10351\n",
       "123  New York City       New York    10337\n",
       "124   Philadelphia   Pennsylvania    10256\n",
       "125      Cleveland           Ohio    10208\n",
       "126  New York City       New York    10188\n",
       "127  New York City       New York    10187\n",
       "128    Los Angeles     California    10178\n",
       "129  New York City     New Jersey    10144\n",
       "130    Los Angeles     California    10126\n",
       "131   Philadelphia   Pennsylvania    10107\n",
       "132        Chicago       Illinois    10094\n",
       "133    Los Angeles     California    10065\n",
       "\n",
       "[133 rows x 3 columns]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#-- Get the most populated cities from wikipedia (Thank you wikipedia library!) --\n",
    "html = wp.page(\"List_of_United_States_cities_by_population_density\").html().encode(\"UTF-8\")\n",
    "df = pd.read_html(html)[1]\n",
    "df=df.drop([0,1,4,5,6,8],axis=1)\n",
    "df=df.rename(columns={2:'state',3:'city',7:'density'})\n",
    "df=df.iloc[1:]\n",
    "#df['population']=[int(city.split('\\xa0')[-1].split('[')[0].replace(',','')) for city in df['population']]\n",
    "df['density']=[int(city.split('\\xa0')[-1].split('[')[0].replace(',','').split('.')[0]) for city in df['density']]\n",
    "df\n",
    "    #--- ---- ----- ----- ---- ---- ----- ---- ---- ----- --- ---- ---- ----- ---- ---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}