diff --git a/DataSearch.ipynb b/DataSearch.ipynb index b0497b7..e5cf0c9 100644 --- a/DataSearch.ipynb +++ b/DataSearch.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -10,211 +10,260 @@ "output_type": "stream", "text": [ "translated word: aso\n", - "\n", "Manila , Philippines : 14.5995124,120.9842195,100mi\n", - "there was an error translating sample tweet: @ryapee Hi Rya order ako ulit polyblender mejo winasak ng aso ko. 😂\n", + "Sample Tweet: I imagine anime character yung shiba inu, breed of dog\n", + "\n", + "\n", + "could not translate Marshallese\n", + "translated word: dog\n", + "Ebeye , Marshall Islands : 8.7814708,167.7373396,100mi\n", + "No tweets returned while searching for 'dog'\n", + "[]\n", + "\n", + "\n", + "translated word: aso\n", + "Pateros (Municipality) , Philippines : 14.5483779,121.070773,100mi\n", + "Sample Tweet: I imagine anime character yung shiba inu, breed of dog\n", + "\n", + "\n", + "translated word: कुत्ता\n", + "Mumbai , India : 19.0759837,72.8776559,100mi\n", + "Sample Tweet: @aajtak The hawked dog runs only to bite everyone, it is wise to avoid it.\n", + "\n", + "\n", + "could not translate Bangla\n", + "translated word: dog\n", + "Dhaka , Bangladesh : 23.810332,90.4125181,100mi\n", + "Sample Tweet: #comeonengland \n", + "Hello There\n", + "Are you looking t-shirt designer?\n", + "I am professional t shirt designer level 2 seller. i… https://t.co/ThRrKrtwD3\n", + "\n", + "\n", + "translated word: כֶּלֶב\n", + "Bnei Brak , Israel : 32.084932,34.835226,100mi\n", + "No tweets returned while searching for 'כֶּלֶב'\n", + "[]\n", + "\n", + "\n", + "translated word: aso\n", + "Caloocan , Philippines : 14.7565784,121.0449768,100mi\n", + "Sample Tweet: I imagine anime character yung shiba inu, breed of dog\n", + "\n", + "\n", + "translated word: chien\n", + "Levallois-Perret , France : 48.89321700000001,2.287864,100mi\n", + "Sample Tweet: RT @robinbinda: good guys, I know that twitter it can go super fast to relay an info,\n", + "\n", + "I have a friend of mine who lost his dog ...\n", + "\n", + "\n", + "translated word: chien\n", + "Le Pré-Saint-Gervais , France : 48.885063,2.40557,100mi\n", + "Sample Tweet: RT @robinbinda: good guys, I know that twitter it can go super fast to relay an info,\n", + "\n", + "I have a friend of mine who lost his dog ...\n", + "\n", + "\n", + "translated word: कुत्ता\n", + "Chennai , India : 13.0826802,80.2707184,100mi\n", + "No tweets returned while searching for 'कुत्ता'\n", + "[]\n", + "\n", + "\n", + "translated word: chien\n", + "Vincennes , France : 48.847759,2.4394969,100mi\n", + "Sample Tweet: RT @robinbinda: good guys, I know that twitter it can go super fast to relay an info,\n", + "\n", + "I have a friend of mine who lost his dog ...\n", + "\n", + "\n", + "translated word: chien\n", + "Saint-Mandé , France : 43.2676808,6.640710899999999,100mi\n", + "Sample Tweet: RT @BouletFrbt: Participate in the solidarity click campaign for Association Protection Animale Vallée Hérault https://t.co/tZtBv9Q21W @ ch ...\n", + "\n", + "\n", + "translated word: कुत्ता\n", + "Bally , India : 22.645734,88.34786,100mi\n", + "there was an error translating sample tweet: हम सारे मिलकर\n", + "एक दोस्त की शादी में\n", + "दिल से\n", + "नाच रहे थे,\n", + "🕺🕺💃💃🕺💃💃\n", + "\n", + "और वो कुत्ता 🐶🐕कमीना😈\n", + "\n", + "अपनी लुगाई👯\n", + "\n", + "को बोला सब\n", + ".\n", + ".\n", + ".\n", + ".\n", + "\n", + " बेवड़े है साले।\n", + "\n", + "\n", + "translated word: कुत्ता\n", + "Kolkata , India : 22.572646,88.36389500000001,100mi\n", + "there was an error translating sample tweet: हम सारे मिलकर\n", + "एक दोस्त की शादी में\n", + "दिल से\n", + "नाच रहे थे,\n", + "🕺🕺💃💃🕺💃💃\n", + "\n", + "और वो कुत्ता 🐶🐕कमीना😈\n", + "\n", + "अपनी लुगाई👯\n", + "\n", + "को बोला सब\n", + ".\n", + ".\n", + ".\n", + ".\n", + "\n", + " बेवड़े है साले।\n", + "\n", + "\n", + "translated word: hond\n", + "Saint-Josse-ten-Noode , Belgium : 50.8537715,4.3704657,100mi\n", + "Sample Tweet: Baasje apologizes to her dog - watch the wonderful reaction of the four-legged friend! https://t.co/WRMK7CEEe9\n", + "\n", + "\n", + "translated word: कुकुर\n", + "Kathmandu , Nepal : 27.7172453,85.3239605,100mi\n", + "Sample Tweet: Bukhra Daraouuda Lajskar Dog is a proud person, never reads 15 acres of lesson, the real farmer is given to Kari Kari Day.\n", + "\n", + "\n", + "could not translate BahasaMelayuinEastMalaysia\n", + "translated word: dog\n", + "Subang Jaya , Malaysia : 3.0567333,101.5851192,100mi\n", + "Sample Tweet: Petition: Petition: Chinese dog eating festival starts in 3 weeks – take action now! https://t.co/d4t9byqeNe\n", + "\n", + "\n", + "translated word: σκύλος\n", + "Neapoli , Greece : 36.5121752,23.0600381,100mi\n", + "No tweets returned while searching for 'σκύλος'\n", + "[]\n", + "\n", + "\n", + "translated word: chien\n", + "Montrouge , France : 48.816363,2.317384,100mi\n", + "Sample Tweet: RT @robinbinda: good guys, I know that twitter it can go super fast to relay an info,\n", + "\n", + "I have a friend of mine who lost his dog ...\n", + "\n", + "\n", + "could not translate MaldivianDhivehi\n", + "translated word: dog\n", + "Malé , Maldives : 4.1754959,73.5093474,100mi\n", + "No tweets returned while searching for 'dog'\n", + "[]\n", + "\n", + "\n", + "translated word: aso\n", + "Malabon , Philippines : 14.6680747,120.9658454,100mi\n", + "Sample Tweet: I imagine anime character yung shiba inu, breed of dog\n", + "\n", + "\n", + "translated word: dog\n", + "Guttenberg, New Jersey , United States : 40.7920454,-74.0037505,100mi\n", + "Sample Tweet: This shit a holiday dog https://t.co/8ACGrqLULC\n", + "\n", + "\n", + "translated word: aso\n", + "Pasig , Philippines : 14.5763768,121.0851097,100mi\n", + "Sample Tweet: I imagine anime character yung shiba inu, breed of dog\n", + "\n", + "\n", + "translated word: chien\n", + "Paris , France : 48.856614,2.3522219,100mi\n", + "Sample Tweet: RT @robinbinda: good guys, I know that twitter it can go super fast to relay an info,\n", + "\n", + "I have a friend of mine who lost his dog ...\n", + "\n", + "\n", + "could not translate CastilianSpanish\n", + "translated word: dog\n", + "Mislata , Spain : 39.4779243,-0.4195876,100mi\n", + "there was an error translating sample tweet: Thought my dog looked cute besides my chubby little face 🤷🏻‍♀️ https://t.co/Z9Tc42vc1i\n", + "\n", + "\n", + "could not translate StandardChinese\n", + "translated word: dog\n", + "Macau , China : 22.198745,113.543873,100mi\n", + "Sample Tweet: For every dog owner with a car. https://t.co/DHgd5gpO15\n", + "\n", + "\n", + "translated word: σκύλος\n", + "Kallithea , Greece : 37.955893,23.7020975,100mi\n", + "Sample Tweet: Learn the Language of a Dog's Body! 15 Ways to Understand What Your Dog Thinks https://t.co/L0K8MOgdJ1\n", + "\n", + "\n", + "translated word: σκύλος\n", + "Nea Smyrni , Greece : 37.9444167,23.7127056,100mi\n", + "Sample Tweet: Learn the Language of a Dog's Body! 15 Ways to Understand What Your Dog Thinks https://t.co/L0K8MOgdJ1\n", + "\n", + "\n", + "translated word: कुत्ता\n", + "Howrah , India : 22.5957689,88.26363940000002,100mi\n", + "there was an error translating sample tweet: हम सारे मिलकर\n", + "एक दोस्त की शादी में\n", + "दिल से\n", + "नाच रहे थे,\n", + "🕺🕺💃💃🕺💃💃\n", + "\n", + "और वो कुत्ता 🐶🐕कमीना😈\n", + "\n", + "अपनी लुगाई👯\n", + "\n", + "को बोला सब\n", + ".\n", + ".\n", + ".\n", + ".\n", + "\n", + " बेवड़े है साले।\n", + "\n", + "\n", + "translated word: aso\n", + "Pasay , Philippines : 14.5377516,121.0013794,100mi\n", + "Sample Tweet: I imagine anime character yung shiba inu, breed of dog\n", + "\n", + "\n", + "translated word: aso\n", + "San Juan , Philippines : 14.5994146,121.0368893,100mi\n", + "Sample Tweet: I imagine anime character yung shiba inu, breed of dog\n", + "\n", + "\n", + "translated word: බල්ලා\n", + "Colombo , Sri Lanka : 6.9270786,79.861243,100mi\n", + "No tweets returned while searching for 'බල්ලා'\n", + "[]\n", + "\n", + "\n", + "could not translate CastilianSpanish\n", + "translated word: dog\n", + "L'Hospitalet de Llobregat , Spain : 41.3661865,2.1164942,100mi\n", + "Sample Tweet: hey @andrewdamitio bud, are you Jewish? just checking to see how unselfaware your ‘ironic’ use of an anti semitic d… https://t.co/tS9fstdbsJ\n", + "\n", + "\n", + "translated word: dog\n", + "Union City, New Jersey , United States : 40.7795455,-74.02375119999999,100mi\n", + "Sample Tweet: RT @dodo: This vet locked himself in a hot car for 30 minutes to show what it’s like for a dog — here’s what happened. https://t.co/83T48En…\n", + "\n", + "\n", + "translated word: الكلب\n", + "Cairo , Egypt : 30.0444196,31.2357116,100mi\n", + "Sample Tweet: RT @ osamasalama500: Our neighbor Deloitte in the middle of a civilizational debate sharp with his times is shaking and say her first reply I said, but once, O daughter of the dog Mtradish high de ...\n", + "\n", + "\n", + "translated word: aso\n", + "Makati , Philippines : 14.554729,121.0244452,100mi\n", + "Sample Tweet: I imagine anime character yung shiba inu, breed of dog\n", "\n", "\n" ] - }, - { - "data": { - "text/plain": [ - "{'Manila,Philippines': followers following geo \\\n", - " 0 154060 False None \n", - " 1 83 False None \n", - " 2 643 False None \n", - " 3 2729 False None \n", - " 4 25 False None \n", - " 5 116 False None \n", - " 6 113 False None \n", - " 7 130 False None \n", - " 8 54 False None \n", - " 9 24 False None \n", - " 10 841 False None \n", - " 11 260 False None \n", - " 12 260 False None \n", - " 13 176 False None \n", - " 14 1357 False None \n", - " 15 422 False None \n", - " 16 422 False None \n", - " 17 365 False None \n", - " 18 75 False None \n", - " 19 1021 False None \n", - " 20 47 False None \n", - " 21 147 False None \n", - " 22 269 False None \n", - " 23 548 False None \n", - " 24 88 False None \n", - " 25 394 False None \n", - " 26 105 False None \n", - " 27 174 False None \n", - " 28 90 False None \n", - " 29 850 False None \n", - " .. ... ... ... \n", - " 70 80 False None \n", - " 71 180 False None \n", - " 72 1280 False None \n", - " 73 74 False None \n", - " 74 72 False None \n", - " 75 365 False None \n", - " 76 301 False None \n", - " 77 561 False None \n", - " 78 180 False None \n", - " 79 329 False None \n", - " 80 5 False None \n", - " 81 797 False None \n", - " 82 293 False None \n", - " 83 293 False None \n", - " 84 84 False None \n", - " 85 293 False None \n", - " 86 440 False None \n", - " 87 476 False None \n", - " 88 35 False None \n", - " 89 183 False None \n", - " 90 62 False None \n", - " 91 44 False None \n", - " 92 167 False None \n", - " 93 536 False None \n", - " 94 3228 False {'type': 'Point', 'coordinates': [14.61941886,... \n", - " 95 242 False None \n", - " 96 473 False None \n", - " 97 220 False None \n", - " 98 141 False None \n", - " 99 200 False None \n", - " \n", - " hashtags statuses_count \\\n", - " 0 [] 22455 \n", - " 1 [] 3030 \n", - " 2 [{'text': 'AdoptDontShop', 'indices': [38, 52]}] 8529 \n", - " 3 [] 28771 \n", - " 4 [] 14 \n", - " 5 [] 1993 \n", - " 6 [] 4495 \n", - " 7 [] 1745 \n", - " 8 [] 727 \n", - " 9 [] 296 \n", - " 10 [] 29025 \n", - " 11 [] 8486 \n", - " 12 [] 8486 \n", - " 13 [] 4682 \n", - " 14 [] 15736 \n", - " 15 [] 21045 \n", - " 16 [] 21045 \n", - " 17 [] 2083 \n", - " 18 [] 243 \n", - " 19 [] 11119 \n", - " 20 [] 905 \n", - " 21 [] 3406 \n", - " 22 [] 21310 \n", - " 23 [] 3553 \n", - " 24 [] 1470 \n", - " 25 [] 67411 \n", - " 26 [] 417 \n", - " 27 [] 4033 \n", - " 28 [] 828 \n", - " 29 [] 11211 \n", - " .. ... ... \n", - " 70 [] 1208 \n", - " 71 [] 6216 \n", - " 72 [] 5483 \n", - " 73 [] 2055 \n", - " 74 [] 456 \n", - " 75 [] 2083 \n", - " 76 [] 4788 \n", - " 77 [] 13927 \n", - " 78 [] 6718 \n", - " 79 [] 1259 \n", - " 80 [] 460 \n", - " 81 [] 42407 \n", - " 82 [] 3881 \n", - " 83 [] 3881 \n", - " 84 [] 15631 \n", - " 85 [] 3881 \n", - " 86 [] 1375 \n", - " 87 [] 7158 \n", - " 88 [] 1902 \n", - " 89 [] 1154 \n", - " 90 [] 756 \n", - " 91 [] 64 \n", - " 92 [] 2591 \n", - " 93 [] 6135 \n", - " 94 [] 10223 \n", - " 95 [] 4463 \n", - " 96 [] 5234 \n", - " 97 [] 10928 \n", - " 98 [] 12781 \n", - " 99 [] 8882 \n", - " \n", - " text \n", - " 0 @ryapee Hi Rya order ako ulit polyblender mejo... \n", - " 1 Yung dating saling pusa naging aso bigla // 🎶 \n", - " 2 sana meron din dito sa Pilipinas yung #AdoptDo... \n", - " 3 Aso nga kasi ako, bantay ako dito hahaha 😂 \n", - " 4 lakas mangahol ng kaklase ko dinaig pa aso nam... \n", - " 5 @LampanoElla Dun sa aso oo HAHA \n", - " 6 Me: labas mo dila para lumabas dila ng aso \\nS... \n", - " 7 cute kong aso https://t.co/twDprm7o5P \n", - " 8 me: pabili pong dog food\\ntindero: alin? ung p... \n", - " 9 @dsgalarpez hahahaha aso ka na ba ngayon? \n", - " 10 @DenniceRoselle Uy kawawa mga aso. Di naman si... \n", - " 11 Nagtanggal tuloy ako nang mga tae nang aso kai... \n", - " 12 Nakakapikon ung aso 😭😭 \n", - " 13 May mga sakit aso namin hanep \n", - " 14 i hate when strangers esp. men look at you str... \n", - " 15 @dnnkthryn Ngek malas. Sa rosewood naman okay ... \n", - " 16 @dnnkthryn Yup. Iba talaga pag bahay super fre... \n", - " 17 Tang ina sobrang iba pala pakiramdam pag namat... \n", - " 18 aso't pusa ❤\\ngoodmorning hubby babe ! https:/... \n", - " 19 RT @akoposimarcelo: Yung buti pa yung mga aso,... \n", - " 20 napaka clingy ng aso ko, nebeyen hehe \n", - " 21 ang laki ng aso!!! panay naka tahol \n", - " 22 Alam ata ng aso ko na birthday ko ngayon. Iba ... \n", - " 23 Nagduduet nanaman yung dalawang aso hays \n", - " 24 Yung aso naming maligalig na palundag lundag pa \n", - " 25 Mukha talaga akong tanga kapag nakikipaglaro a... \n", - " 26 RT @akoposimarcelo: Yung buti pa yung mga aso,... \n", - " 27 nagdisitahulan mga aso ang creepy huhu \n", - " 28 @glbysrcmny aso \n", - " 29 Distemper virus. may ganyan plang sakit nang m... \n", - " .. ... \n", - " 70 Plus two agad aso namen AAHAHAHAHAH saya \n", - " 71 @jhnlstrpgnsn Hahaha hindi ko aso yun sa ate k... \n", - " 72 RT @akoposimarcelo: Yung buti pa yung mga aso,... \n", - " 73 Mama: Tanga Di Mo Pa Pinapakain Yung Aso.\\n\\nT... \n", - " 74 @cescamarii di yan kusa aso yan \n", - " 75 Nasagasaan aso ko 😢 \n", - " 76 Ngayon ko nalang na appreciate ulit yung ganda... \n", - " 77 pag gantong nalulungkot ako imbis na maghanap ... \n", - " 78 creepy ng aso pero mahal ko kayo HAHAHHHAHAHA \n", - " 79 Pagod bebi ko ako inaantok na tagal ni aso HAH... \n", - " 80 So ayun diba may hamster kami si Luxus saka si... \n", - " 81 @Anniefernando6 @aldenAllTheWay Baka kayo ang ... \n", - " 82 hindi ako to pramis. hahahahahahaha hindi ako ... \n", - " 83 HUY ANG WEIRD TALAGA KASI NAGSESAVE AKO NG PIC... \n", - " 84 Fun fact about you — Sobrang love ko yung mga ... \n", - " 85 diko maintindihan bakit ako nagsesave ng pictu... \n", - " 86 Hayaan mo lang na husgahan ka nila.Hindi yung ... \n", - " 87 @pauiicosta lumabas na naman pagka aso mo haha... \n", - " 88 nagttampo ako kay potchi, tangina aso lang yon... \n", - " 89 Ako: pare iiyak ka pag namatay aso mo? \\nRoque... \n", - " 90 Aso ko e pero di na maghahabol🐶 https://t.co/Y... \n", - " 91 hi aso \n", - " 92 Nangapitbahay nako para sa aso. Happy pill! 😊 ... \n", - " 93 NP: Banal na aso, santong kabayo\\n\\nNatatawa a... \n", - " 94 Late post: Buti na lang talaga alert ako..kunc... \n", - " 95 tao,ahas at aso. \n", - " 96 ALAM NIYO BANG MUNTIKAN NG GAWING PAGKAIN NG A... \n", - " 97 Dang kyot ng aso nila Sir huhu i want 😍 \n", - " 98 Hindi sa dinidepensahan pero may umuulol na na... \n", - " 99 RT @Itsmeearlbravo: Di naman siguro ako pinang... \n", - " \n", - " [100 rows x 6 columns]}" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -310,10 +359,26 @@ " api = tweepy.API(auth, parser=tweepy.parsers.JSONParser())\n", " \n", " #--- Calculate geocordinates from cityCountry --- --- ---- ---- --- --- --- --\n", - " try:\n", - " result = Geocoder.geocode(cityCountry)\n", - " except Exception as error:\n", - " raise ValueError(f'Cannot translate to geo coordinates.\\n',error)\n", + " geoConvertTries = 0\n", + " while True:\n", + " try:\n", + " result = Geocoder.geocode(cityCountry)\n", + " except Exception as error:\n", + " if 'OVER_QUERY_LIMIT' in error:\n", + " print('Encountered an error:{0}\\nWaiting 30 seconds and trying again.'.format(error))\n", + " time.sleep(30)\n", + " if geoConvertTries>10\n", + " print(\"Could not convert geo. returning empty list\")\n", + " return []\n", + " elif not (re.search('^\\w+,\\w+$',cityCountry)):\n", + " print(\"cityCountry input format is incorrect. It should be \\'city,Country\\' like \\'paris,france\\'\")\n", + " return []\n", + " else:\n", + " print(\"Could not convert geo. returning empty list\")\n", + " return []\n", + " else:\n", + " break\n", + " geoConvertTries+=1\n", " # 34.0934,56.134,50mi\n", " coords = str(result[0].coordinates).replace('(','').replace(')','') + f',{radius}mi'\n", " coords=coords.replace(' ','')\n", @@ -384,12 +449,11 @@ " print('translated word: ',translatedSearch)\n", " tweets[city+','+country]=SearchForData(translatedSearch, numTweets, cityCountry, 100)\n", " print('\\n')\n", - " time.sleep(4)\n", - " break\n", + " time.sleep(30)\n", " return tweets\n", "\n", "tweets = GetTweetsByPopularCities('dog', 100)\n", - "tweets{'Manila,Philippines'}\n" + "tweets['Manila,Philippines']\n" ] }, { @@ -1909,6 +1973,18 @@ "display_name": "Python 3", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" } }, "nbformat": 4,