DataSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributors': None,\n",
       " 'coordinates': None,\n",
       " 'created_at': 'Mon Jun 25 06:43:18 +0000 2018',\n",
       " 'entities': {'hashtags': [],\n",
       "  'symbols': [],\n",
       "  'urls': [],\n",
       "  'user_mentions': [{'id': 886971481200177153,\n",
       "    'id_str': '886971481200177153',\n",
       "    'indices': [0, 16],\n",
       "    'name': 'Sun Shine',\n",
       "    'screen_name': 'SunShin93735000'},\n",
       "   {'id': 1556419213,\n",
       "    'id_str': '1556419213',\n",
       "    'indices': [17, 29],\n",
       "    'name': 'Kitty_Maggy',\n",
       "    'screen_name': 'Kitty_Maggy'},\n",
       "   {'id': 38142665,\n",
       "    'id_str': '38142665',\n",
       "    'indices': [30, 42],\n",
       "    'name': 'Le Parisien',\n",
       "    'screen_name': 'le_Parisien'}]},\n",
       " 'favorite_count': 0,\n",
       " 'favorited': False,\n",
       " 'geo': None,\n",
       " 'id': 1011137744192638977,\n",
       " 'id_str': '1011137744192638977',\n",
       " 'in_reply_to_screen_name': 'SunShin93735000',\n",
       " 'in_reply_to_status_id': 1010592336487776256,\n",
       " 'in_reply_to_status_id_str': '1010592336487776256',\n",
       " 'in_reply_to_user_id': 886971481200177153,\n",
       " 'in_reply_to_user_id_str': '886971481200177153',\n",
       " 'is_quote_status': False,\n",
       " 'lang': 'fr',\n",
       " 'metadata': {'iso_language_code': 'fr', 'result_type': 'recent'},\n",
       " 'place': None,\n",
       " 'retweet_count': 0,\n",
       " 'retweeted': False,\n",
       " 'source': '<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>',\n",
       " 'text': \"@SunShin93735000 @Kitty_Maggy @le_Parisien Oui ce serait l'idéal mais nos dirigeants n'ont pas assez de c......pour le faire.\",\n",
       " 'truncated': False,\n",
       " 'user': {'contributors_enabled': False,\n",
       "  'created_at': 'Sun May 20 15:51:30 +0000 2012',\n",
       "  'default_profile': True,\n",
       "  'default_profile_image': False,\n",
       "  'description': '',\n",
       "  'entities': {'description': {'urls': []}},\n",
       "  'favourites_count': 17349,\n",
       "  'follow_request_sent': False,\n",
       "  'followers_count': 270,\n",
       "  'following': False,\n",
       "  'friends_count': 195,\n",
       "  'geo_enabled': False,\n",
       "  'has_extended_profile': False,\n",
       "  'id': 585766691,\n",
       "  'id_str': '585766691',\n",
       "  'is_translation_enabled': False,\n",
       "  'is_translator': False,\n",
       "  'lang': 'fr',\n",
       "  'listed_count': 2,\n",
       "  'location': '',\n",
       "  'name': 'Sylvia',\n",
       "  'notifications': False,\n",
       "  'profile_background_color': 'C0DEED',\n",
       "  'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_tile': False,\n",
       "  'profile_image_url': 'http://pbs.twimg.com/profile_images/3622230672/57d2d1e53af65e4b25f72d8346db5098_normal.jpeg',\n",
       "  'profile_image_url_https': 'https://pbs.twimg.com/profile_images/3622230672/57d2d1e53af65e4b25f72d8346db5098_normal.jpeg',\n",
       "  'profile_link_color': '1DA1F2',\n",
       "  'profile_sidebar_border_color': 'C0DEED',\n",
       "  'profile_sidebar_fill_color': 'DDEEF6',\n",
       "  'profile_text_color': '333333',\n",
       "  'profile_use_background_image': True,\n",
       "  'protected': False,\n",
       "  'screen_name': 'sylvia9183',\n",
       "  'statuses_count': 6955,\n",
       "  'time_zone': None,\n",
       "  'translator_type': 'none',\n",
       "  'url': None,\n",
       "  'utc_offset': None,\n",
       "  'verified': False}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Text, location, dates, specified number of results\n",
    "def SearchForData(search_term, nTweets):\n",
    "    # Import and Initialize Sentiment Analyzer\n",
    "    from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "    analyzer = SentimentIntensityAnalyzer()\n",
    "\n",
    "    import tweepy; import json\n",
    "    from apikeys import twitterAccessToken as access_token\n",
    "    from apikeys import twitterAccessTokenSecret as access_token_secret\n",
    "    from apikeys import twitterConsumerKey as consumer_key\n",
    "    from apikeys import twitterConsumerSecretKey as consumer_secret\n",
    "    # Setup Tweepy API Authentication\n",
    "    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "    auth.set_access_token(access_token, access_token_secret)\n",
    "    api = tweepy.API(auth, parser=tweepy.parsers.JSONParser())\n",
    "\n",
    "    maxTweets = 10000000 # Some arbitrary large number\n",
    "    public_tweets=[]\n",
    "    # Counter to keep track of the number of tweets retrieved\n",
    "    counter = 0\n",
    "    oldest_tweet = None\n",
    "    unique_ids = []\n",
    "    # Loop through 5 times (total of 500 tweets)\n",
    "    desiredTweets = []\n",
    "    while len(desiredTweets) < min(nTweets,maxTweets):\n",
    "        # Retrieve 100 most recent tweets -- specifying a max_id\n",
    "        public_tweets = api.search(search_term, count=100, result_type=\"recent\", max_id=oldest_tweet)\n",
    "\n",
    "        for tweet in public_tweets['statuses']:\n",
    "            tweet_id = tweet[\"id\"]\n",
    "\n",
    "            # Append tweet_id to ids list if it doesn't already exist\n",
    "            # This allows checking for duplicate tweets\n",
    "            if tweet_id not in unique_ids :\n",
    "                unique_ids.append(tweet_id)\n",
    "                desiredTweets.append(tweet)\n",
    "                                 \n",
    "            # Reassign the the oldest tweet (i.e. the max_id) subtract 1 so the previous oldest isn't included\n",
    "            oldest_tweet = tweet_id - 1\n",
    "    return desiredTweets\n",
    "\n",
    "\n",
    "               \n",
    "#oneTweet = SearchForData(\"kitty\", 100)\n",
    "#oneTweet[0]\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}