ran tests

code/classes.py

@@ -0,0 +1,40 @@
+from pydantic import BaseModel, validator
+from typing import Union, List, Optional
+from datetime import datetime
+from PIL import Image
+from io import BytesIO
+
+class Tweet(BaseModel):
+    account: str
+    url: str
+    time: datetime
+    text: str
+    images: Optional[Union[List[Image.Image], List[BytesIO]]]
+    video: Optional[str]
+
+    class Config:
+        arbitrary_types_allowed = True
+    
+    @validator('images', each_item=True)
+    def parse_image(cls, v):
+        if isinstance(v, BytesIO):
+            v = Image.open(v)
+        return v
+    
+    def parse_images(self):
+        for i in range(len(self.images)):
+            self.images[i] = Image.open(self.images[i])
+    
+    def as_db_dict(self):
+        """Convert to database dictionary"""
+        d = self.dict()
+        d['time'] = d['time'].strftime('%Y-%m-%d %H:%M:%S')
+        img_list = list()
+        for img in d['images']:
+            if isinstance(img, BytesIO):
+                img = Image.open(img)
+            buf = BytesIO()
+            img.save(buf, format='PNG')
+            img_list.append(buf.getvalue())
+        d['images'] = img_list
+        return d

code/database.py

@@ -7,6 +7,7 @@ import pandas as pd
 import psycopg2
 from dotenv import load_dotenv
 from sqlalchemy import create_engine
+from pymongo import MongoClient
 
 
 class Database(object):
@@ -223,3 +224,9 @@ def get_untranslated(dt):
         'content': str(df['content'][0])
     }
     return val_dict
+
+def connect_mongo(ip_addr: str = '192.168.1.2', db_name: str = 'twitter', collection_name: str = 'tweet'):
+    client = MongoClient(ip_addr)
+    db = client[db_name]
+    coll = db[collection_name]
+    return coll

code/selenium_scraper.ipynb

@@ -0,0 +1,113 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from selenium.webdriver.common.by import By\n",
+    "# import requests\n",
+    "# import pyperclip\n",
+    "from selenium_scraper import Twitter\n",
+    "from datetime import timedelta, datetime\n",
+    "from classes import Tweet\n",
+    "import traceback\n",
+    "import logging\n",
+    "from database import connect_mongo\n",
+    "\n",
+    "account = 'Maersk'\n",
+    "until_year = 2010"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# reset database\n",
+    "db = connect_mongo()\n",
+    "# db.drop()\n",
+    "db.count_documents({'account':f'@{account}'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "ERROR:root:failed extracting post url\n",
+      "WARNING:root:retrying later due to error getting period: [2023-02-23 00:00:00:2023-03-05 00:00:00]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium_scraper import scrape_account\n",
+    "\n",
+    "scrape_account(account, until_year)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

code/selenium_scraper.py

@@ -0,0 +1,329 @@
+from selenium import webdriver
+from selenium.webdriver.common.action_chains import ActionChains
+from selenium.webdriver.chrome.service import Service
+from webdriver_manager.chrome import ChromeDriverManager
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.wait import WebDriverWait
+import selenium.webdriver.support.expected_conditions as EC
+from selenium.webdriver.remote.webelement import WebElement
+from pydantic import BaseModel, validator, parse_obj_as, AnyHttpUrl
+from typing import Union, List
+from datetime import datetime, timedelta
+from PIL import Image
+from io import BytesIO
+import requests
+import pyperclip
+import logging
+import time
+import traceback
+
+from database import connect_mongo
+from classes import Tweet
+
+class Twitter(object):
+    def __init__(self):
+        self.webdriver = None
+        self.actiondriver = None
+        self.timeline_elem = None
+        self._read_posts_timestamp_list = list()
+        self._unread_posts_element_list = list()
+
+    @staticmethod
+    def _url( account: str, date_begin: datetime, date_end: datetime) -> AnyHttpUrl:
+        return parse_obj_as(AnyHttpUrl, f"https://twitter.com/search?q=(from%3A{account})%20until%3A{date_end.strftime('%Y-%m-%d')}%20since%3A{date_begin.strftime('%Y-%m-%d')}%20-filter%3Areplies&src=typed_query&f=top")
+
+    def setup(self) -> None:
+        # setup webdriver
+        driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
+        self.webdriver = driver
+        self.actiondriver = ActionChains(self.webdriver)
+
+    def close(self) -> None:
+        self.webdriver.close()
+
+    def __enter__(self):
+        self.setup()
+        return self
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+    def _update_unread_posts(self) -> None:
+        self._handle_popup()
+        elem_list = self.timeline_elem.find_elements(By.XPATH, """.//article[@data-testid="tweet"]""")
+        # elem_list = self.webdriver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
+        for elem in elem_list:
+            try:
+                elem_list = elem.find_elements(By.XPATH, './/time')
+                assert len(elem_list) > 0, 'failed locating time of post'
+                timestamp_str = elem_list[0].get_attribute('datetime')
+            except Exception:
+                pass
+            else:
+                # print('considering post with timestamp_str: ', timestamp_str)
+                if timestamp_str not in self._read_posts_timestamp_list:
+                    self._unread_posts_element_list.append(elem)
+                    # print('attached post to list')
+                else:
+                    # print('post already in list')
+                    pass
+
+    def unread_posts_available(self) -> bool:
+        if len(self._unread_posts_element_list) == 0:
+            self._update_unread_posts()
+        return len(self._unread_posts_element_list) > 0
+    
+    def number_of_posts_read(self) -> int:
+        return len(self._read_posts_timestamp_list)
+    
+    def _handle_popup(self):
+        elem_list = self.webdriver.find_elements(By.XPATH, "//*[text()='Not now']")
+        if len(elem_list) > 0:
+            elem_list[0].click()
+
+    def _accept_cookies(self):
+        self._handle_popup()
+        try:
+            elem = self.webdriver.find_element(By.XPATH, "//*[text()='Accept all cookies']")
+            elem.click()
+        except Exception:
+            logging.warning('failed accepting cookies')
+            
+    def _open_page(self, account: str, date_begin: datetime, date_end: datetime) -> None:
+        url = self._url(account,date_begin, date_end)
+        self.webdriver.get(str(url))
+        # wait for popup to appear and click on 'not now'-button
+        try:
+            elem = WebDriverWait(
+                driver=self.webdriver, 
+                timeout=60
+            ).until(
+                EC.presence_of_element_located(
+                    (By.XPATH, "//*[text()='Not now']")
+                )
+            )
+        except TimeoutError:
+            pass
+        else:
+            elem.click()
+        # click allow cookies
+        self._accept_cookies()
+        # identify timeline element
+        self.timeline_elem = self.webdriver.find_element(By.XPATH, """//div[@aria-label="Timeline: Search timeline"]""")
+        # update unread posts
+        self._update_unread_posts()
+
+    def _extract_account(self, elem):
+        self._handle_popup()
+        try:
+            elem_list = elem.find_elements(By.XPATH, ".//span[contains(text(), '@')]")
+            assert len(elem_list) > 0, 'failed locating account'
+            self.actiondriver.move_to_element(elem_list[0]).perform()
+            account_str = elem_list[0].text
+        except Exception:
+            logging.error('failed extracting account name')
+            account_str = ''
+        return account_str
+
+    def _extract_time(self, elem):
+        self._handle_popup()
+        try:
+            elem_list = elem.find_elements(By.XPATH, './/time')
+            assert len(elem_list) > 0, 'failed locating time of post'
+            self.actiondriver.move_to_element(elem_list[0]).perform()
+            datetime_str = elem_list[0].get_attribute('datetime')
+        except Exception:
+            logging.error('failed extracting post time')
+            datetime_str = ''
+        return datetime_str
+
+    def _extract_text(self, elem):
+        self._handle_popup()
+        try:
+            elem_list = elem.find_elements(By.XPATH, './/div[@data-testid="tweetText"]')
+            assert len(elem_list) > 0, 'failed locating text of post'
+            self.actiondriver.move_to_element(elem_list[0]).perform()
+            text = elem_list[0].text
+        except Exception:
+            logging.error('failed extracting post text')
+            text = ''
+        return text
+    
+    def _extract_post_url(self, elem):
+        self._handle_popup()
+        try:
+            share_button = elem.find_element(By.XPATH, """.//div[@aria-label="Share Tweet"]""")
+            # WebDriverWait(self.webdriver, 15).until(EC.element_to_be_clickable((By.XPATH, "//div[@aria-label='Share Tweet']"))).click()
+            # self.webdriver.execute_script("arguments[0].scrollIntoView();", elem_list[0])
+            # scroll to element
+            self.webdriver.execute_script(
+                """arguments[0].scrollIntoView({behavior: "smooth", block: "center", inline: "nearest"})""", 
+                share_button
+            )
+            # move mouse to element
+            self.actiondriver.move_to_element(share_button).perform()
+            # click element
+            share_button = elem.find_element(By.XPATH, """.//div[@aria-label="Share Tweet"]""")
+            share_button.click()
+            # wait for popup button to appear
+            share_button = WebDriverWait(
+                driver=self.webdriver, 
+                timeout=10
+            ).until(
+                EC.presence_of_element_located(
+                    (By.XPATH, """//span[contains(text(), "Copy link to Tweet")]""")
+                )
+            )
+            # click button
+            # elem_list = elem.find_elements(By.XPATH, "//span[contains(text(), 'Copy link to Tweet')]")
+            self.actiondriver.move_to_element(share_button).perform()
+            share_button.click()
+            # paste copied url
+            url = pyperclip.paste()
+        except Exception:
+            logging.error('failed extracting post url')
+            url = ''
+        return url
+    
+    def _extract_video_url(self, elem):
+        self._handle_popup()
+        try:
+            elem_list = elem.find_elements(By.TAG_NAME, 'video')
+            if len(elem_list) > 0:
+                video_url = elem_list[0].get_attribute('src')
+            else:
+                video_url = ''
+        except Exception:
+            logging.error('failed extracting post video')
+            video_url = ''
+        return video_url
+    
+    def _extract_post_images(self, elem):
+        self._handle_popup()
+        img_list = list()
+        try:
+            elem_list = elem.find_elements(By.TAG_NAME, 'img')
+            if len(elem_list) == 0:
+                return img_list
+            # get image urls
+            image_url_list = [e.get_attribute('src') for e in elem_list]
+            # keep only media-related urls
+            image_url_list = [url for url in image_url_list if 'media' in url]
+            if len(image_url_list) == 0:
+                return img_list
+            # download image data
+            for url in image_url_list:
+                try:
+                    data = requests.get(url).content
+                except Exception:
+                    logging.error(f'failed downloading image: {url}')
+                    continue
+                img_data = BytesIO(data)
+                img_list.append(img_data)
+        except Exception:
+            logging.error('failed extracting post images')
+        return img_list
+
+    def get_post(self):
+        if len(self._unread_posts_element_list) == 0:
+            raise ValueError('no unread posts to read')
+        # get element
+        elem = self._unread_posts_element_list.pop(0)
+        # prepare response
+        content = dict()
+        try:
+            # look for popup
+            self._handle_popup()
+            # scroll to element
+            self.actiondriver.move_to_element(elem).perform()
+            # get account
+            content['account'] = self._extract_account(elem)
+            # get time
+            datetime_str = self._extract_time(elem)
+            # print(datetime_str)
+            content['time'] = datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S.%fZ')
+            # get text
+            content['text'] = self._extract_text(elem)
+            # get post link
+            content['url'] = self._extract_post_url(elem)
+            # get video url
+            content['video'] = self._extract_video_url(elem)
+            # get image(s)
+            content['images'] = self._extract_post_images(elem)
+        except KeyboardInterrupt:
+            return
+        except Exception:
+            traceback.print_exc()
+        else:
+            # add timestamp_str to list of read posts
+            self._read_posts_timestamp_list.append(datetime_str)
+            # print('sucessfully read post')
+        return content, elem
+
+def generate_date_list(until_year=2010):
+    end_date = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
+    start_date = end_date - timedelta(days=10)
+    end_date_list = [end_date]
+    start_date_list = [start_date]
+    while start_date.year > until_year:
+        end_date = start_date - timedelta(days=1)
+        end_date_list.append(end_date)
+        start_date = end_date - timedelta(days=10)
+        start_date_list.append(start_date)
+    date_list = list(zip(start_date_list, end_date_list))
+    return date_list
+
+def get_period(account: str, start_date: datetime, end_date: datetime):
+    try:
+        # setup
+        twitter = Twitter()
+        twitter.setup()
+        twitter._open_page(account, start_date, end_date)
+        db = connect_mongo()
+        # scrape
+        limit = 500
+        attempts = 0
+        while attempts < limit and twitter.unread_posts_available():
+            content, elem = twitter.get_post()
+            # print(content)
+            t = Tweet(**content)
+            db_id = db.insert_one(t.as_db_dict()).inserted_id
+            # print(f"""inserted tweet from {t.time.strftime('%Y-%m-%d %H:%M:%S')} with db id: {db_id}""")
+            attempts += 1
+        num_posts_read = twitter.number_of_posts_read()
+        # print('number of posts read: ', num_posts_read)
+    except Exception as e:
+        raise e
+    finally:
+        twitter.close()
+    return num_posts_read
+
+def scrape_account(account: str, until_year: int):
+    # generate date list
+    date_list = generate_date_list(until_year)
+    # start scraping
+    i = 0
+    limit = 1000
+    while len(date_list) > 0:
+        if i >= limit:
+            logging.error('scraping attempt limit reached!')
+            return
+        # get period
+        start_date, end_date = date_list.pop(0)
+        try:
+            num_posts_found = get_period(account, start_date, end_date)
+        except KeyboardInterrupt:
+            logging.warning('KeyboardInterrupt')
+            return
+        except:
+            date_list.append((start_date, end_date))
+            logging.warning(f'retrying later due to error getting period {start_date} to {end_date}')
+        else:
+            logging.info(f'found {num_posts_found} posts between {start_date} and {end_date}')
+        i += 1
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    account = 'Maersk'
+    until_year = 2010
+    scrape_account(account, until_year)

code/sntwitter_scraper.ipynb

@@ -2,24 +2,31 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
+     "name": "stderr",
      "output_type": "stream",
      "text": [
-      "found 0 tweets\n",
+      "Error retrieving https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel: non-200 status code\n",
-      "found 100 tweets\n",
+      "4 requests to https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel failed, giving up.\n"
-      "found 200 tweets\n",
+     ]
-      "found 300 tweets\n",
+    },
-      "found 400 tweets\n",
+    {
-      "found 500 tweets\n",
+     "ename": "ScraperException",
-      "found 600 tweets\n",
+     "evalue": "4 requests to https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel failed, giving up.",
-      "found 700 tweets\n",
+     "output_type": "error",
-      "found 800 tweets\n",
+     "traceback": [
-      "found 900 tweets\n",
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "found 1000 tweets\n"
+      "\u001b[0;31mScraperException\u001b[0m                          Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 10\u001b[0m\n\u001b[1;32m      8\u001b[0m tweet_list \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m()\n\u001b[1;32m      9\u001b[0m \u001b[39m# begin scraping\u001b[39;00m\n\u001b[0;32m---> 10\u001b[0m \u001b[39mfor\u001b[39;00m i, tweet \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(sntwitter\u001b[39m.\u001b[39mTwitterSearchScraper(query)\u001b[39m.\u001b[39mget_items()):\n\u001b[1;32m     11\u001b[0m     \u001b[39m# if limit is reached, break out of loop\u001b[39;00m\n\u001b[1;32m     12\u001b[0m     \u001b[39mif\u001b[39;00m limit \u001b[39m>\u001b[39m \u001b[39m0\u001b[39m \u001b[39mand\u001b[39;00m i \u001b[39m>\u001b[39m limit: \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m     13\u001b[0m     \u001b[39m# if another 100 tweets found, tell it\u001b[39;00m\n",
+      "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:680\u001b[0m, in \u001b[0;36mTwitterSearchScraper.get_items\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    677\u001b[0m \t\u001b[39mdel\u001b[39;00m params[\u001b[39m'\u001b[39m\u001b[39mtweet_search_mode\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[1;32m    678\u001b[0m \t\u001b[39mdel\u001b[39;00m paginationParams[\u001b[39m'\u001b[39m\u001b[39mtweet_search_mode\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[0;32m--> 680\u001b[0m \u001b[39mfor\u001b[39;00m obj \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_iter_api_data(\u001b[39m'\u001b[39m\u001b[39mhttps://api.twitter.com/2/search/adaptive.json\u001b[39m\u001b[39m'\u001b[39m, params, paginationParams, cursor \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_cursor):\n\u001b[1;32m    681\u001b[0m \t\u001b[39myield from\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_instructions_to_tweets(obj)\n",
+      "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:369\u001b[0m, in \u001b[0;36m_TwitterAPIScraper._iter_api_data\u001b[0;34m(self, endpoint, params, paginationParams, cursor, direction)\u001b[0m\n\u001b[1;32m    367\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m    368\u001b[0m \t_logger\u001b[39m.\u001b[39minfo(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mRetrieving scroll page \u001b[39m\u001b[39m{\u001b[39;00mcursor\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m)\n\u001b[0;32m--> 369\u001b[0m \tobj \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_get_api_data(endpoint, reqParams)\n\u001b[1;32m    370\u001b[0m \t\u001b[39myield\u001b[39;00m obj\n\u001b[1;32m    372\u001b[0m \t\u001b[39m# No data format test, just a hard and loud crash if anything's wrong :-)\u001b[39;00m\n",
+      "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:339\u001b[0m, in \u001b[0;36m_TwitterAPIScraper._get_api_data\u001b[0;34m(self, endpoint, params)\u001b[0m\n\u001b[1;32m    337\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_get_api_data\u001b[39m(\u001b[39mself\u001b[39m, endpoint, params):\n\u001b[1;32m    338\u001b[0m \t\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ensure_guest_token()\n\u001b[0;32m--> 339\u001b[0m \tr \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_get(endpoint, params \u001b[39m=\u001b[39;49m params, headers \u001b[39m=\u001b[39;49m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_apiHeaders, responseOkCallback \u001b[39m=\u001b[39;49m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_check_api_response)\n\u001b[1;32m    340\u001b[0m \t\u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    341\u001b[0m \t\tobj \u001b[39m=\u001b[39m r\u001b[39m.\u001b[39mjson()\n",
+      "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/base.py:216\u001b[0m, in \u001b[0;36mScraper._get\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    215\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_get\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[0;32m--> 216\u001b[0m \t\u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_request(\u001b[39m'\u001b[39;49m\u001b[39mGET\u001b[39;49m\u001b[39m'\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
+      "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/base.py:212\u001b[0m, in \u001b[0;36mScraper._request\u001b[0;34m(self, method, url, params, data, headers, timeout, responseOkCallback, allowRedirects)\u001b[0m\n\u001b[1;32m    210\u001b[0m \tmsg \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_retries \u001b[39m+\u001b[39m \u001b[39m1\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m requests to \u001b[39m\u001b[39m{\u001b[39;00mreq\u001b[39m.\u001b[39murl\u001b[39m}\u001b[39;00m\u001b[39m failed, giving up.\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m    211\u001b[0m \tlogger\u001b[39m.\u001b[39mfatal(msg)\n\u001b[0;32m--> 212\u001b[0m \t\u001b[39mraise\u001b[39;00m ScraperException(msg)\n\u001b[1;32m    213\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39mReached unreachable code\u001b[39m\u001b[39m'\u001b[39m)\n",
+      "\u001b[0;31mScraperException\u001b[0m: 4 requests to https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel failed, giving up."
      ]
     }
    ],
@@ -28,16 +35,16 @@
     "import pandas as pd\n",
     "\n",
     "# prepare variables\n",
-    "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
+    "# query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
-    "limit = 1000\n",
+    "query = '(from:Maersk) -filter:replies'\n",
+    "limit = 10\n",
     "tweet_list = list()\n",
-    "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
     "# begin scraping\n",
     "for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n",
     "    # if limit is reached, break out of loop\n",
     "    if limit > 0 and i > limit: break\n",
     "    # if another 100 tweets found, tell it\n",
-    "    if i % 100 == 0: \n",
+    "    if i % 10 == 0: \n",
     "        print(f'found {i} tweets')\n",
     "    tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n",
     "# create dataframe\n",
@@ -46,152 +53,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
-     "data": {
+     "ename": "NameError",
-      "text/html": [
+     "evalue": "name 'df' is not defined",
-       "<div>\n",
+     "output_type": "error",
-       "<style scoped>\n",
+     "traceback": [
-       "    .dataframe tbody tr th:only-of-type {\n",
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-       "        vertical-align: middle;\n",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-       "    }\n",
+      "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df\n",
-       "\n",
+      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
-       "    .dataframe tbody tr th {\n",
+     ]
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>Datetime</th>\n",
-       "      <th>Id</th>\n",
-       "      <th>Text</th>\n",
-       "      <th>Username</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>2022-12-16 22:20:26+00:00</td>\n",
-       "      <td>1603877740972752897</td>\n",
-       "      <td>Danimarka’nın Odense takımı bu yaz müthiş bir ...</td>\n",
-       "      <td>EryilmazFa10</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>2022-12-16 21:56:35+00:00</td>\n",
-       "      <td>1603871736579371008</td>\n",
-       "      <td>Highlights: SønderjyskE – Odense https://t.co/...</td>\n",
-       "      <td>trans_rumor</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>2022-12-16 21:39:40+00:00</td>\n",
-       "      <td>1603867481101066241</td>\n",
-       "      <td>“Meta Platforms Inc has halted construction of...</td>\n",
-       "      <td>sakak</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>2022-12-16 21:19:09+00:00</td>\n",
-       "      <td>1603862318647070720</td>\n",
-       "      <td>Sponsoreret: Verdensmestre, europamestre og st...</td>\n",
-       "      <td>otwndk</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>2022-12-16 21:07:09+00:00</td>\n",
-       "      <td>1603859298249019392</td>\n",
-       "      <td>Meta just halted a major data centre project t...</td>\n",
-       "      <td>techosmo</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>996</th>\n",
-       "      <td>2022-11-23 15:42:37+00:00</td>\n",
-       "      <td>1595442703990493211</td>\n",
-       "      <td>Mohamed Djan Fofana går fra banen, og han erst...</td>\n",
-       "      <td>Odense_Boldklub</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>997</th>\n",
-       "      <td>2022-11-23 15:34:36+00:00</td>\n",
-       "      <td>1595440689629859840</td>\n",
-       "      <td>1-1...\\nEfter et hjørnespark passerer bolden t...</td>\n",
-       "      <td>Odense_Boldklub</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>998</th>\n",
-       "      <td>2022-11-23 15:31:12+00:00</td>\n",
-       "      <td>1595439830187798529</td>\n",
-       "      <td>Ud er gået Sebastian Wille, Omar Jebali, Law T...</td>\n",
-       "      <td>Odense_Boldklub</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>999</th>\n",
-       "      <td>2022-11-23 15:21:18+00:00</td>\n",
-       "      <td>1595437338997542912</td>\n",
-       "      <td>Anden halvleg er netop sparket i gang 🔵⚪\\n#obd...</td>\n",
-       "      <td>Odense_Boldklub</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1000</th>\n",
-       "      <td>2022-11-23 15:05:57+00:00</td>\n",
-       "      <td>1595433476135829506</td>\n",
-       "      <td>Efter en scoring af Franco Tongya går vi nu ti...</td>\n",
-       "      <td>Odense_Boldklub</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>1001 rows × 4 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                      Datetime                   Id  \\\n",
-       "0    2022-12-16 22:20:26+00:00  1603877740972752897   \n",
-       "1    2022-12-16 21:56:35+00:00  1603871736579371008   \n",
-       "2    2022-12-16 21:39:40+00:00  1603867481101066241   \n",
-       "3    2022-12-16 21:19:09+00:00  1603862318647070720   \n",
-       "4    2022-12-16 21:07:09+00:00  1603859298249019392   \n",
-       "...                        ...                  ...   \n",
-       "996  2022-11-23 15:42:37+00:00  1595442703990493211   \n",
-       "997  2022-11-23 15:34:36+00:00  1595440689629859840   \n",
-       "998  2022-11-23 15:31:12+00:00  1595439830187798529   \n",
-       "999  2022-11-23 15:21:18+00:00  1595437338997542912   \n",
-       "1000 2022-11-23 15:05:57+00:00  1595433476135829506   \n",
-       "\n",
-       "                                                   Text         Username  \n",
-       "0     Danimarka’nın Odense takımı bu yaz müthiş bir ...     EryilmazFa10  \n",
-       "1     Highlights: SønderjyskE – Odense https://t.co/...      trans_rumor  \n",
-       "2     “Meta Platforms Inc has halted construction of...            sakak  \n",
-       "3     Sponsoreret: Verdensmestre, europamestre og st...           otwndk  \n",
-       "4     Meta just halted a major data centre project t...         techosmo  \n",
-       "...                                                 ...              ...  \n",
-       "996   Mohamed Djan Fofana går fra banen, og han erst...  Odense_Boldklub  \n",
-       "997   1-1...\\nEfter et hjørnespark passerer bolden t...  Odense_Boldklub  \n",
-       "998   Ud er gået Sebastian Wille, Omar Jebali, Law T...  Odense_Boldklub  \n",
-       "999   Anden halvleg er netop sparket i gang 🔵⚪\\n#obd...  Odense_Boldklub  \n",
-       "1000  Efter en scoring af Franco Tongya går vi nu ti...  Odense_Boldklub  \n",
-       "\n",
-       "[1001 rows x 4 columns]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -222,12 +96,12 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
+   "version": "3.8.10"
   },
   "orig_nbformat": 4,
   "vscode": {
    "interpreter": {
-    "hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
+    "hash": "57b41ea9bda188de64800ef31c5cfb4fcfca577257038becb66e4fa0a5ee97a6"
    }
   }
  },

code/sntwitter_scraper.py

@@ -5,13 +5,14 @@ import pandas as pd
 from database import insert_list
 from dotenv import load_dotenv
 
-def scrape():
+def scrape(query=''):
+    assert len(query) > 0, 'query string must be provided'
     # prepare lists to hold information
     username_list = list()
     timestamp_list = list()
     content_list = list()
     # begin scraping
-    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
+    # query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
     for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
         # store information in lists
         username_list.append(tweet.user.username)

requirements.txt

@@ -14,6 +14,7 @@ debugpy==1.6.4
 decorator==5.1.1
 discord-logging==1.0
 discord-webhook==1.0.0
+dnspython==2.3.0
 emoji==1.7.0
 entrypoints==0.4
 exceptiongroup==1.0.4
@@ -32,8 +33,8 @@ ipykernel==6.19.2
 ipython==8.7.0
 jedi==0.18.2
 jsonschema==4.17.3
-jupyter-client==7.4.8
+jupyter_client==7.4.8
-jupyter-core==5.1.0
+jupyter_core==5.1.0
 kiwisolver==1.4.4
 langdetect==1.0.9
 lxml==4.9.2
@@ -49,7 +50,7 @@ parso==0.8.3
 pexpect==4.8.0
 pickleshare==0.7.5
 Pillow==9.3.0
-pkgutil-resolve-name==1.3.10
+pkgutil_resolve_name==1.3.10
 platformdirs==2.6.0
 plotly==5.11.0
 prompt-toolkit==3.0.36
@@ -57,8 +58,11 @@ psutil==5.9.4
 psycopg2-binary==2.9.5
 ptyprocess==0.7.0
 pure-eval==0.2.2
+pydantic==1.10.5
 Pygments==2.13.0
+pymongo==4.3.3
 pyparsing==3.0.9
+pyperclip==1.8.2
 pyrsistent==0.19.2
 PySocks==1.7.1
 python-dateutil==2.8.2
@@ -67,7 +71,7 @@ python-logging-discord-handler==0.1.4
 pytz==2022.6
 pyzmq==24.0.1
 requests==2.28.1
-selenium==4.7.2
+selenium==4.8.2
 six==1.16.0
 sniffio==1.3.0
 snscrape==0.4.3.20220106
@@ -81,9 +85,12 @@ tqdm==4.64.1
 traitlets==5.7.1
 trio==0.22.0
 trio-websocket==0.9.2
+typing_extensions==4.5.0
 Unidecode==1.3.6
 urllib3==1.26.13
 wcwidth==0.2.5
+webdriver-manager==3.8.5
 wordcloud==1.8.2.2
 wsproto==1.2.0
+xtarfile==0.1.0
 zipp==3.11.0