added code and ran tests using sntwitter

This commit is contained in:
simplypower-bbj
2022-12-18 11:26:50 +01:00
parent d2b9f08dea
commit a1d3406ce9
2 changed files with 248 additions and 0 deletions
+18
View File
@@ -0,0 +1,18 @@
import snscrape.modules.twitter as sntwitter
import pandas as pd
def scrape(limit=0):
# prepare variables
tweet_list = list()
query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
# begin scraping
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
# if limit is reached, break out of loop
if limit > 0 and i > limit: break
# if another 100 tweets found, tell it
if i % 100 == 0:
print(f'found {i} tweets')
tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])
# create dataframe
df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])
return df