57 lines
2.2 KiB
Python
57 lines
2.2 KiB
Python
from datetime import datetime
|
|
import time
|
|
import pandas as pd
|
|
import snscrape.modules.twitter as sntwitter
|
|
|
|
def scrapeTweets(handle, slice_data, keywords, td, tweetDFColumns, maxTweets = 5000):
|
|
i = 0
|
|
|
|
currentTime = datetime.now()
|
|
ts_beg = slice_data['beg_time']
|
|
ts_end = slice_data['end_time']
|
|
suffix = slice_data['suffix']
|
|
tweetDataFilePath = td + f"Tweets-{handle}{suffix}.csv"
|
|
|
|
# create empty tweetlist that will be filled with tweets of current sen
|
|
TweetList = []
|
|
|
|
# statusmsg
|
|
print(f'{currentTime:<30} Fetching: {handle:>15}{suffix:<7} - from {ts_beg} to {ts_end}')
|
|
|
|
# Snscrape query:
|
|
query = f'from:{handle} since:{ts_beg} until:{ts_end}'
|
|
for i,tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
|
|
if i >= maxTweets:
|
|
break
|
|
# get tweet vars from tweetDFColumns and append to singleTweetList
|
|
# which will then be appended to TweetList. TweetList contains all tweets of the current slice.
|
|
singleTweetList = []
|
|
for col in tweetDFColumns:
|
|
singleTweetList.append(eval(f'tweet.{col}'))
|
|
TweetList.append(singleTweetList)
|
|
|
|
|
|
# # Check if no tweets fetched for the current time slice. If there are no tweets, skip to next time_slices loop iteration
|
|
# if not TweetList:
|
|
# open(tweetDataFilePath, 'a').close()
|
|
# print(f'return empty in {handle}{suffix} - from {ts_beg} to {ts_end}')
|
|
# return
|
|
|
|
print(f'{i:<6} tweets scraped for: {handle:>15}{suffix:<7}')
|
|
|
|
# convert to dataframe
|
|
tweet_df = pd.DataFrame(TweetList, columns=tweetDFColumns)
|
|
|
|
## Check if tweet-text contains keyword
|
|
tweet_df['contains_keyword'] = ''
|
|
tweet_df['contains_keyword'] = (
|
|
tweet_df['rawContent'].str.findall('|'.join(keywords)).str.join(',').replace('', 'none')
|
|
)
|
|
#return(tweet_df)
|
|
# Save two versions of the dataset, one with all fields and one without dict fields
|
|
# define filepaths
|
|
csv_path = tweetDataFilePath
|
|
# save short csv
|
|
tweet_df.to_csv(csv_path, encoding='utf-8')
|
|
# sleep 1 second to not get blocked because of excessive requests
|
|
time.sleep(0.5) |