mischbeck/CollectUSSenatorTweets
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adds typerror controls

Browse Source
This commit is contained in:
Michael Beck 2023-08-15 14:19:33 +02:00
parent 7c6b618272
commit b89b5969ec
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
funs/CleanTweets.py
View File

@ -1,8 +1,19 @@
import re import re
import string import string
def preprocess_roberta(text): # https://huggingface.co/cardiffnlp/twitter-roberta-base-sep2022
preprocessed_text = []
for t in text.split():
if len(t) > 1:
t = '@user' if t[0] == '@' and t.count('@') == 1 else t
t = 'http' if t.startswith('http') else t
preprocessed_text.append(t)
return ' '.join(preprocessed_text)
def remove_URL(text): def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+') try:
url = re.compile(r'https?://\S+|www\.\S+')
except: print(text)
return url.sub(r'', text) return url.sub(r'', text)
def remove_emoji(text): def remove_emoji(text):