34 lines
1,022 B
Python
34 lines
1,022 B
Python
from argparse import ArgumentParser
|
|
from html import unescape
|
|
from re import sub
|
|
|
|
from psycopg2 import connect
|
|
|
|
|
|
parser = ArgumentParser()
|
|
parser.add_argument('-d', '--database', help='database connection string')
|
|
parser.add_argument('-o', '--output', help='output file', default='data')
|
|
args = parser.parse_args()
|
|
|
|
|
|
# Fetch messages from database since it's way faster than using the API
|
|
conn = connect(args.database)
|
|
cur = conn.cursor()
|
|
cur.execute('SELECT text FROM statuses WHERE language=\'en\'')
|
|
statuses = cur.fetchall()
|
|
|
|
|
|
# Clean up statuses
|
|
for i in range(len(statuses)):
|
|
# Remove HTML stuff
|
|
statuses[i] = unescape(sub(r'<[^>]*>', '', statuses[i][0]))
|
|
# Remove URLs and special characters and convert to lowercase
|
|
# Uncomment for generating data for LSTMs
|
|
#statuses[i] = ' '.join([sub(r'[^a-z0-9]', '', word.lower()) for word in statuses[i].split() if word.find('://') == -1])
|
|
|
|
|
|
# Save to output file
|
|
with open(args.output, 'w') as f:
|
|
for status in statuses:
|
|
print(status, file=f)
|