import json
import os
import pandas as pd
from IPython.display import display
from tqdm.notebook import tqdm

with open('./data/etc/ba_char.txt', 'r+', encoding='utf-8') as f:
    char_list = [char.strip() for char in f.readlines()]
char_list = list(set([char.strip().split(' ')[0].lower() for char in char_list]))

char_list += ['aris', 'arisu']

print(len(char_list))
char_list

file_list = [f for f in os.listdir('./data') if f.endswith('.json')]
file_list.sort()
print(file_list)

data_l = list()
for file in tqdm(file_list):
    data = pd.read_json(f'./data/{file}', orient='index')[['author', 'created_utc', 'title', 'over_18', 'link_flair_text', 'comments']]
    data_l.append(data)

data = pd.concat(data_l)
# display(data)

data.comments = data.comments.map(lambda inp: [comment['body'] for comment in inp])
display(data)

count = 0
for comments in data.comments:
    count += len(comments)

count

flair_data = data.link_flair_text.value_counts()

flair_data_clean = clean_flair(flair_data)

print(flair_data_clean)

y = flair_data_clean
mylabels = flair_data_clean.index

def make_autopct(values):
    def my_autopct(pct):
        total = sum(values)
        val = int(round(pct*total/100.0))
        return '{p:.2f}%  ({v:d})'.format(p=pct,v=val)
    return my_autopct

plt.pie(y, labels = mylabels, startangle = 90, autopct=make_autopct(y), rotatelabels = True, pctdistance = .8)
plt.show() 

flair_data = data.over_18.value_counts()
print(data.over_18.value_counts())

# Calculate percentage for each flair
flair_counts = data['link_flair_text'].value_counts(normalize=True) * 100

# Calculate proportion of over_18 for each flair
over_18_proportion = data.groupby('link_flair_text')['over_18'].mean() * 100

# Calculate proportion of over_18 for total
over_18_proportion_total = over_18_proportion.multiply(flair_counts) / 100

# Merging the results
result = pd.concat([flair_counts, over_18_proportion, over_18_proportion_total], axis=1)
result.columns = ['Flair Percentage', 'Over 18 Proportion', 'Over 18 proportion for total']

display(result.head(10))

result.to_csv('./data/etc/flairs.csv')

from nltk.tokenize import TweetTokenizer # seems suitable for Reddit? also for TW artist sourcing!
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer

import nltk
nltk.download('wordnet')
nltk.download('punkt')

tk = word_tokenize
ttk = TweetTokenizer()
wnl = WordNetLemmatizer()

tks = [ [ wnl.lemmatize(word).lower() for word in ttk.tokenize(title)] for title in tqdm(data.title)]

# tks = [(idx, [ wnl.lemmatize(word).lower() for word in ttk.tokenize(data.iloc[i].title)] ) for i, idx in tqdm(enumerate(data.index))]
# print(len(tks))
# print(tks[:5])
print(tks[:5])

# TweetTokenizer can't tokenize 's (eg: it's -> it + 's) so manual tokenization needed
# tks = [([word[:-2] if word.endswith("'s") else word for word in title]) for post in tks]
tks = [ [word[:-2] if word.endswith("'s") else word for word in title] for title in tks]
print('\n', tks[:5])

data['tks'] = tks
display(data)

def is_there_student(tks):
    return list(set([tk for tk in tks if tk in char_list]))

def find_alias(inp, aliases: list, to: str):
    return list(set([to if i in aliases else i for i in inp]))

data['students'] = data['tks'].map(is_there_student)

# find aliases for aris
data['students'] = data['students'].map(lambda inp: find_alias(inp, ['aris', 'alice', 'arisu'], 'aris'))

display(data)

artworks = data[ data['link_flair_text'].isin(['NON OC ART ', 'OC ART ', 'Comic/TL']) ]
display(artworks)

artworks_student = pd.Series( [student for students in artworks.students for student in students] ).value_counts()
artworks_student.nlargest(25)

sfw_artworks = data[ ~ data['over_18'] & data['link_flair_text'].isin(['NON OC ART ', 'OC ART ', 'Comic/TL']) ]
display(sfw_artworks)

sfw_artworks_student = pd.Series( [student for students in sfw_artworks.students for student in students] ).value_counts()
sfw_artworks_student

nsfw_artworks = data[ data['over_18'] & data['link_flair_text'].isin(['NON OC ART ', 'OC ART ', 'Comic/TL']) ]
display(nsfw_artworks)

nsfw_artworks_student = pd.Series( [student for students in nsfw_artworks.students for student in students] ).value_counts()
nsfw_artworks_student

fl = sfw_artworks_student.to_frame(name='sfw_artworks')
fl['nsfw_artworks'] = nsfw_artworks_student
fl['nsfw_%'] = round(fl.nsfw_artworks / (fl.nsfw_artworks + fl.sfw_artworks) * 100, 2)
fl

fl.nlargest(10, 'nsfw_%')

artists = pd.Series([ tk for tks in artworks.tks for tk in tks if tk.startswith('@') ]).value_counts()
print(len(artists))
artists.nlargest(10)

from nltk.tokenize import TweetTokenizer
ttk = TweetTokenizer()

data_lc = data_l

try:
    # can't run this part twice...
    for data in data_lc:
        data.comments = data.comments.map(lambda inp: [comment['body'] for comment in inp])
except: pass


emoji_data = pd.DataFrame()

# couldn't find an easy way to like is_emoji() or smth
emojis = '😀😁😂🤣😃😄😅😆😉😊😋😎😍😘🥰😗😙🥲😚☺️🙂🤗🤩🤔🫡🤨😐😑😶🫥😶‍🌫️🙄😏😣😥😮🤐😯😪😫🥱😴😌😛😜😝🤤😒😓😔😕🫤🙃🫠🤑😲☹️🙁😖😞😟😤😢😭😦😧😨😩🤯😬😮‍💨😰😱🥵🥶😳🤪😵😵‍💫🥴😠😡🤬😷🤒🤕🤢🤮🤧😇🥳🥸🥺🥹🤠🤡🤥🫨🤫🤭🫢🫣🧐🤓😈👿👹👺💀☠️👻👽👾🤖💩💪🦵🦶👂🦻👃🤏👈👉☝️🫵👆👇✌️🤞🫰🖖🫱🫲🫳🫴🫷🫸🤘🤙🖐️✋👌🤌👍👎✊👊🤛🤜🤚👋🤟✍️👏👐🙌🫶🤲🙏🦀💋✍️🔥💢❤️🩷🧡💛💚💙🩵💜🤎🖤🩶🤍💔❤️‍🔥❤️‍🩹❣️💕💞💓💗💖💘💝💟💌💢💥💤💦💨💫🕳️☮️❌⭕🔞❗❕❓❔‼️⁉️💯'

tks_per_month = [ [ [ttk.tokenize(comment) for comment in comments] for comments in m_data.comments] for m_data in tqdm(data_lc) ]

emote_count = [ pd.Series([tk for p_tk in m_tk for tks in p_tk for tk in tks if tk in emojis]).value_counts() for m_tk in tks_per_month ]
emote_count = [ count.drop('️') for count in emote_count ]
all_emotes = [ [tk for p_tk in m_tk for tks in p_tk for tk in tks if tk in emojis] for m_tk in tks_per_month ]

count = 0
for m in all_emotes:
    count += len(m)
count

for i, m in enumerate(emote_count):
    print(f'\n{i+1}: ')
    print(m.nlargest(10))

ttl_emoji = pd.Series()
for m_count in emote_count:
    ttl_emoji = ttl_emoji.add(m_count, fill_value=0).astype('int64')
print(ttl_emoji.nlargest(10))
top_20 = ttl_emoji.nlargest(20).index

emoji_data