# 한국어 자료
import sys
import os
import numpy as np
import nltk
import konlpy
import pandas as pd
import re
import random
import itertools
import warnings
warnings.filterwarnings(action='ignore')
from sklearn.metrics import classification_report,f1_score,precision_score,recall_score
random.seed(1)

train_data=pd.read_csv('final_train_data.csv')
test_data=pd.read_csv('final_test_data.csv')

# 384 duplicates in content, 240 in title
print(train_data.describe())

train_data = train_data.drop_duplicates(['content'],keep='first')
train_data.duplicated().sum()

train_data['document']=train_data.iloc[:,1]+train_data.iloc[:,2]
train_data['document']
test_data['document']=test_data.iloc[:,1]+test_data.iloc[:,2]

train_data['document'] = train_data['document'].str.replace("[^ㄱ-ㅎㅏ-ㅣ가-힣 ]","")
test_data['document'] = test_data['document'].str.replace("[^ㄱ-ㅎㅏ-ㅣ가-힣 ]","")

# tokenize
from konlpy.tag import Okt
okt = Okt() #형태소 분석기
tokenized_data = []
stopwords = ['의','가','이','은','들','는','좀','잘','걍','과','도','를','으로','자','에','와','한','하다']
for sentence in train_data['document']:
    temp_X = okt.morphs(sentence, norm=True, stem=True) # 형태소 추출
    temp_X = [word for word in temp_X if not word in stopwords] # 불용어 제거
    tokenized_data.append(temp_X)
x_train=tokenized_data

okt = Okt() #형태소 분석기
tokenized_data = []
stopwords = ['의','가','이','은','들','는','좀','잘','걍','과','도','를','으로','자','에','와','한','하다']
for sentence in test_data['document']:
    temp_X = okt.morphs(sentence, norm=True, stem=True) # 형태소 추출 - 토큰화 #norm=True : 근사어
    temp_X = [word for word in temp_X if not word in stopwords] # 불용어 제거 #https://www.ranks.nl/stopwords/korean
    tokenized_data.append(temp_X)
x_test=tokenized_data

import pickle
with open('nlp_final_x_tr.data', 'wb') as f:
    pickle.dump(x_train, f)
with open('nlp_final_x_te.data', 'wb') as f:
    pickle.dump(x_test, f)

import pickle
with open('nlp_final_x_tr.data', 'rb') as f:
    x_train = pickle.load(f)
with open('nlp_final_x_te.data', 'rb') as f:
    x_test = pickle.load(f)

from tensorflow.keras.preprocessing.text import Tokenizer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(x_train)
threshold = 3
total_cnt = len(tokenizer.word_index) # 단어의 수
rare_cnt = 0 # 등장 빈도수가 threshold보다 작은 단어의 개수를 카운트
total_freq = 0 # 훈련 데이터의 전체 단어 빈도수 총 합
rare_freq = 0 # 등장 빈도수가 threshold보다 작은 단어의 등장 빈도수의 총 합

# 단어와 빈도수의 쌍(pair)을 key와 value로 받는다.
for key, value in tokenizer.word_counts.items():
    total_freq = total_freq + value

    # 단어의 등장 빈도수가 threshold보다 작으면
    if(value < threshold):
        rare_cnt = rare_cnt + 1
        rare_freq = rare_freq + value

print('단어 집합(vocabulary)의 크기 :',total_cnt)
print('등장 빈도가 %s번 이하인 희귀 단어의 수: %s'%(threshold - 1, rare_cnt))
print("단어 집합에서 희귀 단어의 비율:", (rare_cnt / total_cnt)*100)
print("전체 등장 빈도에서 희귀 단어 등장 빈도 비율:", (rare_freq / total_freq)*100)

vocab_size = total_cnt - rare_cnt + 1 # 전체 단어 개수 중 빈도수 2이하인 단어 개수는 제거. 0번 패딩 토큰을 고려하여 +1
print('단어 집합의 크기 :',vocab_size)

tokenizer = Tokenizer(vocab_size) 
tokenizer.fit_on_texts(x_train)
X_train = tokenizer.texts_to_sequences(x_train)
X_test = tokenizer.texts_to_sequences(x_test)

y_train=np.array(train_data.category)
y_test=np.array(test_data.category)

drop_train = [index for index, sentence in enumerate(X_train) if len(sentence) < 1]
drop_test = [index for index, sentence in enumerate(X_test) if len(sentence) < 1]
X_train = np.delete(X_train, drop_train, axis=0)
y_train = np.delete(y_train, drop_train, axis=0)
print(len(X_train))
print(len(y_train))

X_test = np.delete(X_test, drop_test, axis=0)
y_test = np.delete(y_test, drop_test, axis=0)
print(len(X_train))
print(len(X_test))
print(len(y_train))
print(len(y_test))

import matplotlib.pyplot as plt
print('리뷰의 최대 길이 :',max(len(l) for l in X_train))
print('리뷰의 평균 길이 :',sum(map(len, X_train))/len(X_train))
plt.hist([len(s) for s in X_train], bins=50)
plt.xlabel('length of samples')
plt.ylabel('number of samples')
plt.show()

def below_threshold_len(max_len, nested_list):
  cnt = 0
  for s in nested_list:
    if(len(s) <= max_len):
        cnt = cnt + 1
  print('전체 샘플 중 길이가 %s 이하인 샘플의 비율: %s'%(max_len, (cnt / len(nested_list))*100))

max_len = 600
below_threshold_len(max_len, X_train)

from tensorflow.keras.preprocessing.sequence import pad_sequences
X_train = pad_sequences(X_train, maxlen = max_len)
X_test = pad_sequences(X_test, maxlen = max_len)

y_train=np.array(train_data.category)
y_test=np.array(test_data.category)
y_train = np.delete(y_train, drop_train, axis=0)
y_test = np.delete(y_test, drop_test, axis=0)
print(len(y_train),len(y_test))

t={'경제민주화': 1, '교통/건축/국토': 2, '보건복지': 3, '육아/교육': 4, '인권/성평등': 5, '일자리': 6, '정치개혁': 7}
print(t['보건복지'])
index1=np.zeros([10301,7])
for i in range(len(y_train)):
  index1[i][t[y_train[i]]-1]=1
y_train=index1

index1=np.zeros([1158,7])
for i in range(len(y_test)):
  index1[i][t[y_test[i]]-1]=1
y_test=index1

print(y_train.shape,y_test.shape)

X_train=np.array(X_train)
X_test=np.array(X_test)
y_train=np.array(y_train)
y_test=np.array(y_test)
print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)
print(vocab_size)

from tensorflow.keras.layers import Embedding, Dense, LSTM
from tensorflow.keras.models import Sequential
from tensorflow.keras.models import load_model
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint

import tensorflow_addons as tfa
f1 = tfa.metrics.F1Score(num_classes=7,threshold=0.5)

from tensorflow import keras
keras.__version__
# '2.6.0'

model = Sequential()
model.add(Embedding(vocab_size, 128))
model.add(LSTM(128))
model.add(Dense(7, activation='sigmoid'))
es = EarlyStopping(monitor='val_loss', mode='min', verbose=2, patience=4)
mc = ModelCheckpoint('best_model.h5', monitor=f1, mode='max', verbose=2, save_best_only=True)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc',f1])
model.fit(X_train, y_train, epochs=15, callbacks=[es, mc], batch_size=512, validation_split=0.2)

# evaluating function: report f1_macro
def evaluate(test_x,test_y,model):
    predictions=model.predict(test_x)
    y_pred=max(predictions)
    print(classification_report(test_y,y_pred))

model.evaluate(X_test, y_test)

# from saved best model
loaded_model = load_model('best_model.h5')
loaded_model.evaluate(X_test, y_test)