Mi solución al reto con 98% de accuracy.
Usé los otros 3 corpus, colocaciones y mayor frecuencia de palabras para obtener atributos.
import os
import nltk
import random
from nltk import word_tokenize
from nltk.collocations import *
import pandas as pd
nltk.download("punkt")
!git clone https://github.com/pachocamacho1990/datasets
! unzip datasets/email/plaintext/corpus1.zip
! unzip datasets/email/plaintext/corpus2.zip
! unzip datasets/email/plaintext/corpus3.zip
Funciones para cargar los datasets
# Get Text and labels from folders with plain text files
def get_text_labels_from_folders(folderBase, folderLabels):
data = []
labels = []
for folderLabel in folderLabels:
for file in os.listdir('{}/{}'.format(folderBase, folderLabel)):
with open('{}/{}/{}'.format(folderBase, folderLabel, file), encoding='latin-1') as f:
data.append(f.read())
labels.append(folderLabel)
return data, labels
def set_label_num(label_str):
if label_str == "spam":
return 1
else:
return 0
dataCorpus1, labelsCorpus1 = get_text_labels_from_folders('corpus1', ["spam", "ham"])
dataCorpus2, labelsCorpus2 = get_text_labels_from_folders('corpus2', ["spam", "ham"])
dataCorpus3, labelsCorpus3 = get_text_labels_from_folders('corpus3', ["spam", "ham"])
data = dataCorpus1 + dataCorpus2 + dataCorpus3
labels = labelsCorpus1 + labelsCorpus2 + labelsCorpus3
dataframe = pd.DataFrame({'text': data, 'labels': labels})
dataframe = dataframe.sample(frac = 1)
dataframe['tokens'] = dataframe['text'].apply(lambda x: word_tokenize(x))
dataframe['labels_num'] = dataframe['labels'].apply(lambda x: set_label_num(x))
Funciones para filtrar palabras y obtener colocaciones de n gramas
def filter_words_by_threshold(text_tokenized, threshold = 3, ):
words = []
words = [word for word in text_tokenized if len(word) > threshold]
return words
def get_n_grams_collocations_from_words(words, freq_filter = 10, n_best= 10,
n_gran_measure = nltk.collocations.BigramAssocMeasures()):
finder = BigramCollocationFinder.from_words(words)
finder.apply_freq_filter(freq_filter)
email_spam_collocations = finder.nbest(n_gran_measure.pmi, n_best)
return email_spam_collocations
Obtener las colocaciones y palabras más comunes en el datasets de spam
spamCorpus1, _ = get_text_labels_from_folders('corpus1', ["spam"])
spamCorpus2, _ = get_text_labels_from_folders('corpus2', ["spam"])
spamCorpus3, _ = get_text_labels_from_folders('corpus3', ["spam"])
spamCorpuses = spamCorpus1 + spamCorpus2 + spamCorpus3
filtered_words = []
for text in spamCorpuses:
filtered_words += filter_words_by_threshold(word_tokenize(text))
filtered_words
email_spam_collocations = get_n_grams_collocations_from_words(filtered_words, 120, 40)
all_spam_words = nltk.FreqDist([w for w in filtered_words])
top_spam_words = all_spam_words.most_common(200)