import pandas as pd
pd.set_option("display.max_columns", 100)
%matplotlib inline

texts = [
    "Penny bought bright blue fishes.",
    "Penny bought bright blue and orange fish.",
    "The cat ate a fish at the store.",
    "Penny went to the store. Penny ate a bug. Penny saw a fish.",
    "It meowed once at the bug, it is still meowing at the bug and the fish",
    "The cat is at the fish store. The cat is orange. The cat is meowing at the fish.",
    "Penny is a fish"
]

# Sentences with Penny
"Penny bought bright blue fishes."
"Penny bought bright blue and orange fish."
"Penny went to the store. Penny ate a bug. Penny saw a fish."
"Penny is a fish"

# Sentences without Penny
"The cat ate a fish at the store."
"It meowed once at the bug, it is still meowing at the bug and the fish"
"The cat is at the fish store. The cat is orange. The cat is meowing at the fish."

# Start with Penny
"Penny bought bright blue fishes."
"Penny bought bright blue and orange fish."
"Penny went to the store. Penny ate a bug. Penny saw a fish."
"Penny is a fish"

# Start with the cat
"The cat ate a fish at the store."
"The cat is at the fish store. The cat is orange. The cat is meowing at the fish."

# Start with it
"It meowed once at the bug, it is still meowing at the bug and the fish"    

from sklearn.feature_extraction.text import CountVectorizer

vec = CountVectorizer()
matrix = vec.fit_transform(texts)
pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())

from sklearn.feature_extraction.text import TfidfVectorizer

vec = TfidfVectorizer(use_idf=False, norm='l1')
matrix = vec.fit_transform(texts)
pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())

from textblob import TextBlob

def textblob_tokenizer(str_input):
    blob = TextBlob(str_input.lower())
    tokens = blob.words
    words = [token.stem() for token in tokens]
    return words

vec = CountVectorizer(tokenizer=textblob_tokenizer)
matrix = vec.fit_transform(texts)
pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())

vec = CountVectorizer(tokenizer=textblob_tokenizer, stop_words='english')
matrix = vec.fit_transform(texts)
pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())

# A custom list if we really wanted to
vec = CountVectorizer(tokenizer=textblob_tokenizer, stop_words=['it', 'and', 'is', 'a'])
matrix = vec.fit_transform(texts)
pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())

texts = [
    "Penny bought bright blue fishes.",
    "Penny bought bright blue and orange fish.",
    "The cat ate a fish at the store.",
    "Penny went to the store. Penny ate a bug. Penny saw a fish.",
    "It meowed once at the bug, it is still meowing at the bug and the fish",
    "The cat is at the fish store. The cat is orange. The cat is meowing at the fish.",
    "Penny is a fish"
]

# We have to use these other parameters because I SAID SO
vec = TfidfVectorizer(tokenizer=textblob_tokenizer,
                      stop_words='english',
                      norm='l1', # ELL - ONE
                      use_idf=False)
matrix = vec.fit_transform(texts)
df = pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())
df

df.sort_values(by='fish', ascending=False)

df[['fish', 'meow']]

# Add them together!
# Percent of words about either 'meow' or 'fish'
df.meow + df.fish

pd.DataFrame({
    'fish': df.fish,
    'meow': df.meow,
    'meow + fish': df.meow + df.fish
})

#    "Penny bought bright blue fishes.",
#    "Penny bought bright blue and orange fish.",
#    "The cat ate a fish at the store.",
#    "Penny went to the store. Penny ate a bug. Penny saw a fish.",
"It meowed once at the bug, it is still meowing at the bug and the fish",
#    "The cat is at the fish store. The cat is orange. The cat is meowing at the fish.",
"Penny is a fish"

# We have to use these other parameters because I SAID SO
vec = TfidfVectorizer(tokenizer=textblob_tokenizer,
                      stop_words='english',
                      norm='l1',
                      use_idf=True)
matrix = vec.fit_transform(texts)
idf_df = pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())
idf_df

# OLD dataframe
pd.DataFrame({
    'fish': df.fish,
    'meow': df.meow,
    'meow + fish': df.meow + df.fish
})

# NEW dataframe
pd.DataFrame({
    'fish': idf_df.fish,
    'meow': idf_df.meow,
    'meow + fish': idf_df.meow + idf_df.fish
})

# We have to *get rid of* norm='l1' because I SAID SO
vec = TfidfVectorizer(tokenizer=textblob_tokenizer,
                      stop_words='english',
                      use_idf=True)
matrix = vec.fit_transform(texts)
idf_df = pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())
idf_df

# OLD dataframe
pd.DataFrame({
    'fish': df.fish,
    'meow': df.meow,
    'meow + fish': df.meow + df.fish
})

# NEW dataframe
pd.DataFrame({
    'fish': idf_df.fish,
    'meow': idf_df.meow,
    'meow + fish': idf_df.meow + idf_df.fish
})

# .5 => .8 "It meowed once at the bug, it is still meowing at the bug and the fish",
# .375 => .56 "The cat is at the fish store. The cat is orange. The cat is meowing at the fish.",
# .5 => .56 "Penny is a fish"

# We have to use these other parameters because I SAID SO
vec = TfidfVectorizer(tokenizer=textblob_tokenizer,
                      stop_words='english',
                      use_idf=True)
matrix = vec.fit_transform(texts)
idf_df = pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())
idf_df

# KMeans clustering a kind of clustering.
from sklearn.cluster import KMeans

number_of_clusters=2
km = KMeans(n_clusters=number_of_clusters)
# Normally people fit the matrix
km.fit(matrix)
# But you could fit the idf_df instead
km.fit
# It just means "HEY TAKE A LOOK AT AND TRY ON MY TEXT STUFF"

print("Top terms per cluster:")
order_centroids = km.cluster_centers_.argsort()[:, ::-1]
terms = vec.get_feature_names()
for i in range(number_of_clusters):
    top_ten_words = [terms[ind] for ind in order_centroids[i, :5]]
    print("Cluster {}: {}".format(i, ' '.join(top_ten_words)))

km.labels_

texts

results = pd.DataFrame()
results['text'] = texts
results['category'] = km.labels_
results

# KMeans clustering a kind of clustering.
from sklearn.cluster import KMeans

number_of_clusters=3
km = KMeans(n_clusters=number_of_clusters)
# Normally people fit the matrix
km.fit(matrix)
# But you could fit the idf_df instead
km.fit
# It just means "HEY TAKE A LOOK AT AND TRY ON MY TEXT STUFF"

km.labels_

results = pd.DataFrame({
    'text': texts,
    'category': km.labels_
})
results

print("Top terms per cluster:")
order_centroids = km.cluster_centers_.argsort()[:, ::-1]
terms = vec.get_feature_names()
for i in range(number_of_clusters):
    top_ten_words = [terms[ind] for ind in order_centroids[i, :3]]
    print("Cluster {}: {}".format(i, ' '.join(top_ten_words)))

texts = [
    'Penny bought bright blue fishes.',
    'Penny bought bright blue and orange bowl.',
    'The cat ate a fish at the store.',
    'Penny went to the store. Penny ate a bug. Penny saw a fish.',
    'It meowed once at the bug, it is still meowing at the bug and the fish',
    'The cat is at the fish store. The cat is orange. The cat is meowing at the fish.',
    'Penny is a fish.',
    'Penny Penny she loves fishes Penny Penny is no cat.',
    'The store is closed now.',
    'How old is that tree?',
    'I do not eat fish I do not eat cats I only eat bugs.'
]

vec = TfidfVectorizer(tokenizer=textblob_tokenizer,
                      stop_words='english',
                      use_idf=True,
                      max_features=2)
matrix = vec.fit_transform(texts)
df = pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())
df

ax = df.plot(kind='scatter', x='fish', y='penni', alpha=0.2, s=200)
ax.set_xlabel("Fish")
ax.set_ylabel("Penny")

number_of_clusters = 3
km = KMeans(n_clusters=number_of_clusters)
km.fit(matrix)

# Move the labels into a column of our dataframe
# the first label matches the first row, second label is second row, etc
df['category'] = km.labels_
df

# Category 0 is red
# Category 1 is green
# Category 2 is blue
colormap = {
    0: 'red',
    1: 'green',
    2: 'blue'
}

# Create a list of colors from every single row
colors = df.apply(lambda row: colormap[row.category], axis=1)

# And plot it!
ax = df.plot(kind='scatter', x='fish', y='penni', alpha=0.1, s=300, c=colors)
ax.set_xlabel("Fish")
ax.set_ylabel("Penny")

km = KMeans(n_clusters=4)
km.fit(matrix)
df['category'] = km.labels_

colormap = { 0: 'red', 1: 'green', 2: 'blue', 3: 'purple'}
colors = df.apply(lambda row: colormap[row.category], axis=1)
ax = df.plot(kind='scatter', x='fish', y='penni', alpha=0.1, s=300, c=colors)
ax.set_xlabel("Fish")
ax.set_ylabel("Penny")

# Vectorize and save into a new dataframe
vec = TfidfVectorizer(tokenizer=textblob_tokenizer, stop_words='english', use_idf=True)
matrix = vec.fit_transform(texts)
df = pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())
df.head(2)

# Cluster with 3 categories
# only using the 'fish' and 'penni' categories
km = KMeans(n_clusters=3)
km.fit(df[['fish', 'penni']])

# Assign the category to the dataframe
df['category'] = km.labels_

# Build our color map
colormap = { 0: 'red', 1: 'green', 2: 'blue' }
colors = df.apply(lambda row: colormap[row.category], axis=1)

# Plot our scatter
ax = df.plot(kind='scatter', x='fish', y='penni', alpha=0.1, s=300, c=colors)
ax.set_xlabel("Fish")
ax.set_ylabel("Penny")

# Vectorize and save into a new dataframe
vec = TfidfVectorizer(tokenizer=textblob_tokenizer, max_features=3, stop_words='english', use_idf=True)
matrix = vec.fit_transform(texts)
df = pd.DataFrame(matrix.toarray(), columns=vec.get_feature_names())
df.head(2)

# Cluster
km = KMeans(n_clusters=4)
km.fit(df)

# Assign the category to the dataframe
df['category'] = km.labels_

# Build our color map
colormap = {0: 'red', 1: 'green', 2: 'blue', 3: 'orange'}
colors = df.apply(lambda row: colormap[row.category], axis=1)

# Plot our scatter
ax = df.plot(kind='scatter', x='fish', y='penni', alpha=0.2, s=300, c=colors)
ax.set_xlabel("Fish")
ax.set_ylabel("Penny")

# Plot our scatter
ax = df.plot(kind='scatter', x='penni', y='cat', alpha=0.2, s=300, c=colors)
ax.set_xlabel("Penni")
ax.set_ylabel("Cat")

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

def draw(ax, df):
    colormap = { 0: 'red', 1: 'green', 2: 'blue', 3: 'orange' }
    colors = df.apply(lambda row: colormap[row.category], axis=1)

    ax.scatter(df['fish'], df['penni'], df['cat'], c=colors, s=100, alpha=0.5)
    ax.set_xlabel('Fish')
    ax.set_ylabel('Penni')
    ax.set_zlabel('Cat')

chart_count_vert = 5
chart_count_horiz = 5
number_of_graphs = chart_count_vert * chart_count_horiz

fig = plt.figure(figsize=(3 * chart_count_horiz, 3 * chart_count_vert))

for i in range(number_of_graphs):
    ax = fig.add_subplot(chart_count_horiz, chart_count_vert, i + 1, projection='3d', azim=(-360 / number_of_graphs) * i)
    draw(ax, df)