# -*- coding: utf-8 -*-
"""20241117_Text Mining by MHR_01.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/12yersEgxWeoIJ7GdA7mPZ_lsXyR5Jf5P

# Text Mining Step by Step by MHR
"""

'''===Import the Necessary Library==='''
import pandas as pd
import numpy as np
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

# Download necessary NLTK resources (run this only once)
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('punkt_tab')

# Step 2: Data Collection
# For demonstration, we will use a simple text string.

text_data = """Text mining is the process of extracting meaningful information from unstructured text data.
It involves various techniques such as tokenization, stop word removal, and lemmatization.
Text mining, also known as text data mining, is the process of transforming unstructured text
into a structured format to identify meaningful patterns and new insights.
It can use text mining to analyze vast collections of textual materials to capture key concepts,
trends, and hidden relationships."""

print(text_data)

from google.colab import drive
drive.mount('/content/drive')

path='/content/drive/MyDrive/Colab Notebooks/Text Mining/Text mining_1.txt'

"""Read the Text File"""

# Read a text file using with open()
with open(path, 'r') as file:  # Open the file in read mode
    content = file.read()  # Read the entire content of the file

print(content)  # Print the content

"""Read the Text File using "open" Code"""

# Assign the contents of a text file to a variable manually
file = open(path, 'r')  # Open the file in read mode
Text_Data = file.read()  # Read the entire content and assign it to 'text'
file.close()  # Close the file

print("File Content:\n", Text_Data)  # Print the content of the variable

"""Write the Text File"""

# Write the Text File
# Specify the path where you want to save the file
path_1='/content/drive/MyDrive/Colab Notebooks/Text Mining/Text mining_2.txt'  # Update with your desired path

# Open the file in write mode and write some content
with open(path_1, 'w') as file:
    file.write(Text_Data)

print("File written successfully!")

"""Tokenization of Words"""

# Tokenization of Words
T=word_tokenize(text_data)
T

"""Tokenization of Sentances"""

# Tokenization of Sentances
from nltk.tokenize import sent_tokenize
S=sent_tokenize(text_data)
print(S)

""""text.lower()": The lower() method is called on the text string to convert all characters to lowercase.
This normalization step ensures that the tokenization process treats
words like "This" and "this" as the same word.
It helps avoid duplication in later analysis and improves consistency.
"""

# Sample text
text =text_data
# Tokenization of Words
Tokens = word_tokenize(text.lower())
print("Tokens:", Tokens)

# Get the list of English stop words
stop_words = set(stopwords.words('english'))
stop_words

# Remove stop words from tokens
filtered_tokens = [word for word in Tokens if word not in stop_words]
print("Filtered Tokens Words:\n", filtered_tokens)

""""if token not in stop_words": This condition checks if the current token is not present in the stop_words set. If it is not a stop word, the token will be included in the filtered_tokens list

"and token.isalpha()": This additional condition checks if the token consists only of alphabetic characters (i.e., it does not contain any numbers or punctuation). The method isalpha() returns True if all characters in the string are alphabetic and there is at least one character; otherwise, it returns False. This helps ensure that only meaningful words are retained, further cleaning the data by excluding any non-alphabetic tokens.
"""

# Step 3: Text Preprocessing Function
def preprocess_text(text):
    # Tokenization
    tokens = word_tokenize(text.lower())

    # Remove stop words
    stop_words = set(stopwords.words('english'))
    filtered_tokens = [token for token in tokens if token not in stop_words and token.isalpha()]

    # Lemmatization
    lemmatizer = WordNetLemmatizer()
    lemmatized_tokens = [lemmatizer.lemmatize(token) for token in filtered_tokens]

    return lemmatized_tokens

# Step 4: Apply Preprocessing
cleaned_tokens = preprocess_text(text_data)
print("Cleaned Tokens:", cleaned_tokens)



# Step 5: Frequency Distribution of Tokens
from nltk.probability import FreqDist
fdist = FreqDist(cleaned_tokens)
print("Frequency Distribution:", fdist.most_common())

# Step 6: Visualization (Optional)
import matplotlib.pyplot as plt
# Plotting frequency distribution
plt.figure(figsize=(6,3))
fdist.plot(10, cumulative=False,color='blue')
plt.title('Frequency Distribution of Tokens')
plt.show()

# Step 6: Visualization (Optional)
import matplotlib.pyplot as plt

# Plotting frequency distribution
plt.figure(figsize=(6,3))
fdist.plot(30, cumulative=False,color='purple')
plt.title('Frequency Distribution of Tokens',fontsize=10)
plt.xlabel('Samples',fontsize=10)
plt.ylabel('Counts',fontsize=10)
plt.xticks(fontsize=9)
plt.yticks(fontsize=9)
plt.show()

"""Word Cloud Plot Overview: A word cloud (or tag cloud) is a visual representation of text data where the size of each word indicates its frequency or relevance within the text. In Python, creating a word cloud can be easily accomplished using the wordcloud library along with matplotlib for visualization."""

from wordcloud import WordCloud

wordcloud=WordCloud(width=1000,height=500).generate(text_data)

np.random.seed(12357)
plt.figure(figsize=(8,8))
plt.imshow(wordcloud,cmap ='Greens')
plt.axis('on')

