add tokenization, stemming & lemmatization tutorial

Author: x4nth055

README.md

@@ -57,6 +57,7 @@ This is a repository of all the tutorials of [The Python Code](https://www.thepy
         - [How to Paraphrase Text using Transformers in Python](https://www.thepythoncode.com/article/paraphrase-text-using-transformers-in-python). ([code](machine-learning/nlp/text-paraphrasing))
         - [Fake News Detection using Transformers in Python](https://www.thepythoncode.com/article/fake-news-classification-in-python). ([code](machine-learning/nlp/fake-news-classification))
         - [Named Entity Recognition using Transformers and Spacy in Python](https://www.thepythoncode.com/article/named-entity-recognition-using-transformers-and-spacy). ([code](machine-learning/nlp/named-entity-recognition))
+        - [Tokenization, Stemming, and Lemmatization in Python](https://www.thepythoncode.com/article/tokenization-stemming-and-lemmatization-in-python). ([code](machine-learning/nlp/tokenization-stemming-lemmatization))
     - ### [Computer Vision](https://www.thepythoncode.com/topic/computer-vision)
         - [How to Detect Human Faces in Python using OpenCV](https://www.thepythoncode.com/article/detect-faces-opencv-python). ([code](machine-learning/face_detection))
         - [How to Make an Image Classifier in Python using TensorFlow and Keras](https://www.thepythoncode.com/article/image-classification-keras-python). ([code](machine-learning/image-classifier))

machine-learning/nlp/tokenization-stemming-lemmatization/README.md

@@ -0,0 +1 @@
+# [Tokenization, Stemming, and Lemmatization in Python](https://www.thepythoncode.com/article/tokenization-stemming-and-lemmatization-in-python)

machine-learning/nlp/tokenization-stemming-lemmatization/example1_splitting_by_whitespace.py

@@ -0,0 +1,3 @@
+s = "Hello I am programmer"
+lst = s.split()
+print(lst)

machine-learning/nlp/tokenization-stemming-lemmatization/example2_splitting_by_comma.py

@@ -0,0 +1,3 @@
+s = "Hello, I am programmer"
+lst = s.split(',')
+print(lst)

machine-learning/nlp/tokenization-stemming-lemmatization/example3_splitting_by_whitespace.py

@@ -0,0 +1,11 @@
+def tokenize(file):
+    tok = []
+    f = open(file, 'r')
+    for l in f:
+        lst = l.split()
+        tok.append(lst)
+    return tok
+
+tokens = tokenize('reviews.txt')
+for e in tokens:
+    print(e)

machine-learning/nlp/tokenization-stemming-lemmatization/part_of_speech_tagging.py

@@ -0,0 +1,28 @@
+import nltk
+from nltk.corpus import wordnet
+from nltk.stem import WordNetLemmatizer
+
+word_lst = []
+def lemmatizer(file):
+    lem_lst = []
+    lem = WordNetLemmatizer()
+    f = open(file, 'r')
+    for l in f:
+        word_lst.append(l.strip())
+        w = lem.lemmatize(str(l.strip()))
+        lem_lst.append(w)
+    return lem_lst
+
+def generate_tag(w):
+    t = nltk.pos_tag([w])[0][1][0].upper()
+    dic = {
+        'N': wordnet.NOUN,
+        'V': wordnet.VERB,
+        'A': wordnet.ADJ,
+        'R': wordnet.ADV
+    }
+    return dic.get(t, wordnet.VERB)
+
+lem_lst = lemmatizer('reviews.txt')
+for i in range(len(word_lst)):
+    print(word_lst[i]+"-->"+lem_lst[i])

machine-learning/nlp/tokenization-stemming-lemmatization/port_stemmer.py

@@ -0,0 +1,16 @@
+from nltk.stem import PorterStemmer
+
+word_lst = []
+def stemmer(file):
+    stm_lst = []
+    stm = PorterStemmer()
+    f = open(file, 'r')
+    for l in f:
+        word_lst.append(l)
+        w = stm.stem(str(l.strip()))
+        stm_lst.append(w)
+    return stm_lst
+
+stm_lst = stemmer('reviews.txt')
+for i in range(len(word_lst)):
+    print(word_lst[i]+"-->"+stm_lst[i])

machine-learning/nlp/tokenization-stemming-lemmatization/requirements.txt

@@ -0,0 +1,5 @@
+textblob
+nltk
+huggingface
+tokenizers
+transformers

machine-learning/nlp/tokenization-stemming-lemmatization/reviews.txt

@@ -0,0 +1,4 @@
+The restaurant has a good staff, good food, and a good environment.
+It is a good place for family outings. Hospitable staff.
+The staff is better than other places, but the food is okay.
+People are great here. I loved this place.

machine-learning/nlp/tokenization-stemming-lemmatization/sentence_tokenization_nltk.py

@@ -0,0 +1,13 @@
+from nltk import sent_tokenize
+
+def tokenize(file):
+    tok = []
+    f = open(file, 'r')
+    for l in f:
+        lst = sent_tokenize(l)
+        tok.append(lst)
+    return tok
+
+tokens = tokenize('reviews.txt')
+for e in tokens:
+    print(e)