TUTORIAL TEXT PREPROSESSING DENGAN PYTHON

 Teks Preprosessing atau 

clean data

INDAH MASULA NUR ANNISA
171080200194

LANGKAH : 1 Mengimport Library yang sudah di install

• import pandas as pd
• import numpy as np
• import nltk 
• import string
• import re

LANGKAH : 2 Memasukkan DataSet yang sudah dimiliki

def load_data():

    data = pd.read_excel('dataku.xlsx')

    return data

LANGKAH :3

tweet_df = load_data()
tweet_df.head()

LANGKAH : 4

df = pd.DataFrame(tweet_df[['id_str', 'text']])#ubah dataframe sesuai column kalian

LANGKAH 5

def remove_pattern(input_txt, pattern):

r = re.findall(pattern, input_txt)

for i in r:

input_txt = re.sub(i, '', input_txt)

return input_txt

df['remove_user'] = np.vectorize(remove_pattern)(df['text'], "@[\w]*")

LANGKAH : 6

def remove(tweet):

tweet = re.sub('[0-9]+', '', tweet)

tweet = re.sub(r'\$\w*', '', tweet)

tweet = re.sub(r'^RT[\s]+', '', tweet)

tweet = re.sub(r'#', '', tweet)

return tweet

df['remove_http'] = df['remove_user'].apply(lambda x: remove(x))

df.sort_values("remove_http", inplace = True)

df.drop_duplicates(subset ="remove_http", keep = 'first', inplace = True)

LANGKAH : 7

from nltk.corpus import stopwords

stopwords_indonesia = stopwords.words('indonesian')

from Sastrawi.Stemmer.StemmerFactory import StemmerFactory

factory = StemmerFactory()

stemmer = factory.create_stemmer()

from nltk.tokenize import TweetTokenizer

emoticons_happy = set([

':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>', '=]', '8)', '=)', ':}',

':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD', 'X-D', 'XD', '=-D', '=D',

'=-3', '=3', ':-))', ":'-)", ":')", ':*', ':^*', '>:P', ':-P', ':P', 'X-P',

'x-p', 'xp', 'XP', ':-p', ':p', '=p', ':-b', ':b', '>:)', '>;)', '>:-)',

'<3'

])

emoticons_sad = set([

':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[', ':-||', '=L', ':<',

':-[', ':-<', '=\\', '=/', '>:(', ':(', '>.<', ":'-(", ":'(", ':\\', ':-c',

':c', ':{', '>:\\', ';('

])

emoticons = emoticons_happy.union(emoticons_sad)

LANGKAH : 8

def clean_tweets(tweet):

tweet = re.sub(r'\$\w*', '', tweet)

tweet = re.sub(r'^RT[\s]+', '', tweet)

tweet = re.sub(r'https?:\/\/.*[\r\n]*', '', tweet)

tweet = re.sub(r'#', '', tweet)

tweet = re.sub(r',','',tweet)

tweet = re.sub('[0-9]+', '', tweet)

tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)

tweet_tokens = tokenizer.tokenize(tweet)

LANGKAH :9

tweets_clean = []

for word in tweet_tokens:

if (word not in stopwords_indonesia and # remove stopwords

word not in emoticons and

word not in string.punctuation):

stem_word = stemmer.stem(word) # stemming word

tweets_clean.append(stem_word)

return tweets_clean

df['tweet_clean'] = df['remove_http'].apply(lambda x: clean_tweets(x))

LANGKAH : 10

def remove_punct(text):

text = " ".join([char for char in text if char not in string.punctuation])

return text

df['Tweet'] = df['tweet_clean'].apply(lambda x: remove_punct(x))

LANGKAH : 11

df.sort_values("Tweet", inplace = True)

df.drop(df.columns[[0,1,2,3,4]], axis = 1, inplace = True)

df.drop_duplicates(subset ="Tweet", keep = 'first', inplace = True)

df.to_csv('output.csv',encoding='utf8', index=False)

df