Tugas 5 : Mencari Dokumen berdasarkan Kalimat

Tugas 5 : Mencari Dokumen berdasarkan Kalimat#

Pada Tugas 5 diminta untuk mencari dokumen berdasarkan kalimat yang dimasukkan. Data dokumen dilakukan proses TFIDF dan mengimplementasikan SVD (Singular Value Decomposition) untuk reduksi dimensi serta cosine similarity untuk mencari kedekatan antara kalimat dicari dengan dokumen yang ada.

Dibuat Oleh:

  • Nama : Sabil Ahmad Hidayat

  • NIM : 220411100058

  • Kelas : PPW A

Link Code : https://colab.research.google.com/drive/19V1Li_F8_euESGe6bhCEAcAFffKpnZ67?usp=sharing

Link Github : meinhere/ppw

Import Library#

!pip install -q Sastrawi

^C

# library awal untuk perhitungan dan pengolahan teks
import numpy as np
import re
import pandas as pd
import pickle
import requests

# monitoring
from tqdm import tqdm

# library untuk praproses teks
import nltk
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('punkt')
from nltk.tokenize import sent_tokenize
from nltk.corpus import stopwords
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics.pairwise import cosine_similarity

# library untuk proses pembentukan vsm
from sklearn.feature_extraction.text import TfidfVectorizer

[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.
[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.

Load Data#

tfidf_df = pd.read_csv('https://raw.githubusercontent.com/meinhere/ppw/master/publish/tugas-2/dataset/train_df_tfidf.csv', delimiter=',')
tfidf_df.drop(columns=['label'], inplace=True)

tfidf_df
aaion aali abadi abai abenkh abnormal absurd ac acara access ... yusdistira yzr za zad zaman zarco zenix zero zigzag zona
0 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.058149 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
1 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
2 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
3 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
4 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
75 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
76 0.053854 0.0 0.0 0.0 0.0 0.0 0.0 0.037974 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
77 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
78 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.121219 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.0
79 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.000000 0.000000 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.059581 0.0

80 rows × 3106 columns

Implementasi SVD (Singular Value Decomposition)#

Proses Reduksi Dimensi#

# Menerapkan untuk LSA
n = 80 # Jumlah component/fitur yang ingin dihasilkan dari svd
svd = TruncatedSVD(n_components=n)
svd_matrix = svd.fit_transform(tfidf_df)

# Mengambil sebagian dari matrix V dan matrix Sigma
v_matrix = svd.components_ # Matrix V (komponen term)
singular_values = svd.singular_values_  # Matrix Sigma (singular values)

# Mengambil berapa jumlah component/fitur yang diinginkan
partial_n = 80

partial_v_matrix = v_matrix[:, :partial_n] # Mengambil matrix v dari jumlah yang diambil
partial_singular_values = singular_values[:partial_n] # Mengambil matrix Sigma dari jumlah yang diambil

# Menampilkan matrix svd
print(f"Shape : {svd_matrix.shape}")
print(svd_matrix)

# Menampilkan matrix v
print(f"Shape : {partial_v_matrix.shape}")
print(partial_v_matrix)

# Menampilkan matrix Sigma
print(f"Shape : {partial_singular_values.shape}")
print(partial_singular_values)

Shape : (80, 80)
[[ 1.54510634e-01 -9.21097057e-02  9.38904274e-03 ... -1.14938909e-03
  -1.51825296e-04  8.67361738e-19]
 [ 1.67615266e-01 -1.60479805e-01  1.23665623e-01 ...  3.11792566e-03
   2.75914213e-04  8.67361738e-18]
 [ 2.80561596e-01 -7.35551633e-02 -6.58748219e-02 ... -1.58720646e-04
  -1.83127832e-03 -9.10729825e-18]
 ...
 [ 2.45615049e-01  3.41531512e-02  2.11698472e-01 ...  1.37748793e-03
  -9.76249092e-04 -3.46944695e-18]
 [ 3.50620184e-01  4.61250396e-01  1.63725318e-01 ... -2.53879878e-03
   4.38598240e-04 -7.26415456e-18]
 [ 2.02074170e-01  4.51572108e-02  2.29484661e-01 ...  3.75707600e-02
   8.40612527e-04 -1.78893358e-18]]
Shape : (80, 80)
[[ 2.01225651e-03  1.89116869e-03  1.89116869e-03 ...  4.84688813e-03
   1.61562938e-03  3.23125875e-03]
 [-2.57111925e-04 -3.92085852e-03 -3.92085852e-03 ... -5.81253948e-03
  -1.93751316e-03 -3.87502632e-03]
 [-5.34058540e-04 -8.34390309e-04 -8.34390309e-04 ...  1.68685095e-03
   5.62283651e-04  1.12456730e-03]
 ...
 [ 7.46625040e-04  7.03061102e-04  7.03061102e-04 ... -1.14463982e-03
  -3.81546605e-04 -7.63093211e-04]
 [ 1.58273886e-04 -3.30449574e-04 -3.30449574e-04 ...  1.02362094e-03
   3.41206981e-04  6.82413962e-04]
 [-5.85262899e-02  4.46086787e-01  1.66679860e-01 ...  7.30632388e-04
   5.01054950e-03 -1.00171660e-03]]
Shape : (80,)
[2.08644723e+00 1.61046945e+00 1.51414718e+00 1.49010163e+00
 1.42892895e+00 1.39172559e+00 1.38121960e+00 1.37914621e+00
 1.33547173e+00 1.29436047e+00 1.24543876e+00 1.23159841e+00
 1.21309281e+00 1.18452936e+00 1.15846926e+00 1.14678667e+00
 1.13286842e+00 1.10834460e+00 1.10569312e+00 1.09306941e+00
 1.07251215e+00 1.06992768e+00 1.04771950e+00 1.04668181e+00
 1.03901828e+00 1.02815257e+00 1.02085022e+00 1.01620363e+00
 1.00760994e+00 1.00682166e+00 1.00015803e+00 9.98080575e-01
 9.90970953e-01 9.81076139e-01 9.74831789e-01 9.69639700e-01
 9.67838856e-01 9.62715192e-01 9.61154589e-01 9.53983034e-01
 9.45392116e-01 9.39711136e-01 9.34676233e-01 9.31641353e-01
 9.28505148e-01 9.13952859e-01 9.12057024e-01 9.06423462e-01
 9.02069972e-01 8.97080939e-01 8.96526698e-01 8.84540253e-01
 8.70195758e-01 8.61985572e-01 8.58068447e-01 8.54400226e-01
 8.36070057e-01 8.27691628e-01 8.09573151e-01 8.07580606e-01
 7.79340481e-01 7.72334579e-01 7.67143915e-01 7.49185508e-01
 7.28650204e-01 7.13854736e-01 7.09908188e-01 6.95030444e-01
 6.93500732e-01 6.58185159e-01 6.52764474e-01 6.29340292e-01
 6.10284846e-01 5.90495438e-01 5.76827008e-01 4.97811913e-01
 3.79346297e-01 3.51456913e-01 2.95308832e-01 9.26568701e-16]

Save DataFrame#

# Convert the svd_matrix to a DataFrame
svd_df = pd.DataFrame(svd_matrix)

# Save the DataFrame to a CSV file
svd_df.to_csv('svd_matrix.csv', index=False)

Save Model SVD#

# Save the SVD model
with open('svd_model.pkl', 'wb') as f:
    pickle.dump(svd, f)

Pencarian Dokumen dengan Kalimat#

Load Model#

github_raw_url = "https://raw.githubusercontent.com/meinhere/ppw/master/publish/tugas-3/model/tfidf_vectorizer.sav"

response = requests.get(github_raw_url)
response.raise_for_status()

vectorizer = pickle.loads(response.content)
vectorizer

TfidfVectorizer()
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
svd = pickle.load(open('svd_model.pkl', 'rb'))
svd

TruncatedSVD(n_components=80)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

Persiapan Preprocessing#

# Case Folding
def clean_lower(lwr):
    lwr = lwr.lower() # lowercase text
    return lwr

# Menghapus tanda baca, angka, dan simbol
def clean_punct(text):
    clean_spcl = re.compile('[/(){}\[\]\|@,;_]')
    clean_symbol = re.compile('[^0-9a-z]')
    clean_number = re.compile('[0-9]')
    text = clean_spcl.sub('', text)
    text = clean_symbol.sub(' ', text)
    text = clean_number.sub('', text)
    return text

# Menghaps double atau lebih whitespace
def _normalize_whitespace(text):
    corrected = str(text)
    corrected = re.sub(r"//t",r"\t", corrected)
    corrected = re.sub(r"( )\1+",r"\1", corrected)
    corrected = re.sub(r"(\n)\1+",r"\1", corrected)
    corrected = re.sub(r"(\r)\1+",r"\1", corrected)
    corrected = re.sub(r"(\t)\1+",r"\1", corrected)
    return corrected.strip(" ")

# Menghapus stopwords
def clean_stopwords(text):
    stopword = set(stopwords.words('indonesian'))
    text = ' '.join(word for word in text.split() if word not in stopword) # hapus stopword dari kolom deskripsi
    return text

# Stemming with Sastrawi
def sastrawistemmer(text):
    factory = StemmerFactory()
    st = factory.create_stemmer()
    text = ' '.join(st.stem(word) for word in tqdm(text.split()) if word in text)
    return text

Definisi Fungsi#

def preprocess_text(text):
    text = clean_lower(text)
    text = clean_punct(text)
    text = _normalize_whitespace(text)
    text = clean_stopwords(text)
    text = sastrawistemmer(text)
    return text

def search_doc(query, top=5):
    query = preprocess_text(query)
    query_tfidf = vectorizer.transform([query])
    query_tfidf = pd.DataFrame(query_tfidf.toarray(), columns=vectorizer.get_feature_names_out())
    query_svd = svd.transform(query_tfidf)

    similarity = cosine_similarity(query_svd, svd_matrix).flatten()

    top_indices = similarity.argsort()[-top:][::-1]
    top_similarity = similarity[top_indices]

    top_df = []

    for i in range(len(top_indices)):
        data = {}
        data['dokumen ke'] = top_indices[i]
        data['similarity'] = top_similarity[i]
        top_df.append(data)

    top_df = pd.DataFrame(top_df)
    top_df = top_df.sort_values(by='similarity', ascending=False)

    return top_df

Pengujian Pencarian#

query = "indonesia"

results = search_doc(query)
results

100%|██████████| 1/1 [00:00<00:00, 279.92it/s]
dokumen ke similarity
0 12 0.649388
1 50 0.441052
2 8 0.384027
3 26 0.336437
4 45 0.300869
Contents