qaznltk 1.3.0.0

A package for working with Kazakh language text processing.

QazNLTK

A Python library for Kazakh language text processing.

pip install qaznltk

---

Features

#	Feature	Method
1	Tokenization by frequency	tokenize(text)
2	Sentence segmentation	sent_tokenize(text)
3	Text similarity score	calc_similarity(a, b)
4	Cyrillic → Latin (ISO-9)	convert2latin_iso9(text)
5	Latin → Cyrillic (ISO-9)	convert2cyrillic_iso9(text)
6	Sentiment analysis	sentimize(text) → -1 / 0 / 1
7	Number to words	num2word(n)
8	IIN parser	get_info_from_iin(iin)
9	Stop words list	get_stop_words()
10	Kazakh alphabet	get_kaz_alphabet()
11	TF-IDF + KNN search	QazNLTKVectorizer + KNN
12	Kazakh LLM (QazPerry)	via HuggingFace

---

Usage

from qaznltk import QazNLTK
qn = QazNLTK()

Tokenize

qn.tokenize("Біздің өміріміз үлкен өзен іспетті.")
# [('өміріміз', 1), ('үлкен', 1), ('өзен', 1), ('іспетті', 1)]

Sentence split

qn.sent_tokenize("Сәлем. Қалайсың?")
# ['Сәлем.', 'Қалайсың?']

Similarity

qn.calc_similarity("Еңбегіне қарай — құрмет.", "Еңбегіне қарай табысы.")
# 0.368

Transliteration

qn.convert2latin_iso9("Бүгін керемет күн!")   # Bùgìn keremet kùn!
qn.convert2cyrillic_iso9("Bùgìn keremet kùn!")  # Бүгін керемет күн!

Sentiment (-1 negative, 0 neutral, 1 positive)

qn.sentimize("Бұл мақала өте нашар жазылған.")  # -1.0

Number to words

qn.num2word(1465)  # 'бір мың төрт жүз алпыс бес'

IIN parser

qn.get_info_from_iin("990408482390")
# {'status': 'success', 'date_of_birth': '08.04.1999', 'gender': 'female', ...}

Stop words

qn.get_stop_words()
# ['біздің', 'бұл', 'және', 'мен', 'не', ...]

Kazakh alphabet

qn.get_kaz_alphabet()
# ['а', 'ә', 'б', 'в', 'г', 'ғ', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й',
#  'к', 'қ', 'л', 'м', 'н', 'ң', 'о', 'ө', 'п', 'р', 'с', 'т', 'у',
#  'ү', 'ұ', 'ф', 'х', 'һ', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'і', 'ь', 'э', 'ю', 'я']

TF-IDF + KNN

from qaznltk import QazNLTKVectorizer
 
vectorizer = QazNLTKVectorizer()
matrix = vectorizer.fit_transform(documents)
knn = vectorizer.KNN(matrix)
 
query_vector = vectorizer.transform(["Еліміздің алтын күні жарық күн."])[0]
results = knn.search(query_vector, k=3)
# [(idx, distance), ...]

QazPerry (Kazakh LLM)

pip install keras-nlp huggingface_hub

from huggingface_hub import hf_hub_download
import keras, keras_nlp
 
model_path = hf_hub_download("silvermete0r/Gemma2_2B_QazPerry", "Gemma2_2B_QazPerry.keras")
model = keras.models.load_model(model_path, custom_objects={"GemmaCausalLM": keras_nlp.models.GemmaCausalLM})
 
print(model.generate("Instruction:\nҚазақша бірдеңе айтшы?\n\nResponse:\n"))

HuggingFace: silvermete0r/Gemma2_2B_QazPerry

---

License

• MIT
• Built at Skillset School