Usage Sample ''''''''''''
.. code:: python
import torch
from sklearn.model_selection import train_test_split
from nlpx.tokenize import Tokenizer
from nlpx.model.classifier import TextCNNClassifier
from nlpx.model.wrapper import ClassifyModelWrapper
from nlpx.dataset import TokenDataset, PaddingTokenCollator
if __name__ == '__main__':
classes = ['class1', 'class2', 'class3'...]
texts = [[str],]
labels = [0, 0, 1, 2, 1...]
tokenizer = Tokenizer.from_texts(texts, min_freq=5)
sent = 'I love you'
tokens = tokenizer.encode(sent, max_length=6)
# [101, 66, 88, 99, 102, 0]
sent = tokenizer.decode(tokens)
# ['<BOS>', 'I', 'love', 'you', '<EOS>', '<PAD>']
tokens = tokenizer.batch_encode(texts, padding=False)
X_train, X_test, y_train, y_test = train_test_split(tokens, labels, test_size=0.2)
train_set = TokenDataset(X_train, y_train)
val_set = TokenDataset(X_test, y_test)
model = TextCNNClassifier(embed_dim=128, vocab_size=tokenizer.vocab_size, num_classes=len(classes))
model_wrapper = ClassifyModelWrapper(model, classes=classes)
model_wrapper.train(train_set, val_set, show_progress=True, collate_fn=PaddingTokenCollator(tokenizer.pad))
result = model_wrapper.evaluate(val_set, collate_fn=PaddingTokenCollator(tokenizer.pad))
# 0.953125
test_inputs = torch.tensor(test_tokens, dtype=torch.long)
result = model_wrapper.predict(test_inputs)
# [0, 1]
result = model_wrapper.predict_classes(test_inputs)
# ['class1', 'class2']
result = model_wrapper.predict_proba(test_inputs)
# ([0, 1], array([0.99439645, 0.99190724], dtype=float32))
result = model_wrapper.predict_classes_proba(test_inputs)
# (['class1', 'class2'], array([0.99439645, 0.99190724], dtype=float32))
1 maintainer
Release files for NLPX 2.0.7
For a detailed explanation of source distributions (sdists) and built distributions (wheels), please see the package formats documentation.
Source distribution (sdist)
| File | Size | Uploaded | |
|---|---|---|---|
| nlpx-2.0.7.tar.gz | 17.5 kB | Details |
Release files / nlpx-2.0.7.tar.gz
| Download URL | nlpx-2.0.7.tar.gz |
|---|---|
| Size | 17.5 kB |
| Tags | Source |
|
SHA-256 checksum How to use checksums |
8c4225f533c521b4393a2738a42879f42cc97301802950a928bcf10d2f45d7f9
|
|
BLAKE2b-256 checksum How to use checksums |
9dc4363c2e767cc32c2428749812f37ae28c6d29e8adeb72bc507d21d4b8ba64
|
| Upload date | |
|
Uploaded using Trusted Publishing? What is trusted publishing? |
No |
| Uploaded via |
twine/5.0.0 CPython/3.9.18
|
