transformers-model 0.0.6

Model hub for transformers.

Usage Sample ''''''''''''

.. code:: python

    import pandas as pd
    from sklearn.model_selection import train_test_split
    import torch
    from transformers import BertTokenizer
    from nlpx.tokenize.utils import get_df_text_labels
    from nlpx.dataset import TextDataset, text_collate
    from transformers_model import AutoCNNTextClassifier, AutoCNNTokenClassifier,BertDataset, BertCollator, BertTokenizeCollator
    from nlpx.model.wrapper import ClassifyModelWrapper

    texts = [[str],]
    labels = [0, 0, 1, 2, 1...]
    pretrained_path = "clue/albert_chinese_tiny"
    classes = ['class1', 'class2', 'class3'...]
    train_texts, test_texts, y_train, y_test = train_test_split(texts, labels, test_size=0.2)
    
    train_set = TextDataset(train_texts, y_train)
    test_set = TextDataset(test_texts, y_test)

    ################################### TextClassifier ##################################
    model = AutoCNNTextClassifier(pretrained_path, len(classes))
    wrapper = ClassifyModelWrapper(model, classes)
    _ = wrapper.train(train_set, test_set, collate_fn=text_collate)

    ################################### TokenClassifier #################################
    tokenizer = BertTokenizer.from_pretrained(pretrained_path)

    ##################### BertTokenizeCollator #########################
    model = AutoCNNTokenClassifier(pretrained_path, len(classes))
    wrapper = ClassifyModelWrapper(model, classes)
    _ = wrapper.train(train_set, test_set, collate_fn=BertTokenizeCollator(tokenizer, 256))

    ##################### BertCollator ##################################
    train_tokenizies = tokenizer.batch_encode_plus(
            train_texts,
            max_length=256,
            padding="max_length",
            truncation=True,
            return_token_type_ids=True,
            return_attention_mask=True,
            return_tensors="pt",
    )

    test_tokenizies = tokenizer.batch_encode_plus(
            test_texts,
            max_length=256,
            padding="max_length",
            truncation=True,
            return_token_type_ids=True,
            return_attention_mask=True,
            return_tensors="pt",
    )

    train_set = BertDataset(train_tokenizies, y_train)
    test_set = BertDataset(test_tokenizies, y_test)

    model = AutoCNNTokenClassifier(pretrained_path, len(classes))
    wrapper = ClassifyModelWrapper(model, classes)
    _ = wrapper.train(train_set, test_set, collate_fn=BertCollator())