maskyrag 0.0.1

RAG implementation pipeline with GPU support

Basic Usage

Create a vector store from a document and query it using semantic search.

Example

import maskyrag

identifier = maskyrag.create_vector("file/path/to/document.pdf")

# Search the vector store
for page_number, result in enumerate(
        maskyrag.search_in_vector_store("What is VA?", identifier)
):
    print(f"\nPage {page_number}")
    print(result.page_content)

Output

Page 0
VA (Value Added) is ...

Page 1
Additional information about VA ...

API Reference

create_vector(path)

Creates a vector store from a supported document.

Parameters

Name	Type	Description
path	str	Path to the input document.

Returns

Type	Description
Vector Store Identifier	Searchable vector index containing document embeddings.

---

search_in_vector_store(query, identifier)

Performs semantic similarity search against a vector store.

Parameters

Name	Type	Description
query	str	Natural language search query.
identifier	Vector Store	Previously created vector store.

Returns

Type	Description
list[Document]	Matching document chunks ranked by relevance.

---

Supported File Types

Type	Extensions
PDF Documents	.pdf
Word Documents	.docx
Excel Spreadsheets	.xlsx
PowerPoint Presentations	.pptx
Text Files	.txt
Markdown Files	.md
CSV Files	.csv

---

---

Document Loader

Helps to load different document types into standard document object

load_from_file(file_object, file_extension)

Loads file from file object which is binary

Parameters

Name	Type	Description
file_object	_io.BufferedReader	File object with 'rb'.
file_extension	str	File extension.

Returns

Type	Description
List	List of documents containing information.

Example

from maskyrag import document_loaders

file_name = "a.pdf"

file = open(file_name, "rb")
documents = document_loaders.load_from_file(file, ".pdf")
file.close()

print(documents)

Output

[Document(metadata={'source': 'file_object', 'page': 1}, page_content='...'), ...]

---

load_from_file_path(file_path)

Loads file from file path

Parameters

Name	Type	Description
file_path	str	File path.

Returns

Type	Description
List	List of documents containing information.

Example

from maskyrag import document_loaders

file_name = "a.pdf"
documents = document_loaders.load_from_file_path(file_name)
print(documents)

Output

[Document(metadata={'source': 'file_object', 'page': 1}, page_content='...'), ...]

---

---

Document Splitter

Helps to split documents into chunks

get_chunks(documents, chunk_size=1000, chunk_overlap=200)

Splits the given documents object based on given chunk size

Parameters

Name	Type	Description
documents	list	List of document objects.
chunk_size	int	Maximum size of data in a single document.
chunk_overlap	int	the number of characters that are shared between consecutive chunks when splitting a document.

Returns

Type	Description
List	List of documents containing information.

Example

from maskyrag import document_loaders
from maskyrag import document_splitter

file_name = "a.pdf"
documents = document_loaders.load_from_file_path(file_name)

chunks = document_splitter.get_chunks(documents, chunk_size=100, chunk_overlap=10)

print(chunks)

Output

[Document(metadata={'source': 'file_object', 'page': 1}, page_content='...'), ...]

---

---

Embedding Generator

Gets embedding vector model

list_embeddings_model()

Lists all Embeddings Models

Returns

Type	Description
List	List of known model names available.

Example

from maskyrag import generate_embeddings

print(generate_embeddings.list_embeddings_model())

Output

['sentence-transformers/all-MiniLM-L6-v2', 'BAAI/bge-small-en-v1.5', 'sentence-transformers/all-mpnet-base-v2', ...]

---

get_embeddings(model)

Creates instance of the embedding vector model

Parameters

Name	Type	Description
model	str	Model name.

Returns

Type	Description
langchain_huggingface.embeddings.huggingface.HuggingFaceEmbeddings	Embedding model instance.

Example

from maskyrag import generate_embeddings

print(generate_embeddings.get_embeddings("sentence-transformers/all-MiniLM-L6-v2"))

Output

Loading weights: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 103/103 [00:00<00:00, 2358.80it/s]
model_name='sentence-transformers/all-MiniLM-L6-v2' cache_folder=None model_kwargs={} encode_kwargs={'normalize_embeddings': True} query_encode_kwargs={} multi_process=False show_progress=False

---

---

Vector Store

Generates Vectors and helps to search between vectors

get_vector_store(chunks, embeddings)

Generates vector from chunks, embeddings

Parameters

Name	Type	Description
chunks	list	List of documents.
embeddings	langchain_huggingface.embeddings.huggingface.HuggingFaceEmbeddings	Embedding model.

Returns

Type	Description
langchain_community.vectorstores.faiss.FAISS	Vector Store Instance.

Example

from maskyrag import generate_embeddings
from maskyrag import document_loaders
from maskyrag import document_splitter
from maskyrag import vector_store

file_name = "a.pdf"
documents = document_loaders.load_from_file_path(file_name)

chunks = document_splitter.get_chunks(documents, chunk_size=100, chunk_overlap=10)

embeddings = generate_embeddings.get_embeddings("sentence-transformers/all-MiniLM-L6-v2")

print(vector_store.get_vector_store(chunks, embeddings))

Output

<langchain_community.vectorstores.faiss.FAISS object at 0x...>

---

save_vector(identifier, vector_store)

Save the vector

Parameters

Name	Type	Description
identifier	str	Identifier to retrive the vector store.
vector_store	langchain_community.vectorstores.faiss.FAISS	Vector store to save.

Example

from maskyrag import generate_embeddings
from maskyrag import document_loaders
from maskyrag import document_splitter
from maskyrag import vector_store

file_name = "a.pdf"
documents = document_loaders.load_from_file_path(file_name)

chunks = document_splitter.get_chunks(documents, chunk_size=100, chunk_overlap=10)

embeddings = generate_embeddings.get_embeddings("sentence-transformers/all-MiniLM-L6-v2")

vector_store.save_vector("a", vector_store.get_vector_store(chunks, embeddings))

---

load_vector_store(identifier, embeddings)

Retrieve the saved vector

Parameters

Name	Type	Description
identifier	str	Identifier to retrive the vector store.
embeddings	langchain_huggingface.embeddings.huggingface.HuggingFaceEmbeddings	Embedding model.

Returns

Type	Description
langchain_community.vectorstores.faiss.FAISS	Vector Store Instance.

Example

from maskyrag import generate_embeddings
from maskyrag import document_loaders
from maskyrag import document_splitter
from maskyrag import vector_store

file_name = "a.pdf"
documents = document_loaders.load_from_file_path(file_name)

chunks = document_splitter.get_chunks(documents, chunk_size=100, chunk_overlap=10)

embeddings = generate_embeddings.get_embeddings("sentence-transformers/all-MiniLM-L6-v2")

vector_store.save_vector("a", vector_store.get_vector_store(chunks, embeddings))

print(vector_store.load_vector_store("a", embeddings))

Output

<langchain_community.vectorstores.faiss.FAISS object at 0x...>