alawymdb 0.2.1

Linear-scaling in-memory database optimized for ML workloads

AlawymDB

Almost Linear Any Way You Measure - A high-performance in-memory database that achieves near-linear O(n) scaling for operations that traditionally suffer from O(n log n) complexity.

🚀 Breakthrough Performance

AlawymDB (pronounced "ah-LAY-wim") lives up to its name - delivering almost linear performance any way you measure it:

Column Scaling Performance

Columns:     10    →    100   →   1000   →   2000
Cells/sec:  18.5M  →   7.5M   →   5.8M   →   5.2M
Scaling:     1×    →   10×    →   100×   →   200× (columns)
             1×    →   4.1×   →   5.3×   →   6.1× (time)

Result: O(n) with minimal logarithmic factor - effectively linear! 🎯

🔧 Installation

pip install alawymdb

💡 Quick Start

import alawymdb as db

# Initialize database
db.create_database()

# Create schema and table
db.create_schema("main")
db.create_table(
    "main", 
    "users",
    [
        ("id", "UINT64", False),
        ("name", "STRING", False),
        ("age", "INT64", True),
        ("email", "STRING", True),
        ("score", "FLOAT64", True)
    ]
)

# Insert data
users = [
    (1, "Alice", 30, "[e-mail]", 95.5),
    (2, "Bob", 25, "[e-mail]", 87.3),
    (3, "Charlie", 35, "[e-mail]", 92.1),
]

for user_id, name, age, email, score in users:
    db.insert_row("main", "users", [
        ("id", user_id),
        ("name", name),
        ("age", age),
        ("email", email),
        ("score", score)
    ])

# Query with SQL
result = db.execute_sql("SELECT * FROM main.users")
print(result)

# SQL with WHERE clause
young_users = db.execute_sql("SELECT * FROM main.users WHERE age = 25")
print(f"Young users:\n{young_users}")

# Direct API queries
all_users = db.select_all("main", "users")
print(f"Total users: {db.count_rows('main', 'users')}")

🎯 Working Example: Analytics with Pandas Integration

import alawymdb as db
import numpy as np
import pandas as pd

# Setup
db.create_database()
db.create_schema("test_schema")

# Create employees table
db.create_table(
    "test_schema", 
    "employees",
    [
        ("id", "UINT64", False),
        ("name", "STRING", False),
        ("age", "UINT64", True),
        ("salary", "FLOAT64", True),
        ("department", "STRING", True)
    ]
)

# Insert test data
employees = [
    (1, "Alice Johnson", 28, 75000.0, "Engineering"),
    (2, "Bob Smith", 35, 85000.0, "Sales"),
    (3, "Charlie Brown", 42, 95000.0, "Engineering"),
    (4, "Diana Prince", 31, 78000.0, "Marketing"),
    (5, "Eve Adams", 26, 72000.0, "Sales"),
]

for emp in employees:
    db.insert_row("test_schema", "employees", [
        ("id", emp[0]),
        ("name", emp[1]),
        ("age", emp[2]),
        ("salary", emp[3]),
        ("department", emp[4])
    ])

# Convert to Pandas DataFrame
df = db.to_pandas("test_schema", "employees")
print("DataFrame shape:", df.shape)
print("\nDataFrame head:")
print(df.head())

# Pandas operations
print(f"\nAverage salary: ${df['salary'].mean():,.2f}")
print("\nSalary by department:")
print(df.groupby('department')['salary'].agg(['mean', 'count']))

# Get as NumPy array
ages = db.to_numpy("test_schema", "employees", "age")
print(f"\nAges array: {ages}")
print(f"Mean age: {np.mean(ages):.1f}")

# Get data as dictionary
data_dict = db.select_as_dict("test_schema", "employees")
print(f"\nColumns available: {list(data_dict.keys())}")

📊 Create Table from Pandas DataFrame

import alawymdb as db
import pandas as pd
import numpy as np

db.create_database()
db.create_schema("data")

# Create a DataFrame
df = pd.DataFrame({
    'product_id': np.arange(1, 101),
    'product_name': [f'Product_{i}' for i in range(1, 101)],
    'price': np.random.uniform(10, 100, 100).round(2),
    'quantity': np.random.randint(1, 100, 100),
    'in_stock': np.random.choice([0, 1], 100)  # Use 0/1 instead of True/False
})

# Import DataFrame to AlawymDB
result = db.from_pandas(df, "data", "products")
print(result)

# Verify by reading back
df_verify = db.to_pandas("data", "products")
print(f"Imported {len(df_verify)} rows with {len(df_verify.columns)} columns")
print(df_verify.head())

# Query the imported data
result = db.execute_sql("SELECT * FROM data.products WHERE price > 50.0")
print(f"Products with price > 50: {result}")

📈 Wide Table Example (Working Version)

import alawymdb as db

db.create_database()
db.create_schema("wide")

# Create table with many columns
num_columns = 100
columns = [("id", "UINT64", False)]
columns += [(f"metric_{i}", "FLOAT64", True) for i in range(num_columns)]

db.create_table("wide", "metrics", columns)

# Insert data
for row_id in range(100):
    values = [("id", row_id)]
    values += [(f"metric_{i}", float(row_id * 0.1 + i)) for i in range(num_columns)]
    db.insert_row("wide", "metrics", values)

# Query using direct API (more reliable for wide tables)
all_data = db.select_all("wide", "metrics")
print(f"Inserted {len(all_data)} rows")

# Convert to Pandas for analysis
df = db.to_pandas("wide", "metrics")
print(f"DataFrame shape: {df.shape}")
print(f"Columns: {df.columns[:5].tolist()} ... {df.columns[-5:].tolist()}")

# Get specific column as NumPy array
metric_0 = db.to_numpy("wide", "metrics", "metric_0")
print(f"Metric_0 stats: mean={metric_0.mean():.2f}, std={metric_0.std():.2f}")

🚀 Performance Test

import alawymdb as db
import pandas as pd
import numpy as np
import time

db.create_database()
db.create_schema("perf")

# Create a large DataFrame
n_rows = 10000
df_large = pd.DataFrame({
    'id': np.arange(n_rows),
    'value1': np.random.randn(n_rows),
    'value2': np.random.randn(n_rows) * 100,
    'category': np.random.choice(['A', 'B', 'C', 'D', 'E'], n_rows),
    'flag': np.random.choice([0, 1], n_rows)
})

# Time the import
start = time.time()
db.from_pandas(df_large, "perf", "large_table")
import_time = time.time() - start
print(f"Import {n_rows} rows: {import_time:.3f}s ({n_rows/import_time:.0f} rows/sec)")

# Time the export
start = time.time()
df_export = db.to_pandas("perf", "large_table")
export_time = time.time() - start
print(f"Export to Pandas: {export_time:.3f}s ({n_rows/export_time:.0f} rows/sec)")

# Verify
print(f"Shape verification: {df_export.shape}")

🏗️ Why "Almost Linear Any Way You Measure"?

The name AlawymDB reflects our core achievement:

• Column scaling: O(n) with tiny logarithmic factor (log₂₅₆)
• Row scaling: Pure O(n) for scans
• Memory usage: Linear with data size
• Wide tables: Tested up to 5000 columns with maintained performance

📊 Current SQL Support

✅ Working SQL Features

• SELECT * FROM table
• SELECT column1, column2 FROM table
• SELECT * FROM table WHERE column = value
• SELECT * FROM table WHERE column > value (for INT64 columns)
• SELECT * FROM table WHERE text_column = 'string'

⚠️ SQL Limitations

• Type matching is strict (use 50.0 for FLOAT64, 50 for INT64)
• GROUP BY and JOINs via SQL coming soon (use direct API)
• Complex expressions not yet supported

🎨 API Reference

# Core operations
db.create_database()
db.create_schema(schema_name)
db.create_table(schema, table, columns)
db.insert_row(schema, table, values)

# Query operations
db.select_all(schema, table)
db.select_where(schema, table, columns, where_col, where_val)
db.count_rows(schema, table)
db.execute_sql(sql_query)  # Basic SQL support

# Data science integrations
db.to_pandas(schema, table)         # Export to DataFrame
db.to_numpy(schema, table, column)  # Export column to NumPy
db.from_pandas(df, schema, table)   # Import from DataFrame
db.select_as_dict(schema, table)    # Get as Python dict

🚦 Performance Characteristics

Operation	Complexity	Verified Scale
INSERT	O(1)	2M rows × 2K columns
SELECT *	O(n)	10K rows × 5K columns
WHERE clause	O(n)	1M rows tested
to_pandas()	O(n)	100K rows tested
from_pandas()	O(n)	100K rows tested
Column scaling	~O(n)	Up to 5000 columns

🤝 Contributing

We welcome contributions! Priority areas:

• SQL enhancements (GROUP BY, JOIN)
• Performance optimizations
• Storage persistence
• Additional data types

📜 License

MIT License

---

AlawymDB: Almost Linear Any Way You Measure - because performance should scale with your data, not against it.