shelchemy 2.240219.2

Shelve-like dict using sqlalchemy as a backend, and lazy scheduler for resuming tasks

shelchemy - Dict-like (shelve-like) storage wrapper for any DBMS (SQLAlchemy)

Python installation

from package

# Set up a virtualenv. 
python3 -m venv venv
source venv/bin/activate

# Install from PyPI
pip install shelchemy

from source

git clone https://github.com/shelchemy/shelchemy
cd shelchemy
poetry install

Examples

This library is more useful when used along with hdict. Here are some possible usages by itself.

Scheduling jobs.

from time import sleep

from shelchemy.scheduler import Scheduler

# Jobs can be distributed across multiple computers/networks.
names1 = ["a", "b"]
names2 = ["c"]
names3 = ["d", "e"]
storage = {}
# `storage` can be: shelve; URI pointing to a database; or, any dict-like object.
#   Example of a local database: storage="sqlite+pysqlite:////tmp/sqla-test.db"
#   Example of a remote database: storage="mysql+pymysql://user1:[e-mail]/db1"
for name in Scheduler(storage, timeout=10) << names1 << names2 << names3:
    print(f"Processing {name}")
    sleep(0.1)
    print(f"{name} processed!")
"""
2023-11-13 00:03:40.115637 'a' is new, starting
Processing a
a processed!
2023-11-13 00:03:40.316130 'a' done
2023-11-13 00:03:40.326101 'b' is new, starting
Processing b
b processed!
2023-11-13 00:03:40.526549 'b' done
2023-11-13 00:03:40.534599 'c' is new, starting
Processing c
c processed!
2023-11-13 00:03:40.735062 'c' done
2023-11-13 00:03:40.745734 'd' is new, starting
Processing d
d processed!
2023-11-13 00:03:40.946152 'd' done
2023-11-13 00:03:40.957263 'e' is new, starting
Processing e
e processed!
2023-11-13 00:03:41.157701 'e' done
"""

Persistent dict.

from shelchemy import sopen
from shelchemy.cache import Cache

d = Cache("sqlite+pysqlite:////tmp/sqla-test.db")
d["x"] = 5
d["b"] = None
print(d["x"], d["b"])
"""
5 None
"""

try:
    d["xxx"]
except KeyError as m:
    print(m)
"""
'xxx'
"""

for k, v in d.items():
    print(k, v)
print("x" in d)
"""
872d417d62b78366a71ab9fee25f14dc None
aed0339093d97301965a4e23dac3424a b'only bytes when autopack=False'
a b'by'
x 5
b None
True
"""

del d["x"]
print("x" in d)
"""
False
"""

print(d)
"""
{'872d417d62b78366a71ab9fee25f14dc': None, 'aed0339093d97301965a4e23dac3424a': b'only bytes when autopack=False', 'a': b'by', 'b': None}
"""

# Using a context manager.
with sopen() as db:
    print("x" in db)
    print(db)

    db["x"] = b"asd"
    print(db)
    print(db["x"] == b"asd")
    print("x" in db)
    print(db.x == b"asd")

    del db["x"]
    print("x" in db)

    db["any string key longer than 40 characters will be hashed depending if the DBMS backend is used"] = None
    print(db)
"""
False
{}
{'x': b'asd'}
True
True
True
False
{'872d417d62b78366a71ab9fee25f14dc': None}
"""

Grants

This work was partially supported by Fapesp under supervision of Prof. André C. P. L. F. de Carvalho at CEPID-CeMEAI (Grants 2013/07375-0 – 2019/01735-0).