An easy way to fetch and store data from and store to key-value databases like Redis.
Project description
DB Transfer
An easy way to manipulate data using key-value databases like Redis.
It is designed to support a number of databases, but currently only Redis is supported.
INSTALL
pip install DB-Transfer
Design
There are an adapter class for every database.
After instantiating Python Transfer using certain adapter_name, we can manipulate the
data from key-value database just like dictionaries: transfer[key] = value
Keys
Keys are created using prefix, namespace and item.
Example: data:USERS:arrrlo:full_name
(data is prefix, USERS is namespace and arrrlo:full_name is item)
Redis Adapter:
Connect to Redis using environment variables
Very handy when using in docker containers.
from db_transfer import Transfer, sent_env
os.environ['REDIS_HOST'] = 'localhost'
os.environ['REDIS_PORT'] = '6379'
os.environ['REDIS_DB'] = '0'
@sent_env('redis', 'HOST', 'REDIS_HOST')
@sent_env('redis', 'PORT', 'REDIS_PORT')
@sent_env('redis', 'DB', 'REDIS_DB')
class RedisTransfer(Transfer):
def __init__(self, prefix=None, namespace=None):
super().__init__(prefix=str(prefix), namespace=namespace, adapter_name='redis')
Store data
rt = RedisTransfer()
rt['my_key'] = 'some_string' # redis: "SET" "data:my_key" "some_string"
rt = RedisTransfer(namespace='my_namespace')
rt['my_key'] = 'some_string' # redis: "SET" "data:my_namespace:my_key" "some_string"
rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace')
rt['my_key'] = 'some_string' # redis: "SET" "my_prefix:my_namespace:my_key" "some_string"
Connect to Redis using class parameters
class RedisTransfer(Transfer):
def __init__(self, prefix, namespace, host, port, db):
super().__init__(prefix=str(prefix), namespace=namespace, adapter_name='redis')
self.set_env('HOST', host)
self.set_env('PORT', port)
self.set_env('DB', db)
Store data
rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
rt['my_key'] = 'some_string' # redis: "SET" "my_prefix:my_name_space:my_key" "some_string"
Fetch data
rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
my_var = rt['my_key'] # redis: "GET" "my_prefix:my_namespace:my_key"
Delete data
rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
del rt['my_key'] # redis: "DEL" "my_prefix:my_namespace:my_key"
Other data types
rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
rt['my_key_1'] = [1,2,3,4] # redis: "RPUSH" "my_prefix:my_namespace:my_key_1" "1" "2" "3" "4"
rt['my_key_2'] = {'foo', 'bar'} # redis: "SADD" "my_prefix:my_namespace:my_key_2" "foo" "bar"
rt['my_key_3'] = {'foo': 'bar'} # redis: "HMSET" "my_prefix:my_namespace:my_key_3" "foo" "bar"
my_var_1 = list(rt['my_key_1']) # redis: "LRANGE" "my_prefix:my_namespace:my_key_1" "0" "-1"
my_var_2 = set(rt['my_key_2']) # redis: "SMEMBERS" "my_prefix:my_namespace:my_key_2"
my_var_3 = dict(rt['my_key_2']) # redis: "HGETALL" "my_prefix:my_namespace:my_key_3"
Redis hash data type
rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
rt['my_key'] = {'foo': 'bar'} # redis: "HMSET" "my_prefix:my_namespace:my_key" "foo" "bar"
my_var = dict(rt['my_key']) # redis: "HGETALL" "my_prefix:my_namespace:my_key"
my_var = rt['my_key']['foo'] # redis: "HGET" "my_prefix:my_namespace:my_key" "foo"
rt['my_key']['boo'] = 'doo' # redis: "HSET" "my_prefix:my_namespace:my_key" "boo" "bar"
Multiple commands execution with context manager (only for set and delete)
with RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0) as rt:
rt['my_key_1'] = 'some_string'
rt['my_key_2'] = [1,2,3,4]
rt['my_key_3'] = {'foo': 'bar'}
# redis:
#
# "MULTI"
# "SET" "my_prefix:my_namespace:my_key_1" "some_string"
# "RPUSH" "my_prefix:my_namespace:my_key_2" "1" "2" "3" "4"
# "HMSET" "my_prefix:my_namespace:my_key_3" "foo" "bar"
# "EXEC"
Using iterators
rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
for key, value in iter(rt):
# yields key and value of every key starting with my_prefix:my_namespace:
rt['my_key'] = {...} # saving a hash data (dict)
for key, value in iter(rt['my_key']):
# yields key and value for every HGET in my_prefix:my_namespace:my_key
Keys
Every key in Redis is stored in set in same Redis.
Example:
rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
rt['key_1'] = 'value'
rt['key_2:key3'] = 'value'
rt['key_2:key4'] = 'value'
rt['key_2:key_5:key_6'] = 'value'
rt['key_2:key_5:key_7'] = 'value'
rt['key_2:key_5:key_8'] = 'value'
So, the keys are "key_1", "key_2:key3", "key_2:key4", "key_2:key5:key_6", "key_2:key5:key_7", "key_2:key5:key_8".
They are not stored in one set, but different keys are stored i different sets:
'my_prefix:my_namespace': set({'key_1', 'key_2:keys'})
'my_prefix:my_namespace:key_2': set({'key_3', 'key_4', 'key_5:keys'})
'my_prefix:my_namespace:key_2:key_5': set({'key_6', 'key_7', 'key_8'})
This is done this way so you can easily access data by keys fom any level recursively:
rt.keys()
# > ['key_1', 'key_2:key3', 'key_2:key4', 'key_2:key_5:key_6', 'key_2:key_5:key_7', 'key_2:key_5:key_8']
rt['key_2'].keys()
# > ['key_3', 'key_4', 'key_5:key_6', 'key_5:key_7', 'key_5:key_8']
rt['key_2:key_5'].keys()
# > ['key_6', 'key_7', 'key_8']
Real life examples
Transfer all data from one Redis database to another:
rt_1 = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
rt_2 = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='some_host', port=6379, db=0)
for key in rt_1.keys():
rt_2[key] = rt_1[key]
or if you want to insert data in one batch (read goes one by one):
with rt_2:
for key in rt_1.keys():
rt_2[key] = rt_1[key]
Transfer data from one user to another:
rt_1 = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
for key in rt_1['arrrlo'].keys():
rt_1['edi:' + key] = rt_1['arrrlo:' + key]
Delete user from database:
rt_1 = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
with rt_1:
for key in rt_1['arrrlo'].keys():
del rt_1['arrrlo:' + key]
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distribution
Built Distribution
Hashes for DB_Transfer-0.4.4-py2.py3-none-any.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 05e345fea2190fae357c066ebd8d7caa254258d79c06925f96accb00194557c0 |
|
MD5 | 486ceb74a8b41e9254e7c4aefd9aec1d |
|
BLAKE2b-256 | 9df761ddb27972a4a7160755b9ffb985b1038c739b20e38746588fccdb596f9b |