DB-Transfer 0.3.2

An easy way to fetch and store data from and store to key-value databases like Redis.

Transfer

An easy way to manipulate data using key-value databases like Redis.
It is designed to support a number of databases, but currently only Redis is supported.

INSTALL (Python 3.x)

pip install git+git://github.com/arrrlo/python-transfer@master

Design

There are an adapter class for every database.
After instantiating Python Transfer using certain adapter_name, we can manipulate the
data from key-value database just like dictionaries: transfer[key] = value

Keys

Keys are created using prefix, namespace and item.
Example: data:USERS:arrrlo:full_name
(data is prefix, USERS is namespace and arrrlo:full_name is item)

Redis Adapter:

Connect to Redis using environment variables

from transfer import Transfer, sent_env

os.environ['REDIS_HOST'] = 'localhost'
os.environ['REDIS_PORT'] = '6379'
os.environ['REDIS_DB'] = '0'

@sent_env('redis', 'HOST', 'REDIS_HOST')
@sent_env('redis', 'PORT', 'REDIS_PORT')
@sent_env('redis', 'DB', 'REDIS_DB')
class RedisTransfer(Transfer):

    def __init__(self, prefix=None, namespace=None):
        super().__init__(prefix=str(prefix), namespace=namespace, adapter_name='redis')

Store data

rt = RedisTransfer()
rt['my_key'] = 'some_string' # redis: "SET" "data:my_key" "some_string"

rt = RedisTransfer(namespace='my_namespace')
rt['my_key'] = 'some_string' # redis: "SET" "data:my_namespace:my_key" "some_string"

rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace')
rt['my_key'] = 'some_string' # redis: "SET" "my_prefix:my_namespace:my_key" "some_string"

Connect to Redis using class parameters

class RedisTransfer(Transfer):

    def __init__(self, prefix, namespace, host, port, db):
        super().__init__(prefix=str(prefix), namespace=namespace, adapter_name='redis')

        self.set_env('HOST', host)
        self.set_env('PORT', port)
        self.set_env('DB', db)

Store data

rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

rt['my_key'] = 'some_string' # redis: "SET" "my_prefix:my_name_space:my_key" "some_string"

Fetch data

rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

my_var = rt['my_key'] # redis: "GET" "my_prefix:my_namespace:my_key"

Delete data

rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

del rt['my_key'] # redis: "DEL" "my_prefix:my_namespace:my_key"

Other data types

rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

rt['my_key_1'] = [1,2,3,4] # redis: "RPUSH" "my_prefix:my_namespace:my_key_1" "1" "2" "3" "4"
rt['my_key_2'] = {'foo', 'bar'} # redis: "SADD" "my_prefix:my_namespace:my_key_2" "foo" "bar"
rt['my_key_3'] = {'foo': 'bar'} # redis: "HMSET" "my_prefix:my_namespace:my_key_3" "foo" "bar"

my_var_1 = list(rt['my_key_1']) # redis: "LRANGE" "my_prefix:my_namespace:my_key_1" "0" "-1"
my_var_2 = set(rt['my_key_2']) # redis: "SMEMBERS" "my_prefix:my_namespace:my_key_2"
my_var_3 = dict(rt['my_key_2']) # redis: "HGETALL" "my_prefix:my_namespace:my_key_3"

Redis hash data type

rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

rt['my_key'] = {'foo': 'bar'} # redis: "HMSET" "my_prefix:my_namespace:my_key" "foo" "bar"

my_var = dict(rt['my_key']) # redis: "HGETALL" "my_prefix:my_namespace:my_key"
my_var = rt['my_key']['foo'] # redis: "HGET" "my_prefix:my_namespace:my_key" "foo"

rt['my_key']['boo'] = 'doo' # redis: "HSET" "my_prefix:my_namespace:my_key" "boo" "bar"

Multiple commands execution with context manager (only for set and delete)

with RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0) as rt:
    rt['my_key_1'] = 'some_string'
    rt['my_key_2'] = [1,2,3,4]
    rt['my_key_3'] = {'foo': 'bar'}

# redis:
#
# "MULTI"
# "SET" "my_prefix:my_namespace:my_key_1" "some_string"
# "RPUSH" "my_prefix:my_namespace:my_key_2" "1" "2" "3" "4"
# "HMSET" "my_prefix:my_namespace:my_key_3" "foo" "bar"
# "EXEC"

Using iterators

rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

for key, value in iter(rt):
    # yields key and value of every key starting with my_prefix:my_namespace:


rt['my_key'] = {...} # saving a hash data (dict)

for key, value in iter(rt['my_key']):
    # yields key and value for every HGET in my_prefix:my_namespace:my_key

Keys

Every key in Redis is stored in set in same Redis.
Example:

rt = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

rt['key_1'] = 'value'
rt['key_2:key3'] = 'value'
rt['key_2:key4'] = 'value'
rt['key_2:key_5:key_6'] = 'value'
rt['key_2:key_5:key_7'] = 'value'
rt['key_2:key_5:key_8'] = 'value'

So, the keys are "key_1", "key_2:key3", "key_2:key4", "key_2:key5:key_6", "key_2:key5:key_7", "key_2:key5:key_8".
They are not stored in one set, but different keys are stored i different sets:
'my_prefix:my_namespace': set({'key_1', 'key_2:keys'})
'my_prefix:my_namespace:key_2': set({'key_3', 'key_4', 'key_5:keys'})
'my_prefix:my_namespace:key_2:key_5': set({'key_6', 'key_7', 'key_8'})

This is done this way so you can easily access data by keys fom any level recursively:

rt.keys()
# > ['key_1', 'key_2:key3', 'key_2:key4', 'key_2:key_5:key_6', 'key_2:key_5:key_7', 'key_2:key_5:key_8']

rt['key_2'].keys()
# > ['key_3', 'key_4', 'key_5:key_6', 'key_5:key_7', 'key_5:key_8']

rt['key_2:key_5'].keys()
# > ['key_6', 'key_7', 'key_8']

Real life examples

Transfer all data from one Redis database to another:

rt_1 = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)
rt_2 = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='some_host', port=6379, db=0)

for key in rt_1.keys():
    rt_2[key] = rt_1[key]

or if you want to insert data in one batch (read goes one by one):

with rt_2:
    for key in rt_1.keys():
        rt_2[key] = rt_1[key]

Transfer data from one user to another:

rt_1 = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

for key in rt_1['arrrlo'].keys():
    rt_1['edi:' + key] = rt_1['arrrlo:' + key]

Delete user from database:

rt_1 = RedisTransfer(prefix='my_prefix', namespace='my_namespace', host='localhost', port=6379, db=0)

with rt_1:
    for key in rt_1['arrrlo'].keys():
        del rt_1['arrrlo:' + key]