tfrecords: simplify and transplant the tfrecords
Project description
tfrecords
simplify and transplant the tfrecord and table
update information
2022-10-30: Add lmdb leveldb read and writer and add record batch write
2022-10-17: Add shared memory for record to read mode with more accelerated Reading.
2022-02-01: simplify and transplant the tfrecord dataset
1. record read and write demo , with_share_memory flags will Accelerated Reading
# -*- coding: utf-8 -*-
# @Time : 2022/9/8 15:49
import tfrecords
options = tfrecords.TFRecordOptions(compression_type=tfrecords.TFRecordCompressionType.NONE)
def test_write(filename, N=3, context='aaa'):
with tfrecords.TFRecordWriter(filename, options=options) as file_writer:
batch_data = []
for i in range(N):
d = context + '____' + str(i)
batch_data.append(d)
if (i + 1) % 100 == 0:
file_writer.write_batch(batch_data)
batch_data.clear()
if len(batch_data):
file_writer.write_batch(batch_data)
batch_data.clear()
def test_record_iterator(example_paths):
print('test_record_iterator')
for example_path in example_paths:
iterator = tfrecords.tf_record_iterator(example_path, options=options,skip_bytes=0,with_share_memory=True)
offset_list = iterator.read_offsets(0)
count = iterator.read_count(0)
print(count)
num = 0
for iter in iterator:
num += 1
print(iter)
def test_random_reader(example_paths):
print('test_random_reader')
for example_path in example_paths:
file_reader = tfrecords.tf_record_random_reader(example_path, options=options,with_share_memory=True)
last_pos = 0
while True:
try:
x,pos = file_reader.read(last_pos)
print(x,pos)
last_pos = pos
except Exception as e:
break
def test_random_reader2(example_paths):
print('test_random_reader2')
for example_path in example_paths:
file_reader = tfrecords.tf_record_random_reader(example_path, options=options,with_share_memory=True)
skip_bytes = 0
offset_list = file_reader.read_offsets(skip_bytes)
for offset,length in offset_list:
x, _ = file_reader.read(offset)
print(x)
test_write('d:/example.tfrecords0',3,'file0')
example_paths = tfrecords.glob('d:/example.tfrecords*')
print(example_paths)
test_record_iterator(example_paths)
print()
test_random_reader(example_paths)
print()
test_random_reader2(example_paths)
print()
2. leveldb read and write demo
# -*- coding: utf-8 -*-
# @Time : 2022/9/8 15:49
from tfrecords import LEVELDB
db_path = 'd:/example_leveldb'
def test_write(db_path):
options = LEVELDB.LeveldbOptions(create_if_missing=True, error_if_exists=False)
file_writer = LEVELDB.Leveldb(db_path,options)
keys,values = [],[]
for i in range(30):
keys.append(b"input_" + str(i).encode())
keys.append(b"label_" + str(i).encode())
values.append(b"xiaoming" + str(i).encode())
values.append(b"zzs" + str(i).encode())
if (i + 1) % 1000 == 0:
file_writer.put_batch(keys,values)
keys.clear()
values.clear()
if len(keys):
file_writer.put_batch(keys, values)
keys.clear()
values.clear()
file_writer.close()
def test_read(db_path):
options = LEVELDB.LeveldbOptions(create_if_missing=False, error_if_exists=False)
reader = LEVELDB.Leveldb(db_path,options)
def show():
it = reader.get_iterater(reverse=False)
i = 0
for item in it:
print(i,item)
i += 1
def test_find(key):
value = reader.get(key)
print('find', type(value), value)
show()
test_find(b'input_0')
test_find(b'input_5')
test_find(b'input_10')
reader.close()
test_write(db_path)
test_read(db_path)
3. lmdb read and write demo
# -*- coding: utf-8 -*-
# @Time : 2022/9/8 15:49
from tfrecords import LMDB
db_path = 'd:/example_lmdb'
def test_write(db_path):
options = LMDB.LmdbOptions(env_open_flag = 0,
env_open_mode = 0o664, # 8进制表示
txn_flag = 0,
dbi_flag = 0,
put_flag = 0)
file_writer = LMDB.Lmdb(db_path,options,map_size=1024 * 1024 * 10)
keys, values = [], []
for i in range(30):
keys.append(b"input_" + str(i).encode())
keys.append(b"label_" + str(i).encode())
values.append(b"xiaoming_" + str(i).encode())
values.append(b"zzs_" + str(i).encode())
if (i + 1) % 1000 == 0:
file_writer.put_batch(keys, values)
keys.clear()
values.clear()
if len(keys):
file_writer.put_batch(keys, values)
file_writer.close()
def test_read(db_path):
options = LMDB.LmdbOptions( env_open_flag = LMDB.LmdbFlag.MDB_RDONLY,
env_open_mode = 0o664, # 8进制表示
txn_flag = LMDB.LmdbFlag.MDB_RDONLY,
dbi_flag = 0,
put_flag = 0)
reader = LMDB.Lmdb(db_path,options,map_size=0)
def show():
it = reader.get_iterater(reverse=False)
i = 0
for item in it:
print(i,item)
i += 1
def test_find(key):
value = reader.get(key)
print('find', type(value), value)
show()
test_find('input0')
test_find('input5')
test_find(b'input10')
reader.close()
test_write(db_path)
test_read(db_path)
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distributions
No source distribution files available for this release.See tutorial on generating distribution archives.
Built Distributions
Close
Hashes for tfrecords-0.2.6-cp311-cp311-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 58d0baf419b0920c5ffe78cebe3e74aec99b0d68cb64df185ad99af52a54dde2 |
|
MD5 | 8fcf8c98a16fbdefeb7124bd3ccb1da3 |
|
BLAKE2b-256 | e1af8737abf84e4b2c023f5c4ae8f0ce755a24847b9e2978d0baad89b97a45f8 |
Close
Hashes for tfrecords-0.2.6-cp311-cp311-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 0f26751791fc51a8c6a79032b6fdd7b5debb6af887e2d531a6095ae5f1d90e31 |
|
MD5 | 902bc5fd4e8ed1bc02e6b5e24e290a51 |
|
BLAKE2b-256 | 94b5586af735a2ae7c296eebfa6c0dce619c0189f170634b452908cbea63ef01 |
Close
Hashes for tfrecords-0.2.6-cp311-cp311-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | eba583dfba09fb719e4ef5daa32cc77f9a9d661520f518e9a0a9cd7cec36182f |
|
MD5 | db2d7d8907bad7adef740baee1e83572 |
|
BLAKE2b-256 | f9120d5087d2c808c5d76d33b73db40ab3c9242dd3aa92327734cde2883015f0 |
Close
Hashes for tfrecords-0.2.6-cp310-cp310-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | a355884ddc2bdb06464a30fe3130f2f32b48dc3e904323b2c95eab92d8c82992 |
|
MD5 | c4b6424b751dfcbe2084c4d7ab1c7cca |
|
BLAKE2b-256 | 5be42be35eb14e5f9fb975abda5c9629aaa626657571b9e4b6ce4387735c74d6 |
Close
Hashes for tfrecords-0.2.6-cp310-cp310-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 8cd9caa78c56daca5e5f0b3f9f3bb43324b86eeca9d94c0cd44f0df4a486f78b |
|
MD5 | 835edf1ac7207d93c642f43eb1895e51 |
|
BLAKE2b-256 | bad1a54890fd3a7a24a10381b9ffc7987a75ee6b0dfa38b47fa1aa9500ae3812 |
Close
Hashes for tfrecords-0.2.6-cp310-cp310-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | b57b6d57d6785e75fa9d5f51968f6ff8fad0b83aa530815ef539d84c112b2a00 |
|
MD5 | e041284efc63bab84a7a408fc1192d96 |
|
BLAKE2b-256 | 08f1fdd4141eb2239dac3e0e37544d52a52e44af213f148b30218875d4365edf |
Close
Hashes for tfrecords-0.2.6-cp39-cp39-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 8c32a50542a25a5165f205bb6a618f09f682d2a5aa1546d017a7e3dbfb75646c |
|
MD5 | 256456c785bc99923aabf89f40f35b28 |
|
BLAKE2b-256 | 0504f392a2049760781234f460f402c6e1ad7bea926e4e7b82d02a95565bf25e |
Close
Hashes for tfrecords-0.2.6-cp39-cp39-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 61c49702d42d059147d85b74286a103fa32bb5eed170df4e6072b32d22daad13 |
|
MD5 | fe697ab7142de177c1cb6c71f7e0585b |
|
BLAKE2b-256 | c3ffe569aa9a5dd5fb616fe8e718c5a537efa535b12b4c1bc6fc77afba0b9a49 |
Close
Hashes for tfrecords-0.2.6-cp39-cp39-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 6fc16311f2e0e3f14e5bb12c9437ca0786a7acb65ec74525c4a7921d79d5874c |
|
MD5 | 174a20665b7925867a4af7d10a3197c6 |
|
BLAKE2b-256 | 1ae64c5cf1f22dc1b71b124d5916ec8f0728de6e729b663b77d7ba91f2dfcd4f |
Close
Hashes for tfrecords-0.2.6-cp38-cp38-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 41dbb5e46987382fea09813ad2e51b2a992e413269e88b7435027a6391fec146 |
|
MD5 | b5f964c208fb2d8e0cfd7fae5acd5a4b |
|
BLAKE2b-256 | ef21cacee942eb357d199c23dc521882f515bcf2ca370b1b327c504de6f39f99 |
Close
Hashes for tfrecords-0.2.6-cp38-cp38-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 9b23b2deaa8b6eee93f19a1ee94b43670d7a775d804a6180e34fdf06d0a6a228 |
|
MD5 | 2c6cd7190e64d186ec36f7c9e2ab7bfb |
|
BLAKE2b-256 | 353a3711293f83e639ff53cd9f280675f87e905631ef6d143c3e891f23c4e1b9 |
Close
Hashes for tfrecords-0.2.6-cp38-cp38-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 54052699e6c5ebea27d8be0f06f701b14e5674cbe1b44147d22c55bb6f5e6b23 |
|
MD5 | 29fc42bd1158dd3b294d697934b5d548 |
|
BLAKE2b-256 | ed15aedacb2cc3ffa01b834a56f3fe7671302fb9fca8d67012df6f65889e67e8 |
Close
Hashes for tfrecords-0.2.6-cp37-cp37m-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 7558233fb92ab82597d643f4dbeea34f8f29b5861c82b610f84fa562500f122a |
|
MD5 | 133e11848e6120c40b0767a9e0066a98 |
|
BLAKE2b-256 | 2dfd630e61d2a6a9844de0dbc84192884059b305c8d36ff1ac2e8745c86464b8 |
Close
Hashes for tfrecords-0.2.6-cp37-cp37m-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 78bd9d04569e9de6a8bbf936b6a27b4455a29575472cdc7431211d571fd6e286 |
|
MD5 | 90effd9b32b75adb527d93795edbc61c |
|
BLAKE2b-256 | 4c7157158c7748990d5e492ae5e3ebb8995f7f5bc1f2778626448668e8b9d4bb |
Close
Hashes for tfrecords-0.2.6-cp37-cp37m-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 83cd822df796923702f9f885f0f5176927dada5e60903ae0f3be204e080ecdff |
|
MD5 | 4a12fc6b2c65e0b3843ac0fe4ee47ccd |
|
BLAKE2b-256 | b0ef937c7d8d445b59e49d4067aada0d8e8b76582edb1cce29d6fde3b771f378 |
Close
Hashes for tfrecords-0.2.6-cp36-cp36m-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 180542e722611783249bdc1f49ad9041a170e075f1b6191466a01b76e4cebcc4 |
|
MD5 | d6f2718353aa46586375f50e7f27c7a0 |
|
BLAKE2b-256 | 816a983643eb279c58bcbb7dfd387746dda5d13649f05b3a332667d82c18c1cc |
Close
Hashes for tfrecords-0.2.6-cp36-cp36m-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | ce2677504bfb80196cde7c22cc932fea4cd5ac58d6b2904a027a34ae37034fa5 |
|
MD5 | 50cbf8ad3b268fdcc9b2e714b350e748 |
|
BLAKE2b-256 | 47eb0cb7d12065e74a195b8926f28f15cdc5fe14d980cdad8a87fc293d8531e7 |
Close
Hashes for tfrecords-0.2.6-cp36-cp36m-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | d8117a5fba398dc9c984be0d571d193c98abd737f846d8b32c355baa40605576 |
|
MD5 | ed4f7fb2760833bf39dae216aa2ecc74 |
|
BLAKE2b-256 | e2da22949924cfb451bfa5048532e3b7268671436c08841e01fb3428584950d1 |