tfrecords: simplify and transplant the tfrecords
Project description
tfrecords
simplify and transplant the tfrecord dataset
update information
2022-10-17: Add shared memory read mode with more accelerated Reading.
2022-02-01: simplify and transplant the tfrecord dataset
record read and write demo , with_share_memory flags will Accelerated Reading
record 读写demo , 建议开启 参数with_share_memory , 将会大大加速文件读取
import tfrecords
options = tfrecords.TFRecordOptions(compression_type=tfrecords.TFRecordCompressionType.NONE)
def test_write(filename, N=3, context='aaa'):
with tfrecords.TFRecordWriter(filename, options=options) as file_writer:
for _ in range(N):
# x, y = np.random.random(), np.random.random()
file_writer.write(context + '____' + str(_))
def test_record_iterator(example_paths):
print('test_record_iterator')
for example_path in example_paths:
iterator = tfrecords.tf_record_iterator(example_path, options=options,with_share_memory=False)
num = 0
for iter in iterator:
num += 1
print(iter)
def test_random_reader(example_paths):
print('test_random_reader')
for example_path in example_paths:
file_reader = tfrecords.tf_record_random_reader(example_path, options=options,with_share_memory=False)
last_pos = 0
while True:
try:
x,pos = file_reader.read(last_pos)
print(x)
last_pos = pos
except:
break
def test_random_reader2(example_paths):
print('test_random_reader2')
for example_path in example_paths:
file_reader = tfrecords.tf_record_random_reader(example_path, options=options,with_share_memory=False)
skip_bytes = 0
offset_list = file_reader.read_offsets(skip_bytes)
for offset,length in offset_list:
x, _ = file_reader.read(offset)
print(x)
test_write('d:/example.tfrecords0',3,'file0')
test_write('d:/example.tfrecords1',10,'file1')
test_write('d:/example.tfrecords2',12,'file2')
example_paths = tfrecords.glob('d:/example.tfrecords*')
test_record_iterator(example_paths)
print()
test_random_reader2(example_paths)
print()
test_random_reader2(example_paths)
print()
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distributions
No source distribution files available for this release.See tutorial on generating distribution archives.
Built Distributions
Close
Hashes for tfrecords-0.0.8-cp310-cp310-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | a589d39165379eac4e8d7c4f65bf3d736b08eb4e6ff7440a7938e7bc4fcd27e3 |
|
MD5 | 62af78c16180c626b86df7682a6bb3eb |
|
BLAKE2b-256 | d97ed910034d3137a761e3909b97c2c5c8c101ca2493e75843ac8e3ee7e09bcc |
Close
Hashes for tfrecords-0.0.8-cp310-cp310-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | df866e8a1d7e4d21211425620cd35900a3cd7aa72954df5207400203ac442dad |
|
MD5 | 0f69c7bb7c04f3852f8833d2a779d95d |
|
BLAKE2b-256 | 310c2febaf54f0f86616cf17a5d13252f8c9daa741d9a006c44c772120ccc713 |
Close
Hashes for tfrecords-0.0.8-cp310-cp310-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 397fce97760e5d3f2b5750a31e5ec50f764d1b71f814c7d48edf1ac58c406ea0 |
|
MD5 | 7349aca04a98b30f5ee38c513d3e6d70 |
|
BLAKE2b-256 | 0e90be25ba3b8912b992344788f1893e2d8ac6db037881ef49bd189555b554e7 |
Close
Hashes for tfrecords-0.0.8-cp39-cp39-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 102023970dbdeb7aa8500cba0d3adf8bc53338a1055f74107cc4f96c704391a3 |
|
MD5 | ec3c1bd9c83abed5f4f0b39a23fa153f |
|
BLAKE2b-256 | d27520f786b88aff2b53c93609287096817b1ab2fe42927858d6df63dec921bc |
Close
Hashes for tfrecords-0.0.8-cp39-cp39-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 04b6e7a5877ce7139a7fb9ec638995622c881894c7a84b05c22bd6c360fd88a7 |
|
MD5 | a34f452e8e3e38448138f29ac902c7d0 |
|
BLAKE2b-256 | 9b42f664daaa46de27381a370491ee063a60fbe91b6ef099c5f9c52a868787ff |
Close
Hashes for tfrecords-0.0.8-cp39-cp39-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 3c584bcd199661216cdc80b797bff7cdf59fd18ac0ed22226a98f8739cc56af7 |
|
MD5 | f8d40579d8fa5fa3a6790ef23d067829 |
|
BLAKE2b-256 | 88cc2df71c2434da7a01d365f8d8f657f319e9707d40eb0a1f4627c4bedfa753 |
Close
Hashes for tfrecords-0.0.8-cp38-cp38-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | cc29960b317269c4611b993e2d83e06caa49dd68f7ded6b39e67566073168d10 |
|
MD5 | f05517ed4a27293f932923bef313a817 |
|
BLAKE2b-256 | 5d1f5fb07228df5214c76d71ddef11012971cc692a8f610adbe09dc7c07a1a03 |
Close
Hashes for tfrecords-0.0.8-cp38-cp38-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 580de1769c48aee9f6bae8bc85ffea19bcc81d915905bb5fa5e46d9571eea1ee |
|
MD5 | d3bbe91fb993337e303e4a41cb77c15b |
|
BLAKE2b-256 | 58d72afcec431e6854243bd126d05c426804e818f05fe128380421c337fdae14 |
Close
Hashes for tfrecords-0.0.8-cp38-cp38-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | cbf0c99fd166a7216e8098e6fb70b9436bcba04716c17913d2ba944e6e5cee67 |
|
MD5 | 35a66739336df9b9b06247ac79a9a4fb |
|
BLAKE2b-256 | c2f13b0519430c41b054d745cfee51c5cfb726416529cb1605262a745fe890d8 |
Close
Hashes for tfrecords-0.0.8-cp37-cp37m-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | d0d4f97e213f079d85b9f7c46995c541472f93372a9f5f7a7b76d61d5e9e5380 |
|
MD5 | 4408fc02b01f50eb22d922024592226d |
|
BLAKE2b-256 | e2d1feddf0cd3d504f77f94aeb512a87b30fecfbf208dff59099623cbf7139d6 |
Close
Hashes for tfrecords-0.0.8-cp37-cp37m-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | f0436f69c72b28214f0b575114eada63507e8ebb028978122e35b2e5b953d71a |
|
MD5 | b1f30d7a98e38c5e1c6a2182cdba3b7a |
|
BLAKE2b-256 | 356d765ecfb76b54f5f6f4e1ff3608b5c4cb58b8ec6925c4e6f2030bc28fe926 |
Close
Hashes for tfrecords-0.0.8-cp37-cp37m-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | f7f3dd1586a8fffcb9182fd824239e68108441a563ff95d2e0fc049522a2f17c |
|
MD5 | a7bf9022265a3e1ac488bf12f1906295 |
|
BLAKE2b-256 | b5be7aa9d4cd37c9e21873c873ed4b04a25620a223c15a2f70fbb9a5b03f761a |
Close
Hashes for tfrecords-0.0.8-cp36-cp36m-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 1acf045e50e989447273553d5562534097b5ed02215196032e005bc79ebf9944 |
|
MD5 | 868fd66708b1f6e8b518402574df338b |
|
BLAKE2b-256 | 3e016924ad3a872b81f78f3a9eac2de554047af380dc34b93fc32b990145be14 |
Close
Hashes for tfrecords-0.0.8-cp36-cp36m-manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 6c5c2f47dca70e0f0200eba6ea30e709d94cffb90e563ae84fe8126eefe6046d |
|
MD5 | 763d96ff38da1b07b70acbd9598a54f8 |
|
BLAKE2b-256 | cbc2ab86f613c0c9e011dd5dd2a0fb604b4da10beab997b0969752d4ba59d613 |
Close
Hashes for tfrecords-0.0.8-cp36-cp36m-manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | e6b9324cc400d39987bcd855d3436c60b54d1a5f0082dea76bba1de086afa979 |
|
MD5 | 0086d637dbf09bfc9ec34926d5e236cc |
|
BLAKE2b-256 | 541255dd282f7a1df102f0040b3fcdec21a14e85f0ada53fd6c377230bc4c25b |