External sort algorithm implementation.
Project description
External sort.
Compatibility
Ext-sort requires 3.6+.
Installation
You can install ext-sort with pip:
$ pip install ext-sort
Quick start
Quick start.
import csv
import io
import ext_sort as es
class CSVSerializer(es.Serializer):
def __init__(self, writer):
super().__init__(csv.writer(io.TextIOWrapper(writer, write_through=True)))
def write(self, item):
return self._writer.writerow(item)
class CSVDeserializer(es.Deserializer):
def __init__(self, reader):
super().__init__(csv.reader(io.TextIOWrapper(reader)))
def read(self):
return next(self._reader)
with open('/home/user/data.csv', 'rb') as unsorted_file, open('/home/user/data.sorted.csv', 'wb') as sorted_file:
# save the csv header
sorted_file.write(unsorted_file.readline())
es.sort(
reader=unsorted_file,
writer=sorted_file,
butch_size=10_000_000,
Serializer=CSVSerializer,
Deserializer=CSVDeserializer,
workers_cnt=4,
)
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distribution
ext-sort-0.1.0.tar.gz
(4.3 kB
view hashes)
Built Distribution
ext_sort-0.1.0-py3.7.egg
(11.3 kB
view hashes)