srcd_smart_open 1.9.0

What?

smart_open is a Python 2 & Python 3 library for efficient streaming of very large files from/to S3, HDFS, WebHDFS, HTTP, or local (compressed) files. It’s a drop-in replacement for Python’s built-in open(): it can do anything open can (100% compatible, falls back to native open wherever possible), plus lots of nifty extra stuff on top.

smart_open is well-tested, well-documented, and has a simple, Pythonic API:

>>> from smart_open import smart_open

>>> # stream lines from an S3 object
>>> for line in smart_open('s3://mybucket/mykey.txt', 'rb'):
...    print(line.decode('utf8'))

>>> # stream from/to compressed files, with transparent (de)compression:
>>> for line in smart_open('./foo.txt.gz', encoding='utf8'):
...    print(line)

>>> # can use context managers too:
>>> with smart_open('/home/radim/foo.txt.bz2', 'wb') as fout:
...    fout.write(u"some content\n".encode('utf8'))

>>> with smart_open('s3://mybucket/mykey.txt', 'rb') as fin:
...     for line in fin:
...         print(line.decode('utf8'))
...     fin.seek(0)  # seek to the beginning
...     b1000 = fin.read(1000)  # read 1000 bytes

>>> # stream from HDFS
>>> for line in smart_open('hdfs://user/hadoop/my_file.txt', encoding='utf8'):
...     print(line)

>>> # stream from HTTP
>>> for line in smart_open('http://example.com/index.html'):
...     print(line)

>>> # stream from WebHDFS
>>> for line in smart_open('webhdfs://host:port/user/hadoop/my_file.txt'):
...     print(line)

>>> # stream content *into* S3 (write mode):
>>> with smart_open('s3://mybucket/mykey.txt', 'wb') as fout:
...     for line in [b'first line\n', b'second line\n', b'third line\n']:
...          fout.write(line)

>>> # stream content *into* HDFS (write mode):
>>> with smart_open('hdfs://host:port/user/hadoop/my_file.txt', 'wb') as fout:
...     for line in [b'first line\n', b'second line\n', b'third line\n']:
...          fout.write(line)

>>> # stream content *into* WebHDFS (write mode):
>>> with smart_open('webhdfs://host:port/user/hadoop/my_file.txt', 'wb') as fout:
...     for line in [b'first line\n', b'second line\n', b'third line\n']:
...          fout.write(line)

>>> # stream using a completely custom s3 server, like s3proxy:
>>> for line in smart_open('s3u://user:secret@host:port@mybucket/mykey.txt', 'rb'):
...    print(line.decode('utf8'))

>>> # you can also use a boto.s3.key.Key instance directly:
>>> key = boto.connect_s3().get_bucket("my_bucket").get_key("my_key")
>>> with smart_open(key, 'rb') as fin:
...     for line in fin:
...         print(line.decode('utf8'))

>>> # Stream to Digital Ocean Spaces bucket providing credentials from boto profile
>>> with smart_open('s3://bucket-for-experiments/file.txt', 'wb', endpoint_url='https://ams3.digitaloceanspaces.com', profile_name='digitalocean') as fout:
...     fout.write(b'here we stand')

pip install smart_open

python setup.py test  # run unit tests
python setup.py install

>>> smart_open('s3://', host='s3.amazonaws.com')
>>> smart_open('s3://', profile_name='my-profile')

>>> smart_open('s3://', s3_session=boto3.Session())

>>> smart_open('s3://', s3_upload={ 'ServerSideEncryption': 'AES256' })

>>> from smart_open import smart_open, s3_iter_bucket
>>> # get all JSON files under "mybucket/foo/"
>>> bucket = boto.connect_s3().get_bucket('mybucket')
>>> for key, content in s3_iter_bucket(bucket, prefix='foo/', accept_key=lambda key: key.endswith('.json')):
...     print(key, len(content))

>>> import smart_open
>>> help(smart_open.smart_open_lib)