DataSae 0.3.0

Data Quality Framework provides by Jabar Digital Service

DataSae

Data Quality Framework provides by Jabar Digital Service

Converter

https://github.com/jabardigitalservice/DataSae/blob/733d16fc8a39d25d56594de63c7ec4ad1fc0225c/tests/data/config.json#L1-L30

https://github.com/jabardigitalservice/DataSae/blob/733d16fc8a39d25d56594de63c7ec4ad1fc0225c/tests/data/config.yaml#L1-L24

Google Spreadsheet

https://github.com/jabardigitalservice/DataSae/blob/4308324d066c6627936773ab2d5b990adaa60100/tests/data/creds.json#L1-L12

pip install 'DataSae[converter,gsheet]'

from datasae.converter import Config

# From JSON
config = Config('DataSae/tests/data/config.json')
gsheet = config('test_gsheet')
df = gsheet('gsheet_id', 'Sheet1')

# From YAML
config = Config('DataSae/tests/data/config.yaml')
gsheet = config('test_gsheet')
df = gsheet('gsheet_id', 'Sheet1')

S3

pip install 'DataSae[converter,s3]'

from datasae.converter import Config

# From JSON
config = Config('DataSae/tests/data/config.json')
s3 = config('test_s3')
df = s3('bucket_name', 'path/file_name.csv')
df = s3('bucket_name', 'path/file_name.json')
df = s3('bucket_name', 'path/file_name.parquet')
df = s3('bucket_name', 'path/file_name.xlsx', sheet_name='Sheet1')

# From YAML
config = Config('DataSae/tests/data/config.yaml')
s3 = config('test_s3')
df = s3('bucket_name', 'path/file_name.csv')
df = s3('bucket_name', 'path/file_name.json')
df = s3('bucket_name', 'path/file_name.parquet')
df = s3('bucket_name', 'path/file_name.xlsx', sheet_name='Sheet1')

SQL

pip install 'DataSae[converter,sql]'

MariaDB or MySQL

from datasae.converter import Config

# From JSON
config = Config('DataSae/tests/data/config.json')
mariadb_or_mysql = config('test_mariadb_or_mysql')
df = mariadb_or_mysql('select 1 column_name from schema_name.table_name;')
df = mariadb_or_mysql('path/file_name.sql')

# From YAML
config = Config('DataSae/tests/data/config.yaml')
mariadb_or_mysql = config('test_mariadb_or_mysql')
df = mariadb_or_mysql('select 1 column_name from schema_name.table_name;')
df = mariadb_or_mysql('path/file_name.sql')

PostgreSQL

from datasae.converter import Config

# From JSON
config = Config('DataSae/tests/data/config.json')
postgresql = config('test_postgresql')
df = postgresql('select 1 column_name from schema_name.table_name;')
df = postgresql('path/file_name.sql')

# From YAML
config = Config('DataSae/tests/data/config.yaml')
postgresql = config('test_postgresql')
df = postgresql('select 1 column_name from schema_name.table_name;')
df = postgresql('path/file_name.sql')