PYGEF

Simple parser for *.GEF and BRO XML files. These are files used for soil property measurements.

If you find a file that doesn't work with pygef, please make an issue about it or a PR :)

Installation

Latest stable version:

pip install pygef

Cutting-edge version (might break):

pip install git+https://github.com/cemsbv/pygef.git

CPT files

>> > from pygef import read_cpt
>> >  # read gef and xml files
>> > cpt_data = read_cpt("./my-cpt.xml")
>> > cpt_data
CPTData: {'bro_id': 'CPT000000099543',
          'cone_diameter': 44,
          'cone_surface_area': 1500,
          'cone_surface_quotient': 0.67,
          'cone_to_friction_sleeve_distance': 100,
          'cone_to_friction_sleeve_surface_area': 22530,
          'cone_to_friction_sleeve_surface_quotient': 1.0,
          ...
              'zlm_pore_pressure_u3_after': None,
'zlm_pore_pressure_u3_before': None}
>> >  # access the underlying polars DataFrame under the `data` attribute
>> > cpt_data.data.head()
shape: (5, 9)
┌────────────┬───────┬───────────┬────────────┬─────┬────────────┬────────────┬────────────┬────────────┐
│ penetratio ┆ depth ┆ elapsedTi ┆ coneResist ┆ ... ┆ inclinatio ┆ inclinatio ┆ localFrict ┆ frictionRa │
│ nLength    ┆ ---   ┆ me        ┆ ance       ┆     ┆ nNS        ┆ nResultant ┆ ion        ┆ tio        │
│ ---        ┆ f64   ┆ ---       ┆ ---        ┆     ┆ ---        ┆ ---        ┆ ---        ┆ ---        │
│ f64        ┆       ┆ f64       ┆ f64        ┆     ┆ i64        ┆ i64        ┆ f64        ┆ f64        │
╞════════════╪═══════╪═══════════╪════════════╪═════╪════════════╪════════════╪════════════╪════════════╡
│ 0.0        ┆ 0.0   ┆ -9.99999
e ┆ -9.99999e5 ┆ ... ┆ -999999    ┆ -999999    ┆ -9.99999e5 ┆ -9.99999e5 │
│            ┆       ┆ 5         ┆            ┆     ┆            ┆            ┆            ┆            │
├╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤
│ 0.02       ┆ 0.02  ┆ 11.0      ┆ 2.708      ┆ ... ┆ 0          ┆ 0          ┆ 0.03       ┆ 0.6        │
├╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤
│ 0.04       ┆ 0.039 ┆ 13.0      ┆ 4.29       ┆ ... ┆ 0          ┆ 0          ┆ 0.039      ┆ 0.8        │
├╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤
│ 0.06       ┆ 0.059 ┆ 15.0      ┆ 5.124      ┆ ... ┆ 0          ┆ 0          ┆ 0.045      ┆ 0.9        │
├╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤
│ 0.08       ┆ 0.079 ┆ 17.0      ┆ 5.45       ┆ ... ┆ 0          ┆ 0          ┆ 0.049      ┆ 1.0        │
└────────────┴───────┴───────────┴────────────┴─────┴────────────┴────────────┴────────────┴────────────┘

Bore files

>> > from pygef import read_bore
>> >  # read gef and xml files
>> > bore_data = read_bore("./my-bore.xml")
>> > bore_data
BoreData: {'bore_hole_completed': True,
           'bore_rock_reached': False,
           'data': (13, 8),
           'delivered_location': Location(srs_name='urn:ogc:def:crs:EPSG::28992', x=158322.139, y=444864.706),
           'delivered_vertical_position_datum': 'nap',
           'delivered_vertical_position_offset': 10.773,
           'delivered_vertical_position_reference_point': 'maaiveld',
           'description_procedure': 'ISO14688d1v2019c2020',
           'final_bore_depth': 12.0,
           'final_sample_depth': 12.0,
           'research_report_date': datetime.date(2021, 10, 19)}
>> >  # access the underlying polars DataFrame under the `data` attribute
>> > bore_data.data.head()
shape: (5, 8)
┌────────────┬────────────┬────────────┬──────────┬────────────┬────────────┬────────────┬─────────┐
│ upper_boun ┆ lower_boun ┆ geotechnic ┆ color    ┆ dispersed_ ┆ organic_ma ┆ sand_media ┆ soil_di │
│ dary       ┆ dary       ┆ al_soil_na ┆ ---      ┆ inhomogeni ┆ tter_conte ┆ n_class    ┆ st      │
│ ---        ┆ ---        ┆ me         ┆ str      ┆ ty         ┆ nt_class   ┆ ---        ┆ ---     │
│ f64        ┆ f64        ┆ ---        ┆          ┆ ---        ┆ ---        ┆ str        ┆ list[f6 │
│            ┆            ┆ str        ┆          ┆ bool       ┆ str        ┆            ┆ 4]      │
╞════════════╪════════════╪════════════╪══════════╪════════════╪════════════╪════════════╪═════════╡
│ 0.0        ┆ 1.0        ┆ zwakGrindi ┆ donkergr ┆ false      ┆ nietOrgani ┆ middelgrof ┆ [0.2,   │
│            ┆            ┆ gZand      ┆ ijs      ┆            ┆ sch        ┆ 420
tot630u ┆ 0.0,    │
│            ┆            ┆            ┆          ┆            ┆            ┆ m          ┆ ...     │
│            ┆            ┆            ┆          ┆            ┆            ┆            ┆ 0.0]    │
├╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
│ 1.0        ┆ 1.1        ┆ zwakGrindi ┆ donkergr ┆ false      ┆ nietOrgani ┆ middelgrof ┆ [0.2,   │
│            ┆            ┆ gZand      ┆ ijs      ┆            ┆ sch        ┆ 420
tot630u ┆ 0.0,    │
│            ┆            ┆            ┆          ┆            ┆            ┆ m          ┆ ...     │
│            ┆            ┆            ┆          ┆            ┆            ┆            ┆ 0.0]    │
├╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
│ 1.1        ┆ 2.0        ┆ zwakZandig ┆ standaar ┆ false      ┆ nietOrgani ┆ null       ┆ [0.0,   │
│            ┆            ┆ eKleiMetGr ┆ dBruin   ┆            ┆ sch        ┆            ┆ 0.1,    │
│            ┆            ┆ ind        ┆          ┆            ┆            ┆            ┆ ...     │
│            ┆            ┆            ┆          ┆            ┆            ┆            ┆ 0.0]    │
├╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
│ 2.0        ┆ 3.0        ┆ zwakZandig ┆ standaar ┆ false      ┆ nietOrgani ┆ null       ┆ [0.0,   │
│            ┆            ┆ eKlei      ┆ dGrijs   ┆            ┆ sch        ┆            ┆ 0.0,    │
│            ┆            ┆            ┆          ┆            ┆            ┆            ┆ ...     │
│            ┆            ┆            ┆          ┆            ┆            ┆            ┆ 0.0]    │
├╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┤
│ 3.0        ┆ 4.0        ┆ zwakZandig ┆ donkergr ┆ false      ┆ nietOrgani ┆ null       ┆ [0.0,   │
│            ┆            ┆ eKlei      ┆ ijs      ┆            ┆ sch        ┆            ┆ 0.0,    │
│            ┆            ┆            ┆          ┆            ┆            ┆            ┆ ...     │
│            ┆            ┆            ┆          ┆            ┆            ┆            ┆ 0.0]    │
└────────────┴────────────┴────────────┴──────────┴────────────┴────────────┴────────────┴─────────┘

Plotting

To use the plotting functionality pygef needs to be installed with the [plot] feature.

pip install pygef[plot]

from pygef import read_cpt, read_bore
from pygef.plotting import plot_cpt, plot_bore

# plot cpt file
plot_cpt(read_cpt("myfile.xml"))

# plot a bore file
plot_bore(read_bore("myfile.xml"))

Documentation

Build the docs:

python -m pip install --upgrade pip setuptools
pip install -r requirements.txt
pip install .

sphinx-build -b html docs public

Format

We format our code with black and isort.

black --config "pyproject.toml" .
isort --settings-path "pyproject.toml" .

Lint

To maintain code quality we use the GitHub super-linter.

To run the linters locally, run the following bash script from the root directory:

docker run \
--env VALIDATE_ALL_CODEBASE=true \
--env DEFAULT_BRANCH=master \
--env RUN_LOCAL=true \
--env FIX_PYTHON_ISORT=true \
--env FIX_PYTHON_BLACK=true \
--env FIX_YAML_PRETTIER=true \
--env VALIDATE_BASH=false \
--env VALIDATE_BIOME_FIX=false \
--env VALIDATE_BIOME_FORMAT=false \
--env VALIDATE_BIOME_LINT=false \
--env VALIDATE_CSS=false \
--env VALIDATE_CSS_PRETTIER=false \
--env VALIDATE_GRAPHQL=false \
--env VALIDATE_GRAPHQL_PRETTIER=false \
--env VALIDATE_HTML=false \
--env VALIDATE_HTML_PRETTIER=false \
--env VALIDATE_JAVASCRIPT_ES=false \
--env VALIDATE_JAVASCRIPT_PRETTIER=false \
--env VALIDATE_JSCPD=false \
--env VALIDATE_JSONC=false \
--env VALIDATE_JSONC_PRETTIER=false \
--env VALIDATE_JSON_ES=false \
--env VALIDATE_JSON_PRETTIER=false \
--env VALIDATE_JSX=false \
--env VALIDATE_JSX_PRETTIER=false \
--env VALIDATE_MARKDOWN=false \
--env VALIDATE_MARKDOWN_PRETTIER=false \
--env VALIDATE_NATURAL_LANGUAGE=false \
--env VALIDATE_PYTHON_PYLINT=false \
--env VALIDATE_PYTHON_RUFF=false \
--env VALIDATE_PYTHON_RUFF_FORMAT=false \
--env VALIDATE_TSX=false \
--env VALIDATE_TYPESCRIPT_ES=false \
--env VALIDATE_TYPESCRIPT_PRETTIER=false \
--env VALIDATE_VUE=false \
--env VALIDATE_VUE_PRETTIER=false \
--env VALIDATE_YAML=false \
--env LINTER_RULES_PATH=/ \
--env PYTHON_BLACK_CONFIG_FILE=pyproject.toml \
--env PYTHON_ISORT_CONFIG_FILE=pyproject.toml \
--env PYTHON_MYPY_CONFIG_FILE=pyproject.toml \
--env PYTHON_FLAKE8_CONFIG_FILE=.flake8 \
-v $(pwd):/tmp/lint ghcr.io/super-linter/super-linter:v8

UnitTest

Test the software with the use of coverage:

python -m pip install --upgrade pip setuptools
pip install -r requirements.txt
pip install -e .[test] .[map] .[plot]

coverage run -m pytest

Requirements

Requirements are autogenerated by pip-compile with python 3.9

pip-compile --extra=plot --extra=test --extra=docs --extra=lint --extra=map --output-file=requirements.txt pyproject.toml

To update the requirements within the defined ranges, run:

pip-compile --upgrade --extra=plot --extra=test --extra=docs --extra=lint --extra=map --output-file=requirements.txt pyproject.toml