pyftdc 0.3.0

A small example package

pyftdc

CI	status
pip builds

A MongoDB FTDC files parser written in C++ that provides Python bindings using pybind11 and scikit-build.

Requisites

To build the source distribution, you will need Python 3.8 or newer, git, python3-dev/python3-devel installed.

Please read the build document for more details.

Installation

Building on Unix (Ubuntu, Centos, macOS)

1. clone this repository and change to the top level directory.

   git clone [e-mail]:jimper/mongo_ftdc.git 
   cd mongo_ftdc
   

2. Install Python libraries to build binaries. Create a virtual environment to make your life easier.

   python3 -m venv venv
   source venv/bin/activate
   pip3 install --user-pep517 .
   

   You will now have built and installed in your virtual environment.

Alternatively, you can use setup.py directly, but for that you will need to manually install the required libraries into your virtual environment by running

 cd mongo_ftdc
 pip install -r requirements.txt

After which you can create a source distribution or a binary wheel:

 python3 setup.py sdist
 python3 setup.py bdist_wheel

These will reside in the dist directory.

Building on Windows

Not tested yet, but 'It should work'(TM)

License

Apache V2

Test call

import pyftdc


def get_prefixed_metrics_names(param, ds):
    ops_counters = []
    for name in ds.metrics_names:
        if name.startswith(param):
            ops = ds.get_metric(name)
            ops_counters.append((name, ops))

    return ops_counters


def open_files_in_dir(dir_path, prefix):
    from os import listdir

    files_read = []
    try:
        dir_list = listdir(dir_path)
        for file_name in dir_list:
            if file_name.startswith(prefix):
                parser = pyftdc.FTDCParser()
                ds = parser.parse_file(dir_path + '/' + file_name)
                if ds:
                    print(f'File: {ds.file}')
                    print(f'{ds.metadata}')
                    ts = ds.get_metric("start")
                    if ts:
                        ts_size = len(ts)

                        print(f'Timestamp count {ts_size}. Start:{ts[0]}  Last: {ts[-1]}')

                        op_counter_names = get_prefixed_metrics_names('serverStatus.opcounters', ds)
                        cpu = get_prefixed_metrics_names('systemMetrics.cpu', ds)
                        disk = get_prefixed_metrics_names('systemMetrics.disks.nvme1n1', ds)

                        xxx = ds.get_metric_list_numpy(['systemMetrics.cpu.iowait_ms', 'xxx', 'systemMetrics.cpu.num_cpus'])
                        disk_n = ds.get_metric_numpy('systemMetrics.disks.nvme1n1.writes')

                        files_read.append(file_name)
                    else:
                        print(f'No timestamps on this dataset.')
    except FileNotFoundError as not_found:
        print('Path not found.')

    return files_read


def open_dir(dir_path):
    multi_parser = pyftdc.FTDCParser()
    datasets = multi_parser.parse_dir(dir_path)
    ds_count = len(datasets)
    print(f'There are {ds_count} datasets from {dir_path}')
    if ds_count > 0:
        for ds in datasets:
            if ds:
                print(f'File: {ds.file}')
                print(f'{ds.metadata}')
                ts = ds.get_metrics("start")
                if ts:
                    ts_size = len(ts)

                    print(f'Timestamp count {ts_size}. Start:{ts[0]}  Last: {ts[-1]}')

                    op_counter_names = get_prefixed_metrics_names('serverStatus.opcounters', ds)
                    cpu = get_prefixed_metrics_names('systemMetrics.cpu', ds)
                    disk = get_prefixed_metrics_names('systemMetrics.disks.nvme1n1', ds)

                    print('')
                else:
                    print(f'No timestamps on this dataset.')
            else:
                print(f'Bad dataset from file {ds}')


if __name__ == "__main__":
    files = open_files_in_dir('/somepath/diagnostic.data/',
                              'metrics.2022-11-13T21')
    print(files)