InsulaWorkflowClient 0.9.1

Insula Workflow CLient

Insula Workflow Client

Example

from InsulaWorkflowClient import InsulaOpenIDConnect
from InsulaWorkflowClient.InsulaApiConfig import InsulaApiConfig
from InsulaWorkflowClient.InsulaClient import InsulaClient

insula_auth: InsulaOpenIDConnect = InsulaOpenIDConnect(
    authorization_endpoint='https://identity.ope.insula.earth//realms/iride-lot3/protocol/openid-connect/auth',
    token_endpoint='https://identity.ope.insula.earth//realms/iride-lot3/protocol/openid-connect/token',
    redirect_uri='http://localhost:9207/auth',
    client_id="api-client"
)
insula_auth.set_user_credentials(username="<USER>", password="<PASSWORD>")

ic = InsulaClient(InsulaApiConfig(
    'https://iride-lot3.ope.insula.earth',
    insula_auth,
    status_polling_interval=30,    # Seconds
    interval_between_requests=5,   # Seconds
    max_processor_attempts=3
))

ic.run_from_file('<THE TEMPLATE PATH>',
                 {
                     'access_key': '<access_key>',
                     'secret_key': '<secret_key>',
                     'elaboration_date': '2023_2024',
                     'region_name': 'Molise'
                 }, '<optional log name>')

type: workflow
language: native
name: work_iride
version: beta/1
description: |
  PLEASE, DO NOT CHANGE THIS FILE 
  pre processing workflow from step 0a to Siam, push SPOT_T0, SPOT_T1

portal_ref: eopaas://refData/33/workflow_pre_processing.yaml

parameters:
  #----dynamic----
  id_tile: '552'
  elaboration_date: '2023_2024'
  time: 'SPOT_T1'
  #----static----
  reticle_definition: 'eopaas://refData/33/reticolo_20x20km_ita_epsg4326.geojson'
  collection_base: '{"out":"iridelot33c67fd6450604bfebfa541474a094a62"}'
  siam_collection_base: '{"calibration_data":"iridelot33c67fd6450604bfebfa541474a094a62","masks":"iridelot33c67fd6450604bfebfa541474a094a62","pct_full":"iridelot33c67fd6450604bfebfa541474a094a62","pct_33":"iridelot33c67fd6450604bfebfa541474a094a62"}'
  s3_platform_ref: 'eopaas://refData/33/obs.eu-de.otc.t-systems.com_bot.json'
  #for s3copy processor without https://
  s3_copy_endpoint: 'hhf.eu-dfd.otc.t-fgfd.com'
  s3_copy_bucket_map_sat: 'hhs.fs-de.ffd.t-systems.com'
  s3_bucket_map_cgi: 'csa-fdf-svc-hh-fg-soil-sealing-map-cgi'

configuration:
  max_parallel_jobs: 5
  continue_on_error: false
  delete_workflow_log: false

requirements:
  connections:
    - name: s3_soil_sealing
      type: s3
      params:
        access_key: ${workflow.param.access_key}
        secret_key: ${workflow.param.secret_key}
        endpoint: 'https://sfd.ff-fgd.otc.t-dgfd.com'
        bucket: '${workflow.param.s3_bucket_map_cgi}'

steps:
  - - name: create_buckets_name
      type: script
      continue_on_error: false
      outputs:
        - s4_03_t0_siam
        - s4_02_t1_siam
        - soil_s4_03_First_processing
        - soil_s4_02_First_processing
        - pid_path
      params:
        - name: id_tile
          values: ${workflow.param.id_tile}
        - name: elaboration_date
          values: ${workflow.param.elaboration_date}
        - name: time
          values: ${workflow.param.time}
      source: |

        s4_03_t0_siam = f'{elaboration_date}/S4_03/{id_tile}/Pre_Processing/siam'
        s4_02_t1_siam = f'{elaboration_date}/S4_02/{id_tile}/Pre_Processing/siam'  
        soil_s4_03_First_processing = f'{elaboration_date}/S4_03/{id_tile}/First_Processing'
        soil_s4_02_First_processing = f'{elaboration_date}/S4_02/{id_tile}/First_Processing'
        pid_path = f'{elaboration_date}/__processing/{id_tile}/First_Processing'

        print(f'Pre processing s4_03 siam s3 path: {s4_03_t0_siam}')    
        print(f'Pre processing s4_02 siam s3 path: {s4_02_t0_siam}')                
        print(f'soil_s4_03_First_processing s3 path: {soil_s4_03_First_processing}')
        print(f'soil_s4_02_First_processing s3 path: {soil_s4_02_First_processing}')


  - - name: IrideLot3ClippingReticle
      type: processor
      service_id: '8'
      params:
        - name: id_tile
          values: '${workflow.param.idtile}'
        - name: elaboration_date
          values: '${workflow.param.elaboration_date}'
        - name: s3_platform_ref
          values: '${workflow.param.s3_platform_ref}'
        - name: reticle_definition
          values: '${workflow.param.reticle_definition}'
        - name: collection
          values: '${workflow.param.collection_base}'

  - - name: IrideLot3Clipping
      type: processor
      service_id: '9'
      params:
        - name: id_tile
          values: '${workflow.param.id_tile}'
        - name: elaboration_date
          values: '${workflow.param.elaboration_date}'
        - name: SPOT_T0
          values: '${workflow.step.IrideLot3ClippingReticle.SPOT_T0}'
        - name: SPOT_T1
          values: '${workflow.step.IrideLot3ClippingReticle.SPOT_T1}'
        - name: collection
          values: '${workflow.param.collection_base}'

  - - name: siam_spot_t0
      template: siam
      params:
        - name: parallelInputs
          values: '${workflow.step.IrideLot3Clipping.SPOT_T0}'
    - name: siam_spot_t1
      template: siam
      params:
        - name: parallelInputs
          values: '${workflow.step.IrideLot3Clipping.SPOT_T1}'

    - name: IrideLot3SoilSealing
      type: processor
      service_id: '10'
      params:
        - name: id_tile
          values: '${workflow.param.id_tile}'
        - name: elaboration_date
          values: '${workflow.param.elaboration_date}'
        - name: reticolo
          values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T0}
        - name: siam
          values: ${workflow.step.siam_spot_t0} $[(.*61SpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
        - name: mapsat
          values: 'none'
        - name: collection
          values: '${workflow.param.collection_base}'

    - name: IrideLot3ChangeDetectionMergeSiam
      type: processor
      service_id: '12'
      params:
        - name: id_tile
          values: '${workflow.param.id_tile}'
        - name: elaboration_date
          values: '${workflow.param.elaboration_date}'
        - name: reticolo_t0
          values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T0}
        - name: reticolo_t1
          values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T1}
        - name: siam_t0
          values: ${workflow.step.siam_spot_t0} $[(.*fRatioGreennessIndex.*)|(.*fRatioCanopyChlorophyllContent.*)|(.*33SharedSpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
        - name: siam_t1
          values: ${workflow.step.siam_spot_t1} $[(.*fRatioGreennessIndex.*)|(.*fRatioCanopyChlorophyllContent.*)|(.*33SharedSpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
        - name: collection
          values: '${workflow.param.collection_base}'

  - - name: SiamChangeDetector
      type: processor
      service_id: '14'
      params:
        - name: T1
          values: ${workflow.step.IrideLot3ChangeDetectionMergeSiam.merge_t0} $[(.*\.dat$)|(.*\.hdr$)]
        - name: T2
          values: ${workflow.step.IrideLot3ChangeDetectionMergeSiam.merge_t1} $[(.*\.dat$)|(.*\.hdr$)]
        - name: collection
          values: '${workflow.param.collection_base}'


  - - name: siam_push_t0
      template: s3_copy
      params:
        - name: source
          values: '${workflow.step.siam_spot_t0}'
        - name: destination
          values: '${workflow.step.create_buckets_name.s4_03_t0_siam}'

    - name: siam_push_t1
      template: s3_copy
      params:
        - name: source
          values: '${workflow.step.siam_spot_t1}'
        - name: destination
          values: '${workflow.step.create_buckets_name.s4_02_t1_siam}'

    - name: IrideLot3SoilSealing_push_t0
      template: s3_copy
      params:
        - name: source
          values: '${workflow.step.IrideLot3SoilSealing}'
        - name: destination
          values: '${workflow.step.create_buckets_name.soil_s4_03_First_processing}'

    - name: SiamChangeDetector_push_t1
      template: s3_copy
      params:
        - name: source
          values: '${workflow.step.SiamChangeDetector}'
        - name: destination
          values: '${workflow.step.create_buckets_name.soil_s4_02_First_processing}'


  - - name: create_pids_file
      type: script
      continue_on_error: true
      outputs:
        - file_pid
      params:
        - name: job_reticle_id
          values: ${status.step.IrideLot3ClippingReticle.job_id}
        - name: job_siam_spot_t0_id
          values: ${status.step.siam_spot_t0.job_id}
        - name: job_siam_spot_t1_id
          values: ${status.step.siam_spot_t1.job_id}
      source: |

        content = f'{{"job_reticle_id":"{job_reticle_id}", "job_siam_spot_t0_id":"{job_siam_spot_t0_id}", "job_siam_spot_t1_id":"{job_siam_spot_t1_id}" }}'
        print(f'{content}')
        with open('jobs_file.json', 'w') as jobs_file:
          jobs_file.write(content)  
        print(f'{content}')
        file_pid = 'jobs_file.json'


  - - name: push_ids
      type: s3
      connection: s3_soil_sealing
      continue_on_error: false
      action: push
      params:
        - name: in
          values: '${workflow.step.create_pids_file.file_pid}'
          save_in: '2023_2024/S4-03/TEST'

templates:

  - name: s3_copy
    service_id: '11'
    type: processor
    params:
      - name: action
        values: upload
      - name: s3_endpoint
        values: '${workflow.param.s3_copy_endpoint}'
      - name: s3_bucket
        values: '${workflow.param.s3_bucket_map_cgi}'
      - name: s3_platform_ref
        values: '${workflow.param.s3_platform_ref}'
      - name: collection
        values: '{"output":"iridelot365b7d9dde6044c00b26abe046aed0556"}'


  - name: siam
    service_id: '3'
    type: processor
    params:
      - name: veg_mask
        values: 'binary'
      - name: cloud_mask
        values: 'binary'
      - name: urban_mask
        values: 'disabled'
      - name: collection
        values: '${workflow.param.siam_collection_base}'
      - name: shadow_mask
        values: 'enabled'
      - name: keep_calibration_data
        values: 'false'
      - name: extra_indexes
        values: 'true'
      - name: water_mask
        values: 'disabled'
      - name: bare_soil_mask
        values: 'disabled'
      - name: burned_area_mask
        values: 'disabled'
      - name: use_no_data_mask
        values: 'true'
      - name: smoke_mask
        values: 'disabled'
      - name: classification_mode
        values: 'crisp'