Insula CLient
Project description
Insula Client
Example
type: workflow
language: native
name: work_iride
version: beta/1
description: |
PLEASE, DO NOT CHANGE THIS FILE
pre processing workflow from step 0a to Siam, push SPOT_T0, SPOT_T1
portal_ref: eopaas://refData/33/workflow_pre_processing.yaml
parameters:
#----dynamic----
id_tile: '552'
elaboration_date: '2023_2024'
time: 'SPOT_T1'
#----static----
reticle_definition: 'eopaas://refData/33/reticolo_20x20km_ita_epsg4326.geojson'
collection_base: '{"out":"iridelot33c67fd6450604bfebfa541474a094a62"}'
siam_collection_base: '{"calibration_data":"iridelot33c67fd6450604bfebfa541474a094a62","masks":"iridelot33c67fd6450604bfebfa541474a094a62","pct_full":"iridelot33c67fd6450604bfebfa541474a094a62","pct_33":"iridelot33c67fd6450604bfebfa541474a094a62"}'
s3_platform_ref: 'eopaas://refData/33/obs.eu-de.otc.t-systems.com_bot.json'
#for s3copy processor without https://
s3_copy_endpoint: 'obs.eu-de.otc.t-systems.com'
s3_copy_bucket_map_sat: 'obs.eu-de.otc.t-systems.com'
s3_bucket_map_cgi: 'ext-iride-svc-s4-03-soil-sealing-map-cgi'
configuration:
max_parallel_jobs: 5
continue_on_error: false
delete_workflow_log: false
requirements:
connections:
- name: s3_soil_sealing
type: s3
params:
access_key: ${workflow.param.access_key}
secret_key: ${workflow.param.secret_key}
endpoint: 'https://obs.eu-de.otc.t-systems.com'
bucket: '${workflow.param.s3_bucket_map_cgi}'
steps:
- - name: create_buckets_name
type: script
continue_on_error: false
outputs:
- s4_03_t0_siam
- s4_02_t1_siam
- soil_s4_03_First_processing
- soil_s4_02_First_processing
- pid_path
params:
- name: id_tile
values: ${workflow.param.id_tile}
- name: elaboration_date
values: ${workflow.param.elaboration_date}
- name: time
values: ${workflow.param.time}
source: |
s4_03_t0_siam = f'{elaboration_date}/S4_03/{id_tile}/Pre_Processing/siam'
s4_02_t1_siam = f'{elaboration_date}/S4_02/{id_tile}/Pre_Processing/siam'
soil_s4_03_First_processing = f'{elaboration_date}/S4_03/{id_tile}/First_Processing'
soil_s4_02_First_processing = f'{elaboration_date}/S4_02/{id_tile}/First_Processing'
pid_path = f'{elaboration_date}/__processing/{id_tile}/First_Processing'
print(f'Pre processing s4_03 siam s3 path: {s4_03_t0_siam}')
print(f'Pre processing s4_02 siam s3 path: {s4_02_t0_siam}')
print(f'soil_s4_03_First_processing s3 path: {soil_s4_03_First_processing}')
print(f'soil_s4_02_First_processing s3 path: {soil_s4_02_First_processing}')
- - name: IrideLot3ClippingReticle
type: processor
service_id: '8'
params:
- name: id_tile
values: '${workflow.param.idtile}'
- name: elaboration_date
values: '${workflow.param.elaboration_date}'
- name: s3_platform_ref
values: '${workflow.param.s3_platform_ref}'
- name: reticle_definition
values: '${workflow.param.reticle_definition}'
- name: collection
values: '${workflow.param.collection_base}'
- - name: IrideLot3Clipping
type: processor
service_id: '9'
params:
- name: id_tile
values: '${workflow.param.id_tile}'
- name: elaboration_date
values: '${workflow.param.elaboration_date}'
- name: SPOT_T0
values: '${workflow.step.IrideLot3ClippingReticle.SPOT_T0}'
- name: SPOT_T1
values: '${workflow.step.IrideLot3ClippingReticle.SPOT_T1}'
- name: collection
values: '${workflow.param.collection_base}'
- - name: siam_spot_t0
template: siam
params:
- name: parallelInputs
values: '${workflow.step.IrideLot3Clipping.SPOT_T0}'
- name: siam_spot_t1
template: siam
params:
- name: parallelInputs
values: '${workflow.step.IrideLot3Clipping.SPOT_T1}'
- name: IrideLot3SoilSealing
type: processor
service_id: '10'
params:
- name: id_tile
values: '${workflow.param.id_tile}'
- name: elaboration_date
values: '${workflow.param.elaboration_date}'
- name: reticolo
values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T0}
- name: siam
values: ${workflow.step.siam_spot_t0} $[(.*61SpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
- name: mapsat
values: 'none'
- name: collection
values: '${workflow.param.collection_base}'
- name: IrideLot3ChangeDetectionMergeSiam
type: processor
service_id: '12'
params:
- name: id_tile
values: '${workflow.param.id_tile}'
- name: elaboration_date
values: '${workflow.param.elaboration_date}'
- name: reticolo_t0
values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T0}
- name: reticolo_t1
values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T1}
- name: siam_t0
values: ${workflow.step.siam_spot_t0} $[(.*fRatioGreennessIndex.*)|(.*fRatioCanopyChlorophyllContent.*)|(.*33SharedSpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
- name: siam_t1
values: ${workflow.step.siam_spot_t1} $[(.*fRatioGreennessIndex.*)|(.*fRatioCanopyChlorophyllContent.*)|(.*33SharedSpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
- name: collection
values: '${workflow.param.collection_base}'
- - name: SiamChangeDetector
type: processor
service_id: '14'
params:
- name: T1
values: ${workflow.step.IrideLot3ChangeDetectionMergeSiam.merge_t0} $[(.*\.dat$)|(.*\.hdr$)]
- name: T2
values: ${workflow.step.IrideLot3ChangeDetectionMergeSiam.merge_t1} $[(.*\.dat$)|(.*\.hdr$)]
- name: collection
values: '${workflow.param.collection_base}'
- - name: siam_push_t0
template: s3_copy
params:
- name: source
values: '${workflow.step.siam_spot_t0}'
- name: destination
values: '${workflow.step.create_buckets_name.s4_03_t0_siam}'
- name: siam_push_t1
template: s3_copy
params:
- name: source
values: '${workflow.step.siam_spot_t1}'
- name: destination
values: '${workflow.step.create_buckets_name.s4_02_t1_siam}'
- name: IrideLot3SoilSealing_push_t0
template: s3_copy
params:
- name: source
values: '${workflow.step.IrideLot3SoilSealing}'
- name: destination
values: '${workflow.step.create_buckets_name.soil_s4_03_First_processing}'
- name: SiamChangeDetector_push_t1
template: s3_copy
params:
- name: source
values: '${workflow.step.SiamChangeDetector}'
- name: destination
values: '${workflow.step.create_buckets_name.soil_s4_02_First_processing}'
- - name: create_pids_file
type: script
continue_on_error: true
outputs:
- file_pid
params:
- name: job_reticle_id
values: ${status.step.IrideLot3ClippingReticle.job_id}
- name: job_siam_spot_t0_id
values: ${status.step.siam_spot_t0.job_id}
- name: job_siam_spot_t1_id
values: ${status.step.siam_spot_t1.job_id}
source: |
content = f'{{"job_reticle_id":"{job_reticle_id}", "job_siam_spot_t0_id":"{job_siam_spot_t0_id}", "job_siam_spot_t1_id":"{job_siam_spot_t1_id}" }}'
print(f'{content}')
with open('jobs_file.json', 'w') as jobs_file:
jobs_file.write(content)
print(f'{content}')
file_pid = 'jobs_file.json'
- - name: push_ids
type: s3
connection: s3_soil_sealing
continue_on_error: false
action: push
params:
- name: in
values: '${workflow.step.create_pids_file.file_pid}'
save_in: '2023_2024/S4-03/TEST'
templates:
- name: s3_copy
service_id: '11'
type: processor
params:
- name: action
values: upload
- name: s3_endpoint
values: '${workflow.param.s3_copy_endpoint}'
- name: s3_bucket
values: '${workflow.param.s3_bucket_map_cgi}'
- name: s3_platform_ref
values: '${workflow.param.s3_platform_ref}'
- name: collection
values: '{"output":"iridelot365b7d9dde6044c00b26abe046aed0556"}'
- name: siam
service_id: '3'
type: processor
params:
- name: veg_mask
values: 'binary'
- name: cloud_mask
values: 'binary'
- name: urban_mask
values: 'disabled'
- name: collection
values: '${workflow.param.siam_collection_base}'
- name: shadow_mask
values: 'enabled'
- name: keep_calibration_data
values: 'false'
- name: extra_indexes
values: 'true'
- name: water_mask
values: 'disabled'
- name: bare_soil_mask
values: 'disabled'
- name: burned_area_mask
values: 'disabled'
- name: use_no_data_mask
values: 'true'
- name: smoke_mask
values: 'disabled'
- name: classification_mode
values: 'crisp'
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distributions
No source distribution files available for this release.See tutorial on generating distribution archives.
Built Distribution
Close
Hashes for insulaclient-0.6.3-py3-none-any.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 0e4d6fbfa6f3cf5c7554a9e3747705ce176eb8c13c991d79899478c0b5c668f7 |
|
MD5 | fead407f5b1d350f53abfb90983f5820 |
|
BLAKE2b-256 | 65bb9ba27d9c0c3d275136c62a8aac86c1bb99753f2a3d24fde2dc5cd2ccad1f |