#!Jinja2

# TEST WORKFLOW ADAPTED FROM examples/satellite/ext-trigger/

[meta]
    title = Real time satellite data processing demo, variant 3 of 3
    description = """
        Successive cycle points retrieve and processes the next arbitrarily
        timed and labelled dataset, in parallel if the data comes in quickly.
        This variant of the workflow has initial get_data tasks with external
        triggers: they do not submit until triggered by an external system.
    """

# Note that the satellite simulator task here that supplies the external event
# trigger happens to be a workflow task - i.e. it is not really "external" - but
# this is only a convenience - an easy route to a self-contained example workflow.

# you can monitor output processing with:
# $ watch -n 1 \
#    "find ~/cylc-run/<WORKFLOW>/share; find ~/cylc-run/<WORKFLOW>/work"

{% set N_DATASETS = 5 %}

# define shared directories (could use runtime namespaces for this)
{% set DATA_IN_DIR = "$CYLC_WORKFLOW_SHARE_DIR/incoming" %}
{% set PRODUCT_DIR = "$CYLC_WORKFLOW_SHARE_DIR/products" %}

[scheduler]
    UTC mode = True

[scheduling]
    cycling mode = integer
    initial cycle point = 1
    final cycle point = {{N_DATASETS}}
    runahead limit = P4
    [[special tasks]]
        external-trigger = get_data("new dataset ready for processing")
    [[graph]]
        R1 = prep => satsim & get_data
        P1 = """
            # Processing chain for each dataset
            get_data => proc1 => proc2 => products
            # As one dataset is retrieved, start waiting on another.
            get_data[-P1] => get_data
        """
        R1//{{N_DATASETS}} = products => collate  # last cycle

[runtime]
    [[prep]]
        script = rm -rf $CYLC_WORKFLOW_SHARE_DIR $CYLC_WORKFLOW_WORK_DIR
        [[[meta]]]
            title = clean the workflow output directories

    [[satsim]]
        pre-script = mkdir -p {{DATA_IN_DIR}}
        script = """
            COUNT=0
            while ((COUNT < {{N_DATASETS}})); do
                # sleep $((RANDOM % 20))
                # Generate datasets very quickly to test parallel processing.
                DATA_ID=$(date +%s).$((RANDOM % 100))
                DATA_FILE=dataset-${DATA_ID}.raw
                touch {{DATA_IN_DIR}}/$DATA_FILE
                ((COUNT += 1))
                # (required to distinguish fast-arriving messages).
                # Trigger downstream processing in the workflow.
                cylc ext-trigger $CYLC_WORKFLOW_ID \
                   "new dataset ready for processing" $DATA_ID
            done
        """
        [[[meta]]]
            title = simulate a satellite data feed
        description = """
            Generates {{N_DATASETS}} arbitrarily labelled
            datasets very quickly, to show parallel processing streams.
        """

    [[WORKDIR]]
        # Define a common cycle-point-specific work-directory for all
        # processing tasks so that they all work on the same dataset.
        work sub-directory = proc-$CYLC_TASK_CYCLE_POINT
        [[[environment]]]
            DATASET = dataset-$CYLC_EXT_TRIGGER_ID

    [[get_data]]
        inherit = WORKDIR
        script = mv {{DATA_IN_DIR}}/${DATASET}.raw $PWD
        [[[meta]]]
            title = retrieve next dataset
            description = just do it - we know it exists already

    [[proc1]]
        inherit = WORKDIR
        script = mv ${DATASET}.raw ${DATASET}.proc1
        [[[meta]]]
            title = convert .raw dataset to .proc1 form

    [[proc2]]
        inherit = WORKDIR
        script = mv ${DATASET}.proc1 ${DATASET}.proc2
        [[[meta]]]
            title = convert .proc1 dataset to .proc2 form

    [[products]]
        inherit = WORKDIR
        script = """
            mkdir -p {{PRODUCT_DIR}}
            mv ${DATASET}.proc2 {{PRODUCT_DIR}}/${DATASET}.prod
        """
        [[[meta]]]
            title = generate products from .proc2 processed dataset

    [[collate]]
        # Note you might want to use "cylc workflow-state" to check that
        # _all_ product tasks have finished before collating results.
        script = """
            echo PRODUCTS:
            ls {{PRODUCT_DIR}}
        """
        [[[meta]]]
            title = collate all products from the workflow run
