#!Jinja2
[meta]
    title = Test workflow based on the satellite data processing example
    description = """
        Each successive integer cycle retrieves and processes the next
        arbitrarily timed and arbitrarily labelled dataset, in parallel
        with previous cycles if the data comes in quickly.
    """

# you can monitor output processing with:
# $ watch -n 1 \
#    "find ~/cylc-run/<WORKFLOW>/share; find ~/cylc-run/<WORKFLOW>/work"

{% set N_DATASETS = 3 %}

# define shared directories (could use runtime namespaces for this)
{% set DATA_IN_DIR = "$CYLC_WORKFLOW_SHARE_DIR/incoming" %}
{% set PRODUCT_DIR = "$CYLC_WORKFLOW_SHARE_DIR/products" %}

[scheduling]
    cycling mode = integer
    initial cycle point = 1
    final cycle point = {{N_DATASETS}}
    runahead limit = P2
    [[graph]]
        R1 = prep => satsim & get_data
        P1 = """
            # Processing chain for each dataset
            get_data => proc1 => proc2 => products
            # As one dataset is retrieved, start waiting on another.
            get_data[-P1] => get_data
        """
        R1//{{N_DATASETS}} = products => collate

[runtime]
    [[prep]]
        script = rm -rf $CYLC_WORKFLOW_SHARE_DIR $CYLC_WORKFLOW_WORK_DIR
        [[[meta]]]
            title = clean the workflow output directories

    [[satsim]]
        pre-script = mkdir -p {{DATA_IN_DIR}}
        script = """
            COUNT=0
            while true; do
                (( COUNT == {{N_DATASETS}} )) && break
                sleep $(( 1 + RANDOM % 5 ))
                touch {{DATA_IN_DIR}}/dataset-$(date +%s).raw
                (( COUNT += 1 ))
            done
        """
        [[[meta]]]
            title = simulate a satellite data feed
            description = """
                Generates {{N_DATASETS}} arbitrarily labelled
                datasets after random durations.
            """

    [[WORKDIR]]
        # Define a common cycle-point-specific work-directory for all
        # processing tasks so that they all work on the same dataset.
        work sub-directory = proc-$CYLC_TASK_CYCLE_POINT
        #pre-script = sleep 10

    [[get_data]]
        inherit = WORKDIR

        script = """
            while ! DATASET=$(ls {{DATA_IN_DIR}}/dataset-*.raw 2>/dev/null \
                | head -n 1)
            do
                sleep 1
            done
            mv "$DATASET" "$PWD"
        """
        [[[meta]]]
            title = grab one new dataset, waiting if necessary

    [[proc1]]
        inherit = WORKDIR
        script = """
            DATASET=$(ls dataset-*.raw)
            mv $DATASET ${DATASET%raw}proc1
        """
        [[[meta]]]
            title = convert .raw dataset to .proc1 form

    [[proc2]]
        inherit = WORKDIR
        script = """
            DATASET=$(ls dataset-*.proc1)
            mv $DATASET ${DATASET%proc1}proc2
        """
        [[[meta]]]
            title = convert .proc1 dataset to .proc2 form

    [[products]]
        inherit = WORKDIR
        pre-script = mkdir -p {{PRODUCT_DIR}}
        script = """
            DATASET=$( ls dataset-*.proc2 )
            mv $DATASET {{PRODUCT_DIR}}/${DATASET%proc2}prod
        """
        [[[meta]]]
            title = generate products from .proc2 processed dataset

    [[collate]]
        # Note you might want to use "cylc workflow-state" to check that
        # _all_ product tasks have finished before collating results.
        script = ls {{PRODUCT_DIR}}
        [[[meta]]]
            title = collate all products from the workflow run
