Async task / queue management

Get Help
,
3

until can’t be used on include_tasks, but that doesn’t mean you can’t redo include_tasks until a condition is true. You just have to use tail recursion.

test.yml:

- hosts: localhost
  gather_facts: false
  tasks:
    - ansible.builtin.set_fact:
        queued_tasks: "{{ range(0, 45, 5) | zip(range(45, 0, -5)) | flatten }}"
        queue_start: "{{ lookup('pipe', 'date +%s') | int }}"

    - ansible.builtin.include_tasks:
        file: tq.yml

tq.yml:

- when: (running_tasks | default([]) | length < 3) and (queued_tasks | length > 0)
  block:
    - name: Launch a new task
      ansible.builtin.command: sleep {{ queued_tasks[0] }}
      async: 60
      poll: 0
      register: result

    - ansible.builtin.debug:
        msg: Launched {{ queued_tasks[0] }} at {{ (lookup('pipe', 'date +%s') | int) - queue_start }}

    - name: Move launched task between queues
      ansible.builtin.set_fact:
        queued_tasks: "{{ queued_tasks[1:] }}"
        running_tasks: "{{ running_tasks | default([]) | union([result.ansible_job_id]) }}"

# consider adding a sleep here to make the loop less busy
# - ansible.builtin.wait_for:
#     timeout: 5
#   when: running_tasks | length >= 3

- name: Check on tasks
  ansible.builtin.async_status:
    jid: "{{ item }}"
  register: result
  loop: "{{ running_tasks }}"

- name: Remove finished tasks from check queue
  ansible.builtin.set_fact:
    running_tasks: "{{ result.results | rejectattr('finished') | map(attribute='item') }}"

- name: Run again if needed
  ansible.builtin.include_tasks:
    file: tq.yml
  when: (queued_tasks | length > 0) or (running_tasks | length > 0)

Output:

$ ansible-playbook test.yml | grep Launched
    msg: Launched 0 at 1
    msg: Launched 45 at 2
    msg: Launched 5 at 3
    msg: Launched 40 at 5
    msg: Launched 10 at 10
    msg: Launched 35 at 24
    msg: Launched 15 at 48
    msg: Launched 30 at 50
    msg: Launched 20 at 62
    msg: Launched 25 at 65
    msg: Launched 25 at 85
    msg: Launched 20 at 89
    msg: Launched 30 at 95
    msg: Launched 15 at 112
    msg: Launched 35 at 115
    msg: Launched 10 at 128
    msg: Launched 40 at 131
    msg: Launched 5 at 143
4 Likes