Airflow¶

This section demonstrates the use of Apache Airflow as backend.

Start by running a local Airflow instance with some generated test DAGs:

commandline
cd eozilla-airflow
pixi install
pixi run airflow standalone

Then run the wraptile gateway server with the local Airflow instance (assuming the local Airflow webserver runs on http://localhost:8080):

commandline
pixi shell
wraptile run -- wraptile.services.airflow:service --airflow-password=a8e7f4bb230

Get the airflow user password from eozilla-airflow/.airflow/simple_auth_manager_passwords.json.generated.

In [1]:

Copied!

from cuiman import Client
from gavicore.models import ProcessRequest
from cuiman import Client
from gavicore.models import ProcessRequest

In [2]:

Copied!

client = Client(server_url="http://127.0.0.1:8008")
client
client = Client(server_url="http://127.0.0.1:8008")
client

Out[2]:

{
  "access_token": null,
  "server_url": "http://127.0.0.1:8008",
  "user_name": null
}

In [3]:

Copied!

client.get_capabilities()
client.get_capabilities()

Out[3]:

{
  "description": "A gateway API compliant with OGC API - Processes that uses an Airflow backend.",
  "links": [
    {
      "href": "http://127.0.0.1:8008/",
      "hreflang": "en",
      "rel": "self",
      "title": "get_capabilities",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/openapi.json",
      "hreflang": "en",
      "rel": "service",
      "title": "openapi",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/docs",
      "hreflang": "en",
      "rel": "service",
      "title": "swagger_ui_html",
      "type": "text/html"
    },
    {
      "href": "http://127.0.0.1:8008/docs/oauth2-redirect",
      "hreflang": "en",
      "rel": "service",
      "title": "swagger_ui_redirect",
      "type": "text/html"
    },
    {
      "href": "http://127.0.0.1:8008/redoc",
      "hreflang": "en",
      "rel": "service",
      "title": "redoc_html",
      "type": "text/html"
    },
    {
      "href": "http://127.0.0.1:8008/",
      "hreflang": "en",
      "rel": "service",
      "title": "get_capabilities",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/conformance",
      "hreflang": "en",
      "rel": "service",
      "title": "get_conformance",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/processes",
      "hreflang": "en",
      "rel": "service",
      "title": "get_processes",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/processes/{processID}",
      "hreflang": "en",
      "rel": "service",
      "title": "get_process",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/processes/{processID}/execution",
      "hreflang": "en",
      "rel": "service",
      "title": "execute_process",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/jobs",
      "hreflang": "en",
      "rel": "service",
      "title": "get_jobs",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/jobs/{jobId}",
      "hreflang": "en",
      "rel": "service",
      "title": "get_job",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/jobs/{jobId}",
      "hreflang": "en",
      "rel": "service",
      "title": "dismiss_job",
      "type": "application/json"
    },
    {
      "href": "http://127.0.0.1:8008/jobs/{jobId}/results",
      "hreflang": "en",
      "rel": "service",
      "title": "get_job_results",
      "type": "application/json"
    }
  ],
  "title": "Airflow Service"
}

In [4]:

Copied!

client.get_conformance()
client.get_conformance()

Out[4]:

{
  "conformsTo": [
    "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core",
    "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description",
    "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json",
    "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30",
    "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/job-list",
    "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/dismiss"
  ]
}

In [5]:

Copied!

client.get_processes()
client.get_processes()

Out[5]:

{
  "links": [
    {
      "href": "http://127.0.0.1:8008/processes",
      "hreflang": "en",
      "rel": "self",
      "title": "get_processes",
      "type": "application/json"
    }
  ],
  "processes": [
    {
      "description": "Returns the list of prime numbers between a `min_val` and `max_val`. ",
      "id": "primes_between",
      "title": "Prime Processor",
      "version": "0.0.0"
    },
    {
      "id": "return_base_model",
      "title": "BaseModel Test",
      "version": "0.0.0"
    },
    {
      "description": "Simulate a set scene images slices for testing. Creates an xarray dataset with `periodicity` time slices and writes it as Zarr into a temporary location. Requires installed `dask`, `xarray`, and `zarr` packages.",
      "id": "simulate_scene",
      "title": "Generate scene for testing",
      "version": "0.0.0"
    },
    {
      "description": "Sleeps for `duration` seconds. Fails on purpose if `fail` is `True`. Returns the effective amount of sleep in seconds.",
      "id": "sleep_a_while",
      "title": "Sleep Processor",
      "version": "0.0.0"
    }
  ]
}

In [6]:

Copied!

client.get_process(process_id="sleep_a_while")
client.get_process(process_id="sleep_a_while")

Out[6]:

{
  "description": "Sleeps for `duration` seconds. Fails on purpose if `fail` is `True`. Returns the effective amount of sleep in seconds.",
  "id": "sleep_a_while",
  "inputs": {},
  "outputs": {},
  "title": "Sleep Processor",
  "version": "0.0.0"
}

In [7]:

Copied!

client.get_jobs()
client.get_jobs()

Out[7]:

{
  "jobs": [
    {
      "created": "2025-07-15T11:12:09.501829Z",
      "finished": "2025-07-15T11:12:13.599646Z",
      "jobID": "manual__2025-07-15T11:12:09.473432+00:00",
      "processID": "primes_between",
      "started": "2025-07-15T11:12:09.689505Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-07-15T11:12:13.592815Z"
    },
    {
      "created": "2025-09-09T13:31:19.770932Z",
      "finished": "2025-09-09T13:31:24.008108Z",
      "jobID": "primes_between__20250909133119_1",
      "processID": "primes_between",
      "started": "2025-09-09T13:31:20.451716Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T13:31:24.003845Z"
    },
    {
      "created": "2025-09-09T14:00:46.601700Z",
      "finished": "2025-09-09T14:00:49.218611Z",
      "jobID": "primes_between__20250909140046_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:00:47.320271Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:00:49.212736Z"
    },
    {
      "created": "2025-09-09T14:08:20.775729Z",
      "finished": "2025-09-09T14:08:23.296110Z",
      "jobID": "primes_between__20250909140820_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:08:22.043965Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:08:23.292599Z"
    },
    {
      "created": "2025-09-09T14:12:53.830201Z",
      "finished": "2025-09-09T14:12:55.669104Z",
      "jobID": "primes_between__20250909141253_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:12:54.336515Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:12:55.662957Z"
    },
    {
      "created": "2025-07-15T12:43:21.762728Z",
      "finished": "2025-07-15T12:44:15.335494Z",
      "jobID": "manual__2025-07-15T12:43:21.705839+00:00",
      "processID": "sleep_a_while",
      "started": "2025-07-15T12:43:23.054525Z",
      "status": "failed",
      "type": "process",
      "updated": "2025-07-15T12:44:15.329219Z"
    },
    {
      "created": "2025-09-09T13:31:22.836061Z",
      "finished": "2025-09-09T13:31:35.360842Z",
      "jobID": "sleep_a_while__20250909133121_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T13:31:23.950645Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T13:31:35.356595Z"
    },
    {
      "created": "2025-09-09T14:00:46.708624Z",
      "finished": "2025-09-09T14:00:59.371320Z",
      "jobID": "sleep_a_while__20250909140046_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:00:47.325045Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:00:59.367748Z"
    },
    {
      "created": "2025-09-09T14:08:20.863568Z",
      "finished": "2025-09-09T14:08:32.877437Z",
      "jobID": "sleep_a_while__20250909140820_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:08:22.045075Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:08:32.873613Z"
    },
    {
      "created": "2025-09-09T14:12:53.960935Z",
      "finished": "2025-09-09T14:13:07.991002Z",
      "jobID": "sleep_a_while__20250909141253_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:12:54.338449Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:13:07.986703Z"
    }
  ],
  "links": [
    {
      "href": "http://127.0.0.1:8008/jobs",
      "hreflang": "en",
      "rel": "self",
      "title": "get_jobs",
      "type": "application/json"
    }
  ]
}

In [8]:

Copied!

client.execute_process(process_id="primes_between", request=ProcessRequest())
client.execute_process(process_id="primes_between", request=ProcessRequest())

Out[8]:

{
  "created": "2025-09-09T14:27:49.355547Z",
  "jobID": "primes_between__20250909142749_1",
  "processID": "primes_between",
  "status": "accepted",
  "type": "process"
}

In [9]:

Copied!

client.execute_process(process_id="sleep_a_while", request=ProcessRequest())
client.execute_process(process_id="sleep_a_while", request=ProcessRequest())

Out[9]:

{
  "created": "2025-09-09T14:27:49.520012Z",
  "jobID": "sleep_a_while__20250909142749_2",
  "processID": "sleep_a_while",
  "status": "accepted",
  "type": "process"
}

In [10]:

Copied!

client.get_jobs()
client.get_jobs()

Out[10]:

{
  "jobs": [
    {
      "created": "2025-07-15T11:12:09.501829Z",
      "finished": "2025-07-15T11:12:13.599646Z",
      "jobID": "manual__2025-07-15T11:12:09.473432+00:00",
      "processID": "primes_between",
      "started": "2025-07-15T11:12:09.689505Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-07-15T11:12:13.592815Z"
    },
    {
      "created": "2025-09-09T13:31:19.770932Z",
      "finished": "2025-09-09T13:31:24.008108Z",
      "jobID": "primes_between__20250909133119_1",
      "processID": "primes_between",
      "started": "2025-09-09T13:31:20.451716Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T13:31:24.003845Z"
    },
    {
      "created": "2025-09-09T14:00:46.601700Z",
      "finished": "2025-09-09T14:00:49.218611Z",
      "jobID": "primes_between__20250909140046_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:00:47.320271Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:00:49.212736Z"
    },
    {
      "created": "2025-09-09T14:08:20.775729Z",
      "finished": "2025-09-09T14:08:23.296110Z",
      "jobID": "primes_between__20250909140820_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:08:22.043965Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:08:23.292599Z"
    },
    {
      "created": "2025-09-09T14:12:53.830201Z",
      "finished": "2025-09-09T14:12:55.669104Z",
      "jobID": "primes_between__20250909141253_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:12:54.336515Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:12:55.662957Z"
    },
    {
      "created": "2025-09-09T14:27:49.355547Z",
      "jobID": "primes_between__20250909142749_1",
      "processID": "primes_between",
      "status": "accepted",
      "type": "process"
    },
    {
      "created": "2025-07-15T12:43:21.762728Z",
      "finished": "2025-07-15T12:44:15.335494Z",
      "jobID": "manual__2025-07-15T12:43:21.705839+00:00",
      "processID": "sleep_a_while",
      "started": "2025-07-15T12:43:23.054525Z",
      "status": "failed",
      "type": "process",
      "updated": "2025-07-15T12:44:15.329219Z"
    },
    {
      "created": "2025-09-09T13:31:22.836061Z",
      "finished": "2025-09-09T13:31:35.360842Z",
      "jobID": "sleep_a_while__20250909133121_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T13:31:23.950645Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T13:31:35.356595Z"
    },
    {
      "created": "2025-09-09T14:00:46.708624Z",
      "finished": "2025-09-09T14:00:59.371320Z",
      "jobID": "sleep_a_while__20250909140046_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:00:47.325045Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:00:59.367748Z"
    },
    {
      "created": "2025-09-09T14:08:20.863568Z",
      "finished": "2025-09-09T14:08:32.877437Z",
      "jobID": "sleep_a_while__20250909140820_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:08:22.045075Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:08:32.873613Z"
    },
    {
      "created": "2025-09-09T14:12:53.960935Z",
      "finished": "2025-09-09T14:13:07.991002Z",
      "jobID": "sleep_a_while__20250909141253_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:12:54.338449Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:13:07.986703Z"
    },
    {
      "created": "2025-09-09T14:27:49.520012Z",
      "jobID": "sleep_a_while__20250909142749_2",
      "processID": "sleep_a_while",
      "status": "accepted",
      "type": "process"
    }
  ],
  "links": [
    {
      "href": "http://127.0.0.1:8008/jobs",
      "hreflang": "en",
      "rel": "self",
      "title": "get_jobs",
      "type": "application/json"
    }
  ]
}

In [11]:

Copied!

client.get_job_results("sleep_a_while__20250909140046_2")
client.get_job_results("sleep_a_while__20250909140046_2")

Out[11]:

{
  "return_value": 10.069934606552124
}

In [15]:

Copied!

client.get_job_results("primes_between__20250909142749_1")
client.get_job_results("primes_between__20250909142749_1")

Out[15]:

{
  "return_value": [
    2,
    3,
    5,
    7,
    11,
    13,
    17,
    19,
    23,
    29,
    31,
    37,
    41,
    43,
    47,
    53,
    59,
    61,
    67,
    71,
    73,
    79,
    83,
    89,
    97
  ]
}

In [12]:

Copied!

# for job in client.get_jobs().jobs:
#     client.dismiss_job(job.jobID)
# for job in client.get_jobs().jobs:
#     client.dismiss_job(job.jobID)

In [13]:

Copied!

client.get_jobs()
client.get_jobs()

Out[13]:

{
  "jobs": [
    {
      "created": "2025-07-15T11:12:09.501829Z",
      "finished": "2025-07-15T11:12:13.599646Z",
      "jobID": "manual__2025-07-15T11:12:09.473432+00:00",
      "processID": "primes_between",
      "started": "2025-07-15T11:12:09.689505Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-07-15T11:12:13.592815Z"
    },
    {
      "created": "2025-09-09T13:31:19.770932Z",
      "finished": "2025-09-09T13:31:24.008108Z",
      "jobID": "primes_between__20250909133119_1",
      "processID": "primes_between",
      "started": "2025-09-09T13:31:20.451716Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T13:31:24.003845Z"
    },
    {
      "created": "2025-09-09T14:00:46.601700Z",
      "finished": "2025-09-09T14:00:49.218611Z",
      "jobID": "primes_between__20250909140046_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:00:47.320271Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:00:49.212736Z"
    },
    {
      "created": "2025-09-09T14:08:20.775729Z",
      "finished": "2025-09-09T14:08:23.296110Z",
      "jobID": "primes_between__20250909140820_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:08:22.043965Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:08:23.292599Z"
    },
    {
      "created": "2025-09-09T14:12:53.830201Z",
      "finished": "2025-09-09T14:12:55.669104Z",
      "jobID": "primes_between__20250909141253_1",
      "processID": "primes_between",
      "started": "2025-09-09T14:12:54.336515Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:12:55.662957Z"
    },
    {
      "created": "2025-09-09T14:27:49.355547Z",
      "jobID": "primes_between__20250909142749_1",
      "processID": "primes_between",
      "status": "accepted",
      "type": "process"
    },
    {
      "created": "2025-07-15T12:43:21.762728Z",
      "finished": "2025-07-15T12:44:15.335494Z",
      "jobID": "manual__2025-07-15T12:43:21.705839+00:00",
      "processID": "sleep_a_while",
      "started": "2025-07-15T12:43:23.054525Z",
      "status": "failed",
      "type": "process",
      "updated": "2025-07-15T12:44:15.329219Z"
    },
    {
      "created": "2025-09-09T13:31:22.836061Z",
      "finished": "2025-09-09T13:31:35.360842Z",
      "jobID": "sleep_a_while__20250909133121_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T13:31:23.950645Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T13:31:35.356595Z"
    },
    {
      "created": "2025-09-09T14:00:46.708624Z",
      "finished": "2025-09-09T14:00:59.371320Z",
      "jobID": "sleep_a_while__20250909140046_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:00:47.325045Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:00:59.367748Z"
    },
    {
      "created": "2025-09-09T14:08:20.863568Z",
      "finished": "2025-09-09T14:08:32.877437Z",
      "jobID": "sleep_a_while__20250909140820_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:08:22.045075Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:08:32.873613Z"
    },
    {
      "created": "2025-09-09T14:12:53.960935Z",
      "finished": "2025-09-09T14:13:07.991002Z",
      "jobID": "sleep_a_while__20250909141253_2",
      "processID": "sleep_a_while",
      "started": "2025-09-09T14:12:54.338449Z",
      "status": "successful",
      "type": "process",
      "updated": "2025-09-09T14:13:07.986703Z"
    },
    {
      "created": "2025-09-09T14:27:49.520012Z",
      "jobID": "sleep_a_while__20250909142749_2",
      "processID": "sleep_a_while",
      "status": "accepted",
      "type": "process"
    }
  ],
  "links": [
    {
      "href": "http://127.0.0.1:8008/jobs",
      "hreflang": "en",
      "rel": "self",
      "title": "get_jobs",
      "type": "application/json"
    }
  ]
}

In [14]:

Copied!

client.get_job("primes_between__20250909133119_1")
client.get_job("primes_between__20250909133119_1")

Out[14]:

{
  "created": "2025-09-09T13:31:19.770932Z",
  "finished": "2025-09-09T13:31:24.008108Z",
  "jobID": "primes_between__20250909133119_1",
  "processID": "primes_between",
  "started": "2025-09-09T13:31:20.451716Z",
  "status": "successful",
  "type": "process",
  "updated": "2025-09-09T13:31:24.003845Z"
}

In [ ]: