{ "openapi": "3.0.1", "info": { "title": "Oleander API", "description": "Oleander implements the api spec as defined by OpenLineage to ingest data. We also provide a RESTful API to extract data within our system.", "version": "1.0.0" }, "servers": [ { "url": "https://oleander.dev" } ], "security": [ { "bearerAuth": [] } ], "paths": { "/api/v1/query": { "post": { "summary": "Query (routed)", "description": "The unified query endpoint. oleander parses the SQL, estimates how much data the referenced tables hold, and picks the engine (DuckDB, Polars, Bloom, or Spark) and machine size to match. Leave `engine` on `auto` unless you need a specific one.\n\nInteractive reads return rows on the call. A query with a `destination`, a statement that names its own target, or a read too large for an interactive engine is submitted asynchronously and returns `state: \"SUBMITTED\"` with a `run_id` to poll.\n\nThis endpoint supersedes `/api/v1/warehouse/query`, which pins DuckDB and is sized by the org default rather than by the query.", "operationId": "query", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "description": "Provide either `query` or `script`, never both.", "properties": { "query": { "type": "string", "description": "The SQL to run.", "example": "SELECT species, count(*) AS n FROM oleander.default.flowers GROUP BY 1" }, "script": { "type": "string", "description": "A Polars DataFrame script that assigns `result`. Mutually exclusive with `query`; forces the Polars engine.", "example": "result = events.group_by('day').len()" }, "tables": { "type": "array", "description": "Tables a script reads, as \"alias=namespace.table\" or {alias, table}. Script mode only.", "items": { "oneOf": [ { "type": "string" }, { "type": "object", "required": [ "alias", "table" ], "properties": { "alias": { "type": "string" }, "table": { "type": "string" } } } ] }, "default": [] }, "engine": { "type": "string", "enum": [ "auto", "duckdb", "polars", "bloom", "spark" ], "default": "auto", "description": "Ask for a specific engine. An impossible combination returns an engine capability error rather than rerouting." }, "destination": { "type": "string", "description": "Table to write the result to, as a dotted identifier such as namespace.table. Presence of a destination makes this a write.", "example": "default.daily_counts" }, "write_mode": { "type": "string", "enum": [ "overwrite", "append" ], "default": "overwrite", "description": "Applies to a `destination` write. A statement that names its own target carries its own semantics." }, "explain": { "type": "boolean", "default": false, "description": "Return the routing decision without executing anything or spending compute." } } } } } }, "responses": { "200": { "description": "Query executed, submitted, or explained.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UnifiedQueryResponse" } } } }, "400": { "description": "Invalid request, or an engine capability error when `engine` was pinned to something that cannot run the query.", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean", "example": false }, "error": { "type": "string", "example": "query and script are mutually exclusive." } } } } } }, "401": { "description": "Unauthorized – request lacks a valid Bearer token or session credentials.", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean", "example": false }, "error": { "type": "string", "example": "Unauthorized" } } } } } }, "402": { "description": "Payment required – metered compute needs a payment method on file. A billing decision, not a transient failure: do not retry the same query.", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean", "example": false }, "error": { "type": "string", "example": "A payment method is required to run this query" } } } } } }, "403": { "description": "Forbidden – the plan does not allow this query. A billing decision, not a transient failure.", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean", "example": false }, "error": { "type": "string", "example": "This query requires a plan upgrade" } } } } } }, "500": { "description": "Query execution failed.", "content": { "application/json": { "schema": { "type": "object", "properties": { "success": { "type": "boolean", "example": false }, "error": { "type": "string", "example": "Query execution failed" } } } } } } } } }, "/api/v1/lineage": { "post": { "summary": "Record event", "description": "Submit an OpenLineage compliant event to record lineage metadata. This is the primary endpoint for ingesting lineage data from your pipelines and integrations.", "operationId": "recordLineageEvent", "requestBody": { "description": "This is an OpenLineage compliant payload", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OpenLineageEvent" } } }, "required": true }, "responses": { "200": { "description": "Success response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Success" } } } }, "401": { "description": "Unauthorized – request either lacks a valid Bearer token.", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string", "example": "Unauthorized" } } } } } }, "500": { "description": "unexpected error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/api/v1/events": { "get": { "summary": "Retrieve events", "description": "List lineage events. Supports filtering by time range, run, state, namespace, job, and integrations. Results can be paginated and sorted.", "operationId": "getLineageEvents", "parameters": [ { "name": "start", "in": "query", "description": "Start timestamp (UNIX epoch). Only events with event_time >= this value are returned.", "required": true, "schema": { "type": "integer", "format": "int64", "example": 1672531200 } }, { "name": "end", "in": "query", "description": "End timestamp (UNIX epoch). Only events with event_time < this value are returned.", "required": true, "schema": { "type": "integer", "format": "int64", "example": 1672617600 } }, { "name": "sort_by", "in": "query", "description": "Field by which results should be sorted. Defaults to EVENT_TIME.", "required": false, "schema": { "type": "string", "enum": [ "EVENT_TIME", "EVENT_SIZE_IN_BYTES" ], "default": "EVENT_TIME" } }, { "name": "order", "in": "query", "description": "Sort order. Defaults to DESC.", "required": false, "schema": { "type": "string", "enum": [ "ASC", "DESC" ], "default": "DESC" } }, { "name": "run", "in": "query", "description": "Filter by run_id (UUID). Only events matching this run_id are returned.", "required": false, "schema": { "type": "string", "format": "uuid", "example": "550e8400-e29b-41d4-a716-446655440000" } }, { "name": "states", "in": "query", "description": "Comma-separated list of run states to filter by. E.g. 'START,RUNNING,COMPLETE'.", "required": false, "schema": { "type": "string", "enum": [ "START", "RUNNING", "COMPLETE", "ABORT", "FAIL", "OTHER" ] } }, { "name": "namespace", "in": "query", "description": "Filter by job namespace, partial matches included.", "required": false, "schema": { "type": "string", "example": "my-scheduler-namespace" } }, { "name": "job", "in": "query", "description": "Filter by job name, partial matches included.", "required": false, "schema": { "type": "string", "example": "myjob.mytask" } }, { "name": "integrations", "in": "query", "description": "Comma-separated list of integration names to filter by.", "required": false, "schema": { "type": "string", "example": "airflow,dbt" } }, { "name": "limit", "in": "query", "description": "Number of results to return. Defaults to 30.", "required": false, "schema": { "type": "integer", "default": 30, "example": 50 } }, { "name": "offset", "in": "query", "description": "Offset for pagination. Defaults to 0.", "required": false, "schema": { "type": "integer", "default": 0, "example": 100 } } ], "responses": { "200": { "description": "A list of lineage events.", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/LineageEvent" } } } } }, "400": { "description": "Invalid request (e.g. parameter validation error).", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string", "example": "Validation error details." } } } } } }, "401": { "description": "Unauthorized – request either lacks a valid Bearer token or session credentials.", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string", "example": "Unauthorized" } } } } } }, "500": { "description": "unexpected error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/api/v1/warehouse/polars": { "post": { "summary": "Polars", "description": "Execute a Polars workload against your Iceberg lake tables. Supports two modes: **query** (Polars SQL run against registered tables) and **script** (user-authored Python that assigns `result` to a LazyFrame or DataFrame). Workloads run in an isolated sandbox; pass `distributed: true` to offload execution to Polars Cloud.\n\n**Beta:** Observability (lineage, traces, logs) is not yet captured for Polars workloads.\n\n**Cost note:** Distributed execution incurs Polars Cloud compute charges in addition to oleander usage. Start small and scale up as needed.", "operationId": "runPolars", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "mode" ], "properties": { "mode": { "type": "string", "enum": [ "query", "script" ], "description": "Execution mode. `query` runs a Polars SQL expression against registered tables. `script` executes a Python script that has `pl`, `scan(table)`, and `params` in scope and must assign `result`." }, "query": { "type": "string", "description": "Polars SQL query string. Required when `mode` is `query`.", "example": "SELECT species, avg(sepal_length) AS avg_sepal_length, avg(petal_length) AS avg_petal_length FROM flowers GROUP BY species ORDER BY avg_sepal_length DESC" }, "tables": { "type": "array", "description": "Tables to register for query mode. Each entry is either `alias=namespace.table` or an object `{alias, table}`. Required when `mode` is `query`.", "items": { "oneOf": [ { "type": "string", "example": "flowers=default.flowers" }, { "type": "object", "required": [ "alias", "table" ], "properties": { "alias": { "type": "string" }, "table": { "type": "string" } } } ] } }, "script": { "type": "string", "description": "Python script source for script mode. The script runs with `pl` (polars), `scan(table)`, `params`, and `catalog` in scope. It must assign `result` to a Polars LazyFrame or DataFrame. Do not call `.collect()` or `.remote()` inside the script; oleander handles execution.", "example": "table = params.get('table', 'default.flowers')\nlimit = int(params.get('limit', 50))\n\nflowers = scan(table)\n\nresult = (\n flowers.group_by('species')\n .agg(\n pl.len().alias('count'),\n pl.col('sepal_length').mean().round(2).alias('avg_sepal_length'),\n pl.col('petal_length').mean().round(2).alias('avg_petal_length'),\n )\n .sort('avg_sepal_length', descending=True)\n .head(limit)\n)" }, "params": { "type": "object", "description": "Key-value parameters available inside a script as the `params` dict.", "additionalProperties": { "type": "string" }, "example": { "table": "default.flowers", "limit": "25" } }, "distributed": { "type": "boolean", "description": "Run the workload on Polars Cloud instead of a local sandbox.", "default": false }, "instanceType": { "type": "string", "description": "Polars Cloud instance type. Only used when `distributed` is `true`.", "example": "t4g.medium" }, "clusterSize": { "type": "integer", "description": "Number of Polars Cloud nodes. Only used when `distributed` is `true`.", "example": 4 }, "vcpus": { "type": "integer", "description": "Sandbox vCPU tier for local (non-distributed) execution. Valid values: 2, 4, 8, 16, 32.", "enum": [ 2, 4, 8, 16, 32 ], "default": 2 }, "catalog": { "type": "string", "description": "Catalog name to query against.", "default": "oleander" }, "destination": { "type": "string", "description": "Write results to a lake table. Format: `namespace.table`. If omitted, results are returned inline.", "example": "default.flower_summary" }, "saveMode": { "type": "string", "enum": [ "overwrite", "append" ], "description": "How to write to `destination`. `overwrite` replaces the table; `append` adds rows.", "default": "overwrite" } } } } } }, "responses": { "200": { "description": "Workload executed successfully", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PolarsResponse" } } } }, "400": { "description": "Invalid request or script/query error.", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string" }, "details": { "type": "string" } } } } } }, "401": { "description": "Unauthorized – missing or invalid Bearer token.", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string", "example": "Unauthorized" } } } } } }, "500": { "description": "Unexpected error.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/api/v2/spark/query": { "post": { "summary": "Spark", "description": "Execute a SQL query against the oleander lake using managed Spark. The query runs as a named job in the specified namespace. Results are written to `output_table` as an Iceberg table. Lineage is captured automatically for every run.", "operationId": "sparkQuery", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "namespace", "name", "query", "output_table" ], "properties": { "namespace": { "type": "string", "description": "Job namespace used to identify this query in lineage and run history.", "example": "oleander.query" }, "name": { "type": "string", "description": "Job name. Combined with `namespace` to form the unique job identity.", "example": "flowers_copy" }, "query": { "type": "string", "description": "SQL query to execute. Supports all Spark SQL syntax and can reference any table in your Iceberg catalogs.", "example": "SELECT * FROM oleander.default.flowers" }, "output_table": { "type": "string", "description": "Destination Iceberg table for query results. Format: `catalog.namespace.table`. The table is created if it does not exist.", "example": "oleander.default.flowers_query_spark_sql" }, "driver_machine_type": { "type": "string", "description": "Machine type for the Spark driver.", "example": "spark.1.b", "default": "spark.1.b" }, "executor_machine_type": { "type": "string", "description": "Machine type for each Spark executor.", "example": "spark.1.b", "default": "spark.1.b" }, "executor_numbers": { "type": "integer", "description": "Number of Spark executors to allocate.", "example": 1, "default": 1 } } } } } }, "responses": { "200": { "description": "Query submitted successfully.", "content": { "application/json": { "schema": { "type": "object", "properties": { "run_id": { "type": "string", "format": "uuid", "description": "Run ID for this Spark query execution. Use this to track the run in the oleander platform." }, "job": { "type": "object", "properties": { "namespace": { "type": "string" }, "name": { "type": "string" } } }, "output_table": { "type": "string", "description": "The destination table results will be written to." }, "status": { "type": "string", "description": "Initial run status.", "example": "RUNNING" } } } } } }, "400": { "description": "Invalid request.", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string" } } } } } }, "401": { "description": "Unauthorized - missing or invalid Bearer token.", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string", "example": "Unauthorized" } } } } } }, "500": { "description": "Unexpected error.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/api/v1/warehouse/query": { "post": { "summary": "DuckDB (legacy)", "description": "Legacy endpoint. Executes SQL against the oleander lake, always on DuckDB, sized by the organization's default sandbox setting rather than by the query. Prefer POST /api/v1/query, which routes to the right engine and machine; use this only for autoSaveByHash. All queries capture lineage metadata automatically. This endpoint is only available on the Pro plan.", "operationId": "queryWarehouse", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "query" ], "properties": { "query": { "type": "string", "description": "The SQL query to execute. Supports all DuckDB SQL syntax.", "example": "WITH\n src AS (\n SELECT\n *\n FROM\n read_json (\n 'https://api.tvmaze.com/search/shows?q=breaking+bad',\n auto_detect = TRUE\n )\n )\nSELECT\n src -> 'show' ->> 'name' AS show_name,\n src -> 'show' ->> 'premiered' AS premiered,\n src -> 'show' -> 'rating' ->> 'average' AS rating,\n src -> 'show' -> 'network' ->> 'name' AS network_name\nFROM\n src;" }, "autoSaveByHash": { "type": "boolean", "description": "Automatically save query results to a table based on the query hash. If a table with the same hash exists, returns cached results instead of re-executing.", "default": false } } } } } }, "responses": { "200": { "description": "Query executed successfully", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/QueryResponse" } } } }, "400": { "description": "Invalid request (e.g. invalid SQL query).", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string", "example": "SQL syntax error" } } } } } }, "401": { "description": "Unauthorized – request either lacks a valid Bearer token or session credentials.", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string", "example": "Unauthorized" } } } } } }, "403": { "description": "Forbidden – Pro plan required.", "content": { "application/json": { "schema": { "type": "object", "properties": { "error": { "type": "string", "example": "This endpoint requires a Pro plan subscription" } } } } } }, "500": { "description": "unexpected error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } } }, "components": { "schemas": { "Error": { "required": [ "status" ], "type": "object", "properties": { "status": { "type": "integer", "format": "int32", "default": 500 } } }, "PolarsResponse": { "type": "object", "properties": { "success": { "type": "boolean", "description": "Whether the workload executed successfully." }, "results": { "type": "object", "description": "Inline results (omitted when `destination` is set and `distributed` is true).", "properties": { "columns": { "type": "array", "items": { "type": "string" }, "description": "Column names in result order." }, "column_types": { "type": "array", "items": { "type": "string" }, "description": "Polars dtype string for each column (e.g. `Int64`, `Utf8`)." }, "rows": { "type": "array", "items": { "type": "array", "items": {} }, "description": "Result rows as arrays of values." } } }, "row_count": { "type": "integer", "description": "Total number of rows in the result." }, "execution_time": { "type": "string", "description": "Wall-clock time for the workload (e.g. `\"340ms\"`)." }, "error": { "type": "string", "description": "Error message if `success` is `false`." }, "saved": { "type": "object", "description": "Present when `destination` was set.", "properties": { "table": { "type": "string", "description": "Full `namespace.table` identifier of the written table." }, "rows_written": { "type": "integer", "description": "Number of rows written." }, "mode": { "type": "string", "description": "`overwrite` or `append`." } } }, "compute": { "type": "object", "description": "Compute environment details.", "properties": { "mode": { "type": "string", "description": "`\"daytona\"` for local sandbox execution, `\"polars-cloud\"` for distributed." }, "distributed": { "type": "boolean" }, "instanceType": { "type": "string", "nullable": true }, "clusterSize": { "type": "integer", "nullable": true } } } } }, "QueryResponse": { "type": "object", "properties": { "success": { "type": "boolean", "description": "Whether the query executed successfully" }, "results": { "type": "object", "description": "Query results with columns, column types, and rows", "properties": { "columns": { "type": "array", "items": { "type": "string" }, "description": "Column names" }, "column_types": { "type": "array", "items": { "type": "string" }, "description": "Column data types" }, "rows": { "type": "array", "items": { "type": "array", "items": {} }, "description": "Query results as an array of row arrays" } }, "required": [ "columns", "column_types", "rows" ] }, "row_count": { "type": "integer", "description": "Number of rows returned" }, "execution_time": { "type": "string", "description": "Query execution time (e.g., '991ms')" }, "query": { "type": "string", "description": "The original SQL query that was executed" }, "explainResult": { "type": "object", "description": "Query execution plan explanation" }, "job": { "type": "object", "description": "Lineage job information", "properties": { "namespace": { "type": "string", "description": "Job namespace (typically 'oleander.lake')" }, "name": { "type": "string", "description": "Job name (query hash identifier)" } }, "required": [ "namespace", "name" ] } }, "required": [ "success", "results", "row_count", "execution_time", "query", "job" ] }, "Success": { "required": [ "status" ], "type": "object", "properties": { "status": { "type": "integer", "format": "int32", "default": 200 } } }, "OpenLineageEvent": { "type": "object", "oneOf": [ { "$ref": "#/components/schemas/RunEvent", "title": "Run Event", "description": "A RunEvent is an event that describes the lifecycle of a run." } ] }, "BaseEvent": { "type": "object", "properties": { "eventTime": { "description": "the time the event occurred at", "type": "string", "format": "date-time" }, "producer": { "description": "URI identifying the producer of this metadata. For example this could be a git url with a given tag or sha", "type": "string", "format": "uri", "example": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client" }, "schemaURL": { "description": "The JSON Pointer (https://tools.ietf.org/html/rfc6901) URL to the corresponding version of the schema definition for this RunEvent", "type": "string", "format": "uri", "example": "https://openlineage.io/spec/0-0-1/OpenLineage.json" } }, "required": [ "eventTime", "producer", "schemaURL" ] }, "RunEvent": { "allOf": [ { "$ref": "#/components/schemas/BaseEvent" }, { "type": "object", "properties": { "eventType": { "description": "the current transition of the run state. It is required to issue 1 START event and 1 of [ COMPLETE, ABORT, FAIL ] event per run. Additional events with OTHER eventType can be added to the same run. For example to send additional metadata after the run is complete", "type": "string", "enum": [ "START", "RUNNING", "COMPLETE", "ABORT", "FAIL", "OTHER" ], "example": "START|RUNNING|COMPLETE|ABORT|FAIL|OTHER" }, "run": { "$ref": "#/components/schemas/Run" }, "job": { "$ref": "#/components/schemas/Job" }, "inputs": { "description": "The set of **input** datasets.", "type": "array", "items": { "$ref": "#/components/schemas/InputDataset" } }, "outputs": { "description": "The set of **output** datasets.", "type": "array", "items": { "$ref": "#/components/schemas/OutputDataset" } } }, "required": [ "run", "job" ] } ] }, "DatasetEvent": { "allOf": [ { "$ref": "#/components/schemas/BaseEvent" }, { "type": "object", "properties": { "dataset": { "$ref": "#/components/schemas/StaticDataset" } }, "required": [ "dataset" ], "not": { "required": [ "job", "run" ] } } ] }, "JobEvent": { "allOf": [ { "$ref": "#/components/schemas/BaseEvent" }, { "type": "object", "properties": { "job": { "$ref": "#/components/schemas/Job" }, "inputs": { "description": "The set of **input** datasets.", "type": "array", "items": { "$ref": "#/components/schemas/InputDataset" } }, "outputs": { "description": "The set of **output** datasets.", "type": "array", "items": { "$ref": "#/components/schemas/OutputDataset" } } }, "required": [ "job" ], "not": { "required": [ "run" ] } } ] }, "Run": { "type": "object", "properties": { "runId": { "description": "The globally unique ID of the run associated with the job.", "type": "string", "format": "uuid" }, "facets": { "description": "The run facets.", "type": "object", "anyOf": [ { "type": "object", "additionalProperties": { "$ref": "#/components/schemas/RunFacet" } } ] } }, "required": [ "runId" ] }, "RunFacet": { "description": "A Run Facet", "type": "object", "allOf": [ { "$ref": "#/components/schemas/BaseFacet" } ] }, "Job": { "type": "object", "properties": { "namespace": { "description": "The namespace containing that job", "type": "string", "example": "my-scheduler-namespace" }, "name": { "description": "The unique name for that job within that namespace", "type": "string", "example": "myjob.mytask" }, "facets": { "description": "The job facets.", "type": "object", "anyOf": [ { "type": "object", "additionalProperties": { "$ref": "#/components/schemas/JobFacet" } } ] } }, "required": [ "namespace", "name" ] }, "JobFacet": { "description": "A Job Facet", "type": "object", "allOf": [ { "$ref": "#/components/schemas/BaseFacet" }, { "type": "object", "properties": { "_deleted": { "description": "set to true to delete a facet", "type": "boolean" } } } ] }, "InputDataset": { "description": "An input dataset", "type": "object", "allOf": [ { "$ref": "#/components/schemas/Dataset" }, { "type": "object", "properties": { "inputFacets": { "description": "The input facets for this dataset.", "type": "object", "anyOf": [ { "type": "object", "additionalProperties": { "$ref": "#/components/schemas/InputDatasetFacet" } } ] } } } ] }, "InputDatasetFacet": { "description": "An Input Dataset Facet", "type": "object", "allOf": [ { "$ref": "#/components/schemas/BaseFacet" } ] }, "OutputDataset": { "description": "An output dataset", "type": "object", "allOf": [ { "$ref": "#/components/schemas/Dataset" }, { "type": "object", "properties": { "outputFacets": { "description": "The output facets for this dataset", "type": "object", "anyOf": [ { "type": "object", "additionalProperties": { "$ref": "#/components/schemas/OutputDatasetFacet" } } ] } } } ] }, "OutputDatasetFacet": { "description": "An Output Dataset Facet", "type": "object", "allOf": [ { "$ref": "#/components/schemas/BaseFacet" } ] }, "Dataset": { "type": "object", "properties": { "namespace": { "description": "The namespace containing that dataset", "type": "string", "example": "my-datasource-namespace" }, "name": { "description": "The unique name for that dataset within that namespace", "type": "string", "example": "instance.schema.table" }, "facets": { "description": "The facets for this dataset", "type": "object", "anyOf": [ { "type": "object", "additionalProperties": { "$ref": "#/components/schemas/DatasetFacet" } } ] } }, "required": [ "namespace", "name" ] }, "StaticDataset": { "description": "A Dataset sent within static metadata events", "type": "object", "allOf": [ { "$ref": "#/components/schemas/Dataset" } ] }, "DatasetFacet": { "description": "A Dataset Facet", "type": "object", "allOf": [ { "$ref": "#/components/schemas/BaseFacet" }, { "type": "object", "properties": { "_deleted": { "description": "set to true to delete a facet", "type": "boolean" } } } ] }, "BaseFacet": { "description": "all fields of the base facet are prefixed with _ to avoid name conflicts in facets", "type": "object", "properties": { "_producer": { "description": "URI identifying the producer of this metadata. For example this could be a git url with a given tag or sha", "type": "string", "format": "uri", "example": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client" }, "_schemaURL": { "description": "The JSON Pointer (https://tools.ietf.org/html/rfc6901) URL to the corresponding version of the schema definition for this facet", "type": "string", "format": "uri", "example": "https://openlineage.io/spec/1-0-2/OpenLineage.json#/$defs/BaseFacet" } }, "additionalProperties": true, "required": [ "_producer", "_schemaURL" ] }, "LineageEvent": { "type": "object", "description": "Corresponds to the lineageEventSchema in your TypeScript code.", "properties": { "id": { "type": "string", "format": "uuid" }, "parent_run_id": { "type": "string", "format": "uuid", "nullable": true, "description": "Optional parent run ID." }, "parent_job_namespace": { "type": "string", "nullable": true, "description": "Optional parent job namespace." }, "parent_job_name": { "type": "string", "nullable": true, "description": "Optional parent job name." }, "run_id": { "type": "string", "format": "uuid" }, "run_state": { "type": "string", "enum": [ "START", "RUNNING", "COMPLETE", "ABORT", "FAIL", "OTHER" ] }, "received_at": { "type": "string", "format": "date-time", "description": "Timestamp with offset, e.g. '2023-07-14T12:34:56Z'." }, "job_name": { "type": "string" }, "job_namespace": { "type": "string" }, "inputs": { "type": "array", "description": "List of input datasets.", "items": { "type": "object", "properties": { "namespace": { "type": "string" }, "name": { "type": "string" } } }, "nullable": true }, "outputs": { "type": "array", "description": "List of output datasets.", "items": { "type": "object", "properties": { "namespace": { "type": "string" }, "name": { "type": "string" } } }, "nullable": true }, "event_time": { "type": "string", "format": "date-time", "description": "Event occurrence time (with offset)." }, "event_schema": { "type": "string", "format": "uri" }, "event_schema_version": { "type": "string" }, "event_size_in_bytes": { "type": "integer", "description": "Size of the raw event payload in bytes." }, "event_raw": { "$ref": "#/components/schemas/OpenLineageEvent", "description": "Arbitrary structure corresponding to rawLineageEventSchema." }, "produced_by": { "type": "string", "format": "uri" }, "integration": { "type": "string" }, "integration_version": { "type": "string" } }, "required": [ "id", "run_id", "run_state", "received_at", "job_name", "job_namespace", "event_time", "event_schema", "event_schema_version", "event_size_in_bytes", "event_raw", "produced_by", "integration", "integration_version" ] }, "EngineDecision": { "type": "object", "description": "How the query router chose the engine and machine for this run.", "properties": { "engine": { "type": "string", "enum": [ "duckdb", "polars", "bloom", "spark" ], "example": "bloom" }, "execution_mode": { "type": "string", "enum": [ "interactive", "async" ], "example": "interactive" }, "reasons": { "type": "array", "items": { "type": "string" }, "description": "Ordered trail of the rules that produced this decision.", "example": [ "not a script", "no external connection tables", "not DDL", "no destination table", "estimated input 1.2 GiB is below the distributed threshold", "sandbox 2 vCPU / 4 GB" ] }, "size_band": { "type": "string", "nullable": true, "example": "small" }, "estimated_input_bytes": { "type": "string", "nullable": true, "description": "Estimated input size from Iceberg snapshot metadata, as a string to preserve precision.", "example": "1288490188" }, "vcpus": { "type": "integer", "nullable": true, "example": 2 }, "sandbox": { "type": "object", "nullable": true, "properties": { "vcpus": { "type": "integer", "example": 2 }, "memory_gb": { "type": "integer", "nullable": true, "example": 4 } } }, "distributed": { "type": "boolean", "default": false }, "worker_machine_type": { "type": "string", "nullable": true, "example": "bloom.2.b" }, "worker_count": { "type": "integer", "nullable": true, "example": 4 }, "input_tables": { "type": "array", "items": { "type": "string" }, "example": [ "oleander.default.events" ] } } }, "UnifiedQueryResponse": { "type": "object", "properties": { "success": { "type": "boolean", "example": true }, "explain": { "type": "boolean", "description": "Present and true when `explain` was requested. No compute was spent and no other result fields are returned." }, "engine_decision": { "$ref": "#/components/schemas/EngineDecision" }, "results": { "type": "object", "nullable": true, "description": "Rows, for interactive reads only. Absent on submitted runs.", "properties": { "columns": { "type": "array", "items": { "type": "string" } }, "column_types": { "type": "array", "items": { "type": "string" } }, "rows": { "type": "array", "items": { "type": "array", "items": {} } } } }, "row_count": { "type": "integer", "nullable": true }, "execution_time": { "type": "string", "nullable": true, "example": "42ms" }, "state": { "type": "string", "enum": [ "COMPLETE", "SUBMITTED" ], "description": "COMPLETE means the write already landed. SUBMITTED means a job is running and `run_id` must be polled." }, "run_id": { "type": "string", "nullable": true, "description": "Run to poll when `state` is SUBMITTED." }, "output_table": { "type": "string", "nullable": true, "description": "Fully qualified destination table for a write." }, "compute": { "type": "object", "nullable": true, "description": "The sandbox or cluster that actually served the run. A warm larger sandbox can serve a smaller request, and that is the machine the org is billed for." }, "job": { "type": "object", "description": "Lineage job identity for this query.", "properties": { "namespace": { "type": "string", "example": "oleander.lake" }, "name": { "type": "string", "example": "query_9f2c1ab4" } } } } } }, "securitySchemes": { "bearerAuth": { "type": "http", "scheme": "bearer" } } } }