import asyncio
from oleander_sdk import (
Oleander,
QuerySubmitOptions,
SubmitSparkJobAndWaitOptions,
)
async def main():
client = Oleander()
# oleander picks the engine and machine size
result = await client.query_run(
"SELECT * FROM oleander.default.flowers LIMIT 10"
)
for row in result.results.rows:
sepal_length, sepal_width = row[0], row[1]
# process each row ...
# Write a result to a table
submitted = await client.query_submit(
QuerySubmitOptions(
sql="SELECT day, count(*) AS n FROM oleander.default.events GROUP BY day",
destination="default.daily_counts",
)
)
jobs = await client.list_spark_jobs()
print([artifact.name for artifact in jobs.artifacts])
run = await client.submit_spark_job_and_wait(
SubmitSparkJobAndWaitOptions(
namespace="my-namespace",
name="daily-etl",
entrypoint="etl_pipeline.py",
)
)
if run.state != "COMPLETE":
raise Exception(f"Run {run.run_id} ended with state: {run.state}")
asyncio.run(main())