From f18cb93bd759666fe94f2fc08d7ec4f3f7014e2e Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Mon, 29 Jan 2024 16:59:42 +0300 Subject: [PATCH 1/9] Add unbounded table creation. --- docs/source/user-guide/sql/ddl.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/source/user-guide/sql/ddl.md b/docs/source/user-guide/sql/ddl.md index b67d323126996..bd362ab357a0f 100644 --- a/docs/source/user-guide/sql/ddl.md +++ b/docs/source/user-guide/sql/ddl.md @@ -56,7 +56,7 @@ file system or remote object store as a named table which can be queried. The supported syntax is: ``` -CREATE EXTERNAL TABLE +CREATE [UNBOUNDED] EXTERNAL TABLE [ IF NOT EXISTS ] [ () ] STORED AS @@ -147,6 +147,16 @@ WITH HEADER ROW LOCATION '/path/to/directory/of/files'; ``` + +With `CREATE UNBOUNDED EXTERNAL TABLE` SQL statement. We can create unbounded data sources such as following: + +```sql +CREATE UNBOUNDED EXTERNAL TABLE taxi +STORED AS PARQUET +LOCATION '/mnt/nyctaxi/tripdata.parquet'; +``` +Datafusion tries to execute queries that refer to unbounded sources in streaming fashion. If this is not possible according to query specifications, datafusion plan generation fails stating it is not possible to execute given query in streaming fashion. + When creating an output from a data source that is already ordered by an expression, you can pre-specify the order of the data using the `WITH ORDER` clause. This applies even if the expression used for From 17cfe36a52903ccd32f96d63a66edfe8c0edc943 Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Mon, 29 Jan 2024 17:01:24 +0300 Subject: [PATCH 2/9] Update comment --- docs/source/user-guide/sql/ddl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/sql/ddl.md b/docs/source/user-guide/sql/ddl.md index bd362ab357a0f..f42cbbae833cf 100644 --- a/docs/source/user-guide/sql/ddl.md +++ b/docs/source/user-guide/sql/ddl.md @@ -155,7 +155,7 @@ CREATE UNBOUNDED EXTERNAL TABLE taxi STORED AS PARQUET LOCATION '/mnt/nyctaxi/tripdata.parquet'; ``` -Datafusion tries to execute queries that refer to unbounded sources in streaming fashion. If this is not possible according to query specifications, datafusion plan generation fails stating it is not possible to execute given query in streaming fashion. +Datafusion tries to execute queries that refer to unbounded sources in streaming fashion. If this is not possible according to query specifications, datafusion plan generation fails stating it is not possible to execute given query in streaming fashion. Please note that queries that can be executed with unbounded sources (e.g. in streaming mode) are a subset of the bounded sources. A query that fail with unbounded source may work in bounded source. When creating an output from a data source that is already ordered by an expression, you can pre-specify the order of the data using the From 06683d74f5267412fe365e27106c3a59c9291d97 Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Tue, 30 Jan 2024 09:17:23 +0300 Subject: [PATCH 3/9] Add new streaming test cases --- .../examples/csv_sql_streaming.rs | 73 +++++++++++++++++++ datafusion/common/src/test_util.rs | 20 +++++ 2 files changed, 93 insertions(+) create mode 100644 datafusion-examples/examples/csv_sql_streaming.rs diff --git a/datafusion-examples/examples/csv_sql_streaming.rs b/datafusion-examples/examples/csv_sql_streaming.rs new file mode 100644 index 0000000000000..ad34dd4ca48a6 --- /dev/null +++ b/datafusion-examples/examples/csv_sql_streaming.rs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion::common::test_util::datafusion_test_data; +use datafusion::error::Result; +use datafusion::prelude::*; +use datafusion_expr::expr::Sort; + +/// This example demonstrates executing a simple query against an Arrow data source (CSV) and +/// fetching results with streaming aggregation and streaming window +#[tokio::main] +async fn main() -> Result<()> { + // create local execution context + let ctx = SessionContext::new(); + + let testdata = datafusion_test_data(); + + // File is ordered by ts ASC. This is an invariant, it is the responsibility of the user to make sure + // that file indeed satisfies this condition. + let sort_expr = vec![Expr::Sort(Sort { + expr: Box::new(Expr::Column(Column::from_name("ts"))), + asc: true, + nulls_first: true, + })]; + // register csv file with the execution context + ctx.register_csv( + "ordered_table", + &format!("{testdata}/window_1.csv"), + CsvReadOptions::new().file_sort_order(vec![sort_expr]), + ) + .await?; + + // execute the query + // Following query can be executed with unbounded sources because group by expressions (e.g ts) is + // already ordered at the source. + let df = ctx + .sql( + "SELECT ts, MIN(inc_col), MAX(inc_col) \ + FROM ordered_table \ + GROUP BY ts", + ) + .await?; + + df.show().await?; + + // execute the query + // Following query can be executed with unbounded sources because window executor can calculate + // its result in streaming, when its required ordering is already satisfied at the source. + let df = ctx + .sql( + "SELECT ts, SUM(inc_col) OVER(ORDER BY ts ASC) \ + FROM ordered_table", + ) + .await?; + + df.show().await?; + + Ok(()) +} diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rs index eeace97eebfa3..239711186cadd 100644 --- a/datafusion/common/src/test_util.rs +++ b/datafusion/common/src/test_util.rs @@ -144,6 +144,26 @@ macro_rules! assert_not_contains { }; } +/// Returns the arrow test data directory, which is by default rooted at `datafusion/core/tests/data`. +/// +/// The default can be overridden by the optional environment +/// variable `DATAFUSION_TEST_DATA` +/// +/// panics when the directory can not be found. +/// +/// Example: +/// ``` +/// let testdata = datafusion_common::test_util::datafusion_test_data(); +/// let csvdata = format!("{}/window_1.csv.csv", testdata); +/// assert!(std::path::PathBuf::from(csvdata).exists()); +/// ``` +pub fn datafusion_test_data() -> String { + match get_data_dir("DATAFUSION_TEST_DATA", "../../datafusion/core/tests/data") { + Ok(pb) => pb.display().to_string(), + Err(err) => panic!("failed to get arrow data dir: {err}"), + } +} + /// Returns the arrow test data directory, which is by default stored /// in a git submodule rooted at `testing/data`. /// From c28a192cd5aee2a963ffd98da4285c0841485f97 Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Tue, 30 Jan 2024 16:04:38 +0300 Subject: [PATCH 4/9] Minor changes --- docs/source/user-guide/sql/ddl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/sql/ddl.md b/docs/source/user-guide/sql/ddl.md index f42cbbae833cf..947083b2257a9 100644 --- a/docs/source/user-guide/sql/ddl.md +++ b/docs/source/user-guide/sql/ddl.md @@ -147,7 +147,6 @@ WITH HEADER ROW LOCATION '/path/to/directory/of/files'; ``` - With `CREATE UNBOUNDED EXTERNAL TABLE` SQL statement. We can create unbounded data sources such as following: ```sql @@ -155,6 +154,7 @@ CREATE UNBOUNDED EXTERNAL TABLE taxi STORED AS PARQUET LOCATION '/mnt/nyctaxi/tripdata.parquet'; ``` + Datafusion tries to execute queries that refer to unbounded sources in streaming fashion. If this is not possible according to query specifications, datafusion plan generation fails stating it is not possible to execute given query in streaming fashion. Please note that queries that can be executed with unbounded sources (e.g. in streaming mode) are a subset of the bounded sources. A query that fail with unbounded source may work in bounded source. When creating an output from a data source that is already ordered by From 1b7e89f42f1df8e013d62adfdf84f8fa55742aa1 Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Tue, 30 Jan 2024 16:13:07 +0300 Subject: [PATCH 5/9] Fix doctest --- datafusion/common/src/test_util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rs index 239711186cadd..e33e640e00092 100644 --- a/datafusion/common/src/test_util.rs +++ b/datafusion/common/src/test_util.rs @@ -154,7 +154,7 @@ macro_rules! assert_not_contains { /// Example: /// ``` /// let testdata = datafusion_common::test_util::datafusion_test_data(); -/// let csvdata = format!("{}/window_1.csv.csv", testdata); +/// let csvdata = format!("{}/window_1.csv", testdata); /// assert!(std::path::PathBuf::from(csvdata).exists()); /// ``` pub fn datafusion_test_data() -> String { From 997d77df6a4bd360d9cca3b5ce2d97769ecbf9f4 Mon Sep 17 00:00:00 2001 From: Mustafa Akur <106137913+mustafasrepo@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:48:59 +0300 Subject: [PATCH 6/9] Update docs/source/user-guide/sql/ddl.md Co-authored-by: Mehmet Ozan Kabak --- docs/source/user-guide/sql/ddl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/sql/ddl.md b/docs/source/user-guide/sql/ddl.md index 947083b2257a9..81e62efbeb25e 100644 --- a/docs/source/user-guide/sql/ddl.md +++ b/docs/source/user-guide/sql/ddl.md @@ -155,7 +155,7 @@ STORED AS PARQUET LOCATION '/mnt/nyctaxi/tripdata.parquet'; ``` -Datafusion tries to execute queries that refer to unbounded sources in streaming fashion. If this is not possible according to query specifications, datafusion plan generation fails stating it is not possible to execute given query in streaming fashion. Please note that queries that can be executed with unbounded sources (e.g. in streaming mode) are a subset of the bounded sources. A query that fail with unbounded source may work in bounded source. +Note that this statement actually reads data from a fixed-size file, so a better example would involve reading from a FIFO file. Nevertheless, once Datafusion sees the `UNBOUNDED` keyword in a data source, it tries to execute queries that refer to this unbounded source in streaming fashion. If this is not possible according to query specifications, plan generation fails stating it is not possible to execute given query in streaming fashion. Note that queries that can run with unbounded sources (i.e. in streaming mode) are a subset of those that can with bounded sources. A query that fails with unbounded source(s) may work with bounded source(s). When creating an output from a data source that is already ordered by an expression, you can pre-specify the order of the data using the From 620e55d9d32daf7fc3d0a1ada755d707e3e5ef81 Mon Sep 17 00:00:00 2001 From: Mustafa Akur <106137913+mustafasrepo@users.noreply.github.com> Date: Wed, 31 Jan 2024 09:04:18 +0300 Subject: [PATCH 7/9] Comment Updates, minor changes Co-authored-by: Andrew Lamb --- .../examples/csv_sql_streaming.rs | 18 ++++++++++-------- datafusion/common/src/test_util.rs | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/datafusion-examples/examples/csv_sql_streaming.rs b/datafusion-examples/examples/csv_sql_streaming.rs index ad34dd4ca48a6..b1676b4f42aff 100644 --- a/datafusion-examples/examples/csv_sql_streaming.rs +++ b/datafusion-examples/examples/csv_sql_streaming.rs @@ -29,13 +29,12 @@ async fn main() -> Result<()> { let testdata = datafusion_test_data(); - // File is ordered by ts ASC. This is an invariant, it is the responsibility of the user to make sure - // that file indeed satisfies this condition. - let sort_expr = vec![Expr::Sort(Sort { - expr: Box::new(Expr::Column(Column::from_name("ts"))), - asc: true, - nulls_first: true, - })]; + // Register a table source and tell DataFusion the file is ordered by `ts ASC`. + // Note it is the responsibility of the user to make sure + // that file indeed satisfies this condition or else incorrect answers may be produced. + let asc = true; + let nulls_first = true; + let sort_expr = vec![col("ts").sort(asc, nulls_first)]; // register csv file with the execution context ctx.register_csv( "ordered_table", @@ -47,6 +46,9 @@ async fn main() -> Result<()> { // execute the query // Following query can be executed with unbounded sources because group by expressions (e.g ts) is // already ordered at the source. + // + // Unbounded sources means that if the input came from a "never ending" source (such as a FIFO + // file on unix) the query could produce results incrementally as data was read. let df = ctx .sql( "SELECT ts, MIN(inc_col), MAX(inc_col) \ @@ -59,7 +61,7 @@ async fn main() -> Result<()> { // execute the query // Following query can be executed with unbounded sources because window executor can calculate - // its result in streaming, when its required ordering is already satisfied at the source. + // its result in streaming fashion, because its required ordering is already satisfied at the source. let df = ctx .sql( "SELECT ts, SUM(inc_col) OVER(ORDER BY ts ASC) \ diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rs index e33e640e00092..9c3dfe62e119a 100644 --- a/datafusion/common/src/test_util.rs +++ b/datafusion/common/src/test_util.rs @@ -144,7 +144,7 @@ macro_rules! assert_not_contains { }; } -/// Returns the arrow test data directory, which is by default rooted at `datafusion/core/tests/data`. +/// Returns the datafusion test data directory, which is by default rooted at `datafusion/core/tests/data`. /// /// The default can be overridden by the optional environment /// variable `DATAFUSION_TEST_DATA` From c65268185fcf42bb0c9749048131f74b70471421 Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Wed, 31 Jan 2024 09:07:53 +0300 Subject: [PATCH 8/9] Add link of streaming example to readme page --- datafusion-examples/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md index 298ee9364efef..053ae8d1b26e6 100644 --- a/datafusion-examples/README.md +++ b/datafusion-examples/README.md @@ -44,6 +44,7 @@ cargo run --example csv_sql - [`avro_sql.rs`](examples/avro_sql.rs): Build and run a query plan from a SQL statement against a local AVRO file - [`csv_sql.rs`](examples/csv_sql.rs): Build and run a query plan from a SQL statement against a local CSV file +- [`csv_sql_streaming.rs`](examples/csv_sql_streaming.rs): Build and run a streaming query plan from a SQL statement against a local CSV file - [`catalog.rs`](examples/external_dependency/catalog.rs): Register the table into a custom catalog - [`custom_datasource.rs`](examples/custom_datasource.rs): Run queries against a custom datasource (TableProvider) - [`dataframe.rs`](examples/dataframe.rs): Run a query using a DataFrame against a local parquet file From cc1ad9afc90ef5895bddfad9a2c24a0aeb602420 Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Wed, 31 Jan 2024 09:09:41 +0300 Subject: [PATCH 9/9] Fix formatting --- datafusion-examples/examples/csv_sql_streaming.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datafusion-examples/examples/csv_sql_streaming.rs b/datafusion-examples/examples/csv_sql_streaming.rs index b1676b4f42aff..99264bbcb486d 100644 --- a/datafusion-examples/examples/csv_sql_streaming.rs +++ b/datafusion-examples/examples/csv_sql_streaming.rs @@ -18,7 +18,6 @@ use datafusion::common::test_util::datafusion_test_data; use datafusion::error::Result; use datafusion::prelude::*; -use datafusion_expr::expr::Sort; /// This example demonstrates executing a simple query against an Arrow data source (CSV) and /// fetching results with streaming aggregation and streaming window @@ -29,10 +28,10 @@ async fn main() -> Result<()> { let testdata = datafusion_test_data(); - // Register a table source and tell DataFusion the file is ordered by `ts ASC`. + // Register a table source and tell DataFusion the file is ordered by `ts ASC`. // Note it is the responsibility of the user to make sure // that file indeed satisfies this condition or else incorrect answers may be produced. - let asc = true; + let asc = true; let nulls_first = true; let sort_expr = vec![col("ts").sort(asc, nulls_first)]; // register csv file with the execution context @@ -48,7 +47,7 @@ async fn main() -> Result<()> { // already ordered at the source. // // Unbounded sources means that if the input came from a "never ending" source (such as a FIFO - // file on unix) the query could produce results incrementally as data was read. + // file on unix) the query could produce results incrementally as data was read. let df = ctx .sql( "SELECT ts, MIN(inc_col), MAX(inc_col) \