From 8a4b5796e826dad31ed7033d29faf094487d66bd Mon Sep 17 00:00:00 2001 From: Chunchun <14298407+appletreeisyellow@users.noreply.github.com> Date: Tue, 30 Jan 2024 15:08:00 -0600 Subject: [PATCH 1/4] test(placeholder): add positive test --- datafusion/core/tests/sql/select.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs index 667d3eeab31ed..c27ba73df1f86 100644 --- a/datafusion/core/tests/sql/select.rs +++ b/datafusion/core/tests/sql/select.rs @@ -251,3 +251,29 @@ async fn test_parameter_invalid_types() -> Result<()> { ); Ok(()) } + +#[tokio::test] +async fn test_parameter_empty_table_with_param_values() -> Result<()> { + let ctx = SessionContext::new(); + + let df = ctx + .read_empty() + .unwrap() + .with_column("a", lit(1)) + .unwrap() + .filter(col("a").eq(placeholder("$0"))) + .unwrap() + .with_param_values(vec![("0", ScalarValue::from(3i32))]) + .unwrap(); + + let logical_plan = df.logical_plan(); + assert_eq!( + "Filter: a = Int32(3)\n Projection: Int32(1) AS a\n EmptyRelation", + logical_plan.display_indent().to_string() + ); + + let optimized_plan = ctx.state().optimize(logical_plan).unwrap(); + assert_eq!("EmptyRelation", optimized_plan.display_indent().to_string()); + + Ok(()) +} From 5696528d4e787499c2c51cffdd221d67b9f4c40d Mon Sep 17 00:00:00 2001 From: Chunchun <14298407+appletreeisyellow@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:40:38 -0600 Subject: [PATCH 2/4] test(placeholder): add negative test and update error message --- datafusion/core/tests/sql/select.rs | 40 ++++++++++++++++++++++++++--- datafusion/expr/src/expr_schema.rs | 2 +- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs index c27ba73df1f86..3b82208d6df37 100644 --- a/datafusion/core/tests/sql/select.rs +++ b/datafusion/core/tests/sql/select.rs @@ -266,14 +266,48 @@ async fn test_parameter_empty_table_with_param_values() -> Result<()> { .with_param_values(vec![("0", ScalarValue::from(3i32))]) .unwrap(); + let logical_plan = df.logical_plan(); + let expected = r#" +Filter: a = Int32(3) + Projection: Int32(1) AS a + EmptyRelation + "# + .trim(); + assert_eq!(expected, logical_plan.display_indent().to_string()); + + let optimized_plan = ctx.state().optimize(logical_plan).unwrap(); + assert_eq!("EmptyRelation", optimized_plan.display_indent().to_string()); + + Ok(()) +} + +#[tokio::test] +async fn test_parameter_empty_table_missing_param_values() -> Result<()> { + let ctx = SessionContext::new(); + + let df = ctx + .read_empty() + .unwrap() + .with_column("a", lit(1)) + .unwrap() + .filter(col("a").eq(placeholder("$0"))) + .unwrap(); + let logical_plan = df.logical_plan(); assert_eq!( - "Filter: a = Int32(3)\n Projection: Int32(1) AS a\n EmptyRelation", + "Filter: a = $0\n Projection: Int32(1) AS a\n EmptyRelation", logical_plan.display_indent().to_string() ); - let optimized_plan = ctx.state().optimize(logical_plan).unwrap(); - assert_eq!("EmptyRelation", optimized_plan.display_indent().to_string()); + // The placeholder is not replaced with a value, + // so the filter data type is not know, i.e. a = $0. + // Therefore, the optimization fails. + let optimized_plan = ctx.state().optimize(logical_plan); + assert!(optimized_plan.is_err()); + assert!(optimized_plan + .unwrap_err() + .to_string() + .contains("Placeholder type could not be resolved. Make sure that the placeholder is bound to a concrete type, e.g. by providing parameter values.")); Ok(()) } diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs index 4967e66fed40a..865279ab9aaff 100644 --- a/datafusion/expr/src/expr_schema.rs +++ b/datafusion/expr/src/expr_schema.rs @@ -159,7 +159,7 @@ impl ExprSchemable for Expr { Expr::Like { .. } | Expr::SimilarTo { .. } => Ok(DataType::Boolean), Expr::Placeholder(Placeholder { data_type, .. }) => { data_type.clone().ok_or_else(|| { - plan_datafusion_err!("Placeholder type could not be resolved") + plan_datafusion_err!("Placeholder type could not be resolved. Make sure that the placeholder is bound to a concrete type, e.g. by providing parameter values.") }) } Expr::Wildcard { qualifier } => { From 1544aaa5c796c61520a792191dd2a15eb2243971 Mon Sep 17 00:00:00 2001 From: Chunchun <14298407+appletreeisyellow@users.noreply.github.com> Date: Tue, 30 Jan 2024 17:17:15 -0600 Subject: [PATCH 3/4] test(sql_integration): update error message --- datafusion/sql/tests/sql_integration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index 9e6d46a7b3ee9..3c72bf334e7f4 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -465,7 +465,7 @@ Dml: op=[Insert Into] table=[test_decimal] )] #[case::placeholder_type_unresolved( "INSERT INTO person (id, first_name, last_name) VALUES ($2, $4, $6)", - "Error during planning: Placeholder type could not be resolved" + "Error during planning: Placeholder type could not be resolved. Make sure that the placeholder is bound to a concrete type, e.g. by providing parameter values." )] #[case::placeholder_type_unresolved( "INSERT INTO person (id, first_name, last_name) VALUES ($id, $first_name, $last_name)", From 3ad62dcc340f98fc4f3d7b69759c345571b794ae Mon Sep 17 00:00:00 2001 From: Chunchun <14298407+appletreeisyellow@users.noreply.github.com> Date: Wed, 31 Jan 2024 13:08:56 -0600 Subject: [PATCH 4/4] test(placeholder): move test from sql to dataframe chore: add positive test and remove println --- datafusion/core/tests/dataframe/mod.rs | 74 +++++++++++++++++++++++++- datafusion/core/tests/sql/select.rs | 60 --------------------- 2 files changed, 72 insertions(+), 62 deletions(-) diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs index 89ab04dfee89c..dc347ed9c8765 100644 --- a/datafusion/core/tests/dataframe/mod.rs +++ b/datafusion/core/tests/dataframe/mod.rs @@ -45,8 +45,9 @@ use datafusion_execution::runtime_env::RuntimeEnv; use datafusion_expr::expr::{GroupingSet, Sort}; use datafusion_expr::{ array_agg, avg, cast, col, count, exists, expr, in_subquery, lit, max, out_ref_col, - scalar_subquery, sum, when, wildcard, AggregateFunction, Expr, ExprSchemable, - WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, + placeholder, scalar_subquery, sum, when, wildcard, AggregateFunction, Expr, + ExprSchemable, WindowFrame, WindowFrameBound, WindowFrameUnits, + WindowFunctionDefinition, }; use datafusion_physical_expr::var_provider::{VarProvider, VarType}; @@ -1759,3 +1760,72 @@ async fn test_array_agg() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn test_dataframe_placeholder_missing_param_values() -> Result<()> { + let ctx = SessionContext::new(); + + let df = ctx + .read_empty() + .unwrap() + .with_column("a", lit(1)) + .unwrap() + .filter(col("a").eq(placeholder("$0"))) + .unwrap(); + + let logical_plan = df.logical_plan(); + let formatted = logical_plan.display_indent_schema().to_string(); + let actual: Vec<&str> = formatted.trim().lines().collect(); + let expected = vec![ + "Filter: a = $0 [a:Int32]", + " Projection: Int32(1) AS a [a:Int32]", + " EmptyRelation []", + ]; + assert_eq!( + expected, actual, + "\n\nexpected:\n\n{expected:#?}\nactual:\n\n{actual:#?}\n\n" + ); + + // The placeholder is not replaced with a value, + // so the filter data type is not know, i.e. a = $0. + // Therefore, the optimization fails. + let optimized_plan = ctx.state().optimize(logical_plan); + assert!(optimized_plan.is_err()); + assert!(optimized_plan + .unwrap_err() + .to_string() + .contains("Placeholder type could not be resolved. Make sure that the placeholder is bound to a concrete type, e.g. by providing parameter values.")); + + // Prodiving a parameter value should resolve the error + let df = ctx + .read_empty() + .unwrap() + .with_column("a", lit(1)) + .unwrap() + .filter(col("a").eq(placeholder("$0"))) + .unwrap() + .with_param_values(vec![("0", ScalarValue::from(3i32))]) // <-- provide parameter value + .unwrap(); + + let logical_plan = df.logical_plan(); + let formatted = logical_plan.display_indent_schema().to_string(); + let actual: Vec<&str> = formatted.trim().lines().collect(); + let expected = vec![ + "Filter: a = Int32(3) [a:Int32]", + " Projection: Int32(1) AS a [a:Int32]", + " EmptyRelation []", + ]; + assert_eq!( + expected, actual, + "\n\nexpected:\n\n{expected:#?}\nactual:\n\n{actual:#?}\n\n" + ); + + let optimized_plan = ctx.state().optimize(logical_plan); + assert!(optimized_plan.is_ok()); + + let actual = optimized_plan.unwrap().display_indent_schema().to_string(); + let expected = "EmptyRelation [a:Int32]"; + assert_eq!(expected, actual); + + Ok(()) +} diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs index 3b82208d6df37..667d3eeab31ed 100644 --- a/datafusion/core/tests/sql/select.rs +++ b/datafusion/core/tests/sql/select.rs @@ -251,63 +251,3 @@ async fn test_parameter_invalid_types() -> Result<()> { ); Ok(()) } - -#[tokio::test] -async fn test_parameter_empty_table_with_param_values() -> Result<()> { - let ctx = SessionContext::new(); - - let df = ctx - .read_empty() - .unwrap() - .with_column("a", lit(1)) - .unwrap() - .filter(col("a").eq(placeholder("$0"))) - .unwrap() - .with_param_values(vec![("0", ScalarValue::from(3i32))]) - .unwrap(); - - let logical_plan = df.logical_plan(); - let expected = r#" -Filter: a = Int32(3) - Projection: Int32(1) AS a - EmptyRelation - "# - .trim(); - assert_eq!(expected, logical_plan.display_indent().to_string()); - - let optimized_plan = ctx.state().optimize(logical_plan).unwrap(); - assert_eq!("EmptyRelation", optimized_plan.display_indent().to_string()); - - Ok(()) -} - -#[tokio::test] -async fn test_parameter_empty_table_missing_param_values() -> Result<()> { - let ctx = SessionContext::new(); - - let df = ctx - .read_empty() - .unwrap() - .with_column("a", lit(1)) - .unwrap() - .filter(col("a").eq(placeholder("$0"))) - .unwrap(); - - let logical_plan = df.logical_plan(); - assert_eq!( - "Filter: a = $0\n Projection: Int32(1) AS a\n EmptyRelation", - logical_plan.display_indent().to_string() - ); - - // The placeholder is not replaced with a value, - // so the filter data type is not know, i.e. a = $0. - // Therefore, the optimization fails. - let optimized_plan = ctx.state().optimize(logical_plan); - assert!(optimized_plan.is_err()); - assert!(optimized_plan - .unwrap_err() - .to_string() - .contains("Placeholder type could not be resolved. Make sure that the placeholder is bound to a concrete type, e.g. by providing parameter values.")); - - Ok(()) -}