test: characterize correlated NOT IN null-aware behaviour

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q05.benchmark

@@ -3,6 +3,22 @@ group null_aware_join
 
 load sql_benchmarks/null_aware_join/init/load.sql
 
+# Correctness canary: the NOT IN result must match a reference count
+# that does not use NOT IN. It holds for every NAJ_ROWS / NAJ_LARGE_ROWS.
+# As Q04, with NULL outer keys excluded unless the subquery is empty.
+assert I
+SELECT count(*) = (
+  SELECT count(*) FROM small_outer o
+  WHERE o.z <= (SELECT min(z) FROM small_inner)
+     OR (o.id_n1 IS NOT NULL AND NOT (o.id % 2 = 0 AND (o.id / 2) % 1000 < o.z))
+)
+FROM small_outer o
+WHERE o.id_n1 NOT IN (SELECT i.id_n0 FROM small_inner i WHERE i.z < o.z);
+----
+# Pinned to today's behaviour, which is incorrect. See
+# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
+false
+
 expect_plan HashJoinExec
 expect_plan null_aware
 

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q06.benchmark

@@ -3,6 +3,22 @@ group null_aware_join
 
 load sql_benchmarks/null_aware_join/init/load.sql
 
+# Correctness canary: the NOT IN result must match a reference count
+# that does not use NOT IN. It holds for every NAJ_ROWS / NAJ_LARGE_ROWS.
+# As Q04, with NULL outer keys excluded unless the subquery is empty.
+assert I
+SELECT count(*) = (
+  SELECT count(*) FROM small_outer o
+  WHERE o.z <= (SELECT min(z) FROM small_inner)
+     OR (o.id_n50 IS NOT NULL AND NOT (o.id % 2 = 0 AND (o.id / 2) % 1000 < o.z))
+)
+FROM small_outer o
+WHERE o.id_n50 NOT IN (SELECT i.id_n0 FROM small_inner i WHERE i.z < o.z);
+----
+# Pinned to today's behaviour, which is incorrect. See
+# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
+false
+
 expect_plan HashJoinExec
 expect_plan null_aware
 

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q07.benchmark

@@ -3,6 +3,24 @@ group null_aware_join
 
 load sql_benchmarks/null_aware_join/init/load.sql
 
+# Correctness canary: the NOT IN result must match a reference count
+# that does not use NOT IN. It holds for every NAJ_ROWS / NAJ_LARGE_ROWS.
+# A row is TRUE when the subquery is empty, or when the subquery holds no NULL
+# and the key is not in it. A NULL is in scope when its z is below o.z.
+assert I
+SELECT count(*) = (
+  SELECT count(*) FROM small_outer o
+  WHERE o.z <= (SELECT min(z) FROM small_inner)
+     OR (o.z <= (SELECT min(z) FROM small_inner WHERE id_n50 IS NULL)
+         AND NOT (o.id % 2 = 0 AND (o.id / 2) % 1000 < o.z))
+)
+FROM small_outer o
+WHERE o.id_n0 NOT IN (SELECT i.id_n50 FROM small_inner i WHERE i.z < o.z);
+----
+# Pinned to today's behaviour, which is incorrect. See
+# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
+false
+
 expect_plan HashJoinExec
 expect_plan null_aware
 

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark

@@ -3,6 +3,30 @@ group null_aware_join
 
 load sql_benchmarks/null_aware_join/init/load.sql
 
+# Correctness canary: the NOT IN result must match a reference count
+# that does not use NOT IN. It holds for every NAJ_ROWS / NAJ_LARGE_ROWS.
+# A row is TRUE when o.z > 900, when the subquery for its k is empty, or when
+# its key is not NULL and not in that subquery.
+assert I
+SELECT count(*) = (
+  SELECT count(*)
+  FROM small_outer o
+  JOIN (SELECT k, min(z) AS min_z FROM small_inner GROUP BY k) m ON m.k = o.k
+  WHERE o.z > 900
+     OR o.z <= m.min_z
+     OR (o.id_n50 IS NOT NULL
+         AND NOT (o.id % 2 = 0 AND (o.id / 2) % 16 = o.k AND (o.id / 2) % 1000 < o.z))
+)
+FROM small_outer o
+WHERE o.z > 900
+   OR o.id_n50 NOT IN (
+        SELECT i.id_n0 FROM small_inner i WHERE i.k = o.k AND i.z < o.z
+      );
+----
+# Pinned to today's behaviour, which is incorrect. See
+# https://github.com/apache/datafusion/issues/25336 -- the fix flips this.
+false
+
 expect_plan HashJoinExec
 
 run

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q09.benchmark

@@ -0,0 +1,45 @@
+name Q09
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+# Correctness canary: the positive IN result must match a reference count that
+# does not use IN. It holds for every NAJ_ROWS. A row is TRUE when o.z > 900 or
+# when some in-scope subquery row equals its key; a NULL mark and a FALSE mark
+# keep the same rows here, which is what makes the plain mark join correct.
+assert I
+SELECT count(*) = (
+  SELECT count(*)
+  FROM small_outer o
+  WHERE o.z > 900
+     OR EXISTS (
+          SELECT 1 FROM small_inner i
+          WHERE i.k = o.k AND i.z < o.z AND i.id_n0 = o.id_n50
+        )
+)
+FROM small_outer o
+WHERE o.z > 900
+   OR o.id_n50 IN (
+        SELECT i.id_n0 FROM small_inner i WHERE i.k = o.k AND i.z < o.z
+      );
+----
+true
+
+expect_plan HashJoinExec
+
+run
+-- Q9: the same correlated shape as Q08, but a NON-negated IN. A Filter keeps a
+-- row only when the predicate is TRUE, and AND/OR make TRUE only out of TRUE,
+-- so a NULL mark behaves exactly like a FALSE mark and the mark join must NOT
+-- be null-aware. Every other query in this suite covers the direction where
+-- null-aware handling is required; this one covers the direction where taking
+-- it anyway is pure cost. Widening the "needs null-aware" test until this shape
+-- is included does not change any result, so only this timing shows it.
+SELECT count(*)
+FROM small_outer o
+WHERE o.z > 900
+   OR o.id_n50 IN (
+        SELECT i.id_n0 FROM small_inner i WHERE i.k = o.k AND i.z < o.z
+      );
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite

@@ -1,4 +1,4 @@
-description = "Null-aware (NOT IN) hash join benchmarks: uncorrelated, non-equality-correlated, and equality-correlated NOT IN across NULL fractions"
+description = "Null-aware (NOT IN) hash join benchmarks: uncorrelated, non-equality-correlated, and equality-correlated NOT IN across NULL fractions, plus the positive IN shape that must not be null-aware"
 
 query_pattern = "q{QUERY_ID_PADDED}.benchmark"
 
@@ -8,7 +8,7 @@ short = "r"
 env = "NAJ_ROWS"
 default = "10000"
 values = ["10000", "..."]
-help = "Rows per table for the correlated NOT IN queries (Q04-Q08). These evaluate the join filter over candidate build x probe pairs, so their cost grows with the square of this value."
+help = "Rows per table for the correlated queries (Q04-Q09). These evaluate the join filter over candidate build x probe pairs, so their cost grows with the square of this value."
 
 [[options]]
 name = "large-rows"