fix: fix non-null constraint

src/query/service/src/pipelines/processors/transforms/transform_resort_addon_without_source_schema.rs

@@ -15,6 +15,7 @@
 use std::sync::Arc;
 
 use common_catalog::table_context::TableContext;
+use common_exception::ErrorCode;
 use common_exception::Result;
 use common_expression::BlockMetaInfoDowncast;
 use common_expression::DataBlock;
@@ -61,6 +62,22 @@ pub fn build_expression_transform(
                 }
                 expr
             } else {
+                // #issue13932
+                // if there is a non-null constraint, we should return an error
+                // although we will give a valid default value. for example:
+                // 1. (a int not null), we give zero
+                // 2. (a int), we give null
+                // but for pg or snowflake, if a field is non-null, it will return
+                // a non-null error (it means they will give a null as the default value).
+                if !f.is_nullable() {
+                    // if we have a user-specified default expr, it must satisfy the non-null constraint
+                    // in table-create phase. So we just consider default_expr is none.
+                    return Err(ErrorCode::BadArguments(format!(
+                        "null value in column `{}` of table `{}` violates not-null constraint",
+                        f.name(),
+                        table.name()
+                    )));
+                }
                 let default_value = Scalar::default_value(f.data_type());
                 Expr::Constant {
                     span: None,

src/query/service/src/test_kits/fixture.rs

@@ -278,13 +278,15 @@ impl TestFixture {
 
     pub fn default_schema() -> DataSchemaRef {
         let tuple_inner_data_types = vec![
-            DataType::Number(NumberDataType::Int32),
-            DataType::Number(NumberDataType::Int32),
+            DataType::Nullable(Box::new(DataType::Number(NumberDataType::Int32))),
+            DataType::Nullable(Box::new(DataType::Number(NumberDataType::Int32))),
         ];
         let tuple_data_type = DataType::Tuple(tuple_inner_data_types);
+        let field_id = DataField::new("id", DataType::Number(NumberDataType::Int32));
+        let field_t = DataField::new("t", tuple_data_type);
         DataSchemaRefExt::create(vec![
-            DataField::new("id", DataType::Number(NumberDataType::Int32)),
-            DataField::new("t", tuple_data_type),
+            field_id.with_default_expr(Some("0".to_string())),
+            field_t.with_default_expr(Some("(0,0)".to_string())),
         ])
     }
 

tests/sqllogictests/suites/base/03_common/03_0032_select_null_count.test

@@ -13,9 +13,12 @@ drop table if exists t
 statement ok
 create table t(c0 tuple(int, int) not null, c1 string null)
 
-statement ok
+statement error 1006
 insert into t(c1) values(null), (null), (null)
 
+statement ok
+insert into t(c0,c1) values((0,0),null), ((0,0),null), ((0,0),null)
+
 query I
 select count(c1) from t
 ----
@@ -26,5 +29,21 @@ select count(c0) from t
 ----
 3
 
+## test #issue13932
+statement ok
+create table t3(a int not null,b text);
+
+statement error 1006
+insert into t3(b) values('a');
+
+statement error 1006
+insert into t3(a) values(null);
+
+statement ok
+insert into t3(a) values(1);
+
+statement ok
+insert into t3(a,b) values(1,'a');
+
 statement ok
 drop table if exists t

tests/sqllogictests/suites/base/03_common/03_0033_select_list.test

@@ -19,15 +19,24 @@ create table db.t1 (id int not null, c1 tuple(int, int) not null)
 statement ok
 create table db.t2 (id2 int not null, c1 tuple(int, int) not null)
 
-statement ok
+statement error 1006
 insert into db.t(id) values(1)
 
 statement ok
+insert into db.t(id,c1) values(1,(0,0))
+
+statement error 1006
 insert into db.t1(id) values(2)
 
 statement ok
+insert into db.t1(id,c1) values(2,(0,0))
+
+statement error 1006
 insert into db.t2(id2) values(3)
 
+statement ok
+insert into db.t2(id2,c1) values(3,(0,0))
+
 query TTTT
 select t.*, t.id, t.c1 from db.t
 ----

tests/sqllogictests/suites/base/03_common/03_0034_select_star_columns.test

@@ -25,25 +25,29 @@ create database db
 statement ok
 create table db.t (id int not null, c1 tuple(int, int) not null)
 
-
 statement ok
 create table db.t1 (id int not null, c1 tuple(int, int) not null)
 
-
 statement ok
 create table db.t2 (id2 int not null, c1 tuple(int, int) not null)
 
+statement error 1006
+insert into db.t(id) values(1)
 
 statement ok
-insert into db.t(id) values(1)
+insert into db.t(id,c1) values(1,(0,0))
 
+statement error 1006
+insert into db.t1(id) values(2)
 
 statement ok
-insert into db.t1(id) values(2)
+insert into db.t1(id,c1) values(2,(0,0))
 
+statement error 1006
+insert into db.t2(id2) values(3)
 
 statement ok
-insert into db.t2(id2) values(3)
+insert into db.t2(id2,c1) values(3,(0,0))
 
 
 statement ok

tests/sqllogictests/suites/base/03_common/03_0039_insert_into_bitmap.test

@@ -29,14 +29,17 @@ SELECT id, to_string(v) FROM t1 order by id
 statement ok
 CREATE TABLE IF NOT EXISTS t2(a int not null, b bitmap not null) Engine = Fuse
 
-statement ok
+statement error 1006
 insert into t2 (a) values (1),(2);
 
+statement ok
+insert into t2 (a,b) values (1,to_bitmap('0')),(2,to_bitmap('1'));
+
 query IT
 SELECT a, to_string(b) FROM t2
 ----
-1 (empty)
-2 (empty)
+1 0
+2 1
 
 statement ok
 DROP DATABASE db_bitmap

tests/sqllogictests/suites/base/09_fuse_engine/09_0004_remote_insert_into_select.test

@@ -84,15 +84,24 @@ select sum(cast(a as uint64)), sum(cast(b as uint64)) from s1
 statement ok
 create table d1(n String not null, a UInt8 not null, b Int16 not null default 4, c String not null default 'c')
 
-statement ok
+statement error 1006
 insert into d1(a) values (1)
 
 statement ok
+insert into d1(n,a) values ('',1)
+
+statement error 1006
 insert into d1(b) values (2)
 
 statement ok
+insert into d1(n,a,b) values ('',0,2)
+
+statement error 1006
 insert into d1(b) select b from d1
 
+statement ok
+insert into d1(n,a,b) select '',0,b from d1
+
 query TIIT
 select * from d1 order by a, b
 ----

tests/sqllogictests/suites/base/09_fuse_engine/09_0023_replace_into.test

@@ -211,6 +211,18 @@ DROP TABLE test;
 statement ok
 CREATE TABLE test(c1 int not null, c2 int not null, a int not null);
 
+statement error 1006
+replace into test(c1, a) on(c1) values(1, 1), (2, 2);
+
+statement error 1006
+replace into test(c2, a) on(c2) values(3, 3), (4, 4);
+
+statement ok
+drop table test;
+
+statement ok
+CREATE TABLE test(c1 int not null default 0, c2 int not null default 0, a int not null default 0);
+
 statement ok
 replace into test(c1, a) on(c1) values(1, 1), (2, 2);
 

tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test

@@ -1024,5 +1024,43 @@ select id,c1,to_date(c2),c3 from tb_01;
 ----
 1 abc 2023-11-29 {"a":1}
 
+## test #issue13932
+statement ok
+create table null_target(a int not null,b text);
+
+statement ok
+create table null_source(a int not null,b text);
+
+statement ok
+insert into null_target values(1,'a1');
+
+statement ok
+insert into null_target values(2,'a2');
+
+statement ok
+insert into null_source values(1,'a3');
+
+statement ok
+insert into null_source values(3,'a4');
+
+statement error 1006
+merge into null_target using null_source on null_target.a = null_source.a when matched then update *
+when not matched then insert (b) values(null_source.b);
+
+statement ok
+delete from null_source where a = 3;
+
+query TT
+merge into null_target using null_source on null_target.a = null_source.a when matched then update *
+when not matched then insert (b) values(null_source.b);
+----
+0 1
+
+query TT
+select * from null_target order by a,b;
+----
+1 a3
+2 a2
+
 statement ok
 set enable_experimental_merge_into = 0;

tests/sqllogictests/suites/base/11_data_type/11_0006_data_type_decimal.test

@@ -815,6 +815,15 @@ select * from t order by c1
 statement ok
 select * from t1 order by c1
 
+statement error 1006
+insert into t1(c1) values(1.2)
+
+statement ok
+drop table t1;
+
+statement ok
+create table t1(c0 int not null default 0, c1 decimal(39,2) null)
+
 statement ok
 insert into t1(c1) values(1.2)
 

tests/sqllogictests/suites/crdb/select.test

@@ -521,10 +521,10 @@ drop table if exists wide
 statement ok
 CREATE TABLE wide (id INT8 NOT NULL, a INT8 NOT NULL, b TEXT NOT NULL, c INT8 NOT NULL, d TEXT NOT NULL, e TEXT NOT NULL, f INT8 NOT NULL, g TEXT NOT NULL, h TEXT NOT NULL, i TEXT NOT NULL, j TEXT NOT NULL, k INT8 NOT NULL, l FLOAT NOT NULL, m FLOAT NOT NULL, n INT8 NOT NULL)
 
-statement ok
+statement error 1006
 INSERT INTO wide(id, n) VALUES(0, 10)
 
-query IITITTITTTTI??I
-SELECT * FROM wide
+query T
+SELECT count(*) FROM wide
 ----
-0 0 (empty) 0 (empty) (empty) 0 (empty) (empty) (empty) (empty) 0 0.0 0.0 10
+0

tests/suites/0_stateless/14_clickhouse_http_handler/14_0007_http_clickhouse_input_format_diagnostic.sh

@@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../../../shell_env.sh
 
 curl -s -u 'root:' -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d "drop table if exists a"
-curl -s -u 'root:' -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d "create table a ( a int not null, b string not null, c int not null)"
+curl -s -u 'root:' -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/" -d "create table a ( a int not null, b string not null, c int )"
 
 cat << EOF > /tmp/databend_test_csv_error.txt
 insert into a(a,b) format CSV 1,"Hello",1

tests/suites/0_stateless/17_altertable/17_0005_alter_table_modify_column_type.result

@@ -8,6 +8,7 @@ b	VARCHAR	NO	''
 c	INT	NO	0	
 Error: APIError: ResponseError with 1006: invalid float literal while evaluating function `to_float32('a')`
 Error: APIError: ResponseError with 1058: Cannot find column b
+Error: APIError: ResponseError with 1006: null value in column `a` of table `c` violates not-null constraint.
 0	1
 Error: APIError: ResponseError with 1006: invalid float literal while evaluating function `to_float32('a')`
 0	1
@@ -24,6 +25,7 @@ begin test default column
 not	2
 begin test not NULL column
 1	1
+Error: APIError: ResponseError with 1006: null value in column `a` of table `f` violates not-null constraint.
 1	1
 	2
 a	VARCHAR	NO	''	

tests/suites/0_stateless/17_altertable/17_0005_alter_table_modify_column_type.sh

@@ -22,6 +22,7 @@ echo "alter table test_modify_column_type.b modify column b float not null"  | $
 
 echo "CREATE table test_modify_column_type.c(a int not null, b int not null)"  | $BENDSQL_CLIENT_CONNECT
 echo "INSERT INTO test_modify_column_type.c (b) values(1)"  | $BENDSQL_CLIENT_CONNECT
+echo "INSERT INTO test_modify_column_type.c (a,b) values(0,1)"  | $BENDSQL_CLIENT_CONNECT
 echo "SELECT a,b from test_modify_column_type.c"  | $BENDSQL_CLIENT_CONNECT
 echo "alter table test_modify_column_type.c modify column a float not null default 'a'"  | $BENDSQL_CLIENT_CONNECT
 echo "alter table test_modify_column_type.c modify column a float not null default 1.2"  | $BENDSQL_CLIENT_CONNECT
@@ -54,6 +55,7 @@ echo "INSERT INTO test_modify_column_type.f values(1,1)"  | $BENDSQL_CLIENT_CONN
 echo "SELECT a,b from test_modify_column_type.f order by b"  | $BENDSQL_CLIENT_CONNECT
 echo "alter table test_modify_column_type.f modify column a VARCHAR(10) NOT NULL COMMENT 'new column'"  | $BENDSQL_CLIENT_CONNECT
 echo "INSERT INTO test_modify_column_type.f (b) values(2)"  | $BENDSQL_CLIENT_CONNECT
+echo "INSERT INTO test_modify_column_type.f (a,b) values('',2)"  | $BENDSQL_CLIENT_CONNECT
 echo "SELECT a,b from test_modify_column_type.f order by b"  | $BENDSQL_CLIENT_CONNECT
 echo "DESC test_modify_column_type.f"  | $BENDSQL_CLIENT_CONNECT