Merge branch 'main' into fix-agg-index

.github/workflows/dev.yml

@@ -16,8 +16,6 @@ on:
       - "scripts/setup/**"
       - ".devcontainer/**"
   merge_group:
-    types:
-      - checks_requested
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
@@ -37,3 +35,11 @@ jobs:
     with:
       build_profile: debug
       runner_provider: gcp
+
+  ready:
+    runs-on: ubuntu-latest
+    needs:
+      - linux
+      - linux_hive
+    steps:
+      - run: echo "ready"

.github/workflows/pr.yml

@@ -9,6 +9,7 @@ on:
       - edited
       - ready_for_review
       - converted_to_draft
+  merge_group:
 
 permissions:
   pull-requests: write

.github/workflows/release.yml

@@ -5,8 +5,7 @@ on:
     tags:
       - "v*"
   schedule:
-    # Release at 00:00 UTC+8
-    - cron: "0 16 * * *"
+    - cron: "0 22 * * *"
   workflow_dispatch:
     inputs:
       tags:

.github/workflows/reuse.linux.yml

@@ -170,7 +170,6 @@ jobs:
           - "standalone"
           - "crdb"
           - "duckdb"
-          - "tpch"
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_sqllogic_standalone_linux
@@ -222,6 +221,7 @@ jobs:
           - "base"
           - "ydb"
           - "tpcds"
+          - "tpch"
         format:
           - "parquet"
           - "native"
@@ -280,10 +280,8 @@ jobs:
         dirs:
           - "base"
           - "query"
-          - "cluster"
           - "crdb"
           - "duckdb"
-          - "tpch"
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_sqllogic_cluster_linux
@@ -299,13 +297,15 @@ jobs:
 
   sqllogic_cluster_large:
     name: sqllogic_cluster_${{ matrix.dirs }}
-    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    runs-on: [self-hosted, X64, Linux, 8c16g, "${{ inputs.runner_provider }}"]
     needs: [build, check]
     strategy:
       matrix:
         dirs:
           - "ydb"
           - "tpcds"
+          - "tpch"
+          - "cluster"
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_sqllogic_cluster_linux

src/query/ee/tests/it/aggregating_index/index_refresh.rs

@@ -305,6 +305,7 @@ async fn test_refresh_agg_index_with_limit() -> Result<()> {
     Ok(())
 }
 
+#[ignore = "flaky"]
 #[tokio::test(flavor = "multi_thread")]
 async fn test_sync_agg_index() -> Result<()> {
     let (_guard, ctx, root) = create_ee_query_context(None).await.unwrap();

src/query/ee/tests/it/aggregating_index/index_scan.rs

@@ -34,18 +34,21 @@ use databend_query::test_kits::TestFixture;
 use enterprise_query::test_kits::context::create_ee_query_context;
 use futures_util::TryStreamExt;
 
+#[ignore = "flaky"]
 #[tokio::test(flavor = "multi_thread")]
 async fn test_index_scan() -> Result<()> {
     test_index_scan_impl("parquet").await?;
     test_index_scan_impl("native").await
 }
 
+#[ignore = "flaky"]
 #[tokio::test(flavor = "multi_thread")]
 async fn test_index_scan_two_agg_funcs() -> Result<()> {
     test_index_scan_two_agg_funcs_impl("parquet").await?;
     test_index_scan_two_agg_funcs_impl("native").await
 }
 
+#[ignore = "flaky"]
 #[tokio::test(flavor = "multi_thread")]
 async fn test_projected_index_scan() -> Result<()> {
     test_projected_index_scan_impl("parquet").await?;
@@ -58,12 +61,14 @@ async fn test_index_scan_with_count() -> Result<()> {
     test_index_scan_with_count_impl("native").await
 }
 
+#[ignore = "flaky"]
 #[tokio::test(flavor = "multi_thread")]
 async fn test_index_scan_agg_args_are_expression() -> Result<()> {
     test_index_scan_agg_args_are_expression_impl("parquet").await?;
     test_index_scan_agg_args_are_expression_impl("native").await
 }
 
+#[ignore = "flaky"]
 #[tokio::test(flavor = "multi_thread")]
 async fn test_fuzz() -> Result<()> {
     test_fuzz_impl("parquet", false).await?;
@@ -268,6 +273,7 @@ async fn test_index_scan_impl(format: &str) -> Result<()> {
     Ok(())
 }
 
+#[ignore = "flaky"]
 async fn test_index_scan_two_agg_funcs_impl(format: &str) -> Result<()> {
     let (_guard, ctx, _) = create_ee_query_context(None).await.unwrap();
     let fixture = TestFixture::new_with_ctx(_guard, ctx).await;
@@ -408,6 +414,7 @@ async fn test_index_scan_two_agg_funcs_impl(format: &str) -> Result<()> {
     Ok(())
 }
 
+#[ignore = "flaky"]
 async fn test_projected_index_scan_impl(format: &str) -> Result<()> {
     let (_guard, ctx, _) = create_ee_query_context(None).await.unwrap();
     let fixture = TestFixture::new_with_ctx(_guard, ctx).await;

src/query/expression/src/utils/date_helper.rs

@@ -427,12 +427,20 @@ pub struct ToMinute;
 pub struct ToSecond;
 pub struct ToUnixTimestamp;
 
+pub struct ToWeekOfYear;
+
 impl ToNumber<u32> for ToYYYYMM {
     fn to_number(dt: &DateTime<Tz>) -> u32 {
         dt.year() as u32 * 100 + dt.month()
     }
 }
 
+impl ToNumber<u32> for ToWeekOfYear {
+    fn to_number(dt: &DateTime<Tz>) -> u32 {
+        dt.iso_week().week()
+    }
+}
+
 impl ToNumber<u32> for ToYYYYMMDD {
     fn to_number(dt: &DateTime<Tz>) -> u32 {
         dt.year() as u32 * 10_000 + dt.month() * 100 + dt.day()

src/query/functions/src/scalars/datetime.rs

@@ -943,6 +943,13 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) {
             ToNumberImpl::eval_date::<ToDayOfWeek, _>(val, ctx.func_ctx.tz)
         }),
     );
+    registry.register_passthrough_nullable_1_arg::<DateType, UInt32Type, _, _>(
+        "to_week_of_year",
+        |_, _| FunctionDomain::Full,
+        vectorize_1_arg::<DateType, UInt32Type>(|val, ctx| {
+            ToNumberImpl::eval_date::<ToWeekOfYear, _>(val, ctx.func_ctx.tz)
+        }),
+    );
     // timestamp
     registry.register_passthrough_nullable_1_arg::<TimestampType, UInt32Type, _, _>(
         "to_yyyymm",
@@ -1014,6 +1021,13 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) {
             ToNumberImpl::eval_timestamp::<ToDayOfWeek, _>(val, ctx.func_ctx.tz)
         }),
     );
+    registry.register_passthrough_nullable_1_arg::<TimestampType, UInt32Type, _, _>(
+        "to_week_of_year",
+        |_, _| FunctionDomain::Full,
+        vectorize_1_arg::<TimestampType, UInt32Type>(|val, ctx| {
+            ToNumberImpl::eval_timestamp::<ToWeekOfYear, _>(val, ctx.func_ctx.tz)
+        }),
+    );
     registry.register_passthrough_nullable_1_arg::<TimestampType, Int64Type, _, _>(
         "to_unix_timestamp",
         |_, _| FunctionDomain::Full,

src/query/functions/tests/it/scalars/datetime.rs

@@ -475,6 +475,7 @@ fn test_to_number(file: &mut impl Write) {
     run_ast(file, "to_day_of_year(to_date(18875))", &[]);
     run_ast(file, "to_day_of_month(to_date(18875))", &[]);
     run_ast(file, "to_day_of_week(to_date(18875))", &[]);
+    run_ast(file, "to_week_of_year(to_date(18875))", &[]);
     run_ast(file, "to_yyyymm(a)", &[(
         "a",
         DateType::from_data(vec![-100, 0, 100]),
@@ -511,6 +512,10 @@ fn test_to_number(file: &mut impl Write) {
         "a",
         DateType::from_data(vec![-100, 0, 100]),
     )]);
+    run_ast(file, "to_week_of_year(a)", &[(
+        "a",
+        DateType::from_data(vec![-100, 0, 100]),
+    )]);
 
     // timestamp
     run_ast(file, "to_yyyymm(to_timestamp(1630812366))", &[]);
@@ -522,6 +527,7 @@ fn test_to_number(file: &mut impl Write) {
     run_ast(file, "to_day_of_year(to_timestamp(1630812366))", &[]);
     run_ast(file, "to_day_of_month(to_timestamp(1630812366))", &[]);
     run_ast(file, "to_day_of_week(to_timestamp(1630812366))", &[]);
+    run_ast(file, "to_week_of_year(to_timestamp(1630812366))", &[]);
     run_ast(file, "to_hour(to_timestamp(1630812366))", &[]);
     run_ast(file, "to_minute(to_timestamp(1630812366))", &[]);
     run_ast(file, "to_second(to_timestamp(1630812366))", &[]);
@@ -561,6 +567,10 @@ fn test_to_number(file: &mut impl Write) {
         "a",
         TimestampType::from_data(vec![-100, 0, 100]),
     )]);
+    run_ast(file, "to_week_of_year(a)", &[(
+        "a",
+        TimestampType::from_data(vec![-100, 0, 100]),
+    )]);
     run_ast(file, "to_hour(a)", &[(
         "a",
         TimestampType::from_data(vec![-100, 0, 100]),

src/query/functions/tests/it/scalars/mod.rs

@@ -37,6 +37,8 @@ mod comparison;
 mod control;
 mod datetime;
 mod geo;
+// NOTE:(everpcpc) result different on macos
+#[cfg(not(target_os = "macos"))]
 mod geo_h3;
 mod hash;
 mod map;

src/query/functions/tests/it/scalars/testdata/datetime.txt

@@ -2620,6 +2620,15 @@ output domain  : {7..=7}
 output         : 7
 
 
+ast            : to_week_of_year(to_date(18875))
+raw expr       : to_week_of_year(to_date(18875))
+checked expr   : to_week_of_year<Date>(to_date<Int64>(to_int64<UInt16>(18875_u16)))
+optimized expr : 35_u32
+output type    : UInt32
+output domain  : {35..=35}
+output         : 35
+
+
 ast            : to_yyyymm(a)
 raw expr       : to_yyyymm(a::Date)
 checked expr   : to_yyyymm<Date>(a)
@@ -2818,6 +2827,28 @@ evaluation (internal):
 +--------+------------------+
 
 
+ast            : to_week_of_year(a)
+raw expr       : to_week_of_year(a::Date)
+checked expr   : to_week_of_year<Date>(a)
+evaluation:
++--------+--------------+------------------+
+|        | a            | Output           |
++--------+--------------+------------------+
+| Type   | Date         | UInt32           |
+| Domain | {-100..=100} | {0..=4294967295} |
+| Row 0  | '1969-09-23' | 39               |
+| Row 1  | '1970-01-01' | 1                |
+| Row 2  | '1970-04-11' | 15               |
++--------+--------------+------------------+
+evaluation (internal):
++--------+---------------------+
+| Column | Data                |
++--------+---------------------+
+| a      | [-100, 0, 100]      |
+| Output | UInt32([39, 1, 15]) |
++--------+---------------------+
+
+
 ast            : to_yyyymm(to_timestamp(1630812366))
 raw expr       : to_yyyymm(to_timestamp(1630812366))
 checked expr   : to_yyyymm<Timestamp>(to_timestamp<Int64>(to_int64<UInt32>(1630812366_u32)))
@@ -2899,6 +2930,15 @@ output domain  : {7..=7}
 output         : 7
 
 
+ast            : to_week_of_year(to_timestamp(1630812366))
+raw expr       : to_week_of_year(to_timestamp(1630812366))
+checked expr   : to_week_of_year<Timestamp>(to_timestamp<Int64>(to_int64<UInt32>(1630812366_u32)))
+optimized expr : 35_u32
+output type    : UInt32
+output domain  : {35..=35}
+output         : 35
+
+
 ast            : to_hour(to_timestamp(1630812366))
 raw expr       : to_hour(to_timestamp(1630812366))
 checked expr   : to_hour<Timestamp>(to_timestamp<Int64>(to_int64<UInt32>(1630812366_u32)))
@@ -3124,6 +3164,28 @@ evaluation (internal):
 +--------+------------------+
 
 
+ast            : to_week_of_year(a)
+raw expr       : to_week_of_year(a::Timestamp)
+checked expr   : to_week_of_year<Timestamp>(a)
+evaluation:
++--------+------------------------------+------------------+
+|        | a                            | Output           |
++--------+------------------------------+------------------+
+| Type   | Timestamp                    | UInt32           |
+| Domain | {-100..=100}                 | {0..=4294967295} |
+| Row 0  | '1969-12-31 23:59:59.999900' | 1                |
+| Row 1  | '1970-01-01 00:00:00.000000' | 1                |
+| Row 2  | '1970-01-01 00:00:00.000100' | 1                |
++--------+------------------------------+------------------+
+evaluation (internal):
++--------+-------------------+
+| Column | Data              |
++--------+-------------------+
+| a      | [-100, 0, 100]    |
+| Output | UInt32([1, 1, 1]) |
++--------+-------------------+
+
+
 ast            : to_hour(a)
 raw expr       : to_hour(a::Timestamp)
 checked expr   : to_hour<Timestamp>(a)

src/query/functions/tests/it/scalars/testdata/function_list.txt

@@ -3567,6 +3567,10 @@ Functions overloads:
 1 to_unix_timestamp(Timestamp NULL) :: Int64 NULL
 0 to_variant(T0) :: Variant
 1 to_variant(T0 NULL) :: Variant NULL
+0 to_week_of_year(Date) :: UInt32
+1 to_week_of_year(Date NULL) :: UInt32 NULL
+2 to_week_of_year(Timestamp) :: UInt32
+3 to_week_of_year(Timestamp NULL) :: UInt32 NULL
 0 to_year(Date) :: UInt16
 1 to_year(Date NULL) :: UInt16 NULL
 2 to_year(Timestamp) :: UInt16

src/query/service/tests/it/servers/http/http_query_handlers.rs

@@ -111,6 +111,7 @@ async fn check_final(ep: &EndpointType, final_uri: &str) -> Result<()> {
     Ok(())
 }
 
+#[ignore = "flaky"]
 #[tokio::test(flavor = "current_thread")]
 async fn test_simple_sql() -> Result<()> {
     let _guard = TestGlobalServices::setup(ConfigBuilder::create().build()).await?;
@@ -179,6 +180,7 @@ async fn test_simple_sql() -> Result<()> {
     Ok(())
 }
 
+#[ignore = "flaky"]
 #[tokio::test(flavor = "current_thread")]
 async fn test_show_databases() -> Result<()> {
     let _guard = TestGlobalServices::setup(ConfigBuilder::create().build()).await?;

tests/sqllogictests/suites/query/02_function/02_0012_function_datetimes

@@ -993,3 +993,18 @@ query T
 select to_timestamp('2022年02月04日，8时58分59秒,时区：+0800', '%Y年%m月%d日，%H时%M分%S秒,时区：%z');
 ----
 2022-02-04 00:58:59.000000
+
+query I
+select to_week_of_year('2017-01-01');
+----
+52
+
+query I
+SELECT to_week_of_year('1900-12-31 23:59:59.999900');
+----
+1
+
+query I
+SELECT to_week_of_year('2016-01-02T23:39:20.123-07:00');
+----
+53