diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index f3d59a96750c..56f549ff0f68 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -449,6 +451,13 @@ void Client::connect() UInt64 server_version_minor = 0; UInt64 server_version_patch = 0; + /// Capture the client local time zone before the branch below may switch the process default + /// to the server time zone. `serverTimezoneInstance()` reads the process default directly and + /// ignores `session_timezone`; `instance()` would fold in an explicit `--session_timezone` and + /// cache the wrong zone. `connect()` can run again on reconnect, so only capture once. + if (client_local_timezone.empty()) + client_local_timezone = DateLUT::serverTimezoneInstance().getTimeZone(); + if (hosts_and_ports.empty()) { String host = config().getString("host", "localhost"); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 6ced3cd4114c..90445efda0fb 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -127,6 +127,8 @@ namespace Setting extern const SettingsString promql_database; extern const SettingsString promql_table; extern const SettingsFloatAuto evaluation_time; + extern const SettingsBool use_client_time_zone; + extern const SettingsTimezone session_timezone; } namespace ErrorCodes @@ -2257,6 +2259,16 @@ void ClientBase::processParsedSingleQuery( applySettingsFromServerIfNeeded(); // after connect() and applySettingsFromQuery() + /// With `use_client_time_zone`, DateTime string literals must be interpreted in the client time + /// zone. The client parses synchronous INSERT literals itself, but literals interpreted server-side + /// (asynchronous INSERT, SELECT) rely on `session_timezone`. Seed it with the client time zone unless + /// the user set `session_timezone` explicitly. This is transient (reverted with the other query + /// settings below), so it tracks per-query `use_client_time_zone` changes in both directions. + if (!client_local_timezone.empty() + && client_context->getSettingsRef()[Setting::use_client_time_zone] + && !client_context->getSettingsRef().isChanged("session_timezone")) + client_context->setSetting("session_timezone", client_local_timezone); + ASTPtr input_function; const auto * insert = parsed_query->as(); if (insert && insert->select) diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index ef44dd610ce0..8f6be4ec8798 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -308,6 +308,11 @@ class ClientBase ContextMutablePtr global_context; ContextMutablePtr client_context; + /// The client local time zone, captured on the first connect() before it may switch the + /// process default to the server time zone. Used to seed `session_timezone` per query when + /// `use_client_time_zone` is set, so server-side literal parsing matches the client side. + String client_local_timezone; + String default_database; String query_id; Int32 suggestion_limit; diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp index dbc7904e7c27..c4083b70f855 100644 --- a/src/DataTypes/DataTypeDateTime.cpp +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -38,6 +39,17 @@ bool DataTypeDateTime::equals(const IDataType & rhs) const SerializationPtr DataTypeDateTime::doGetDefaultSerialization() const { + if (!has_explicit_time_zone) + { + /// When no explicit timezone, resolve the effective timezone (respects session_timezone). + /// This is called once per formatter (not per row), so the cost of DateLUT::instance() is negligible. + const auto & effective_tz = DateLUT::instance(); + if (&effective_tz != &time_zone) + { + TimezoneMixin overridden(effective_tz.getTimeZone()); + return std::make_shared(overridden); + } + } return std::make_shared(*this); } diff --git a/src/DataTypes/DataTypeDateTime64.cpp b/src/DataTypes/DataTypeDateTime64.cpp index d03abbaab075..37f03a10b2b6 100644 --- a/src/DataTypes/DataTypeDateTime64.cpp +++ b/src/DataTypes/DataTypeDateTime64.cpp @@ -63,6 +63,17 @@ bool DataTypeDateTime64::equals(const IDataType & rhs) const SerializationPtr DataTypeDateTime64::doGetDefaultSerialization() const { + if (!has_explicit_time_zone) + { + /// When no explicit timezone, resolve the effective timezone (respects session_timezone). + /// This is called once per formatter (not per row), so the cost of DateLUT::instance() is negligible. + const auto & effective_tz = DateLUT::instance(); + if (&effective_tz != &time_zone) + { + TimezoneMixin overridden(effective_tz.getTimeZone()); + return std::make_shared(scale, overridden); + } + } return std::make_shared(scale, *this); } diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index cb09483cc660..b7a1090482b8 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ namespace Setting extern const SettingsSeconds lock_acquire_timeout; extern const SettingsAlterUpdateMode alter_update_mode; extern const SettingsBool enable_lightweight_update; + extern const SettingsTimezone session_timezone; } namespace ServerSetting @@ -188,6 +190,26 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) } } + /// When session_timezone is set, string literals compared to DateTime columns + /// must be wrapped with explicit timezone to avoid misinterpretation in the + /// background mutation thread which lacks the session context. + const auto & session_tz = settings[Setting::session_timezone].value; + if (!session_tz.empty()) + { + const auto & source_ast = *mut_command->ast->as(); + auto tz_rewritten_ast = rewriteDateTimeLiteralsWithTimezone( + source_ast, table->getInMemoryMetadata().columns, session_tz); + if (tz_rewritten_ast) + { + auto * tz_alter_command = tz_rewritten_ast->as(); + mut_command = MutationCommand::parse(tz_alter_command); + if (!mut_command) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Alter command '{}' is rewritten to invalid command '{}'", + source_ast.formatForErrorMessage(), tz_rewritten_ast->formatForErrorMessage()); + } + } + mutation_commands.emplace_back(std::move(*mut_command)); } else diff --git a/src/Interpreters/MutationsDateTimeLiteralVisitor.cpp b/src/Interpreters/MutationsDateTimeLiteralVisitor.cpp new file mode 100644 index 000000000000..b29507e17194 --- /dev/null +++ b/src/Interpreters/MutationsDateTimeLiteralVisitor.cpp @@ -0,0 +1,229 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace +{ + +/// Returns the DateTime/DateTime64 column type if `identifier_name` refers to +/// a DateTime column without an explicit timezone. Returns nullptr otherwise. +DataTypePtr getDateTimeColumnType(const String & identifier_name, const ColumnsDescription & columns) +{ + const auto * desc = columns.tryGet(identifier_name); + if (!desc) + return nullptr; + + auto unwrapped = removeNullable(removeLowCardinality(desc->type)); + + if (const auto * dt = typeid_cast(unwrapped.get())) + { + if (!dt->hasExplicitTimeZone()) + return unwrapped; + } + else if (const auto * dt64 = typeid_cast(unwrapped.get())) + { + if (!dt64->hasExplicitTimeZone()) + return unwrapped; + } + return nullptr; +} + +/// Wraps a string literal AST with toDateTime('...', 'tz') or toDateTime64('...', scale, 'tz'). +ASTPtr wrapWithTimezone(const ASTPtr & literal_ast, const DataTypePtr & datetime_type, const String & timezone) +{ + auto tz_literal = std::make_shared(Field(timezone)); + + if (const auto * dt64 = typeid_cast(datetime_type.get())) + { + auto scale_literal = std::make_shared(Field(static_cast(dt64->getScale()))); + return makeASTFunction("toDateTime64", literal_ast, std::move(scale_literal), std::move(tz_literal)); + } + + return makeASTFunction("toDateTime", literal_ast, std::move(tz_literal)); +} + +/// For a comparison like `column >= 'datetime-string'`, try to wrap the string +/// literal with an explicit timezone cast. Modifies the AST in place. +/// Returns true if any literal was wrapped. +bool tryWrapComparisonLiteral(ASTFunction & function, const ColumnsDescription & columns, const String & timezone) +{ + if (!function.arguments || function.arguments->children.size() != 2) + return false; + + bool wrapped = false; + auto & left = function.arguments->children[0]; + auto & right = function.arguments->children[1]; + + /// Check left=identifier, right=string-literal + if (const auto * id = left->as()) + { + if (auto dt = getDateTimeColumnType(id->name(), columns)) + { + if (const auto * lit = right->as(); lit && lit->value.getType() == Field::Types::String) + { + right = wrapWithTimezone(right, dt, timezone); + wrapped = true; + } + } + } + + /// Check right=identifier, left=string-literal (e.g. '2000-01-01' <= time) + if (const auto * id = right->as()) + { + if (auto dt = getDateTimeColumnType(id->name(), columns)) + { + if (const auto * lit = left->as(); lit && lit->value.getType() == Field::Types::String) + { + left = wrapWithTimezone(left, dt, timezone); + wrapped = true; + } + } + } + + return wrapped; +} + +/// For an IN expression like `column IN ('dt1', 'dt2')`, wrap each string literal. +/// Returns true if any literal was wrapped. +bool tryWrapInLiterals(ASTFunction & function, const ColumnsDescription & columns, const String & timezone) +{ + if (!function.arguments || function.arguments->children.size() != 2) + return false; + + const auto & left = function.arguments->children[0]; + auto & right = function.arguments->children[1]; + + const auto * id = left->as(); + if (!id) + return false; + + auto dt = getDateTimeColumnType(id->name(), columns); + if (!dt) + return false; + + bool wrapped = false; + + /// The right side of IN can be an ASTFunction (tuple) or ASTExpressionList. + /// Walk its children and wrap string literals. + auto wrap_children = [&](ASTs & children) + { + for (auto & child : children) + { + if (const auto * lit = child->as(); lit && lit->value.getType() == Field::Types::String) + { + child = wrapWithTimezone(child, dt, timezone); + wrapped = true; + } + } + }; + + if (auto * tuple_func = right->as(); tuple_func && tuple_func->name == "tuple" && tuple_func->arguments) + wrap_children(tuple_func->arguments->children); + else if (auto * expr_list = right->as()) + wrap_children(expr_list->children); + + return wrapped; +} + +const std::unordered_set comparison_functions = { + "equals", "notEquals", "less", "greater", "lessOrEquals", "greaterOrEquals", +}; + +const std::unordered_set in_functions = { + "in", "notIn", "globalIn", "globalNotIn", +}; + +class RewriteDateTimeLiteralsMatcher +{ +public: + struct Data + { + const ColumnsDescription & columns; + const String & session_timezone; + bool modified = false; + }; + + static bool needChildVisit(const ASTPtr & ast, const ASTPtr & /*child*/) + { + return !ast->as(); + } + + static void visit(ASTPtr & ast, Data & data) + { + if (auto * function = ast->as()) + visit(*function, data); + } + + static void visit(ASTFunction & function, Data & data) + { + if (comparison_functions.contains(function.name)) + { + if (tryWrapComparisonLiteral(function, data.columns, data.session_timezone)) + data.modified = true; + } + else if (in_functions.contains(function.name)) + { + if (tryWrapInLiterals(function, data.columns, data.session_timezone)) + data.modified = true; + } + } +}; + +using RewriteDateTimeLiteralsVisitor = InDepthNodeVisitor; + +} + +ASTPtr rewriteDateTimeLiteralsWithTimezone( + const ASTAlterCommand & alter_command, + const ColumnsDescription & columns, + const String & session_timezone) +{ + if (session_timezone.empty()) + return nullptr; + + auto query = alter_command.clone(); + auto & new_command = *query->as(); + + auto remove_child = [](auto & children, IAST *& erase_ptr) + { + auto it = std::find_if(children.begin(), children.end(), [&](const auto & ptr) { return ptr.get() == erase_ptr; }); + erase_ptr = nullptr; + children.erase(it); + }; + + RewriteDateTimeLiteralsMatcher::Data data{columns, session_timezone, false}; + RewriteDateTimeLiteralsVisitor visitor(data); + + if (new_command.update_assignments) + { + ASTPtr update_assignments = new_command.update_assignments->clone(); + remove_child(new_command.children, new_command.update_assignments); + visitor.visit(update_assignments); + new_command.update_assignments = new_command.children.emplace_back(std::move(update_assignments)).get(); + } + if (new_command.predicate) + { + ASTPtr predicate = new_command.predicate->clone(); + remove_child(new_command.children, new_command.predicate); + visitor.visit(predicate); + new_command.predicate = new_command.children.emplace_back(std::move(predicate)).get(); + } + + if (!data.modified) + return nullptr; + + return query; +} + +} diff --git a/src/Interpreters/MutationsDateTimeLiteralVisitor.h b/src/Interpreters/MutationsDateTimeLiteralVisitor.h new file mode 100644 index 000000000000..90709ad747a1 --- /dev/null +++ b/src/Interpreters/MutationsDateTimeLiteralVisitor.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class ASTAlterCommand; + +/// When `session_timezone` is set, string literals compared to DateTime/DateTime64 +/// columns in mutation predicates must be interpreted in that timezone. +/// However, mutations execute in a background thread that lacks the original +/// session context. This function rewrites the mutation AST at ALTER time: +/// +/// time >= '2000-01-01 02:00:00' +/// becomes +/// time >= toDateTime('2000-01-01 02:00:00', 'America/Mazatlan') +/// +/// This makes the timezone explicit so the background thread evaluates it correctly. +/// Returns rewritten AST if any literals were wrapped, nullptr otherwise. +ASTPtr rewriteDateTimeLiteralsWithTimezone( + const ASTAlterCommand & alter_command, + const ColumnsDescription & columns, + const String & session_timezone); + +} diff --git a/tests/queries/0_stateless/02737_session_timezone.reference b/tests/queries/0_stateless/02737_session_timezone.reference index 6c6fc2aa93ad..e2caa3ed777d 100644 --- a/tests/queries/0_stateless/02737_session_timezone.reference +++ b/tests/queries/0_stateless/02737_session_timezone.reference @@ -1,8 +1,13 @@ Pacific/Pitcairn Pacific/Pitcairn Asia/Novosibirsk Asia/Novosibirsk +test simple queries 2022-12-12 17:23:23 2022-12-13 07:23:23.123 +subquery shall use main query session_timezone 2022-12-13 07:23:23 2022-12-13 07:23:23 +test proper serialization 2002-12-12 23:23:23 2002-12-12 23:23:23 2002-12-12 23:23:23.123 2002-12-12 23:23:23.123 +Create a table and test that DateTimes are processed correctly on insert +Test parsing in WHERE filter, shall have the same logic as insert 2000-01-01 01:00:00 diff --git a/tests/queries/0_stateless/02737_session_timezone.sql b/tests/queries/0_stateless/02737_session_timezone.sql index 1afadbde6df6..2499ae1d7e38 100644 --- a/tests/queries/0_stateless/02737_session_timezone.sql +++ b/tests/queries/0_stateless/02737_session_timezone.sql @@ -5,23 +5,23 @@ SELECT timezone(), timezoneOf(now()) SETTINGS session_timezone = 'Pacific/Pitcai SET session_timezone = 'Asia/Novosibirsk'; SELECT timezone(), timezoneOf(now()); --- test simple queries +SELECT 'test simple queries'; SELECT toDateTime(toDateTime('2022-12-12 23:23:23'), 'Europe/Zurich'); SELECT toDateTime64(toDateTime64('2022-12-12 23:23:23.123', 3), 3, 'Europe/Zurich') SETTINGS session_timezone = 'America/Denver'; --- subquery shall use main query's session_timezone +SELECT 'subquery shall use main query session_timezone'; SELECT toDateTime(toDateTime('2022-12-12 23:23:23'), 'Europe/Zurich'), (SELECT toDateTime(toDateTime('2022-12-12 23:23:23'), 'Europe/Zurich') SETTINGS session_timezone = 'Europe/Helsinki') SETTINGS session_timezone = 'America/Denver'; --- test proper serialization +SELECT 'test proper serialization'; SELECT toDateTime('2002-12-12 23:23:23') AS dt, toString(dt) SETTINGS session_timezone = 'Asia/Phnom_Penh'; SELECT toDateTime64('2002-12-12 23:23:23.123', 3) AS dt64, toString(dt64) SETTINGS session_timezone = 'Asia/Phnom_Penh'; --- Create a table and test that DateTimes are processed correctly on insert +SELECT 'Create a table and test that DateTimes are processed correctly on insert'; CREATE TABLE test_tz_setting (d DateTime('UTC')) Engine=Memory AS SELECT toDateTime('2000-01-01 00:00:00'); INSERT INTO test_tz_setting VALUES ('2000-01-01 01:00:00'); -- this is parsed using timezone from `d` column INSERT INTO test_tz_setting VALUES (toDateTime('2000-01-02 02:00:00')); -- this is parsed using `session_timezone` --- Test parsing in WHERE filter, shall have the same logic as insert +SELECT 'Test parsing in WHERE filter, shall have the same logic as insert'; SELECT d FROM test_tz_setting WHERE d == '2000-01-01 01:00:00'; -- 1 row expected SELECT d FROM test_tz_setting WHERE d == toDateTime('2000-01-01 02:00:00'); -- 0 rows expected diff --git a/tests/queries/0_stateless/04056_mutation_session_timezone_datetime_literals.reference b/tests/queries/0_stateless/04056_mutation_session_timezone_datetime_literals.reference new file mode 100644 index 000000000000..179770dad9a3 --- /dev/null +++ b/tests/queries/0_stateless/04056_mutation_session_timezone_datetime_literals.reference @@ -0,0 +1,18 @@ +before delete 1 2000-01-01 01:02:03 +before delete 2 2000-01-01 04:05:06 +after delete 1 2000-01-01 01:02:03 +before delete dt64 1 2000-01-01 01:02:03.123 +before delete dt64 2 2000-01-01 04:05:06.456 +after delete dt64 1 2000-01-01 01:02:03.123 +after update 1 old +after update 2 new +before delete nullable 1 2000-01-01 01:02:03 +before delete nullable 2 2000-01-01 04:05:06 +after delete nullable 1 2000-01-01 01:02:03 +before delete lc 1 2000-01-01 01:02:03 +before delete lc 2 2000-01-01 04:05:06 +after delete lc 1 2000-01-01 01:02:03 +before delete nullable dt64 1 2000-01-01 01:02:03.123 +before delete nullable dt64 2 2000-01-01 04:05:06.456 +after delete nullable dt64 1 2000-01-01 01:02:03.123 +explicit tz 1 2000-01-01 01:02:03 diff --git a/tests/queries/0_stateless/04056_mutation_session_timezone_datetime_literals.sql b/tests/queries/0_stateless/04056_mutation_session_timezone_datetime_literals.sql new file mode 100644 index 000000000000..bd48cc2b1ad7 --- /dev/null +++ b/tests/queries/0_stateless/04056_mutation_session_timezone_datetime_literals.sql @@ -0,0 +1,83 @@ +-- Verify that ALTER DELETE/UPDATE with DateTime comparisons works correctly +-- when session_timezone differs from server timezone. String literals in the +-- mutation predicate must be interpreted in the session timezone, not the +-- server default. The fix wraps them with toDateTime('...', '') at ALTER +-- time so the background mutation thread evaluates them consistently. + +SET session_timezone = 'America/Denver'; -- UTC-7 (far from typical UTC server default) +SET mutations_sync = 2; + +-- DateTime (no explicit timezone) +DROP TABLE IF EXISTS test_mutation_tz SYNC; +CREATE TABLE test_mutation_tz (id UInt32, time DateTime) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_mutation_tz VALUES (1, '2000-01-01 01:02:03'), (2, '2000-01-01 04:05:06'); +SELECT 'before delete', id, time FROM test_mutation_tz ORDER BY id; + +ALTER TABLE test_mutation_tz DELETE WHERE time >= '2000-01-01 02:00:00'; +SELECT 'after delete', id, time FROM test_mutation_tz ORDER BY id; + +-- DateTime64 (no explicit timezone) +DROP TABLE IF EXISTS test_mutation_tz64 SYNC; +CREATE TABLE test_mutation_tz64 (id UInt32, time DateTime64(3)) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_mutation_tz64 VALUES (1, '2000-01-01 01:02:03.123'), (2, '2000-01-01 04:05:06.456'); +SELECT 'before delete dt64', id, time FROM test_mutation_tz64 ORDER BY id; + +ALTER TABLE test_mutation_tz64 DELETE WHERE time >= '2000-01-01 02:00:00'; +SELECT 'after delete dt64', id, time FROM test_mutation_tz64 ORDER BY id; + +-- ALTER UPDATE with DateTime comparison in WHERE +DROP TABLE IF EXISTS test_mutation_tz_upd SYNC; +CREATE TABLE test_mutation_tz_upd (id UInt32, time DateTime, val String) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_mutation_tz_upd VALUES (1, '2000-01-01 01:02:03', 'old'), (2, '2000-01-01 04:05:06', 'old'); +ALTER TABLE test_mutation_tz_upd UPDATE val = 'new' WHERE time >= '2000-01-01 02:00:00'; +SELECT 'after update', id, val FROM test_mutation_tz_upd ORDER BY id; + +-- Nullable(DateTime) — the wrapper must be unwrapped before the timezone check +DROP TABLE IF EXISTS test_mutation_tz_nullable SYNC; +CREATE TABLE test_mutation_tz_nullable (id UInt32, time Nullable(DateTime)) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_mutation_tz_nullable VALUES (1, '2000-01-01 01:02:03'), (2, '2000-01-01 04:05:06'); +SELECT 'before delete nullable', id, time FROM test_mutation_tz_nullable ORDER BY id; + +ALTER TABLE test_mutation_tz_nullable DELETE WHERE time >= '2000-01-01 02:00:00'; +SELECT 'after delete nullable', id, time FROM test_mutation_tz_nullable ORDER BY id; + +-- LowCardinality(DateTime) — same unwrapping logic +SET allow_suspicious_low_cardinality_types = 1; +DROP TABLE IF EXISTS test_mutation_tz_lc SYNC; +CREATE TABLE test_mutation_tz_lc (id UInt32, time LowCardinality(DateTime)) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_mutation_tz_lc VALUES (1, '2000-01-01 01:02:03'), (2, '2000-01-01 04:05:06'); +SELECT 'before delete lc', id, time FROM test_mutation_tz_lc ORDER BY id; + +ALTER TABLE test_mutation_tz_lc DELETE WHERE time >= '2000-01-01 02:00:00'; +SELECT 'after delete lc', id, time FROM test_mutation_tz_lc ORDER BY id; + +-- Nullable(DateTime64) — unwrapping for DateTime64 +DROP TABLE IF EXISTS test_mutation_tz64_nullable SYNC; +CREATE TABLE test_mutation_tz64_nullable (id UInt32, time Nullable(DateTime64(3))) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_mutation_tz64_nullable VALUES (1, '2000-01-01 01:02:03.123'), (2, '2000-01-01 04:05:06.456'); +SELECT 'before delete nullable dt64', id, time FROM test_mutation_tz64_nullable ORDER BY id; + +ALTER TABLE test_mutation_tz64_nullable DELETE WHERE time >= '2000-01-01 02:00:00'; +SELECT 'after delete nullable dt64', id, time FROM test_mutation_tz64_nullable ORDER BY id; + +-- DateTime with explicit timezone should NOT be rewritten (timezone is already determined) +DROP TABLE IF EXISTS test_mutation_tz_explicit SYNC; +CREATE TABLE test_mutation_tz_explicit (id UInt32, time DateTime('UTC')) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_mutation_tz_explicit VALUES (1, '2000-01-01 01:02:03'), (2, '2000-01-01 04:05:06'); +ALTER TABLE test_mutation_tz_explicit DELETE WHERE time >= '2000-01-01 02:00:00'; +SELECT 'explicit tz', id, time FROM test_mutation_tz_explicit ORDER BY id; + +DROP TABLE test_mutation_tz SYNC; +DROP TABLE test_mutation_tz64 SYNC; +DROP TABLE test_mutation_tz_upd SYNC; +DROP TABLE test_mutation_tz_nullable SYNC; +DROP TABLE test_mutation_tz_lc SYNC; +DROP TABLE test_mutation_tz64_nullable SYNC; +DROP TABLE test_mutation_tz_explicit SYNC; diff --git a/tests/queries/0_stateless/04057_async_insert_session_timezone.reference b/tests/queries/0_stateless/04057_async_insert_session_timezone.reference new file mode 100644 index 000000000000..52779586bacc --- /dev/null +++ b/tests/queries/0_stateless/04057_async_insert_session_timezone.reference @@ -0,0 +1,36 @@ +== DateTime (no explicit timezone) == +--- TCP sync +946666800 +--- TCP async +946666800 +--- HTTP sync +946666800 +--- HTTP async +946666800 +== DateTime('America/New_York') == +--- TCP sync +946706400 +--- TCP async +946706400 +--- HTTP sync +946706400 +--- HTTP async +946706400 +== DateTime64(3) == +--- TCP sync +946666800 +--- TCP async +946666800 +--- HTTP sync +946666800 +--- HTTP async +946666800 +== DateTime64(3, 'America/New_York') == +--- TCP sync +946706400 +--- TCP async +946706400 +--- HTTP sync +946706400 +--- HTTP async +946706400 diff --git a/tests/queries/0_stateless/04057_async_insert_session_timezone.sh b/tests/queries/0_stateless/04057_async_insert_session_timezone.sh new file mode 100755 index 000000000000..71b9b8b0c1b0 --- /dev/null +++ b/tests/queries/0_stateless/04057_async_insert_session_timezone.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-random-settings +# https://github.com/ClickHouse/ClickHouse/issues/100614 +# +# session_timezone must be respected when parsing DateTime/DateTime64 from +# text on the server side (async inserts over TCP, all inserts over HTTP). +# Covers columns without and with an explicit timezone, and DateTime64. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Strip any randomized session_timezone from the base URL. +CLEAN_URL=$(echo "${CLICKHOUSE_URL}" \ + | sed 's/\&session_timezone=[A-Za-z0-9\/\%\_\-\+]*//g' \ + | sed 's/\?session_timezone=[A-Za-z0-9\/\%\_\-\+]*\&/\?/g') + +URL_TZ="${CLEAN_URL}&session_timezone=Asia%2FNovosibirsk" + +run_cases() +{ + local table=$1 + local select_expr=$2 + + echo "--- TCP sync" + ${CLICKHOUSE_CLIENT} --session_timezone='Asia/Novosibirsk' -q \ + "INSERT INTO ${table} SETTINGS async_insert=0 VALUES ('2000-01-01 01:00:00')" + ${CLICKHOUSE_CLIENT} --session_timezone='Asia/Novosibirsk' -q \ + "SELECT ${select_expr} FROM ${table}" + ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE ${table}" + + echo "--- TCP async" + ${CLICKHOUSE_CLIENT} --session_timezone='Asia/Novosibirsk' -q \ + "INSERT INTO ${table} SETTINGS async_insert=1, wait_for_async_insert=1 VALUES ('2000-01-01 01:00:00')" + ${CLICKHOUSE_CLIENT} --session_timezone='Asia/Novosibirsk' -q \ + "SELECT ${select_expr} FROM ${table}" + ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE ${table}" + + echo "--- HTTP sync" + ${CLICKHOUSE_CURL} -sS "${URL_TZ}" -d \ + "INSERT INTO ${table} SETTINGS async_insert=0 VALUES ('2000-01-01 01:00:00')" + ${CLICKHOUSE_CURL} -sS "${URL_TZ}" -d \ + "SELECT ${select_expr} FROM ${table}" + ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE ${table}" + + echo "--- HTTP async" + ${CLICKHOUSE_CURL} -sS "${URL_TZ}&async_insert=1&wait_for_async_insert=1" -d \ + "INSERT INTO ${table} VALUES ('2000-01-01 01:00:00')" + ${CLICKHOUSE_CURL} -sS "${URL_TZ}" -d \ + "SELECT ${select_expr} FROM ${table}" + ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE ${table}" +} + +# ── Case 1: DateTime without explicit timezone ────────────────────────── +TABLE1="test_async_tz_plain_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE1}" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ${TABLE1} (d DateTime) ENGINE = Memory" + +echo "== DateTime (no explicit timezone) ==" +run_cases "${TABLE1}" "toUnixTimestamp(d)" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE1}" + +# ── Case 2: DateTime with explicit timezone ───────────────────────────── +# The column's explicit timezone is used for parsing plain string literals. +TABLE2="test_async_tz_explicit_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE2}" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ${TABLE2} (d DateTime('America/New_York')) ENGINE = Memory" + +echo "== DateTime('America/New_York') ==" +run_cases "${TABLE2}" "toUnixTimestamp(d)" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE2}" + +# ── Case 3: DateTime64 without explicit timezone ──────────────────────── +TABLE3="test_async_tz_dt64_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE3}" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ${TABLE3} (d DateTime64(3)) ENGINE = Memory" + +echo "== DateTime64(3) ==" +run_cases "${TABLE3}" "toUnixTimestamp(d)" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE3}" + +# ── Case 4: DateTime64 with explicit timezone ─────────────────────────── +TABLE4="test_async_tz_dt64_explicit_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE4}" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ${TABLE4} (d DateTime64(3, 'America/New_York')) ENGINE = Memory" + +echo "== DateTime64(3, 'America/New_York') ==" +run_cases "${TABLE4}" "toUnixTimestamp(d)" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE4}" diff --git a/tests/queries/0_stateless/04401_async_insert_use_client_time_zone.reference b/tests/queries/0_stateless/04401_async_insert_use_client_time_zone.reference new file mode 100644 index 000000000000..77ba76c9816d --- /dev/null +++ b/tests/queries/0_stateless/04401_async_insert_use_client_time_zone.reference @@ -0,0 +1,7 @@ +flag_async 1500036000 +flag_sync 1500036000 +reset_default_async 1500036000 +set_async 1500036000 +set_sync 1500036000 +settings_async 1500036000 +reset_async matches server tz 1 diff --git a/tests/queries/0_stateless/04401_async_insert_use_client_time_zone.sh b/tests/queries/0_stateless/04401_async_insert_use_client_time_zone.sh new file mode 100755 index 000000000000..b6acb91c90a8 --- /dev/null +++ b/tests/queries/0_stateless/04401_async_insert_use_client_time_zone.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-random-settings +# ^ no-random-settings: the runner must not inject a randomized `session_timezone`; an explicit +# `session_timezone` (even empty) is an intentional user override and disables the client-time-zone +# propagation this test exercises. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# With use_client_time_zone=1, a DateTime string literal must be interpreted in the client time zone +# regardless of whether the INSERT is synchronous or asynchronous. The async path parses the VALUES +# block on the server, so the client has to propagate its local time zone as session_timezone, and it +# must do so for every query (not only at connect time), tracking use_client_time_zone changes in both +# directions. America/Hermosillo is a fixed UTC-7 zone (no DST), so 2017-07-14 05:40:00 there is the +# stable instant 1500036000. + +TZC="env TZ=America/Hermosillo ${CLICKHOUSE_CLIENT}" + +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ${CLICKHOUSE_DATABASE}.dt (a DateTime, kind String) ENGINE = Memory" + +# use_client_time_zone via the command line flag (async and sync). +$TZC --use_client_time_zone=1 -q \ + "INSERT INTO ${CLICKHOUSE_DATABASE}.dt SETTINGS async_insert = 1, wait_for_async_insert = 1 VALUES ('2017-07-14 05:40:00', 'flag_async')" +$TZC --use_client_time_zone=1 -q \ + "INSERT INTO ${CLICKHOUSE_DATABASE}.dt SETTINGS async_insert = 0 VALUES ('2017-07-14 05:40:00', 'flag_sync')" + +# use_client_time_zone turned on mid-session with SET, on an already-open connection. +$TZC -mn -q " +SET use_client_time_zone = 1; +INSERT INTO ${CLICKHOUSE_DATABASE}.dt SETTINGS async_insert = 1, wait_for_async_insert = 1 VALUES ('2017-07-14 05:40:00', 'set_async'); +INSERT INTO ${CLICKHOUSE_DATABASE}.dt SETTINGS async_insert = 0 VALUES ('2017-07-14 05:40:00', 'set_sync'); +" + +# use_client_time_zone set only per query via SETTINGS, without the command line flag. +$TZC -q \ + "INSERT INTO ${CLICKHOUSE_DATABASE}.dt SETTINGS use_client_time_zone = 1, async_insert = 1, wait_for_async_insert = 1 VALUES ('2017-07-14 05:40:00', 'settings_async')" + +# Starting with an explicit --session_timezone override and then clearing it with +# SET session_timezone = DEFAULT must fall back to the client time zone, not to the override value. +$TZC --use_client_time_zone=1 --session_timezone=UTC -mn -q " +SET session_timezone = DEFAULT; +INSERT INTO ${CLICKHOUSE_DATABASE}.dt SETTINGS async_insert = 1, wait_for_async_insert = 1 VALUES ('2017-07-14 05:40:00', 'reset_default_async'); +" + +# All of the above interpret the literal in the client time zone: 2017-07-14 05:40:00 = 1500036000. +${CLICKHOUSE_CLIENT} -q "SELECT kind, toUnixTimestamp(a) FROM ${CLICKHOUSE_DATABASE}.dt ORDER BY kind" + +# Turning use_client_time_zone back off must restore server-side parsing (the stored instant no longer +# depends on the client time zone). The exact value depends on the server time zone, so compare it with +# a plain default insert instead of hard-coding it. +$TZC -q \ + "INSERT INTO ${CLICKHOUSE_DATABASE}.dt SETTINGS async_insert = 1, wait_for_async_insert = 1 VALUES ('2017-07-14 05:40:00', 'server_ref')" +$TZC --use_client_time_zone=1 -mn -q " +SET use_client_time_zone = 0; +INSERT INTO ${CLICKHOUSE_DATABASE}.dt SETTINGS async_insert = 1, wait_for_async_insert = 1 VALUES ('2017-07-14 05:40:00', 'reset_async'); +" +${CLICKHOUSE_CLIENT} -q " +SELECT 'reset_async matches server tz', ( + (SELECT toUnixTimestamp(a) FROM ${CLICKHOUSE_DATABASE}.dt WHERE kind = 'reset_async') + = (SELECT toUnixTimestamp(a) FROM ${CLICKHOUSE_DATABASE}.dt WHERE kind = 'server_ref')) +" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${CLICKHOUSE_DATABASE}.dt"