From 7e2cd3faa930cedb6ce54887f67e4f7845e2f0b7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 8 Dec 2025 09:41:15 -0800 Subject: [PATCH] [chaotic-good] Deadline fixes (#41190) * Increase test connection deadline to account for CI slowness * Add experiment to use handshaker deadline instead of hard coded deadline (since this is likely a bug) Closes #41190 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/41190 from ctiller:flake-cg e7852678fe0982f42f386ee1ffb421d334721f5b PiperOrigin-RevId: 841802046 --- bazel/experiments.bzl | 4 ++++ .../client/chaotic_good_connector.cc | 5 ++++- src/core/lib/experiments/experiments.cc | 18 ++++++++++++++++++ src/core/lib/experiments/experiments.h | 11 +++++++++++ src/core/lib/experiments/experiments.yaml | 5 +++++ src/core/lib/experiments/rollouts.yaml | 2 ++ test/core/transport/chaotic_good/BUILD | 1 + .../chaotic_good/chaotic_good_server_test.cc | 5 +++-- 8 files changed, 48 insertions(+), 3 deletions(-) diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index ed8b42bd78..a9a28c7677 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -20,6 +20,7 @@ EXPERIMENT_ENABLES = { "call_tracer_in_transport": "call_tracer_in_transport", "channelz_use_v2_for_v1_api": "channelz_use_v2_for_v1_api", "channelz_use_v2_for_v1_service": "channelz_use_v2_for_v1_service", + "chaotic_good_connect_deadline": "chaotic_good_connect_deadline", "chaotic_good_framing_layer": "chaotic_good_framing_layer", "chttp2_bound_write_size": "chttp2_bound_write_size", "error_flatten": "error_flatten", @@ -161,6 +162,7 @@ EXPERIMENTS = { "event_engine_dns", ], "core_end2end_test": [ + "chaotic_good_connect_deadline", "chaotic_good_framing_layer", "event_engine_client", "event_engine_dns_non_client_channel", @@ -268,6 +270,7 @@ EXPERIMENTS = { "event_engine_dns", ], "core_end2end_test": [ + "chaotic_good_connect_deadline", "chaotic_good_framing_layer", "event_engine_client", "event_engine_dns_non_client_channel", @@ -375,6 +378,7 @@ EXPERIMENTS = { "event_engine_dns", ], "core_end2end_test": [ + "chaotic_good_connect_deadline", "chaotic_good_framing_layer", "event_engine_client", "event_engine_dns_non_client_channel", diff --git a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc index eb86a4a9d9..678070ca8d 100644 --- a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc +++ b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc @@ -231,7 +231,10 @@ void ChaoticGoodConnector::Connect(const Args& args, Result* result, return TrySeq( ConnectChaoticGood( resolved_addr, result_notifier_ptr->args.channel_args, - Timestamp::Now() + Duration::FromSecondsAsDouble(kTimeoutSecs), + IsChaoticGoodConnectDeadlineEnabled() + ? result_notifier_ptr->args.deadline + : Timestamp::Now() + + Duration::FromSecondsAsDouble(kTimeoutSecs), std::move(client_settings)), [resolved_addr, result_notifier_ptr](ConnectChaoticGoodResult result) { diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index 4522706897..012d5a826b 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -32,6 +32,9 @@ const char* const additional_constraints_channelz_use_v2_for_v1_api = "{}"; const char* const description_channelz_use_v2_for_v1_service = "Use the v2 channelz service for the v1 channelz service."; const char* const additional_constraints_channelz_use_v2_for_v1_service = "{}"; +const char* const description_chaotic_good_connect_deadline = + "Use the deadline from the connect args in chaotic good connector"; +const char* const additional_constraints_chaotic_good_connect_deadline = "{}"; const char* const description_chaotic_good_framing_layer = "Enable the chaotic good framing layer."; const char* const additional_constraints_chaotic_good_framing_layer = "{}"; @@ -240,6 +243,9 @@ const ExperimentMetadata g_experiment_metadata[] = { description_channelz_use_v2_for_v1_service, additional_constraints_channelz_use_v2_for_v1_service, nullptr, 0, false, true}, + {"chaotic_good_connect_deadline", description_chaotic_good_connect_deadline, + additional_constraints_chaotic_good_connect_deadline, nullptr, 0, true, + true}, {"chaotic_good_framing_layer", description_chaotic_good_framing_layer, additional_constraints_chaotic_good_framing_layer, nullptr, 0, true, false}, @@ -400,6 +406,9 @@ const char* const additional_constraints_channelz_use_v2_for_v1_api = "{}"; const char* const description_channelz_use_v2_for_v1_service = "Use the v2 channelz service for the v1 channelz service."; const char* const additional_constraints_channelz_use_v2_for_v1_service = "{}"; +const char* const description_chaotic_good_connect_deadline = + "Use the deadline from the connect args in chaotic good connector"; +const char* const additional_constraints_chaotic_good_connect_deadline = "{}"; const char* const description_chaotic_good_framing_layer = "Enable the chaotic good framing layer."; const char* const additional_constraints_chaotic_good_framing_layer = "{}"; @@ -608,6 +617,9 @@ const ExperimentMetadata g_experiment_metadata[] = { description_channelz_use_v2_for_v1_service, additional_constraints_channelz_use_v2_for_v1_service, nullptr, 0, false, true}, + {"chaotic_good_connect_deadline", description_chaotic_good_connect_deadline, + additional_constraints_chaotic_good_connect_deadline, nullptr, 0, true, + true}, {"chaotic_good_framing_layer", description_chaotic_good_framing_layer, additional_constraints_chaotic_good_framing_layer, nullptr, 0, true, false}, @@ -768,6 +780,9 @@ const char* const additional_constraints_channelz_use_v2_for_v1_api = "{}"; const char* const description_channelz_use_v2_for_v1_service = "Use the v2 channelz service for the v1 channelz service."; const char* const additional_constraints_channelz_use_v2_for_v1_service = "{}"; +const char* const description_chaotic_good_connect_deadline = + "Use the deadline from the connect args in chaotic good connector"; +const char* const additional_constraints_chaotic_good_connect_deadline = "{}"; const char* const description_chaotic_good_framing_layer = "Enable the chaotic good framing layer."; const char* const additional_constraints_chaotic_good_framing_layer = "{}"; @@ -976,6 +991,9 @@ const ExperimentMetadata g_experiment_metadata[] = { description_channelz_use_v2_for_v1_service, additional_constraints_channelz_use_v2_for_v1_service, nullptr, 0, false, true}, + {"chaotic_good_connect_deadline", description_chaotic_good_connect_deadline, + additional_constraints_chaotic_good_connect_deadline, nullptr, 0, true, + true}, {"chaotic_good_framing_layer", description_chaotic_good_framing_layer, additional_constraints_chaotic_good_framing_layer, nullptr, 0, true, false}, diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index a8011daf2b..c78344f8b0 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -61,6 +61,8 @@ namespace grpc_core { inline bool IsCallTracerInTransportEnabled() { return true; } inline bool IsChannelzUseV2ForV1ApiEnabled() { return false; } inline bool IsChannelzUseV2ForV1ServiceEnabled() { return false; } +#define GRPC_EXPERIMENT_IS_INCLUDED_CHAOTIC_GOOD_CONNECT_DEADLINE +inline bool IsChaoticGoodConnectDeadlineEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_CHAOTIC_GOOD_FRAMING_LAYER inline bool IsChaoticGoodFramingLayerEnabled() { return true; } inline bool IsChttp2BoundWriteSizeEnabled() { return false; } @@ -125,6 +127,8 @@ inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { return false; } inline bool IsCallTracerInTransportEnabled() { return true; } inline bool IsChannelzUseV2ForV1ApiEnabled() { return false; } inline bool IsChannelzUseV2ForV1ServiceEnabled() { return false; } +#define GRPC_EXPERIMENT_IS_INCLUDED_CHAOTIC_GOOD_CONNECT_DEADLINE +inline bool IsChaoticGoodConnectDeadlineEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_CHAOTIC_GOOD_FRAMING_LAYER inline bool IsChaoticGoodFramingLayerEnabled() { return true; } inline bool IsChttp2BoundWriteSizeEnabled() { return false; } @@ -189,6 +193,8 @@ inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { return false; } inline bool IsCallTracerInTransportEnabled() { return true; } inline bool IsChannelzUseV2ForV1ApiEnabled() { return false; } inline bool IsChannelzUseV2ForV1ServiceEnabled() { return false; } +#define GRPC_EXPERIMENT_IS_INCLUDED_CHAOTIC_GOOD_CONNECT_DEADLINE +inline bool IsChaoticGoodConnectDeadlineEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_CHAOTIC_GOOD_FRAMING_LAYER inline bool IsChaoticGoodFramingLayerEnabled() { return true; } inline bool IsChttp2BoundWriteSizeEnabled() { return false; } @@ -254,6 +260,7 @@ enum ExperimentIds { kExperimentIdCallTracerInTransport, kExperimentIdChannelzUseV2ForV1Api, kExperimentIdChannelzUseV2ForV1Service, + kExperimentIdChaoticGoodConnectDeadline, kExperimentIdChaoticGoodFramingLayer, kExperimentIdChttp2BoundWriteSize, kExperimentIdErrorFlatten, @@ -314,6 +321,10 @@ inline bool IsChannelzUseV2ForV1ApiEnabled() { inline bool IsChannelzUseV2ForV1ServiceEnabled() { return IsExperimentEnabled(); } +#define GRPC_EXPERIMENT_IS_INCLUDED_CHAOTIC_GOOD_CONNECT_DEADLINE +inline bool IsChaoticGoodConnectDeadlineEnabled() { + return IsExperimentEnabled(); +} #define GRPC_EXPERIMENT_IS_INCLUDED_CHAOTIC_GOOD_FRAMING_LAYER inline bool IsChaoticGoodFramingLayerEnabled() { return IsExperimentEnabled(); diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 78da41d893..35ba6714f4 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -57,6 +57,11 @@ expiry: 2026/02/01 owner: ctiller@google.com test_tags: [channelz_test] +- name: chaotic_good_connect_deadline + description: Use the deadline from the connect args in chaotic good connector + expiry: 2026/02/01 + owner: ctiller@google.com + test_tags: [core_end2end_test] - name: chaotic_good_framing_layer description: Enable the chaotic good framing layer. expiry: 2026/02/01 diff --git a/src/core/lib/experiments/rollouts.yaml b/src/core/lib/experiments/rollouts.yaml index 4a2d986e1d..cff515db11 100644 --- a/src/core/lib/experiments/rollouts.yaml +++ b/src/core/lib/experiments/rollouts.yaml @@ -42,6 +42,8 @@ - name: call_tracer_in_transport default: true +- name: chaotic_good_connect_deadline + default: true - name: chaotic_good_framing_layer default: true - name: error_flatten diff --git a/test/core/transport/chaotic_good/BUILD b/test/core/transport/chaotic_good/BUILD index 17de67b994..18e0ee8a4d 100644 --- a/test/core/transport/chaotic_good/BUILD +++ b/test/core/transport/chaotic_good/BUILD @@ -257,6 +257,7 @@ grpc_cc_test( "//src/core:default_event_engine", "//src/core:endpoint_transport", "//src/core:event_engine_tcp_socket_utils", + "//src/core:experiments", "//src/core:grpc_check", "//src/core:grpc_fake_credentials", "//src/core:notification", diff --git a/test/core/transport/chaotic_good/chaotic_good_server_test.cc b/test/core/transport/chaotic_good/chaotic_good_server_test.cc index ec894be91f..dc2d99794c 100644 --- a/test/core/transport/chaotic_good/chaotic_good_server_test.cc +++ b/test/core/transport/chaotic_good/chaotic_good_server_test.cc @@ -32,6 +32,8 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/event_engine/default_event_engine.h" #include "src/core/lib/event_engine/tcp_socket_utils.h" +#include "src/core/lib/experiments/config.h" +#include "src/core/lib/experiments/experiments.h" #include "src/core/server/server.h" #include "src/core/transport/endpoint_transport.h" #include "src/core/util/grpc_check.h" @@ -117,7 +119,7 @@ class ChaoticGoodServerTest : public ::testing::TestWithParam { GRPC_CHECK_OK(uri); GRPC_CHECK(grpc_parse_uri(*uri, &resolved_addr_)); args_.address = &resolved_addr_; - args_.deadline = Timestamp::Now() + Duration::Seconds(5); + args_.deadline = Timestamp::Now() + Duration::Seconds(15); args_.channel_args = channel_args(); grpc_channel_credentials* channel_creds = nullptr; switch (GetParam()) { @@ -152,7 +154,6 @@ class ChaoticGoodServerTest : public ::testing::TestWithParam { } grpc_server* server_; - Server* core_server_; ChaoticGoodConnector::Args args_; ChaoticGoodConnector::Result connecting_result_; bool connecting_successful_ = false; -- 2.43.0