From 526819e02300834b86a7bc4aa28850f41b2f1c5a Mon Sep 17 00:00:00 2001 From: Jeff Widman Date: Wed, 1 Mar 2017 16:41:30 -0800 Subject: [PATCH] Bump KafkaConsumer's request_timeout to 305000 The Java Consumer [changed the value to 305000 in `0.10.1.0`](https://kafka.apache.org/documentation/#upgrade_1010_notable): > The new Java Consumer now supports heartbeating from a background thread. There is a new configuration max.poll.interval.ms which controls the maximum time between poll invocations before the consumer will proactively leave the group (5 minutes by default). The value of the configuration request.timeout.ms must always be larger than max.poll.interval.ms because this is the maximum time that a JoinGroup request can block on the server while the consumer is rebalancing, so we have changed its default value to just above 5 minutes. Finally, the default value of session.timeout.ms has been adjusted down to 10 seconds, and the default value of max.poll.records has been changed to 500. See the current upstream value here: https://kafka.apache.org/documentation/#configuration --- kafka/consumer/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka/consumer/group.py b/kafka/consumer/group.py index 8c2ab2de9..490fe9de4 100644 --- a/kafka/consumer/group.py +++ b/kafka/consumer/group.py @@ -74,7 +74,7 @@ class KafkaConsumer(six.Iterator): happens, the consumer can get stuck trying to fetch a large message on a certain partition. Default: 1048576. request_timeout_ms (int): Client request timeout in milliseconds. - Default: 40000. + Default: 305000. retry_backoff_ms (int): Milliseconds to backoff when retrying on errors. Default: 100. reconnect_backoff_ms (int): The amount of time in milliseconds to @@ -213,7 +213,7 @@ class KafkaConsumer(six.Iterator): 'fetch_max_wait_ms': 500, 'fetch_min_bytes': 1, 'max_partition_fetch_bytes': 1 * 1024 * 1024, - 'request_timeout_ms': 40 * 1000, + 'request_timeout_ms': 305000, 'retry_backoff_ms': 100, 'reconnect_backoff_ms': 50, 'max_in_flight_requests_per_connection': 5,