AgentServer connections ignore VSTS_HTTP_TIMEOUT environment variable and use hardcoded timeout values (#5432)

* Fix: Respect VSTS_HTTP_TIMEOUT environment variable in AgentServer

The agent listener was ignoring the VSTS_HTTP_TIMEOUT environment variable
and using hardcoded timeout values (60s for message/request connections,
100s for generic connection) when establishing server connections.

This caused timeout errors even when users set VSTS_HTTP_TIMEOUT to higher
values like 300 seconds, making the environment variable non-functional.

Changes:
- Modified AgentServer.ConnectAsync() to read timeout from AgentKnobs.HttpTimeout
- Applied valid range constraint [100, 1200] seconds as documented
- All three connections now use the same configurable timeout
- Worker connections already respect this variable via VssUtil.CreateConnection()

Fixes issue where agents in slow network environments experienced frequent
timeouts despite setting VSTS_HTTP_TIMEOUT environment variable.

Related logs showing the issue:
[INFO EnvironmentCapabilitiesProvider] Adding 'VSTS_HTTP_TIMEOUT': '300'
[INFO AgentServer] Establish connection with 60 seconds timeout.
[ERR  MessageListener] System.TimeoutException: HTTP request timed out after 00:01:00

* Fix: Respect VSTS_HTTP_TIMEOUT environment variable for agent connections

- Add GetDefaultConnectionTimeout() helper to centralize timeout logic
- Update ConnectAsync to use environment variable timeout
- Update RefreshConnectionAsync to accept optional timeout (defaults to env var)
- Update ResetConnectionTimeout to accept optional timeout (defaults to env var)
- Rename SetConnectionTimeout to ResetConnectionTimeout for clarity
- Keep 30s timeout for error recovery scenarios (intentional fast failure)
- Update JobDispatcherL0 tests to handle nullable TimeSpan parameters

Fixes agent server ignoring VSTS_HTTP_TIMEOUT setting, causing unexpected
60-second timeouts instead of configured values (100-1200 second range).

* reverting to 60 sec for JobRequest connection type

* fixing override

---------

Co-authored-by: azure-pipelines-bot <[email protected]>

Author: tarunramsinghani

src/Agent.Listener/JobDispatcher.cs

@@ -401,10 +401,10 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDisp
                                                  // Because an agent can be idle for a long time between jobs, it is possible that in that time
                                                  // a firewall has closed the connection. For that reason, forcibly reestablish this connection at the
                                                  // start of a new job
-                    Trace.Info(StringUtil.Format("Refreshing server connection before job execution [ConnectionType:JobRequest, Timeout:30s, JobId:{0}]",
+                    Trace.Info(StringUtil.Format("Refreshing server connection before job execution [ConnectionType:JobRequest, JobId:{0}]",
                         message.JobId));
                     var agentServer = HostContext.GetService<IAgentServer>();
-                    await agentServer.RefreshConnectionAsync(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(30));
+                    await agentServer.RefreshConnectionAsync(AgentConnectionType.JobRequest);
 
                     // start renew job request
                     Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
@@ -816,9 +816,9 @@ public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToke
                     if (encounteringError > 0)
                     {
                         encounteringError = 0;
-                        agentServer.SetConnectionTimeout(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(60));
+                        agentServer.ResetConnectionTimeout(AgentConnectionType.JobRequest);
                         HostContext.WritePerfCounter("JobRenewRecovered");
-                        Trace.Info(StringUtil.Format("Job renewal error recovery completed [RequestId:{0}, ErrorsCleared:True, ConnectionTimeout:60s, Status:Recovered]",
+                        Trace.Info(StringUtil.Format("Job renewal error recovery completed [RequestId:{0}, ErrorsCleared:True, Status:Recovered]",
                             requestId));
                     }
 
@@ -888,7 +888,7 @@ public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToke
                         }
 
                         // Re-establish connection to server in order to avoid affinity with server.
-                        // Reduce connection timeout to 30 seconds (from 60s)
+                        // Reduce connection timeout to 30 seconds for faster failure detection during retries
                         HostContext.WritePerfCounter("ResetJobRenewConnection");
                         Trace.Info(StringUtil.Format("Job renewal connection refresh initiated [RequestId:{0}, Timeout:30s, Reason:RetryRecovery, ErrorCount:{1}]",
                             requestId, encounteringError));

src/Agent.Listener/MessageListener.cs

@@ -244,7 +244,7 @@ public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
                         }
 
                         // re-create VssConnection before next retry
-                        await _agentServer.RefreshConnectionAsync(AgentConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
+                        await _agentServer.RefreshConnectionAsync(AgentConnectionType.MessageQueue);
 
                         Trace.Info(StringUtil.Format("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds));
                         await HostContext.Delay(_getNextMessageRetryInterval, token);

src/Microsoft.VisualStudio.Services.Agent/AgentServer.cs

@@ -11,6 +11,7 @@
 using Microsoft.VisualStudio.Services.WebApi;
 using Microsoft.VisualStudio.Services.Common;
 using Agent.Sdk.Util;
+using Agent.Sdk.Knob;
 
 namespace Microsoft.VisualStudio.Services.Agent
 {
@@ -26,9 +27,9 @@ public interface IAgentServer : IAgentService
     {
         Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
 
-        Task RefreshConnectionAsync(AgentConnectionType connectionType, TimeSpan timeout);
+        Task RefreshConnectionAsync(AgentConnectionType connectionType, TimeSpan? timeout = null);
 
-        void SetConnectionTimeout(AgentConnectionType connectionType, TimeSpan timeout);
+        void ResetConnectionTimeout(AgentConnectionType connectionType, TimeSpan? timeout = null);
 
         // Configuration
         Task<TaskAgent> AddAgentAsync(Int32 agentPoolId, TaskAgent agent);
@@ -73,12 +74,22 @@ public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
 
             // Establish the first connection before doing the rest in parallel to eliminate the redundant 401s.
             // issue: https://github.com/microsoft/azure-pipelines-agent/issues/3149
-            Task<VssConnection> task1 = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            
+            // Read timeout from environment variable (VSTS_HTTP_TIMEOUT), default to 100 seconds
+            TimeSpan connectionTimeout = GetConnectionTimeout(AgentConnectionType.Generic, null);
+            Task<VssConnection> task1 = EstablishVssConnection(serverUrl, credentials, connectionTimeout);
 
             _genericConnection = await task1;
 
-            Task<VssConnection> task2 = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
-            Task<VssConnection> task3 = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+            // MessageQueue connection for long-polling - uses full timeout from environment variable
+            TimeSpan messageQueueTimeout = GetConnectionTimeout(AgentConnectionType.MessageQueue, null);
+            Task<VssConnection> task2 = EstablishVssConnection(serverUrl, credentials, messageQueueTimeout);
+            
+            // JobRequest connection uses capped timeout (max 100s) to ensure job lock renewals
+            // happen within the ~5 minute lock window. With 60s between renewals, timeout must be
+            // < 240s to avoid lock expiration, so we cap at 100s for safety.
+            TimeSpan jobRequestTimeout = GetConnectionTimeout(AgentConnectionType.JobRequest, null);
+            Task<VssConnection> task3 = EstablishVssConnection(serverUrl, credentials, jobRequestTimeout);
 
             await Task.WhenAll(task2, task3);
 
@@ -95,8 +106,10 @@ public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
         }
 
         // Refresh connection is best effort. it should never throw exception
-        public async Task RefreshConnectionAsync(AgentConnectionType connectionType, TimeSpan timeout)
+        public async Task RefreshConnectionAsync(AgentConnectionType connectionType, TimeSpan? timeout = null)
         {
+            TimeSpan actualTimeout = GetConnectionTimeout(connectionType, timeout);
+            
             Trace.Info($"Refresh {connectionType} VssConnection to get on a different AFD node.");
             VssConnection newConnection = null;
             switch (connectionType)
@@ -105,7 +118,7 @@ public async Task RefreshConnectionAsync(AgentConnectionType connectionType, Tim
                     try
                     {
                         _hasMessageConnection = false;
-                        newConnection = await EstablishVssConnection(_messageConnection.Uri, _messageConnection.Credentials, timeout);
+                        newConnection = await EstablishVssConnection(_messageConnection.Uri, _messageConnection.Credentials, actualTimeout);
                         var client = newConnection.GetClient<TaskAgentHttpClient>();
                         _messageConnection = newConnection;
                         _messageTaskAgentClient = client;
@@ -130,7 +143,7 @@ public async Task RefreshConnectionAsync(AgentConnectionType connectionType, Tim
                     try
                     {
                         _hasRequestConnection = false;
-                        newConnection = await EstablishVssConnection(_requestConnection.Uri, _requestConnection.Credentials, timeout);
+                        newConnection = await EstablishVssConnection(_requestConnection.Uri, _requestConnection.Credentials, actualTimeout);
                         var client = newConnection.GetClient<TaskAgentHttpClient>();
                         _requestConnection = newConnection;
                         _requestTaskAgentClient = client;
@@ -155,7 +168,7 @@ public async Task RefreshConnectionAsync(AgentConnectionType connectionType, Tim
                     try
                     {
                         _hasGenericConnection = false;
-                        newConnection = await EstablishVssConnection(_genericConnection.Uri, _genericConnection.Credentials, timeout);
+                        newConnection = await EstablishVssConnection(_genericConnection.Uri, _genericConnection.Credentials, actualTimeout);
                         var client = newConnection.GetClient<TaskAgentHttpClient>();
                         _genericConnection = newConnection;
                         _genericTaskAgentClient = client;
@@ -182,19 +195,21 @@ public async Task RefreshConnectionAsync(AgentConnectionType connectionType, Tim
             }
         }
 
-        public void SetConnectionTimeout(AgentConnectionType connectionType, TimeSpan timeout)
+        public void ResetConnectionTimeout(AgentConnectionType connectionType, TimeSpan? timeout = null)
         {
-            Trace.Info($"Set {connectionType} VssConnection's timeout to {timeout.TotalSeconds} seconds.");
+            TimeSpan actualTimeout = GetConnectionTimeout(connectionType, timeout);
+            
+            Trace.Info($"Set {connectionType} VssConnection's timeout to {actualTimeout.TotalSeconds} seconds.");
             switch (connectionType)
             {
                 case AgentConnectionType.JobRequest:
-                    _requestConnection.Settings.SendTimeout = timeout;
+                    _requestConnection.Settings.SendTimeout = actualTimeout;
                     break;
                 case AgentConnectionType.MessageQueue:
-                    _messageConnection.Settings.SendTimeout = timeout;
+                    _messageConnection.Settings.SendTimeout = actualTimeout;
                     break;
                 case AgentConnectionType.Generic:
-                    _genericConnection.Settings.SendTimeout = timeout;
+                    _genericConnection.Settings.SendTimeout = actualTimeout;
                     break;
                 default:
                     Trace.Error($"Unexpected connection type: {connectionType}.");
@@ -229,6 +244,40 @@ private async Task<VssConnection> EstablishVssConnection(Uri serverUrl, VssCrede
             throw new InvalidOperationException(nameof(EstablishVssConnection));
         }
 
+        /// <summary>
+        /// Gets the connection timeout for the specified connection type.
+        /// Reads from VSTS_HTTP_TIMEOUT environment variable with valid range [100, 1200] seconds.
+        /// JobRequest connections are capped at 60s to ensure job lock renewals complete within
+        /// the ~5 minute lock window (60s timeout + 60s delay = 120s < 300s).
+        /// </summary>
+        /// <param name="connectionType">Type of connection</param>
+        /// <param name="timeout">Optional explicit timeout (overrides environment variable but not type-specific caps)</param>
+        /// <returns>Timeout value to use</returns>
+        private TimeSpan GetConnectionTimeout(AgentConnectionType connectionType, TimeSpan? timeout)
+        {
+            TimeSpan actualTimeout;
+            
+            if (timeout.HasValue)
+            {
+                actualTimeout = timeout.Value;
+            }
+            else
+            {
+                // Read from environment variable, clamped to valid range [100, 1200]
+                int httpRequestTimeoutSeconds = AgentKnobs.HttpTimeout.GetValue(HostContext).AsInt();
+                actualTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(httpRequestTimeoutSeconds, 100), 1200));
+            }
+            
+            // Cap JobRequest timeout to 60s to ensure renewals complete within job lock window
+            // This applies to both explicit and environment variable timeouts
+            if (connectionType == AgentConnectionType.JobRequest)
+            {
+                return TimeSpan.FromSeconds(Math.Min(actualTimeout.TotalSeconds, 60));
+            }
+            
+            return actualTimeout;
+        } 
+
         private void CheckConnection(AgentConnectionType connectionType)
         {
             switch (connectionType)

src/Test/L0/Listener/JobDispatcherL0.cs

@@ -330,8 +330,8 @@ public async void DispatcherRenewJobRequestRecoverFromExceptions()
                 Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed.");
                 Assert.True(cancellationTokenSource.IsCancellationRequested);
                 _agentServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny<int>(), It.IsAny<long>(), It.IsAny<Guid>(), It.IsAny<CancellationToken>()), Times.Exactly(8));
-                _agentServer.Verify(x => x.RefreshConnectionAsync(AgentConnectionType.JobRequest, It.IsAny<TimeSpan>()), Times.Exactly(3));
-                _agentServer.Verify(x => x.SetConnectionTimeout(AgentConnectionType.JobRequest, It.IsAny<TimeSpan>()), Times.Once);
+                _agentServer.Verify(x => x.RefreshConnectionAsync(AgentConnectionType.JobRequest, It.IsAny<TimeSpan?>()), Times.Exactly(3));
+                _agentServer.Verify(x => x.ResetConnectionTimeout(AgentConnectionType.JobRequest, It.IsAny<TimeSpan?>()), Times.Once);
             }
         }
 
@@ -453,8 +453,8 @@ public async void DispatcherRenewJobRequestStopOnExpiredRequest()
                 Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed.");
                 Assert.False(cancellationTokenSource.IsCancellationRequested);
                 _agentServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny<int>(), It.IsAny<long>(), It.IsAny<Guid>(), It.IsAny<CancellationToken>()), Times.Exactly(5));
-                _agentServer.Verify(x => x.RefreshConnectionAsync(AgentConnectionType.JobRequest, It.IsAny<TimeSpan>()), Times.Exactly(3));
-                _agentServer.Verify(x => x.SetConnectionTimeout(AgentConnectionType.JobRequest, It.IsAny<TimeSpan>()), Times.Never);
+                _agentServer.Verify(x => x.RefreshConnectionAsync(AgentConnectionType.JobRequest, It.IsAny<TimeSpan?>()), Times.Exactly(3));
+                _agentServer.Verify(x => x.ResetConnectionTimeout(AgentConnectionType.JobRequest, It.IsAny<TimeSpan?>()), Times.Never);
             }
         }