Merge pull request #1278 from ably/fix/broken-fallbacks

Fix broken REST host fallbacks

.github/workflows/run-tests-macos-xamarin.yml

@@ -21,6 +21,7 @@ jobs:
         with:
           dotnet-version: |
             3.1.x
+            7.0.408
 
       - name: Download dotnet build-script tools
         run:  dotnet tool restore

.github/workflows/run-tests-windows-netframework.yml

@@ -16,6 +16,12 @@ jobs:
         with: 
           submodules: 'recursive'
 
+      - name: Download dotnet framework
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: |
+            7.0.408
+            
       - name: Download dotnet build-script tools
         run: dotnet tool restore
 

build-script/build-script.fsproj

@@ -2,7 +2,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <RootNamespace>build_script</RootNamespace>
     <WarnOn>3390;$(WarnOn)</WarnOn>
     <NoWarn>0020</NoWarn>
@@ -12,4 +12,4 @@
     <Compile Include="build.fs" />
   </ItemGroup>
   <Import Project=".paket\Paket.Restore.targets" />
-</Project>
+</Project>

build-script/paket.references

@@ -3,4 +3,4 @@ Microsoft.Build.Tasks.Core
 Fake.Core.Target
 Fake.DotNet.AssemblyInfoFile
 Fake.DotNet.Cli
-Fake.DotNet.Testing.XUnit2
+Fake.DotNet.Testing.XUnit2

src/IO.Ably.NETFramework/IO.Ably.NETFramework.csproj

@@ -108,4 +108,4 @@
     </PropertyGroup>
     <Error Condition="!Exists('..\packages\Microsoft.CodeQuality.Analyzers.2.9.6\build\Microsoft.CodeQuality.Analyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.CodeQuality.Analyzers.2.9.6\build\Microsoft.CodeQuality.Analyzers.props'))" />
   </Target>
-</Project>
+</Project>

src/IO.Ably.Shared/ClientOptions.cs

@@ -147,10 +147,6 @@ public string[] FallbackHosts
 
         internal bool IsProductionEnvironment => Environment.IsEmpty() || Environment.Equals("production", StringComparison.OrdinalIgnoreCase);
 
-        internal bool IsDefaultRestHost => FullRestHost() == Defaults.RestHost;
-
-        internal bool IsDefaultRealtimeHost => FullRealtimeHost() == Defaults.RealtimeHost;
-
         internal bool IsDefaultPort => Tls ? TlsPort == Defaults.TlsPort : Port == Defaults.Port;
 
         /// <summary>
@@ -159,20 +155,17 @@ public string[] FallbackHosts
         /// <returns>RestHost.</returns>
         public string FullRestHost()
         {
-            var restHost = _restHost;
-            if (restHost.IsEmpty())
+            if (_restHost.IsNotEmpty())
             {
-                restHost = Defaults.RestHost;
+                return _restHost;
             }
 
-            if (restHost == Defaults.RestHost)
+            if (!IsProductionEnvironment)
             {
-                return IsProductionEnvironment
-                    ? restHost
-                    : $"{Environment}-{restHost}";
+                return $"{Environment}-{Defaults.RestHost}";
             }
 
-            return restHost;
+            return Defaults.RestHost;
         }
 
         /// <summary>
@@ -181,27 +174,26 @@ public string FullRestHost()
         /// <returns>RealtimeHost.</returns>
         public string FullRealtimeHost()
         {
-            var realtimeHost = _realtimeHost;
-            if (realtimeHost.IsEmpty())
+            if (_realtimeHost.IsNotEmpty())
             {
-                if (_restHost.IsNotEmpty())
-                {
-                    Logger.Warning(
-                        $@"restHost is set to {_restHost} but realtimeHost is not set,
-                                     so setting realtimeHost to {_restHost} too. If this is not what you want,
-                                     please set realtimeHost explicitly.");
-                    return _restHost;
-                }
+                return _realtimeHost;
+            }
 
-                realtimeHost = Defaults.RealtimeHost;
+            if (_restHost.IsNotEmpty())
+            {
+                Logger.Warning(
+                    $@"restHost is set to {_restHost} but realtimeHost is not set,
+                    so setting realtimeHost to {_restHost} too. If this is not what you want,
+                    please set realtimeHost explicitly.");
+                return _restHost;
             }
 
-            if (realtimeHost == Defaults.RealtimeHost)
+            if (!IsProductionEnvironment)
             {
-                return IsProductionEnvironment ? realtimeHost : $"{Environment}{'-'}{realtimeHost}";
+                return $"{Environment}-{Defaults.RealtimeHost}";
             }
 
-            return realtimeHost;
+            return Defaults.RealtimeHost;
         }
 
         /// <summary>
@@ -220,7 +212,7 @@ public string[] GetFallbackHosts()
                     throw new AblyException(new ErrorInfo(msg, ErrorCodes.BadRequest));
                 }
 
-                if (Port != Defaults.Port || TlsPort != Defaults.TlsPort)
+                if (!IsDefaultPort)
                 {
                     const string msg = "fallbackHostsUseDefault cannot be set when port or tlsPort are set";
                     throw new AblyException(new ErrorInfo(msg, ErrorCodes.BadRequest));
@@ -230,15 +222,12 @@ public string[] GetFallbackHosts()
                 {
                     Logger.Warning("Deprecated fallbackHostsUseDefault : There is no longer a need to set this when the environment option is also set since the library will now generate the correct fallback hosts using the environment option.");
                 }
-                else
-                {
-                    Logger.Warning("Deprecated fallbackHostsUseDefault : fallbackHosts: Ably.Defaults.FALLBACK_HOSTS");
-                }
 
+                Logger.Warning("Deprecated fallbackHostsUseDefault : fallbackHosts: Ably.Defaults.FALLBACK_HOSTS");
                 return Defaults.FallbackHosts;
             }
 
-            if (_fallbackHosts is null && _restHost is null && _realtimeHost is null && IsDefaultPort)
+            if (_fallbackHosts is null && _restHost.IsEmpty() && _realtimeHost.IsEmpty() && IsDefaultPort)
             {
                 return IsProductionEnvironment
                     ? Defaults.FallbackHosts

src/IO.Ably.Shared/Http/AblyHttpClient.cs

@@ -52,8 +52,6 @@ internal string RealtimeConnectedFallbackHost
 
         private DateTimeOffset? FallbackHostUsedFrom { get; set; }
 
-        private bool CanRetry => Options.IsDefaultHost || Options.FallbackHostsUseDefault;
-
         internal void SetPreferredHost(string currentHost)
         {
             // If we are retrying the default host we need to clear the preferred one
@@ -90,7 +88,7 @@ public async Task<AblyResponse> Execute(AblyRequest request)
             int currentTry = 0;
             var startTime = Now();
 
-            var numberOfRetries = Options.HttpMaxRetryCount; // One for the first request
+            var maxNumberOfRetries = Options.HttpMaxRetryCount;
             var host = GetHost();
 
             request.Headers.TryGetValue("request_id", out var requestId);
@@ -115,22 +113,20 @@ public async Task<AblyResponse> Execute(AblyRequest request)
                         break;
                     }
 
-                    if (CanRetry && response.CanRetry)
+                    if (response.CanRetry)
                     {
                         currentTry++;
-
                         Logger.Warning(WrapWithRequestId("Failed response. " + response.GetFailedMessage() + ". Retrying..."));
-                        var (success, newHost) = HandleHostChangeForRetryableFailure();
+                        var (success, newHost) = HandleHostChangeForRetryableFailure(currentTry);
                         if (success)
                         {
                             Logger.Debug(WrapWithRequestId($"Retrying using host: {newHost}"));
-
                             host = newHost;
                             continue;
                         }
                     }
 
-                    // We only return the request if there is no exception
+                    // We only return the response if there is no exception
                     if (request.NoExceptionOnHttpError && response.Exception == null)
                     {
                         return response.AblyResponse;
@@ -140,19 +136,21 @@ public async Task<AblyResponse> Execute(AblyRequest request)
                     // in that case we need to create a new AblyException
                     throw response.Exception ?? AblyException.FromResponse(response.AblyResponse);
                 }
-                catch (AblyException)
+                catch (AblyException ex)
                 {
-                    throw;
+                    var errInfo = ex.ErrorInfo;
+                    errInfo.Message = WrapWithRequestId("Error executing request. " + errInfo.Message);
+                    throw new AblyException(errInfo, ex);
                 }
                 catch (Exception ex)
                 {
                     // TODO: Sentry logging here
                     throw new AblyException(new ErrorInfo(WrapWithRequestId("Error executing request. " + ex.Message), ErrorCodes.InternalError), ex);
                 }
             }
-            while (currentTry < numberOfRetries);
+            while (currentTry <= maxNumberOfRetries); // 1 primary host and remaining fallback hosts
 
-            throw new AblyException(new ErrorInfo(WrapWithRequestId("Error executing request"), ErrorCodes.InternalError));
+            throw new AblyException(new ErrorInfo(WrapWithRequestId("Error executing request, exceeded max no. of retries"), ErrorCodes.InternalError));
 
             List<string> GetFallbackHosts()
             {
@@ -251,15 +249,15 @@ async Task<HttpResponseWrapper> MakeRequest(string requestHost)
                 }
             }
 
-            (bool success, string host) HandleHostChangeForRetryableFailure()
+            (bool success, string host) HandleHostChangeForRetryableFailure(int attempt)
             {
                 if (fallbackHosts.Count == 0)
                 {
                     Logger.Debug(WrapWithRequestId("No more hosts left to retry. Cannot assign a new fallback host."));
                     return (false, null);
                 }
 
-                bool isFirstTryForRequest = currentTry == 1;
+                bool isFirstTryForRequest = attempt == 1;
 
                 // If there is a Preferred fallback host already set
                 // and it failed we should try the RealtimeConnected fallback host first

src/IO.Ably.Shared/Http/AblyHttpOptions.cs

@@ -25,8 +25,6 @@ internal class AblyHttpOptions
 
         internal TimeSpan FallbackRetryTimeOut { get; set; }
 
-        public bool IsDefaultHost { get; set; } = true;
-
         internal string[] FallbackHosts { get; set; }
 
         internal bool FallbackHostsUseDefault { get; set; }
@@ -64,12 +62,11 @@ public AblyHttpOptions(ClientOptions options)
             IsSecure = options.Tls;
             Port = options.Tls ? options.TlsPort : options.Port;
             Host = options.FullRestHost();
-            IsDefaultHost = options.IsDefaultRestHost;
             DisconnectedRetryTimeout = options.DisconnectedRetryTimeout;
             SuspendedRetryTimeout = options.SuspendedRetryTimeout;
             HttpOpenTimeout = options.HttpOpenTimeout;
             HttpRequestTimeout = options.HttpRequestTimeout;
-            HttpMaxRetryCount = options.IsDefaultRestHost ? options.HttpMaxRetryCount : 1;
+            HttpMaxRetryCount = options.HttpMaxRetryCount;
             HttpMaxRetryDuration = options.HttpMaxRetryDuration;
             FallbackRetryTimeOut = options.FallbackRetryTimeout;
             FallbackHosts = options.GetFallbackHosts();

src/IO.Ably.Shared/Realtime/AttemptsHelpers.cs

@@ -9,14 +9,7 @@ internal static class AttemptsHelpers
     {
         public static async Task<bool> CanFallback(this AblyRest restClient, ErrorInfo error)
         {
-            return IsDefaultHost() &&
-                   error != null && error.IsRetryableStatusCode() &&
-                   await restClient.CanConnectToAbly();
-
-            bool IsDefaultHost()
-            {
-                return restClient.Options.IsDefaultRealtimeHost;
-            }
+            return error != null && error.IsRetryableStatusCode() && await restClient.CanConnectToAbly();
         }
 
         public static bool ShouldSuspend(this RealtimeState state, Func<DateTimeOffset> now = null)

src/IO.Ably.Tests.NETFramework/IO.Ably.Tests.NETFramework.csproj

@@ -153,4 +153,4 @@
   </ItemGroup>
   <Import Project="..\IO.Ably.Tests.Shared\IO.Ably.Tests.Shared.projitems" Label="Shared" />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
-</Project>
+</Project>

src/IO.Ably.Tests.Shared/Infrastructure/TestHelpers.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using IO.Ably.Tests.Infrastructure;
 using Xunit;
@@ -7,6 +9,11 @@ namespace IO.Ably.Tests
 {
     internal static class TestHelpers
     {
+        public static bool IsSubsetOf<T>(this IEnumerable<T> a, IEnumerable<T> b)
+        {
+            return !a.Except(b).Any();
+        }
+
         public static void AssertContainsParameter(this AblyRequest request, string key, string value)
         {
             Assert.True(

src/IO.Ably.Tests.Shared/Realtime/ConnectionAttemptsInfoSpecs.cs

@@ -57,15 +57,6 @@ public void ShouldSuspend_WhenFirstAttemptEqualOrGreaterThanConnectionStateTtl_S
             state.ShouldSuspend(now.ValueFn).Should().BeTrue("When time is greater than"); // >
         }
 
-        [Fact]
-        public async Task CanAttemptFallback_ShouldBeFalseWithNonDefaultHost()
-        {
-            var client = GetRealtime(opts => opts.RealtimeHost = "test.test.com");
-
-            var result = await client.RestClient.CanFallback(null);
-            result.Should().BeFalse();
-        }
-
         [Theory]
         [InlineData(500)]
         [InlineData(501)]
@@ -84,7 +75,7 @@ public async Task CanAttemptFallback_WithDefaultHostAndAppropriateError_ShouldBe
         public async Task CanAttemptFallback_WhenInternetCheckFails_ShouldBeFalse()
         {
             var client = GetRealtime(internetCheckOk: false);
-            var result = await client.RestClient.CanFallback(null);
+            var result = await client.RestClient.CanFallback(ErrorInfo.ReasonUnknown);
             result.Should().BeFalse();
         }
 

src/IO.Ably.Tests.Shared/Rest/ChannelSandboxSpecs.cs

@@ -244,7 +244,6 @@ public async Task IdempotentPublishing_SimulateErrorAndRetry(Protocol protocol)
                 {
                     // setting IsDefaultHost and raising a TaskCanceledException
                     // will cause the request to retry
-                    client.HttpClient.Options.IsDefaultHost = true;
                     throw new TaskCanceledException("faked exception to cause retry");
                 }
 
@@ -301,7 +300,6 @@ public async Task IdempotentPublishing_WithClientSpecificMessage_ShouldRetry(Pro
                 {
                     // setting IsDefaultHost and raising a TaskCanceledException
                     // will cause the request to retry
-                    client.HttpClient.Options.IsDefaultHost = true;
                     throw new TaskCanceledException("faked exception to cause retry");
                 }