Merge branch 'feature/fix-unstable-raft-tests' into develop

src/Ocelot/Configuration/FileConfigurationController.cs

@@ -42,28 +42,35 @@ namespace Ocelot.Configuration
         [HttpPost]
         public async Task<IActionResult> Post([FromBody]FileConfiguration fileConfiguration)
         {
-            //todo - this code is a bit shit sort it out..
+            try
-            var test = _provider.GetService(typeof(INode));
-            if (test != null)
             {
-                var node = (INode)test;
+                //todo - this code is a bit shit sort it out..
-                var result = node.Accept(new UpdateFileConfiguration(fileConfiguration));
+                var test = _provider.GetService(typeof(INode));
-                if (result.GetType() == typeof(Rafty.Concensus.ErrorResponse<UpdateFileConfiguration>))
+                if (test != null)
                 {
-                    return new BadRequestObjectResult("There was a problem. This error message sucks raise an issue in GitHub.");
+                    var node = (INode)test;
+                    var result = node.Accept(new UpdateFileConfiguration(fileConfiguration));
+                    if (result.GetType() == typeof(Rafty.Concensus.ErrorResponse<UpdateFileConfiguration>))
+                    {
+                        return new BadRequestObjectResult("There was a problem. This error message sucks raise an issue in GitHub.");
+                    }
+
+                    return new OkObjectResult(result.Command.Configuration);
                 }
 
-                return new OkObjectResult(result.Command.Configuration);
+                var response = await _setter.Set(fileConfiguration);
+
+                if (response.IsError)
+                {
+                    return new BadRequestObjectResult(response.Errors);
+                }
+
+                return new OkObjectResult(fileConfiguration);
             }
-
+            catch(Exception e)
-            var response = await _setter.Set(fileConfiguration);
-
-            if (response.IsError)
             {
-                return new BadRequestObjectResult(response.Errors);
+                return new BadRequestObjectResult($"{e.Message}:{e.StackTrace}");
             }
-
-            return new OkObjectResult(fileConfiguration);
         }
     }
 }

src/Ocelot/Configuration/Repository/ConsulFileConfigurationPoller.cs

@@ -40,7 +40,7 @@ namespace Ocelot.Configuration.Repository
                 _polling = true;
                 await Poll();
                 _polling = false;
-            }, null, 0, _config.Delay);
+            }, null, _config.Delay, _config.Delay);
         }
         
         private async Task Poll()

src/Ocelot/Infrastructure/DelayedMessage.cs

@@ -0,0 +1,15 @@
+namespace Ocelot.Infrastructure
+{
+    internal class DelayedMessage<T>
+    {
+        public DelayedMessage(T message, int delay)
+        {
+            Delay = delay;
+            Message = message;
+        }
+
+        public T Message { get; set; }
+
+        public int Delay { get; set; }
+    }
+}

src/Ocelot/Infrastructure/IBus.cs

@@ -0,0 +1,10 @@
+using System;
+
+namespace Ocelot.Infrastructure
+{
+    public interface IBus<T>
+    {
+        void Subscribe(Action<T> action);   
+        void Publish(T message, int delay);
+    }
+}

src/Ocelot/Infrastructure/InMemoryBus.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Ocelot.Infrastructure
+{
+    public class InMemoryBus<T> : IBus<T>
+    {
+        private readonly BlockingCollection<DelayedMessage<T>> _queue;
+        private readonly List<Action<T>> _subscriptions;
+        private Thread _processing;
+
+        public InMemoryBus()
+        {
+            _queue = new BlockingCollection<DelayedMessage<T>>();
+            _subscriptions = new List<Action<T>>();
+            _processing = new Thread(async () => await Process());
+            _processing.Start();
+        }
+
+        public void Subscribe(Action<T> action)
+        {
+            _subscriptions.Add(action);
+        }
+
+        public void Publish(T message, int delay)
+        {
+            var delayed = new DelayedMessage<T>(message, delay);
+            _queue.Add(delayed);
+        }
+
+        private async Task Process()
+        {
+            foreach(var delayedMessage in _queue.GetConsumingEnumerable())
+            {
+                await Task.Delay(delayedMessage.Delay);
+
+                foreach (var subscription in _subscriptions)
+                {
+                    subscription(delayedMessage.Message);
+                }
+            }
+        }
+    }
+}

src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs

@@ -3,61 +3,64 @@ namespace Ocelot.LoadBalancer.LoadBalancers
     using System;
     using System.Collections.Concurrent;
     using System.Collections.Generic;
-    using System.Linq;
     using System.Threading;
     using System.Threading.Tasks;
+    using Ocelot.Infrastructure;
     using Ocelot.Middleware;
     using Responses;
     using Values;
 
-    public class CookieStickySessions : ILoadBalancer, IDisposable
+    public class CookieStickySessions : ILoadBalancer
     {
-        private readonly int _expiryInMs;
+        private readonly int _keyExpiryInMs;
         private readonly string _key;
         private readonly ILoadBalancer _loadBalancer;
         private readonly ConcurrentDictionary<string, StickySession> _stored;
-        private readonly Timer _timer;
+        private readonly IBus<StickySession> _bus;
-        private bool _expiring;
+        private readonly object _lock = new object();
 
-        public CookieStickySessions(ILoadBalancer loadBalancer, string key, int expiryInMs)
+        public CookieStickySessions(ILoadBalancer loadBalancer, string key, int keyExpiryInMs, IBus<StickySession> bus)
         {
+            _bus = bus;
             _key = key;
-            _expiryInMs = expiryInMs;
+            _keyExpiryInMs = keyExpiryInMs;
             _loadBalancer = loadBalancer;
             _stored = new ConcurrentDictionary<string, StickySession>();
-            _timer = new Timer(x =>
+            _bus.Subscribe(ss =>
             {
-                if (_expiring)
+                //todo - get test coverage for this.
+                if (_stored.TryGetValue(ss.Key, out var stickySession))
                 {
-                    return;
+                    lock (_lock)
+                    {
+                        if (stickySession.Expiry < DateTime.UtcNow)
+                        {
+                            _stored.Remove(stickySession.Key, out _);
+                            _loadBalancer.Release(stickySession.HostAndPort);
+                        }
+                    }
                 }
-
+            });
-                _expiring = true;
-
-                Expire();
-
-                _expiring = false;
-            }, null, 0, 50);
-        }
-
-        public void Dispose()
-        {
-            _timer?.Dispose();
         }
 
         public async Task<Response<ServiceHostAndPort>> Lease(DownstreamContext context)
         {
-            var value = context.HttpContext.Request.Cookies[_key];
+            var key = context.HttpContext.Request.Cookies[_key];
 
-            if (!string.IsNullOrEmpty(value) && _stored.ContainsKey(value))
+            lock (_lock)
             {
-                var cached = _stored[value];
+                if (!string.IsNullOrEmpty(key) && _stored.ContainsKey(key))
+                {
+                    var cached = _stored[key];
 
-                var updated = new StickySession(cached.HostAndPort, DateTime.UtcNow.AddMilliseconds(_expiryInMs));
+                    var updated = new StickySession(cached.HostAndPort, DateTime.UtcNow.AddMilliseconds(_keyExpiryInMs), key);
 
-                _stored[value] = updated;
+                    _stored[key] = updated;
 
-                return new OkResponse<ServiceHostAndPort>(updated.HostAndPort);
+                    _bus.Publish(updated, _keyExpiryInMs);
+
+                    return new OkResponse<ServiceHostAndPort>(updated.HostAndPort);
+                }
             }
 
             var next = await _loadBalancer.Lease(context);
@@ -67,9 +70,14 @@ namespace Ocelot.LoadBalancer.LoadBalancers
                 return new ErrorResponse<ServiceHostAndPort>(next.Errors);
             }
 
-            if (!string.IsNullOrEmpty(value) && !_stored.ContainsKey(value))
+            lock (_lock)
             {
-                _stored[value] = new StickySession(next.Data, DateTime.UtcNow.AddMilliseconds(_expiryInMs));
+                if (!string.IsNullOrEmpty(key) && !_stored.ContainsKey(key))
+                {
+                    var ss = new StickySession(next.Data, DateTime.UtcNow.AddMilliseconds(_keyExpiryInMs), key);
+                    _stored[key] = ss;
+                    _bus.Publish(ss, _keyExpiryInMs);
+                }
             }
 
             return new OkResponse<ServiceHostAndPort>(next.Data);
@@ -78,16 +86,5 @@ namespace Ocelot.LoadBalancer.LoadBalancers
         public void Release(ServiceHostAndPort hostAndPort)
         {
         }
-
-        private void Expire()
-        {
-            var expired = _stored.Where(x => x.Value.Expiry < DateTime.UtcNow);
-
-            foreach (var expire in expired)
-            {
-                _stored.Remove(expire.Key, out _);
-                _loadBalancer.Release(expire.Value.HostAndPort);
-            }
-        }
     }
 }

src/Ocelot/LoadBalancer/LoadBalancers/LoadBalancerFactory.cs

@@ -1,5 +1,6 @@
 using System.Threading.Tasks;
 using Ocelot.Configuration;
+using Ocelot.Infrastructure;
 using Ocelot.ServiceDiscovery;
 
 namespace Ocelot.LoadBalancer.LoadBalancers
@@ -25,7 +26,8 @@ namespace Ocelot.LoadBalancer.LoadBalancers
                     return new LeastConnection(async () => await serviceProvider.Get(), reRoute.ServiceName);
                 case nameof(CookieStickySessions):
                     var loadBalancer = new RoundRobin(async () => await serviceProvider.Get());
-                    return new CookieStickySessions(loadBalancer, reRoute.LoadBalancerOptions.Key, reRoute.LoadBalancerOptions.ExpiryInMs);
+                    var bus = new InMemoryBus<StickySession>();
+                    return new CookieStickySessions(loadBalancer, reRoute.LoadBalancerOptions.Key, reRoute.LoadBalancerOptions.ExpiryInMs, bus);
                 default:
                     return new NoLoadBalancer(await serviceProvider.Get());
             }

src/Ocelot/LoadBalancer/LoadBalancers/StickySession.cs

@@ -5,14 +5,17 @@ namespace Ocelot.LoadBalancer.LoadBalancers
 {
     public class StickySession
     {
-        public StickySession(ServiceHostAndPort hostAndPort, DateTime expiry)
+        public StickySession(ServiceHostAndPort hostAndPort, DateTime expiry, string key)
         {
             HostAndPort = hostAndPort;
             Expiry = expiry;
+            Key = key;
         }
 
         public ServiceHostAndPort HostAndPort { get; }
         
         public DateTime Expiry { get; }
+
+        public string Key {get;}
     }
 }

src/Ocelot/Raft/HttpPeer.cs

@@ -88,6 +88,7 @@ namespace Ocelot.Raft
         public Response<T> Request<T>(T command)
             where T : ICommand
         {
+            Console.WriteLine("SENDING REQUEST....");
             if(_token == null)
             {
                 SetToken();
@@ -99,10 +100,13 @@ namespace Ocelot.Raft
             var response = _httpClient.PostAsync($"{_hostAndPort}/administration/raft/command", content).GetAwaiter().GetResult();
             if(response.IsSuccessStatusCode)
             {
-                return JsonConvert.DeserializeObject<OkResponse<T>>(response.Content.ReadAsStringAsync().GetAwaiter().GetResult(), _jsonSerializerSettings);
+                Console.WriteLine("REQUEST OK....");
+                var okResponse = JsonConvert.DeserializeObject<OkResponse<ICommand>>(response.Content.ReadAsStringAsync().GetAwaiter().GetResult(), _jsonSerializerSettings);
+                return new OkResponse<T>((T)okResponse.Command);
             }
             else 
             {
+                Console.WriteLine("REQUEST NOT OK....");
                 return new ErrorResponse<T>(response.Content.ReadAsStringAsync().GetAwaiter().GetResult(), command);
             }
         }

test/Ocelot.AcceptanceTests/Steps.cs

@@ -701,7 +701,7 @@ namespace Ocelot.AcceptanceTests
             Task.WaitAll(tasks);
         }
 
-        public async Task WhenIGetUrlOnTheApiGatewayMultipleTimes(string url, int times, string cookie, string value)
+        public void WhenIGetUrlOnTheApiGatewayMultipleTimes(string url, int times, string cookie, string value)
         {
             var tasks = new Task[times];
 

test/Ocelot.Benchmarks/AllTheThingsBenchmarks.cs

@@ -76,16 +76,16 @@ namespace Ocelot.Benchmarks
             response.EnsureSuccessStatusCode();
         }
 
-       //  * Summary*
+/*         * Summary*
-       //  BenchmarkDotNet = v0.10.13, OS = macOS 10.12.6 (16G1212) [Darwin 16.7.0]
+         BenchmarkDotNet = v0.10.13, OS = macOS 10.12.6 (16G1212) [Darwin 16.7.0]
-       // Intel Core i5-4278U CPU 2.60GHz(Haswell), 1 CPU, 4 logical cores and 2 physical cores
+        Intel Core i5-4278U CPU 2.60GHz(Haswell), 1 CPU, 4 logical cores and 2 physical cores
-       //.NET Core SDK = 2.1.4
+       .NET Core SDK = 2.1.4
 
-       //  [Host]     : .NET Core 2.0.6 (CoreCLR 4.6.0.0, CoreFX 4.6.26212.01), 64bit RyuJIT
+         [Host]     : .NET Core 2.0.6 (CoreCLR 4.6.0.0, CoreFX 4.6.26212.01), 64bit RyuJIT
-       //    DefaultJob : .NET Core 2.0.6 (CoreCLR 4.6.0.0, CoreFX 4.6.26212.01), 64bit RyuJIT
+           DefaultJob : .NET Core 2.0.6 (CoreCLR 4.6.0.0, CoreFX 4.6.26212.01), 64bit RyuJIT
-       //     Method |     Mean |     Error |    StdDev |    StdErr |      Min |       Q1 |   Median |       Q3 |      Max |  Op/s | Scaled |   Gen 0 |  Gen 1 | Allocated |
+            Method |     Mean |     Error |    StdDev |    StdErr |      Min |       Q1 |   Median |       Q3 |      Max |  Op/s | Scaled |   Gen 0 |  Gen 1 | Allocated |
-       //  --------- |---------:|----------:|----------:|----------:|---------:|---------:|---------:|---------:|---------:|------:|-------:|--------:|-------:|----------:|
+         --------- |---------:|----------:|----------:|----------:|---------:|---------:|---------:|---------:|---------:|------:|-------:|--------:|-------:|----------:|
-       //  Baseline | 2.102 ms | 0.0292 ms | 0.0273 ms | 0.0070 ms | 2.063 ms | 2.080 ms | 2.093 ms | 2.122 ms | 2.152 ms | 475.8 |   1.00 | 31.2500 | 3.9063 |   1.63 KB |
+        Baseline | 2.102 ms | 0.0292 ms | 0.0273 ms | 0.0070 ms | 2.063 ms | 2.080 ms | 2.093 ms | 2.122 ms | 2.152 ms | 475.8 |   1.00 | 31.2500 | 3.9063 |   1.63 KB |*/
 
         private void GivenOcelotIsRunning(string url)
         {

test/Ocelot.IntegrationTests/RaftTests.cs

@@ -23,6 +23,8 @@ using Ocelot.Middleware;
 
 namespace Ocelot.IntegrationTests
 {
+    using Xunit.Abstractions;
+
     public class RaftTests : IDisposable
     {
         private readonly List<IWebHost> _builders;
@@ -34,9 +36,11 @@ namespace Ocelot.IntegrationTests
         private BearerToken _token;
         private HttpResponseMessage _response;
         private static readonly object _lock = new object();
+        private ITestOutputHelper _output;
 
-        public RaftTests()
+        public RaftTests(ITestOutputHelper output)
         {
+            _output = output;
             _httpClientForAssertions = new HttpClient();
             _httpClient = new HttpClient();
             var ocelotBaseUrl = "http://localhost:5000";
@@ -46,21 +50,7 @@ namespace Ocelot.IntegrationTests
             _threads = new List<Thread>();
         }
         
-        public void Dispose()
+        [Fact]
-        {
-            foreach (var builder in _builders)
-            {
-                builder?.Dispose();
-            }
-
-            foreach (var peer in _peers.Peers)
-            {
-                File.Delete(peer.HostAndPort.Replace("/","").Replace(":",""));
-                File.Delete($"{peer.HostAndPort.Replace("/","").Replace(":","")}.db");
-            }
-        }
-
-        [Fact(Skip = "This tests is flakey at the moment so ignoring will be fixed long term see https://github.com/TomPallister/Ocelot/issues/245")]
         public void should_persist_command_to_five_servers()
         {
              var configuration = new FileConfiguration
@@ -113,13 +103,13 @@ namespace Ocelot.IntegrationTests
             var command = new UpdateFileConfiguration(updatedConfiguration);
             GivenThereIsAConfiguration(configuration);
             GivenFiveServersAreRunning();
-            GivenALeaderIsElected();
             GivenIHaveAnOcelotToken("/administration");
             WhenISendACommandIntoTheCluster(command);
+            Thread.Sleep(5000);
             ThenTheCommandIsReplicatedToAllStateMachines(command);
         }
 
-        [Fact(Skip = "This tests is flakey at the moment so ignoring will be fixed long term see https://github.com/TomPallister/Ocelot/issues/245")]
+        [Fact]
         public void should_persist_command_to_five_servers_when_using_administration_api()
         {
              var configuration = new FileConfiguration
@@ -166,7 +156,6 @@ namespace Ocelot.IntegrationTests
             var command = new UpdateFileConfiguration(updatedConfiguration);
             GivenThereIsAConfiguration(configuration);
             GivenFiveServersAreRunning();
-            GivenALeaderIsElected();
             GivenIHaveAnOcelotToken("/administration");
             GivenIHaveAddedATokenToMyRequest();
             WhenIPostOnTheApiGateway("/administration/configuration", updatedConfiguration);
@@ -175,37 +164,54 @@ namespace Ocelot.IntegrationTests
 
         private void WhenISendACommandIntoTheCluster(UpdateFileConfiguration command)
         {
-            var p = _peers.Peers.First();
+            bool SendCommand()
-            var json = JsonConvert.SerializeObject(command,new JsonSerializerSettings() { 
-                TypeNameHandling = TypeNameHandling.All
-            });
-            var httpContent = new StringContent(json);
-            httpContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
-            using(var httpClient = new HttpClient())
             {
-                httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _token.AccessToken);
+                try
-                var response = httpClient.PostAsync($"{p.HostAndPort}/administration/raft/command", httpContent).GetAwaiter().GetResult();
+                {
-                response.EnsureSuccessStatusCode();
+                    var p = _peers.Peers.First();
-                var content = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+                    var json = JsonConvert.SerializeObject(command, new JsonSerializerSettings()
-                var result = JsonConvert.DeserializeObject<OkResponse<UpdateFileConfiguration>>(content);
+                    {
-                result.Command.Configuration.ReRoutes.Count.ShouldBe(2);
+                        TypeNameHandling = TypeNameHandling.All
+                    });
+                    var httpContent = new StringContent(json);
+                    httpContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
+                    using (var httpClient = new HttpClient())
+                    {
+                        httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _token.AccessToken);
+                        var response = httpClient.PostAsync($"{p.HostAndPort}/administration/raft/command", httpContent).GetAwaiter().GetResult();
+                        response.EnsureSuccessStatusCode();
+                        var content = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+
+                        var errorResult = JsonConvert.DeserializeObject<ErrorResponse<UpdateFileConfiguration>>(content);
+
+                        if (!string.IsNullOrEmpty(errorResult.Error))
+                        {
+                            return false;
+                        }
+
+                        var okResult = JsonConvert.DeserializeObject<OkResponse<UpdateFileConfiguration>>(content);
+
+                        if (okResult.Command.Configuration.ReRoutes.Count == 2)
+                        {
+                            return true;
+                        }
+                    }
+
+                    return false;
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                    return false;
+                }
             }
 
-            //dirty sleep to make sure command replicated...
+            var commandSent = WaitFor(20000).Until(() => SendCommand());
-            var stopwatch = Stopwatch.StartNew();
+            commandSent.ShouldBeTrue();   
-            while(stopwatch.ElapsedMilliseconds < 10000)
-            {
-            }
         }
 
         private void ThenTheCommandIsReplicatedToAllStateMachines(UpdateFileConfiguration expecteds)
         {            
-            //dirty sleep to give a chance to replicate...
-            var stopwatch = Stopwatch.StartNew();
-            while(stopwatch.ElapsedMilliseconds < 2000)
-            {
-            }
-            
             bool CommandCalledOnAllStateMachines()
             {
                 try
@@ -256,7 +262,8 @@ namespace Ocelot.IntegrationTests
                 }
                 catch(Exception e)
                 {
-                    Console.WriteLine(e);
+                    //_output.WriteLine($"{e.Message}, {e.StackTrace}");
+                    //Console.WriteLine(e);
                     return false;
                 }
             }
@@ -267,10 +274,29 @@ namespace Ocelot.IntegrationTests
 
         private void WhenIPostOnTheApiGateway(string url, FileConfiguration updatedConfiguration)
         {
-            var json = JsonConvert.SerializeObject(updatedConfiguration);
+            bool SendCommand()
-            var content = new StringContent(json);
+            {
-            content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+                var json = JsonConvert.SerializeObject(updatedConfiguration);
-            _response = _httpClient.PostAsync(url, content).Result;
+                var content = new StringContent(json);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+                _response = _httpClient.PostAsync(url, content).Result;
+                var responseContent = _response.Content.ReadAsStringAsync().Result;
+
+                if(responseContent == "There was a problem. This error message sucks raise an issue in GitHub.")
+                {
+                    return false;
+                }
+
+                if(string.IsNullOrEmpty(responseContent))
+                {
+                    return false;
+                }
+
+                return _response.IsSuccessStatusCode;
+            }
+
+            var commandSent = WaitFor(20000).Until(() => SendCommand());
+            commandSent.ShouldBeTrue();  
         }
 
         private void GivenIHaveAddedATokenToMyRequest()
@@ -280,23 +306,40 @@ namespace Ocelot.IntegrationTests
 
         private void GivenIHaveAnOcelotToken(string adminPath)
         {
-            var tokenUrl = $"{adminPath}/connect/token";
+            bool AddToken()
-            var formData = new List<KeyValuePair<string, string>>
             {
-                new KeyValuePair<string, string>("client_id", "admin"),
+                try
-                new KeyValuePair<string, string>("client_secret", "secret"),
+                {
-                new KeyValuePair<string, string>("scope", "admin"),
+                    var tokenUrl = $"{adminPath}/connect/token";
-                new KeyValuePair<string, string>("grant_type", "client_credentials")
+                    var formData = new List<KeyValuePair<string, string>>
-            };
+                    {
-            var content = new FormUrlEncodedContent(formData);
+                        new KeyValuePair<string, string>("client_id", "admin"),
+                        new KeyValuePair<string, string>("client_secret", "secret"),
+                        new KeyValuePair<string, string>("scope", "admin"),
+                        new KeyValuePair<string, string>("grant_type", "client_credentials")
+                    };
+                    var content = new FormUrlEncodedContent(formData);
 
-            var response = _httpClient.PostAsync(tokenUrl, content).Result;
+                    var response = _httpClient.PostAsync(tokenUrl, content).Result;
-            var responseContent = response.Content.ReadAsStringAsync().Result;
+                    var responseContent = response.Content.ReadAsStringAsync().Result;
-            response.EnsureSuccessStatusCode();
+                    if(!response.IsSuccessStatusCode)
-            _token = JsonConvert.DeserializeObject<BearerToken>(responseContent);
+                    {
-            var configPath = $"{adminPath}/.well-known/openid-configuration";
+                        return false;
-            response = _httpClient.GetAsync(configPath).Result;
+                    }
-            response.EnsureSuccessStatusCode();
+
+                    _token = JsonConvert.DeserializeObject<BearerToken>(responseContent);
+                    var configPath = $"{adminPath}/.well-known/openid-configuration";
+                    response = _httpClient.GetAsync(configPath).Result;
+                    return response.IsSuccessStatusCode;
+                }
+                catch(Exception)
+                {
+                    return false;
+                }
+            }
+
+            var addToken = WaitFor(20000).Until(() => AddToken());
+            addToken.ShouldBeTrue();   
         }
 
         private void GivenThereIsAConfiguration(FileConfiguration fileConfiguration)
@@ -375,18 +418,32 @@ namespace Ocelot.IntegrationTests
 
             foreach (var peer in _peers.Peers)
             {
+                File.Delete(peer.HostAndPort.Replace("/", "").Replace(":", ""));
+                File.Delete($"{peer.HostAndPort.Replace("/", "").Replace(":", "")}.db");
                 var thread = new Thread(() => GivenAServerIsRunning(peer.HostAndPort));
                 thread.Start();
                 _threads.Add(thread);
             }
         }
 
-        private void GivenALeaderIsElected()
+        public void Dispose()
         {
-            //dirty sleep to make sure we have a leader
+            foreach (var builder in _builders)
-            var stopwatch = Stopwatch.StartNew();
-            while(stopwatch.ElapsedMilliseconds < 20000)
             {
+                builder?.Dispose();
+            }
+
+            foreach (var peer in _peers.Peers)
+            {
+                try
+                {
+                    File.Delete(peer.HostAndPort.Replace("/", "").Replace(":", ""));
+                    File.Delete($"{peer.HostAndPort.Replace("/", "").Replace(":", "")}.db");
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                }
             }
         }
     }

test/Ocelot.UnitTests/Configuration/ConsulFileConfigurationPollerTests.cs

@@ -17,11 +17,11 @@ namespace Ocelot.UnitTests.Configuration
 {
     public class ConsulFileConfigurationPollerTests : IDisposable
     {
-        private ConsulFileConfigurationPoller _poller;
+        private readonly ConsulFileConfigurationPoller _poller;
         private Mock<IOcelotLoggerFactory> _factory;
-        private Mock<IFileConfigurationRepository> _repo;
+        private readonly Mock<IFileConfigurationRepository> _repo;
-        private Mock<IFileConfigurationSetter> _setter;
+        private readonly Mock<IFileConfigurationSetter> _setter;
-        private FileConfiguration _fileConfig;
+        private readonly FileConfiguration _fileConfig;
         private Mock<IConsulPollerConfiguration> _config;
 
         public ConsulFileConfigurationPollerTests()
@@ -70,7 +70,7 @@ namespace Ocelot.UnitTests.Configuration
             };
 
             this.Given(x => WhenTheConfigIsChangedInConsul(newConfig, 0))
-                .Then(x => ThenTheSetterIsCalled(newConfig, 1))
+                .Then(x => ThenTheSetterIsCalledAtLeast(newConfig, 1))
                 .BDDfy();
         }
 
@@ -154,5 +154,21 @@ namespace Ocelot.UnitTests.Configuration
             });
             result.ShouldBeTrue();
         }
+
+        private void ThenTheSetterIsCalledAtLeast(FileConfiguration fileConfig, int times)
+        {
+            var result = WaitFor(2000).Until(() => {
+                try
+                {
+                    _setter.Verify(x => x.Set(fileConfig), Times.AtLeast(times));
+                    return true;
+                }
+                catch(Exception)
+                {
+                    return false;
+                }
+            });
+            result.ShouldBeTrue();
+        }
     }
 }

test/Ocelot.UnitTests/Infrastructure/InMemoryBusTests.cs

@@ -0,0 +1,40 @@
+using System.Threading.Tasks;
+using Ocelot.Infrastructure;
+using Shouldly;
+using Xunit;
+
+namespace Ocelot.UnitTests.Infrastructure
+{
+    public class InMemoryBusTests
+    {
+        private readonly InMemoryBus<object> _bus;
+
+        public InMemoryBusTests()
+        {
+            _bus = new InMemoryBus<object>();
+        }
+
+        [Fact]
+        public async Task should_publish_with_delay()
+        {
+            var called = false;
+            _bus.Subscribe(x => {
+                called = true;
+            });
+            _bus.Publish(new object(), 1);
+            await Task.Delay(10);
+            called.ShouldBeTrue();
+        }
+
+        [Fact]
+        public void should_not_be_publish_yet_as_no_delay_in_caller()
+        {
+            var called = false;
+            _bus.Subscribe(x => {
+                called = true;
+            });
+            _bus.Publish(new object(), 1);
+            called.ShouldBeFalse();
+        }
+    }
+}

test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs

@@ -1,5 +1,6 @@
 namespace Ocelot.UnitTests.LoadBalancer
 {
+    using System;
     using System.Threading.Tasks;
     using Ocelot.LoadBalancer.LoadBalancers;
     using Ocelot.Responses;
@@ -10,28 +11,43 @@ namespace Ocelot.UnitTests.LoadBalancer
     using Microsoft.AspNetCore.Http;
     using System.Collections.Generic;
     using System.Collections;
-    using System.Threading;
     using Ocelot.Middleware;
     using Ocelot.UnitTests.Responder;
     using TestStack.BDDfy;
+    using Ocelot.Infrastructure;
 
     public class CookieStickySessionsTests
     {
         private readonly CookieStickySessions _stickySessions;
         private readonly Mock<ILoadBalancer> _loadBalancer;
+        private readonly int _defaultExpiryInMs;
         private DownstreamContext _downstreamContext;
         private Response<ServiceHostAndPort> _result;
         private Response<ServiceHostAndPort> _firstHostAndPort;
         private Response<ServiceHostAndPort> _secondHostAndPort;
+        private readonly FakeBus<StickySession> _bus;
 
         public CookieStickySessionsTests()
         {
+            _bus = new FakeBus<StickySession>();
             _loadBalancer = new Mock<ILoadBalancer>();
-            const int defaultExpiryInMs = 100;
+            _defaultExpiryInMs = 0;
-            _stickySessions = new CookieStickySessions(_loadBalancer.Object, "sessionid", defaultExpiryInMs);
+            _stickySessions = new CookieStickySessions(_loadBalancer.Object, "sessionid", _defaultExpiryInMs, _bus);
             _downstreamContext = new DownstreamContext(new DefaultHttpContext());
         }
 
+        [Fact]
+        public void should_expire_sticky_session()
+        {
+            this.Given(_ => GivenTheLoadBalancerReturns())
+                .And(_ => GivenTheDownstreamRequestHasSessionId("321"))
+                .And(_ => GivenIHackAMessageInWithAPastExpiry())
+                .And(_ => WhenILease())
+                .When(_ => WhenTheMessagesAreProcessed())
+                .Then(_ => ThenTheLoadBalancerIsCalled())
+                .BDDfy();
+        }
+
         [Fact]
         public void should_return_host_and_port()
         {
@@ -48,6 +64,7 @@ namespace Ocelot.UnitTests.LoadBalancer
                 .And(_ => GivenTheDownstreamRequestHasSessionId("321"))
                 .When(_ => WhenILeaseTwiceInARow())
                 .Then(_ => ThenTheFirstAndSecondResponseAreTheSame())
+                .And(_ => ThenTheStickySessionWillTimeout())
                 .BDDfy();
         }
 
@@ -69,92 +86,26 @@ namespace Ocelot.UnitTests.LoadBalancer
                 .BDDfy();
         }
 
-        [Fact]
-        public void should_expire_sticky_session()
-        {
-            this.Given(_ => GivenTheLoadBalancerReturnsSequence())
-                .When(_ => WhenTheStickySessionExpires())
-                .Then(_ => ThenANewHostAndPortIsReturned())
-                .BDDfy();
-        }
-
-        [Fact]
-        public void should_refresh_sticky_session()
-        {
-            this.Given(_ => GivenTheLoadBalancerReturnsSequence())
-                .When(_ => WhenIMakeRequestsToKeepRefreshingTheSession())
-                .Then(_ => ThenTheSessionIsRefreshed())
-                .BDDfy();
-        }
-
-        [Fact]
-        public void should_dispose()
-        {
-            _stickySessions.Dispose();
-        }
-
         [Fact]
         public void should_release()
         {
             _stickySessions.Release(new ServiceHostAndPort("", 0));
         }
 
-        private async Task ThenTheSessionIsRefreshed()
+        private void ThenTheLoadBalancerIsCalled()
         {
-            var postExpireHostAndPort = await _stickySessions.Lease(_downstreamContext);
+            _loadBalancer.Verify(x => x.Release(It.IsAny<ServiceHostAndPort>()), Times.Once);
-            postExpireHostAndPort.Data.DownstreamHost.ShouldBe("one");
-            postExpireHostAndPort.Data.DownstreamPort.ShouldBe(80);
-
-            _loadBalancer
-                .Verify(x => x.Lease(It.IsAny<DownstreamContext>()), Times.Once);
         }
 
-        private async Task WhenIMakeRequestsToKeepRefreshingTheSession()
+        private void WhenTheMessagesAreProcessed()
         {
-            var context = new DefaultHttpContext();
+            _bus.Process();
-            var cookies = new FakeCookies();
-            cookies.AddCookie("sessionid", "321");
-            context.Request.Cookies = cookies;
-            _downstreamContext = new DownstreamContext(context);
-
-            var firstHostAndPort = await _stickySessions.Lease(_downstreamContext);
-            firstHostAndPort.Data.DownstreamHost.ShouldBe("one");
-            firstHostAndPort.Data.DownstreamPort.ShouldBe(80);
-
-            Thread.Sleep(80);
-
-            var secondHostAndPort = await _stickySessions.Lease(_downstreamContext);
-            secondHostAndPort.Data.DownstreamHost.ShouldBe("one");
-            secondHostAndPort.Data.DownstreamPort.ShouldBe(80);
-
-            Thread.Sleep(80);
         }
 
-        private async Task ThenANewHostAndPortIsReturned()
+        private void GivenIHackAMessageInWithAPastExpiry()
         {
-            var postExpireHostAndPort = await _stickySessions.Lease(_downstreamContext);
+            var hostAndPort = new ServiceHostAndPort("999", 999);
-            postExpireHostAndPort.Data.DownstreamHost.ShouldBe("two");
+            _bus.Publish(new StickySession(hostAndPort, DateTime.UtcNow.AddDays(-1), "321"), 0);
-            postExpireHostAndPort.Data.DownstreamPort.ShouldBe(80);
-        }
-
-        private async Task WhenTheStickySessionExpires()
-        {
-            var context = new DefaultHttpContext();
-            var cookies = new FakeCookies();
-            cookies.AddCookie("sessionid", "321");
-            context.Request.Cookies = cookies;
-            _downstreamContext = new DownstreamContext(context);
-
-            var firstHostAndPort = await _stickySessions.Lease(_downstreamContext);
-            var secondHostAndPort = await _stickySessions.Lease(_downstreamContext);
-
-            firstHostAndPort.Data.DownstreamHost.ShouldBe("one");
-            firstHostAndPort.Data.DownstreamPort.ShouldBe(80);
-
-            secondHostAndPort.Data.DownstreamHost.ShouldBe("one");
-            secondHostAndPort.Data.DownstreamPort.ShouldBe(80);
-
-            Thread.Sleep(150);
         }
 
         private void ThenAnErrorIsReturned()
@@ -236,9 +187,14 @@ namespace Ocelot.UnitTests.LoadBalancer
         {
             _result.Data.ShouldNotBeNull();
         }
+
+        private void ThenTheStickySessionWillTimeout()
+        {
+            _bus.Messages.Count.ShouldBe(2);
+        }
     }
 
-    class FakeCookies : IRequestCookieCollection
+    internal class FakeCookies : IRequestCookieCollection
     {
         private readonly Dictionary<string, string> _cookies = new Dictionary<string, string>();
 
@@ -273,4 +229,37 @@ namespace Ocelot.UnitTests.LoadBalancer
             return _cookies.GetEnumerator();
         }
     }
+
+    internal class FakeBus<T> : IBus<T>
+    {
+        public FakeBus()
+        {
+            Messages = new List<T>();
+            Subscriptions = new List<Action<T>>();
+        }
+
+        public List<T> Messages { get; }
+        public List<Action<T>> Subscriptions { get; }
+
+        public void Subscribe(Action<T> action)
+        {
+            Subscriptions.Add(action);
+        }
+
+        public void Publish(T message, int delay)
+        {
+            Messages.Add(message);
+        }
+
+        public void Process()
+        {
+            foreach (var message in Messages)
+            {
+                foreach (var subscription in Subscriptions)
+                {
+                    subscription(message);
+                }
+            }
+        }
+    }
 }