Improving logging and request id (#189)

* hacking around to work out why logging and request id isnt working

* pass request id into logger so it can be structured, removed a bunch of debug logging we dont need because diagnostic trace gets it

* changed config dependency

* always have tracing available

* made it so we dont need to pass config into services.AddOcelot anymore with .net core 2.0

* add test

* lots of changes relating to logging and request ids, also updated documentation

* fixed failing test i missed

test/Ocelot.UnitTests/Configuration/FileConfigurationSetterTests.cs

@@ -38,7 +38,7 @@ namespace Ocelot.UnitTests.Configuration
         {
             var fileConfig = new FileConfiguration();
             var serviceProviderConfig = new ServiceProviderConfigurationBuilder().Build();
-            var config = new OcelotConfiguration(new List<ReRoute>(), string.Empty, serviceProviderConfig);
+            var config = new OcelotConfiguration(new List<ReRoute>(), string.Empty, serviceProviderConfig, "asdf");
 
             this.Given(x => GivenTheFollowingConfiguration(fileConfig))
                 .And(x => GivenTheRepoReturns(new OkResponse()))

test/Ocelot.UnitTests/Configuration/InMemoryConfigurationRepositoryTests.cs

@@ -94,6 +94,8 @@ namespace Ocelot.UnitTests.Configuration
             public string AdministrationPath {get;}
 
             public ServiceProviderConfiguration ServiceProviderConfiguration => throw new NotImplementedException();
+
+            public string RequestId {get;}
         }
     }
 }

test/Ocelot.UnitTests/Configuration/OcelotConfigurationProviderTests.cs

@@ -30,9 +30,9 @@ namespace Ocelot.UnitTests.Configuration
         {
             var serviceProviderConfig = new ServiceProviderConfigurationBuilder().Build();
 
-            this.Given(x => x.GivenTheRepoReturns(new OkResponse<IOcelotConfiguration>(new OcelotConfiguration(new List<ReRoute>(), string.Empty, serviceProviderConfig))))
+            this.Given(x => x.GivenTheRepoReturns(new OkResponse<IOcelotConfiguration>(new OcelotConfiguration(new List<ReRoute>(), string.Empty, serviceProviderConfig, ""))))
                 .When(x => x.WhenIGetTheConfig())
-                .Then(x => x.TheFollowingIsReturned(new OkResponse<IOcelotConfiguration>(new OcelotConfiguration(new List<ReRoute>(), string.Empty, serviceProviderConfig))))
+                .Then(x => x.TheFollowingIsReturned(new OkResponse<IOcelotConfiguration>(new OcelotConfiguration(new List<ReRoute>(), string.Empty, serviceProviderConfig, ""))))
                 .BDDfy();
         }
 

test/Ocelot.UnitTests/Configuration/RequestIdKeyCreatorTests.cs

@@ -51,14 +51,15 @@ namespace Ocelot.UnitTests.Configuration
         }
 
         [Fact]
-        public void should_use_global_cofiguration_over_re_route_specific()
+        public void should_use_re_route_over_global_specific()
         {
             var reRoute = new FileReRoute
             {
                 RequestIdKey = "cheese"
-            };            var globalConfig = new FileGlobalConfiguration
+            };            
+            var globalConfig = new FileGlobalConfiguration
             {
-                RequestIdKey = "cheese"
+                RequestIdKey = "test"
             };
 
             this.Given(x => x.GivenTheFollowingReRoute(reRoute))

test/Ocelot.UnitTests/DependencyInjection/OcelotBuilderTests.cs

@@ -24,7 +24,7 @@ namespace Ocelot.UnitTests.DependencyInjection
     {
         private IServiceCollection _services;
         private IServiceProvider _serviceProvider;
-        private IConfigurationRoot _configRoot;
+        private IConfiguration _configRoot;
         private IOcelotBuilder _ocelotBuilder;
         private int _maxRetries;
 
@@ -35,6 +35,7 @@ namespace Ocelot.UnitTests.DependencyInjection
                 _services = new ServiceCollection();
                 _services.AddSingleton(builder);
                 _services.AddSingleton<IHostingEnvironment, HostingEnvironment>();
+                _services.AddSingleton<IConfiguration>(_configRoot);
                 _maxRetries = 100;
         }
         private Exception _ex;
@@ -95,6 +96,14 @@ namespace Ocelot.UnitTests.DependencyInjection
                 .BDDfy();
         }
 
+        [Fact]
+        public void should_set_up_without_passing_in_config()
+        {
+            this.When(x => WhenISetUpOcelotServicesWithoutConfig())
+                .Then(x => ThenAnExceptionIsntThrown())
+                .BDDfy();
+        }
+
         private void ThenTheCorrectAdminPathIsRegitered()
         {
             _serviceProvider = _services.BuildServiceProvider();
@@ -156,6 +165,19 @@ namespace Ocelot.UnitTests.DependencyInjection
                 _ex = e;
             }
         }
+
+        private void WhenISetUpOcelotServicesWithoutConfig()
+        {
+            try
+            {
+                _ocelotBuilder = _services.AddOcelot();
+            }
+            catch (Exception e)
+            {
+                _ex = e;
+            }
+        }
+
         private void WhenISetUpCacheManager()
         {
             try

test/Ocelot.UnitTests/DownstreamRouteFinder/DownstreamRouteFinderMiddlewareTests.cs

@@ -34,7 +34,7 @@
         [Fact]
         public void should_call_scoped_data_repository_correctly()
         {
-            var config = new OcelotConfiguration(null, null, new ServiceProviderConfigurationBuilder().Build());
+            var config = new OcelotConfiguration(null, null, new ServiceProviderConfigurationBuilder().Build(), "");
 
             this.Given(x => x.GivenTheDownStreamRouteFinderReturns(
                 new DownstreamRoute(

test/Ocelot.UnitTests/DownstreamRouteFinder/DownstreamRouteFinderTests.cs

@@ -422,7 +422,7 @@ namespace Ocelot.UnitTests.DownstreamRouteFinder
         private void GivenTheConfigurationIs(List<ReRoute> reRoutesConfig, string adminPath, ServiceProviderConfiguration serviceProviderConfig)
         {
             _reRoutesConfig = reRoutesConfig;
-            _config = new OcelotConfiguration(_reRoutesConfig, adminPath, serviceProviderConfig);
+            _config = new OcelotConfiguration(_reRoutesConfig, adminPath, serviceProviderConfig, "");
         }
 
         private void GivenThereIsAnUpstreamUrlPath(string upstreamUrlPath)

test/Ocelot.UnitTests/Errors/ExceptionHandlerMiddlewareTests.cs

@@ -11,39 +11,96 @@ namespace Ocelot.UnitTests.Errors
     using TestStack.BDDfy;
     using Xunit;
     using Microsoft.AspNetCore.Http;
+    using Ocelot.Configuration.Provider;
+    using Moq;
+    using Ocelot.Configuration;
+    using Rafty.Concensus;
 
     public class ExceptionHandlerMiddlewareTests : ServerHostedMiddlewareTest
     {
         bool _shouldThrowAnException = false;
+        private Mock<IOcelotConfigurationProvider> _provider;
 
         public ExceptionHandlerMiddlewareTests()
         {
+            _provider = new Mock<IOcelotConfigurationProvider>();
             GivenTheTestServerIsConfigured();
         }
         
         [Fact]
         public void NoDownstreamException()
         {
+            var config = new OcelotConfiguration(null, null, null, null);
+
             this.Given(_ => GivenAnExceptionWillNotBeThrownDownstream())
+                .And(_ => GivenTheConfigurationIs(config))
                 .When(_ => WhenICallTheMiddleware())
                 .Then(_ => ThenTheResponseIsOk())
+                .And(_ => TheRequestIdIsNotSet())
                 .BDDfy();
         }
 
+        private void TheRequestIdIsNotSet()
+        {
+            ScopedRepository.Verify(x => x.Add<string>(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
+        }
+
         [Fact]
         public void DownstreamException()
         {
+            var config = new OcelotConfiguration(null, null, null, null);
+
             this.Given(_ => GivenAnExceptionWillBeThrownDownstream())
+                .And(_ => GivenTheConfigurationIs(config))
                 .When(_ => WhenICallTheMiddleware())
                 .Then(_ => ThenTheResponseIsError())
                 .BDDfy();
         }
 
+        [Fact]
+        public void ShouldSetRequestId()
+        {
+            var config = new OcelotConfiguration(null, null, null, "requestidkey");
+
+            this.Given(_ => GivenAnExceptionWillNotBeThrownDownstream())
+                .And(_ => GivenTheConfigurationIs(config))
+                .When(_ => WhenICallTheMiddlewareWithTheRequestIdKey("requestidkey", "1234"))
+                .Then(_ => ThenTheResponseIsOk())
+                .And(_ => TheRequestIdIsSet("RequestId", "1234"))
+                .BDDfy();
+        }
+
+        [Fact]
+        public void ShouldNotSetRequestId()
+        {
+            var config = new OcelotConfiguration(null, null, null, null);
+
+            this.Given(_ => GivenAnExceptionWillNotBeThrownDownstream())
+                .And(_ => GivenTheConfigurationIs(config))
+                .When(_ => WhenICallTheMiddlewareWithTheRequestIdKey("requestidkey", "1234"))
+                .Then(_ => ThenTheResponseIsOk())
+                .And(_ => TheRequestIdIsNotSet())
+                .BDDfy();
+        }
+
+        private void TheRequestIdIsSet(string key, string value)
+        {
+            ScopedRepository.Verify(x => x.Add<string>(key, value), Times.Once);
+        }
+
+        private void GivenTheConfigurationIs(IOcelotConfiguration config)
+        {
+            var response = new Ocelot.Responses.OkResponse<IOcelotConfiguration>(config);
+            _provider
+                .Setup(x => x.Get()).ReturnsAsync(response);
+        }
+
         protected override void GivenTheTestServerServicesAreConfigured(IServiceCollection services)
         {
             services.AddSingleton<IOcelotLoggerFactory, AspDotNetLoggerFactory>();
             services.AddLogging();
             services.AddSingleton(ScopedRepository.Object);
+            services.AddSingleton<IOcelotConfigurationProvider>(_provider.Object);
         }
 
         protected override void GivenTheTestServerPipelineIsConfigured(IApplicationBuilder app)

test/Ocelot.UnitTests/Infrastructure/HttpDataRepositoryTests.cs

@@ -25,7 +25,7 @@ namespace Ocelot.UnitTests.Infrastructure
         //TODO - Additional tests -> HttpContent null. This should never happen
 
         [Fact]
-        public void Get_returns_correct_key_from_http_context()
+        public void get_returns_correct_key_from_http_context()
         {
 
             this.Given(x => x.GivenAHttpContextContaining("key", "string"))
@@ -35,7 +35,7 @@ namespace Ocelot.UnitTests.Infrastructure
         }
 
         [Fact]
-        public void Get_returns_error_response_if_the_key_is_not_found() //Therefore does not return null
+        public void get_returns_error_response_if_the_key_is_not_found() //Therefore does not return null
         {
             this.Given(x => x.GivenAHttpContextContaining("key", "string"))
                 .When(x => x.GetIsCalledWithKey<string>("keyDoesNotExist"))
@@ -43,6 +43,21 @@ namespace Ocelot.UnitTests.Infrastructure
                 .BDDfy();
         }
 
+        [Fact]
+        public void should_update()
+        {
+            this.Given(x => x.GivenAHttpContextContaining("key", "string"))
+                .And(x => x.UpdateIsCalledWith<string>("key", "new string"))
+                .When(x => x.GetIsCalledWithKey<string>("key"))
+                .Then(x => x.ThenTheResultIsAnOkResponse<string>("new string"))
+                .BDDfy();
+        }
+
+        private void UpdateIsCalledWith<T>(string key, string value)
+        {
+            _httpDataRepository.Update(key, value);
+        }
+
         private void GivenAHttpContextContaining(string key, object o)
         {
             _httpContext.Items.Add(key, o);

test/Ocelot.UnitTests/RequestId/ReRouteRequestIdMiddlewareTests.cs

@@ -19,14 +19,14 @@
     using TestStack.BDDfy;
     using Xunit;
 
-    public class RequestIdMiddlewareTests : ServerHostedMiddlewareTest
+    public class ReRouteRequestIdMiddlewareTests : ServerHostedMiddlewareTest
     {
         private readonly HttpRequestMessage _downstreamRequest;
         private Response<DownstreamRoute> _downstreamRoute;
         private string _value;
         private string _key;
 
-        public RequestIdMiddlewareTests()
+        public ReRouteRequestIdMiddlewareTests()
         {
             _downstreamRequest = new HttpRequestMessage();
 
@@ -50,6 +50,7 @@
             var requestId = Guid.NewGuid().ToString();
 
             this.Given(x => x.GivenTheDownStreamRouteIs(downstreamRoute))
+                .And(x => GivenThereIsNoGlobalRequestId())
                 .And(x => x.GivenTheRequestIdIsAddedToTheRequest("LSRequestId", requestId))
                 .When(x => x.WhenICallTheMiddleware())
                 .Then(x => x.ThenTheTraceIdIs(requestId))
@@ -67,11 +68,74 @@
                 .Build());
 
             this.Given(x => x.GivenTheDownStreamRouteIs(downstreamRoute))
+                .And(x => GivenThereIsNoGlobalRequestId())
                 .When(x => x.WhenICallTheMiddleware())
                 .Then(x => x.ThenTheTraceIdIsAnything())
                 .BDDfy();
         }
 
+        [Fact]
+        public void should_add_request_id_scoped_repo_for_logging_later()
+        {
+            var downstreamRoute = new DownstreamRoute(new List<PlaceholderNameAndValue>(),
+                new ReRouteBuilder()
+                .WithDownstreamPathTemplate("any old string")
+                .WithRequestIdKey("LSRequestId")
+                .WithUpstreamHttpMethod(new List<string> { "Get" })
+                .Build());
+
+            var requestId = Guid.NewGuid().ToString();
+
+            this.Given(x => x.GivenTheDownStreamRouteIs(downstreamRoute))
+                .And(x => GivenThereIsNoGlobalRequestId())
+                .And(x => x.GivenTheRequestIdIsAddedToTheRequest("LSRequestId", requestId))
+                .When(x => x.WhenICallTheMiddleware())
+                .Then(x => x.ThenTheTraceIdIs(requestId))
+                .And(x => ThenTheRequestIdIsSaved())
+                .BDDfy();
+        }
+
+        [Fact]
+        public void should_update_request_id_scoped_repo_for_logging_later()
+        {
+            var downstreamRoute = new DownstreamRoute(new List<PlaceholderNameAndValue>(),
+                new ReRouteBuilder()
+                .WithDownstreamPathTemplate("any old string")
+                .WithRequestIdKey("LSRequestId")
+                .WithUpstreamHttpMethod(new List<string> { "Get" })
+                .Build());
+
+            var requestId = Guid.NewGuid().ToString();
+
+            this.Given(x => x.GivenTheDownStreamRouteIs(downstreamRoute))
+                .And(x => GivenTheRequestIdWasSetGlobally())
+                .And(x => x.GivenTheRequestIdIsAddedToTheRequest("LSRequestId", requestId))
+                .When(x => x.WhenICallTheMiddleware())
+                .Then(x => x.ThenTheTraceIdIs(requestId))
+                .And(x => ThenTheRequestIdIsUpdated())
+                .BDDfy();
+        }
+
+        private void GivenThereIsNoGlobalRequestId()
+        {
+            ScopedRepository.Setup(x => x.Get<string>("RequestId")).Returns(new OkResponse<string>(null));
+        }
+
+        private void GivenTheRequestIdWasSetGlobally()
+        {
+            ScopedRepository.Setup(x => x.Get<string>("RequestId")).Returns(new OkResponse<string>("alreadyset"));
+        }
+
+        private void ThenTheRequestIdIsSaved()
+        {
+            ScopedRepository.Verify(x => x.Add<string>("RequestId", _value), Times.Once);
+        }
+
+        private void ThenTheRequestIdIsUpdated()
+        {
+            ScopedRepository.Verify(x => x.Update<string>("RequestId", _value), Times.Once);
+        }
+
         protected override void GivenTheTestServerServicesAreConfigured(IServiceCollection services)
         {
             services.AddSingleton<IOcelotLoggerFactory, AspDotNetLoggerFactory>();

test/Ocelot.UnitTests/ServerHostedMiddlewareTest.cs

@@ -53,6 +53,12 @@
             ResponseMessage = Client.GetAsync(Url).Result;
         }
 
+        protected void WhenICallTheMiddlewareWithTheRequestIdKey(string requestIdKey, string value)
+        {
+            Client.DefaultRequestHeaders.Add(requestIdKey, value);
+            ResponseMessage = Client.GetAsync(Url).Result;
+        }
+
         public void Dispose()
         {
             Client.Dispose();