Monitoring (#219)

* feat：  use Https://github.com/ButterflyAPM to monitor each API request monitoring metrics

* feat: using DiagnosticSource and Butterfly.OpenTracing

* refactor：refactor Ocelot tracing， merge code into OcelotDiagnosticListener

* refactor: move OcelotHttpTracingHandler to Requester

* fix： Requester\HttpClientBuilder.cs(10,14): error CS0234: The type or namespace name 'Tracing' does not exist in the namespace

* feat: add test should_set_up_tracing

* feat : Remove extraneous code

* feat: remove unused DiagnosticSource diagnostic

* fix : test UseTracing

* add test should_call_scoped_data_repository_QosProviderError

* add test should_return_any_errors

* add test HttpClientHttpRequesterTest

*  it should keep it can not be deleted

Ocelot.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
-VisualStudioVersion = 15.0.26730.15
+VisualStudioVersion = 15.0.27130.2024
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{5CFB79B7-C9DC-45A4-9A75-625D92471702}"
 EndProject

src/Ocelot/Configuration/Creator/HttpHandlerOptionsCreator.cs

@@ -7,7 +7,7 @@ namespace Ocelot.Configuration.Creator
         public HttpHandlerOptions Create(FileReRoute fileReRoute)
         {
             return new HttpHandlerOptions(fileReRoute.HttpHandlerOptions.AllowAutoRedirect,
-                fileReRoute.HttpHandlerOptions.UseCookieContainer);
+                fileReRoute.HttpHandlerOptions.UseCookieContainer, fileReRoute.HttpHandlerOptions.UseTracing);
         }
     }
 }

src/Ocelot/Configuration/File/FileHttpHandlerOptions.cs

@@ -11,5 +11,7 @@
         public bool AllowAutoRedirect { get; set; }
 
         public bool UseCookieContainer { get; set; }
+
+        public bool UseTracing { get; set; }
     }
 }

src/Ocelot/Configuration/HttpHandlerOptions.cs

@@ -6,10 +6,11 @@
     /// </summary>
     public class HttpHandlerOptions
     {
-        public HttpHandlerOptions(bool allowAutoRedirect, bool useCookieContainer)
+        public HttpHandlerOptions(bool allowAutoRedirect, bool useCookieContainer, bool useTracing)
         {
             AllowAutoRedirect = allowAutoRedirect;
             UseCookieContainer = useCookieContainer;
+            UseTracing = useTracing;
         }
 
         /// <summary>
@@ -21,5 +22,10 @@
         /// Specify is handler has to use a cookie container
         /// </summary>
         public bool UseCookieContainer { get; private set; }
+
+        // <summary>
+        /// Specify is handler has to use a opentracing
+        /// </summary>
+        public bool UseTracing { get; private set; }
     }
 }

src/Ocelot/DependencyInjection/IOcelotBuilder.cs

@@ -1,3 +1,4 @@
+using Butterfly.Client.AspNetCore;
 using CacheManager.Core;
 using System;
 
@@ -7,6 +8,7 @@ namespace Ocelot.DependencyInjection
     {
         IOcelotBuilder AddStoreOcelotConfigurationInConsul();
         IOcelotBuilder AddCacheManager(Action<ConfigurationBuilderCachePart> settings);
+        IOcelotBuilder AddOpenTracing(Action<ButterflyOptions> settings);      
         IOcelotAdministrationBuilder AddAdministration(string path, string secret);
     }
 }

src/Ocelot/DependencyInjection/OcelotBuilder.cs

@@ -1,12 +1,19 @@
+using Butterfly.Client.AspNetCore;
 using CacheManager.Core;
+using IdentityServer4.AccessTokenValidation;
 using IdentityServer4.Models;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Ocelot.Authorisation;
 using Ocelot.Cache;
 using Ocelot.Claims;
+using Ocelot.Configuration;
 using Ocelot.Configuration.Authentication;
+using Ocelot.Configuration.Builder;
 using Ocelot.Configuration.Creator;
 using Ocelot.Configuration.File;
 using Ocelot.Configuration.Parser;
@@ -25,6 +32,7 @@ using Ocelot.LoadBalancer.LoadBalancers;
 using Ocelot.Logging;
 using Ocelot.Middleware;
 using Ocelot.QueryStrings;
+using Ocelot.Raft;
 using Ocelot.RateLimit;
 using Ocelot.Request.Builder;
 using Ocelot.Request.Mapper;
@@ -32,26 +40,17 @@ using Ocelot.Requester;
 using Ocelot.Requester.QoS;
 using Ocelot.Responder;
 using Ocelot.ServiceDiscovery;
-using System;
-using System.Collections.Generic;
-using System.IdentityModel.Tokens.Jwt;
-using System.Net.Http;
-using System.Reflection;
-using System.Security.Cryptography.X509Certificates;
-using IdentityServer4.AccessTokenValidation;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Hosting;
-using Ocelot.Configuration;
-using Ocelot.Configuration.Builder;
-using FileConfigurationProvider = Ocelot.Configuration.Provider.FileConfigurationProvider;
-using Microsoft.Extensions.DependencyInjection.Extensions;
-using System.Linq;
-using Ocelot.Raft;
 using Rafty.Concensus;
 using Rafty.FiniteStateMachine;
 using Rafty.Infrastructure;
 using Rafty.Log;
-using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.IdentityModel.Tokens.Jwt;
+using System.Linq;
+using System.Reflection;
+using System.Security.Cryptography.X509Certificates;
+using FileConfigurationProvider = Ocelot.Configuration.Provider.FileConfigurationProvider;
 
 namespace Ocelot.DependencyInjection
 {
@@ -204,6 +203,13 @@ namespace Ocelot.DependencyInjection
             return this;
         }
 
+        public IOcelotBuilder AddOpenTracing(Action<ButterflyOptions> settings)
+        {
+            _services.AddTransient<OcelotHttpTracingHandler>();
+            _services.AddButterfly(settings);   
+            return this;
+        }
+
         private void AddIdentityServer(IIdentityServerConfiguration identityServerConfiguration, IAdministrationPath adminPath) 
         {
             _services.TryAddSingleton<IIdentityServerConfiguration>(identityServerConfiguration);

src/Ocelot/Logging/OcelotDiagnosticListener.cs

@@ -1,6 +1,8 @@
 using System;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.DiagnosticAdapter;
+using Butterfly.Client.AspNetCore;
+using Butterfly.OpenTracing;
 
 namespace Ocelot.Logging
 {
@@ -17,6 +19,7 @@ namespace Ocelot.Logging
         public virtual void OnMiddlewareStarting(HttpContext httpContext, string name)
         {
             _logger.LogTrace($"MiddlewareStarting: {name}; {httpContext.Request.Path}");
+            Event(httpContext, $"MiddlewareStarting: {name}; {httpContext.Request.Path}");
         }
 
         [DiagnosticName("Microsoft.AspNetCore.MiddlewareAnalysis.MiddlewareException")]
@@ -29,6 +32,13 @@ namespace Ocelot.Logging
         public virtual void OnMiddlewareFinished(HttpContext httpContext, string name)
         {
             _logger.LogTrace($"MiddlewareFinished: {name}; {httpContext.Response.StatusCode}");
+            Event(httpContext, $"MiddlewareFinished: {name}; {httpContext.Response.StatusCode}");
+        }
+
+        private void Event(HttpContext httpContext, string @event)
+        {
+            var span = httpContext.GetSpan();
+            span?.Log(LogField.CreateNew().Event(@event));
         }
     }
 }

src/Ocelot/Ocelot.csproj

@@ -23,24 +23,25 @@
     <DebugSymbols>True</DebugSymbols>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="FluentValidation" Version="7.2.1"/>
+    <PackageReference Include="Butterfly.Client.AspNetCore" Version="0.0.5" />
-    <PackageReference Include="IdentityServer4.AccessTokenValidation" Version="2.1.0"/>
+    <PackageReference Include="FluentValidation" Version="7.2.1" />
-    <PackageReference Include="Microsoft.AspNetCore.All" Version="2.0.0"/>
+    <PackageReference Include="IdentityServer4.AccessTokenValidation" Version="2.1.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.0.0"/>
+    <PackageReference Include="Microsoft.AspNetCore.All" Version="2.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.0.0" />
-    <PackageReference Include="Microsoft.Extensions.DiagnosticAdapter" Version="2.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="2.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.DiagnosticAdapter" Version="2.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="2.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.0.0" />
-    <PackageReference Include="System.Text.RegularExpressions" Version="4.3.0"/>
+    <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.0.0" />
-    <PackageReference Include="CacheManager.Core" Version="1.1.1"/>
+    <PackageReference Include="System.Text.RegularExpressions" Version="4.3.0" />
-    <PackageReference Include="CacheManager.Microsoft.Extensions.Configuration" Version="1.1.1"/>
+    <PackageReference Include="CacheManager.Core" Version="1.1.1" />
-    <PackageReference Include="CacheManager.Microsoft.Extensions.Logging" Version="1.1.1"/>
+    <PackageReference Include="CacheManager.Microsoft.Extensions.Configuration" Version="1.1.1" />
-    <PackageReference Include="Consul" Version="0.7.2.3"/>
+    <PackageReference Include="CacheManager.Microsoft.Extensions.Logging" Version="1.1.1" />
-    <PackageReference Include="Polly" Version="5.3.1"/>
+    <PackageReference Include="Consul" Version="0.7.2.3" />
-    <PackageReference Include="IdentityServer4" Version="2.0.2"/>
+    <PackageReference Include="Polly" Version="5.3.1" />
-    <PackageReference Include="Rafty" Version="0.4.2"/>
+    <PackageReference Include="IdentityServer4" Version="2.0.2" />
+    <PackageReference Include="Rafty" Version="0.4.2" />
   </ItemGroup>
 </Project>

src/Ocelot/Request/Builder/HttpRequestCreator.cs

@@ -12,9 +12,10 @@ namespace Ocelot.Request.Builder
             bool isQos,
             IQoSProvider qosProvider,
             bool useCookieContainer,
-            bool allowAutoRedirect)
+            bool allowAutoRedirect,
+            bool isTracing)
         {
-            return new OkResponse<Request>(new Request(httpRequestMessage, isQos, qosProvider, allowAutoRedirect, useCookieContainer));
+            return new OkResponse<Request>(new Request(httpRequestMessage, isQos, qosProvider, allowAutoRedirect, useCookieContainer, isTracing));
         }
     }
 }

src/Ocelot/Request/Builder/IRequestCreator.cs

@@ -13,6 +13,7 @@
             bool isQos,
             IQoSProvider qosProvider,
             bool useCookieContainer,
-            bool allowAutoRedirect);
+            bool allowAutoRedirect,
+            bool isTracing);
     }
 }

src/Ocelot/Request/Middleware/HttpRequestBuilderMiddleware.cs

@@ -46,16 +46,15 @@ namespace Ocelot.Request.Middleware
                     DownstreamRoute.ReRoute.IsQos,
                     qosProvider.Data,
                     DownstreamRoute.ReRoute.HttpHandlerOptions.UseCookieContainer,
-                    DownstreamRoute.ReRoute.HttpHandlerOptions.AllowAutoRedirect);
+                    DownstreamRoute.ReRoute.HttpHandlerOptions.AllowAutoRedirect,
-
+                    DownstreamRoute.ReRoute.HttpHandlerOptions.UseTracing);
             if (buildResult.IsError)
             {
                 _logger.LogDebug("IRequestCreator returned an error, setting pipeline error");
-
                 SetPipelineError(buildResult.Errors);
-
                 return;
             }
+
             _logger.LogDebug("setting upstream request");
 
             SetUpstreamRequestForThisRequest(buildResult.Data);

src/Ocelot/Request/Request.cs

@@ -10,17 +10,21 @@ namespace Ocelot.Request
             bool isQos,
             IQoSProvider qosProvider, 
             bool allowAutoRedirect,
-            bool useCookieContainer)
+            bool useCookieContainer,
+            bool isTracing
+            )
         {
             HttpRequestMessage = httpRequestMessage;
             IsQos = isQos;
             QosProvider = qosProvider;
             AllowAutoRedirect = allowAutoRedirect;
             UseCookieContainer = useCookieContainer;
+            IsTracing = isTracing;
         }
 
         public HttpRequestMessage HttpRequestMessage { get; private set; }
         public bool IsQos { get; private set; }
+        public bool IsTracing { get; private set; }
         public IQoSProvider QosProvider { get; private set; }
         public bool AllowAutoRedirect { get; private set; }
         public bool UseCookieContainer { get; private set; }

src/Ocelot/Requester/HttpClientBuilder.cs

@@ -1,11 +1,11 @@
-using System;
+using Microsoft.Extensions.DependencyInjection;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Threading.Tasks;
 using Ocelot.Logging;
 using Ocelot.Requester.QoS;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
 
 namespace Ocelot.Requester
 {
@@ -20,10 +20,19 @@ namespace Ocelot.Requester
             return this;
         }
 
-        public IHttpClient Create(bool useCookies, bool allowAutoRedirect)
+        private IHttpClientBuilder WithTracing(IServiceProvider provider)
         {
-            var httpclientHandler = new HttpClientHandler { AllowAutoRedirect = allowAutoRedirect, UseCookies = useCookies};
+            _handlers.Add(6000, () => provider.GetService<OcelotHttpTracingHandler>());
+            return this;
+        }
 
+        public IHttpClient Create(bool useCookies, bool allowAutoRedirect, bool isTracing, IServiceProvider provider)
+        {
+            var httpclientHandler = new HttpClientHandler { AllowAutoRedirect = allowAutoRedirect, UseCookies = useCookies };
+            if (isTracing)
+            {
+                WithTracing(provider);
+            }
             var client = new HttpClient(CreateHttpMessageHandler(httpclientHandler));                
             
             return new HttpClientWrapper(client);

src/Ocelot/Requester/HttpClientHttpRequester.cs

@@ -14,12 +14,13 @@ namespace Ocelot.Requester
     {
         private readonly IHttpClientCache _cacheHandlers;
         private readonly IOcelotLogger _logger;
+        private readonly IServiceProvider _serviceProvider;
 
-        public HttpClientHttpRequester(IOcelotLoggerFactory loggerFactory, 
+        public HttpClientHttpRequester(IOcelotLoggerFactory loggerFactory, IHttpClientCache cacheHandlers, IServiceProvider provider)
-            IHttpClientCache cacheHandlers)
         {
             _logger = loggerFactory.CreateLogger<HttpClientHttpRequester>();
             _cacheHandlers = cacheHandlers;
+            _serviceProvider = provider;
         }
 
         public async Task<Response<HttpResponseMessage>> GetResponse(Request.Request request)
@@ -28,7 +29,7 @@ namespace Ocelot.Requester
 
             var cacheKey = GetCacheKey(request, builder);
             
-            var httpClient = GetHttpClient(cacheKey, builder, request.UseCookieContainer, request.AllowAutoRedirect);
+            var httpClient = GetHttpClient(cacheKey, builder, request.UseCookieContainer, request.AllowAutoRedirect, request.IsTracing);
 
             try
             {
@@ -56,13 +57,13 @@ namespace Ocelot.Requester
 
         }
 
-        private IHttpClient GetHttpClient(string cacheKey, IHttpClientBuilder builder, bool useCookieContainer, bool allowAutoRedirect)
+        private IHttpClient GetHttpClient(string cacheKey, IHttpClientBuilder builder, bool useCookieContainer, bool allowAutoRedirect,bool isTracing)
         {
             var httpClient = _cacheHandlers.Get(cacheKey);
 
             if (httpClient == null)
             {
-                httpClient = builder.Create(useCookieContainer, allowAutoRedirect);
+                httpClient = builder.Create(useCookieContainer, allowAutoRedirect, isTracing, _serviceProvider);
             }
             return httpClient;
         }

src/Ocelot/Requester/IHttpClientBuilder.cs

@@ -22,6 +22,6 @@ namespace Ocelot.Requester
         /// </summary>
         /// <param name="useCookies">Defines if http client should use cookie container</param>
         /// <param name="allowAutoRedirect">Defines if http client should allow auto redirect</param>
-        IHttpClient Create(bool useCookies, bool allowAutoRedirect);
+        IHttpClient Create(bool useCookies, bool allowAutoRedirect, bool isTracing, IServiceProvider provider);
     }
 }

src/Ocelot/Requester/IHttpRequester.cs

@@ -7,7 +7,5 @@ namespace Ocelot.Requester
     public interface IHttpRequester
     {
         Task<Response<HttpResponseMessage>> GetResponse(Request.Request request);
-
-
     }
 }

src/Ocelot/Requester/Middleware/HttpRequesterMiddleware.cs

@@ -1,9 +1,8 @@
-using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.Logging;
 using Ocelot.Infrastructure.RequestData;
 using Ocelot.Logging;
 using Ocelot.Middleware;
+using System.Threading.Tasks;
 
 namespace Ocelot.Requester.Middleware
 {
@@ -16,7 +15,8 @@ namespace Ocelot.Requester.Middleware
         public HttpRequesterMiddleware(RequestDelegate next,
             IOcelotLoggerFactory loggerFactory,
             IHttpRequester requester, 
-            IRequestScopedDataRepository requestScopedDataRepository)
+            IRequestScopedDataRepository requestScopedDataRepository
+)
             :base(requestScopedDataRepository)
         {
             _next = next;

src/Ocelot/Requester/OcelotHttpTracingHandler.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Butterfly.Client.Tracing;
+using Butterfly.OpenTracing;
+
+namespace Ocelot.Requester
+{
+    public class OcelotHttpTracingHandler : DelegatingHandler
+    {
+        private readonly IServiceTracer _tracer;
+        private const string prefix_spanId = "ot-spanId";
+
+        public OcelotHttpTracingHandler(IServiceTracer tracer, HttpMessageHandler httpMessageHandler = null)
+        {
+            _tracer = tracer ?? throw new ArgumentNullException(nameof(tracer));
+            InnerHandler = httpMessageHandler ?? new HttpClientHandler();
+        }
+
+        protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            return _tracer.ChildTraceAsync($"httpclient {request.Method}", DateTimeOffset.UtcNow, span => TracingSendAsync(span, request, cancellationToken));
+        }
+
+        protected virtual async Task<HttpResponseMessage> TracingSendAsync(ISpan span, HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            IEnumerable<string> traceIdVals = null;
+            if (request.Headers.TryGetValues(prefix_spanId, out traceIdVals))
+            {
+                request.Headers.Remove(prefix_spanId);
+                request.Headers.TryAddWithoutValidation(prefix_spanId, span.SpanContext.SpanId);
+            };
+
+            span.Tags.Client().Component("HttpClient")
+                .HttpMethod(request.Method.Method)
+                .HttpUrl(request.RequestUri.OriginalString)
+                .HttpHost(request.RequestUri.Host)
+                .HttpPath(request.RequestUri.PathAndQuery)
+                .PeerAddress(request.RequestUri.OriginalString)
+                .PeerHostName(request.RequestUri.Host)
+                .PeerPort(request.RequestUri.Port);
+
+            _tracer.Tracer.Inject(span.SpanContext, request.Headers, (c, k, v) =>
+            {
+                if (!c.Contains(k))
+                {
+                    c.Add(k, v);
+                };
+            });
+
+            span.Log(LogField.CreateNew().ClientSend());
+
+            var responseMessage = await base.SendAsync(request, cancellationToken);
+
+            span.Log(LogField.CreateNew().ClientReceive());
+
+            return responseMessage;
+        }
+    }
+}

src/Ocelot/Responder/Middleware/ResponderMiddleware.cs

@@ -1,10 +1,10 @@
-using System.Collections.Generic;
-using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Ocelot.Errors;
 using Ocelot.Infrastructure.RequestData;
 using Ocelot.Logging;
 using Ocelot.Middleware;
+using System.Collections.Generic;
+using System.Threading.Tasks;
 
 namespace Ocelot.Responder.Middleware
 {
@@ -22,7 +22,8 @@ namespace Ocelot.Responder.Middleware
             IHttpResponder responder,
             IOcelotLoggerFactory loggerFactory,
             IRequestScopedDataRepository requestScopedDataRepository, 
-            IErrorsToHttpStatusCodeMapper codeMapper)
+            IErrorsToHttpStatusCodeMapper codeMapper
+           )
             :base(requestScopedDataRepository)
         {
             _next = next;
@@ -40,7 +41,6 @@ namespace Ocelot.Responder.Middleware
             {
                 var errors = PipelineErrors;
                 _logger.LogError($"{PipelineErrors.Count} pipeline errors found in {MiddlewareName}. Setting error response status code");
-
                 SetErrorResponse(context, errors);
             }
             else

test/Ocelot.IntegrationTests/RaftStartup.cs

@@ -41,7 +41,8 @@ namespace Ocelot.IntegrationTests
             services
                 .AddOcelot(Configuration)
                 .AddAdministration("/administration", "secret")
-                .AddRafty();
+                .AddRafty()
+                ;
         }
 
         public virtual void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory)

test/Ocelot.ManualTest/ManualTestStartup.cs

@@ -29,6 +29,11 @@ namespace Ocelot.ManualTest
 
             services.AddOcelot()
                     .AddCacheManager(settings)
+                    .AddOpenTracing(option =>
+                    {
+                        option.CollectorUrl = "http://localhost:9618";
+                        option.Service = "Ocelot.ManualTest";
+                    })
                     .AddAdministration("/administration", "secret");
         }
 

test/Ocelot.ManualTest/configuration.json

@@ -1,5 +1,22 @@
 {
   "ReRoutes": [
+    {
+      "DownstreamPathTemplate": "/api/values",
+      "DownstreamScheme": "http",
+      "UpstreamPathTemplate": "/api/values",
+      "UpstreamHttpMethod": [ "Get" ],
+      "DownstreamHostAndPorts": [
+        {
+          "Host": "localhost",
+          "Port": 5002
+        }
+      ],
+      "HttpHandlerOptions": {
+        "AllowAutoRedirect": true,
+        "UseCookieContainer": true,
+        "UseTracing": true
+      }
+    },
     {
       "DownstreamPathTemplate": "/",
       "DownstreamScheme": "http",
@@ -282,6 +299,6 @@
   ],
 
   "GlobalConfiguration": {
-    "RequestIdKey": "OcRequestId"
+    "RequestIdKey": "ot-traceid"
   }
 }

test/Ocelot.UnitTests/Configuration/FileConfigurationCreatorTests.cs

@@ -471,7 +471,7 @@ namespace Ocelot.UnitTests.Configuration
         {
             var reRouteOptions = new ReRouteOptionsBuilder()
                 .Build();
-            var httpHandlerOptions = new HttpHandlerOptions(true, true);
+            var httpHandlerOptions = new HttpHandlerOptions(true, true,false);
 
             this.Given(x => x.GivenTheConfigIs(new FileConfiguration
             {

test/Ocelot.UnitTests/Configuration/HttpHandlerOptionsCreatorTests.cs

@@ -23,7 +23,7 @@ namespace Ocelot.UnitTests.Configuration
         public void should_create_options_with_useCookie_and_allowAutoRedirect_true_as_default()
         {
             var fileReRoute = new FileReRoute();
-            var expectedOptions = new HttpHandlerOptions(true, true);
+            var expectedOptions = new HttpHandlerOptions(true, true, false);
 
             this.Given(x => GivenTheFollowing(fileReRoute))
                 .When(x => WhenICreateHttpHandlerOptions())
@@ -39,11 +39,12 @@ namespace Ocelot.UnitTests.Configuration
                 HttpHandlerOptions = new FileHttpHandlerOptions
                 {
                     AllowAutoRedirect = false,
-                    UseCookieContainer = false
+                    UseCookieContainer = false,
+                    UseTracing = false
                 }
             };
 
-            var expectedOptions = new HttpHandlerOptions(false, false);
+            var expectedOptions = new HttpHandlerOptions(false, false, false);
 
             this.Given(x => GivenTheFollowing(fileReRoute))
                 .When(x => WhenICreateHttpHandlerOptions())
@@ -66,6 +67,7 @@ namespace Ocelot.UnitTests.Configuration
             _httpHandlerOptions.ShouldNotBeNull();
             _httpHandlerOptions.AllowAutoRedirect.ShouldBe(options.AllowAutoRedirect);
             _httpHandlerOptions.UseCookieContainer.ShouldBe(options.UseCookieContainer);
+            _httpHandlerOptions.UseTracing.ShouldBe(options.UseTracing);
         }
     }
 }

test/Ocelot.UnitTests/DependencyInjection/OcelotBuilderTests.cs

@@ -1,9 +1,4 @@
-using System;
+using CacheManager.Core;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net.Http;
-using System.Text;
-using CacheManager.Core;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Hosting.Internal;
 using Microsoft.Extensions.Configuration;
@@ -13,8 +8,11 @@ using Ocelot.Configuration;
 using Ocelot.Configuration.File;
 using Ocelot.Configuration.Setter;
 using Ocelot.DependencyInjection;
-using Ocelot.Logging;
+using Ocelot.Requester;
 using Shouldly;
+using System;
+using System.Collections.Generic;
+using System.Linq;
 using TestStack.BDDfy;
 using Xunit;
 
@@ -96,6 +94,16 @@ namespace Ocelot.UnitTests.DependencyInjection
                 .BDDfy();
         }
 
+        [Fact]
+        public void should_set_up_tracing()
+        {
+            this.Given(x => WhenISetUpOcelotServices())
+                .When(x => WhenISetUpOpentracing())
+                .When(x => WhenIAccessOcelotHttpTracingHandler())
+                .BDDfy();
+        }
+
+
         [Fact]
         public void should_set_up_without_passing_in_config()
         {
@@ -193,6 +201,24 @@ namespace Ocelot.UnitTests.DependencyInjection
             }
         }
 
+        private void WhenISetUpOpentracing()
+        {
+            try
+            {
+                _ocelotBuilder.AddOpenTracing(
+                    option =>
+                    {
+                        option.CollectorUrl = "http://localhost:9618";
+                        option.Service = "Ocelot.ManualTest";
+                    }
+               );
+            }
+            catch (Exception e)
+            {
+                _ex = e;
+            }
+        }
+
         private void WhenIAccessLoggerFactory()
         {
             try
@@ -205,6 +231,18 @@ namespace Ocelot.UnitTests.DependencyInjection
             }
         }
 
+        private void WhenIAccessOcelotHttpTracingHandler()
+        {
+            try
+            {
+                var tracingHandler = _serviceProvider.GetService<OcelotHttpTracingHandler>();
+            }
+            catch (Exception e)
+            {
+                _ex = e;
+            }
+        }
+
         private void WhenIValidateScopes()
         {
             try

test/Ocelot.UnitTests/Request/HttpRequestBuilderMiddlewareTests.cs

@@ -17,6 +17,7 @@
     using Ocelot.Requester.QoS;
     using Ocelot.Configuration;
     using Microsoft.AspNetCore.Builder;
+    using Ocelot.Errors;
 
     public class HttpRequestBuilderMiddlewareTests : ServerHostedMiddlewareTest
     {
@@ -51,18 +52,39 @@
                 new ReRouteBuilder()
                     .WithRequestIdKey("LSRequestId")
                     .WithUpstreamHttpMethod(new List<string> { "Get" })
-                    .WithHttpHandlerOptions(new HttpHandlerOptions(true, true))
+                    .WithHttpHandlerOptions(new HttpHandlerOptions(true, true,false))
                     .Build());
 
             this.Given(x => x.GivenTheDownStreamUrlIs("any old string"))
                 .And(x => x.GivenTheQosProviderHouseReturns(new OkResponse<IQoSProvider>(new NoQoSProvider())))
                 .And(x => x.GivenTheDownStreamRouteIs(downstreamRoute))
-                .And(x => x.GivenTheRequestBuilderReturns(new Ocelot.Request.Request(new HttpRequestMessage(), true, new NoQoSProvider(), false, false)))
+                .And(x => x.GivenTheRequestBuilderReturns(new Ocelot.Request.Request(new HttpRequestMessage(), true, new NoQoSProvider(), false, false,false)))
                 .When(x => x.WhenICallTheMiddleware())
                 .Then(x => x.ThenTheScopedDataRepositoryIsCalledCorrectly())
                 .BDDfy();
         }
 
+        [Fact]
+        public void should_call_scoped_data_repository_QosProviderError()
+        {
+
+            var downstreamRoute = new DownstreamRoute(new List<PlaceholderNameAndValue>(),
+                new ReRouteBuilder()
+                    .WithRequestIdKey("LSRequestId")
+                    .WithUpstreamHttpMethod(new List<string> { "Get" })
+                    .WithHttpHandlerOptions(new HttpHandlerOptions(true, true, true))
+                    .Build());
+
+            this.Given(x => x.GivenTheDownStreamUrlIs("any old string"))
+                .And(x => x.GivenTheQosProviderHouseReturns(new ErrorResponse<IQoSProvider>(It.IsAny<Error>())))
+                .And(x => x.GivenTheDownStreamRouteIs(downstreamRoute))
+                .And(x => x.GivenTheRequestBuilderReturns(new Ocelot.Request.Request(new HttpRequestMessage(), true, new NoQoSProvider(), false, false, false)))
+                .When(x => x.WhenICallTheMiddleware())
+                .Then(x => x.ThenTheScopedDataRepositoryQosProviderError())
+                .BDDfy();
+        }
+
+
         protected override void GivenTheTestServerServicesAreConfigured(IServiceCollection services)
         {
             services.AddSingleton<IOcelotLoggerFactory, AspDotNetLoggerFactory>();
@@ -109,7 +131,9 @@
                                     It.IsAny<bool>(),
                                     It.IsAny<IQoSProvider>(),
                                     It.IsAny<bool>(),
-                                    It.IsAny<bool>()))
+                                    It.IsAny<bool>(),
+                                    It.IsAny<bool>()
+                                    ))
                 .ReturnsAsync(_request);
         }
 
@@ -118,5 +142,11 @@
             _scopedRepository
                 .Verify(x => x.Add("Request", _request.Data), Times.Once());
         }
+
+        private void ThenTheScopedDataRepositoryQosProviderError()
+        {
+            _scopedRepository
+                .Verify(x => x.Add("OcelotMiddlewareError", true), Times.Once());
+        }
     }
 }

test/Ocelot.UnitTests/Request/HttpRequestCreatorTests.cs

@@ -17,6 +17,7 @@
         private readonly HttpRequestMessage _requestMessage;
         private readonly bool _useCookieContainer;
         private readonly bool _allowAutoRedirect;
+        private readonly bool _useTracing;
 
         private Response<Ocelot.Request.Request> _response;
 
@@ -27,7 +28,7 @@
             _qoSProvider = new NoQoSProvider();
             _useCookieContainer = false;
             _allowAutoRedirect = false;
-
+            _useTracing = false;
             _requestMessage = new HttpRequestMessage();
         }
 
@@ -39,6 +40,7 @@
                 .Then(x => x.ThenTheRequestContainsTheIsQos())
                 .Then(x => x.ThenTheRequestContainsTheQosProvider())
                 .Then(x => x.ThenTheRequestContainsUseCookieContainer())
+                .Then(x => x.ThenTheRequestContainsUseTracing())
                 .Then(x => x.ThenTheRequestContainsAllowAutoRedirect())
                 .BDDfy();
         }
@@ -46,7 +48,7 @@
         private void WhenIBuildARequest()
         {
             _response = _requestCreator.Build(_requestMessage,
-                    _isQos, _qoSProvider, _useCookieContainer, _allowAutoRedirect)
+                    _isQos, _qoSProvider, _useCookieContainer, _allowAutoRedirect, _useTracing)
                 .GetAwaiter()
                 .GetResult();
         }
@@ -71,6 +73,11 @@
             _response.Data.UseCookieContainer.ShouldBe(_useCookieContainer);
         }
 
+        private void ThenTheRequestContainsUseTracing()
+        {
+            _response.Data.IsTracing.ShouldBe(_useTracing);
+        }
+
         private void ThenTheRequestContainsAllowAutoRedirect()
         {
             _response.Data.AllowAutoRedirect.ShouldBe(_allowAutoRedirect);

test/Ocelot.UnitTests/Requester/HttpClientHttpRequesterTest.cs

@@ -0,0 +1,76 @@
+using Microsoft.Extensions.DependencyInjection;
+using Moq;
+using Ocelot.Logging;
+using Ocelot.Requester;
+using Ocelot.Requester.QoS;
+using Ocelot.Responses;
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Text;
+using TestStack.BDDfy;
+using Xunit;
+
+namespace Ocelot.UnitTests.Requester
+{
+    public class HttpClientHttpRequesterTest
+    {
+        private readonly Mock<IHttpClientCache> _cacheHandlers;
+        private Mock<IServiceProvider> _serviceProvider;
+        private Response<HttpResponseMessage> _response;
+        private readonly HttpClientHttpRequester _httpClientRequester;
+        private Ocelot.Request.Request _request;
+        private Mock<IOcelotLoggerFactory> _loggerFactory;
+        private Mock<IOcelotLogger> _logger;
+
+        public HttpClientHttpRequesterTest()
+        {
+            _serviceProvider = new Mock<IServiceProvider>();
+            _logger = new Mock<IOcelotLogger>();
+            _loggerFactory = new Mock<IOcelotLoggerFactory>();
+            _loggerFactory
+                .Setup(x => x.CreateLogger<HttpClientHttpRequester>())
+                .Returns(_logger.Object);
+            _cacheHandlers = new Mock<IHttpClientCache>();
+            _httpClientRequester = new HttpClientHttpRequester(_loggerFactory.Object, _cacheHandlers.Object, _serviceProvider.Object);            
+        }
+
+        [Fact]
+        public void should_call_request_correctly()
+        {
+            this.Given(x=>x.GivenTheRequestIs(new Ocelot.Request.Request(new HttpRequestMessage() {  RequestUri = new Uri("http://www.bbc.co.uk") }, false, new NoQoSProvider(), false, false, false)))
+                .When(x=>x.WhenIGetResponse())
+                .Then(x => x.ThenTheResponseIsCalledCorrectly())
+                .BDDfy();
+        }
+
+        [Fact]
+        public void should_call_request_UnableToCompleteRequest()
+        {
+            this.Given(x => x.GivenTheRequestIs(new Ocelot.Request.Request(new HttpRequestMessage() { RequestUri = new Uri("http://localhost:60080") }, false, new NoQoSProvider(), false, false, false)))
+                .When(x => x.WhenIGetResponse())
+                .Then(x => x.ThenTheResponseIsCalledError())
+                .BDDfy();
+        }
+
+        private void GivenTheRequestIs(Ocelot.Request.Request request)
+        {
+            _request = request;            
+        }
+
+        private void WhenIGetResponse()
+        {
+            _response = _httpClientRequester.GetResponse(_request).Result;
+        }
+
+        private void ThenTheResponseIsCalledCorrectly()
+        {
+            Assert.True(_response.IsError == false);
+        }
+
+        private void ThenTheResponseIsCalledError()
+        {
+            Assert.True(_response.IsError == true);
+        }
+    }  
+}

test/Ocelot.UnitTests/Requester/HttpRequesterMiddlewareTests.cs

@@ -28,7 +28,7 @@
         [Fact]
         public void should_call_scoped_data_repository_correctly()
         {
-            this.Given(x => x.GivenTheRequestIs(new Ocelot.Request.Request(new HttpRequestMessage(),true, new NoQoSProvider(), false, false)))
+            this.Given(x => x.GivenTheRequestIs(new Ocelot.Request.Request(new HttpRequestMessage(),true, new NoQoSProvider(), false, false,false)))
                 .And(x => x.GivenTheRequesterReturns(new HttpResponseMessage()))
                 .And(x => x.GivenTheScopedRepoReturns())
                 .When(x => x.WhenICallTheMiddleware())

test/Ocelot.UnitTests/Responder/ResponderMiddlewareTests.cs

@@ -1,10 +1,14 @@
 namespace Ocelot.UnitTests.Responder
 {
+    using System.Collections.Generic;
     using System.Net.Http;
     using Microsoft.AspNetCore.Builder;
     using Microsoft.Extensions.DependencyInjection;
     using Moq;
+    using Ocelot.DownstreamRouteFinder.Finder;
+    using Ocelot.Errors;
     using Ocelot.Logging;
+    using Ocelot.Requester;
     using Ocelot.Responder;
     using Ocelot.Responder.Middleware;
     using Ocelot.Responses;
@@ -35,6 +39,17 @@
                 .BDDfy();
         }
 
+
+        [Fact]
+        public void should_return_any_errors()
+        {
+            this.Given(x => x.GivenTheHttpResponseMessageIs(new HttpResponseMessage()))
+                .And(x => x.GivenThereArePipelineErrors(new UnableToFindDownstreamRouteError()))
+                .When(x => x.WhenICallTheMiddleware())
+                .Then(x => x.ThenThereAreNoErrors())
+                .BDDfy();
+        }
+
         protected override void GivenTheTestServerServicesAreConfigured(IServiceCollection services)
         {
             services.AddSingleton<IOcelotLoggerFactory, AspDotNetLoggerFactory>();
@@ -68,5 +83,14 @@
         {
             //todo a better assert?
         }
+
+        private void GivenThereArePipelineErrors(Error error)
+        {
+            ScopedRepository
+                .Setup(x => x.Get<bool>("OcelotMiddlewareError"))
+                .Returns(new OkResponse<bool>(true));
+            ScopedRepository.Setup(x => x.Get<List<Error>>("OcelotMiddlewareErrors"))
+                .Returns(new OkResponse<List<Error>>(new List<Error>() { error }));
+        }  
     }
 }