Added request id functionality and general refactoring..also turned out i wasnt returning headers....sigh

src/Ocelot/Configuration/Builder/ReRouteBuilder.cs

@@ -20,6 +20,7 @@ namespace Ocelot.Configuration.Builder
         private Dictionary<string, string> _routeClaimRequirement;
         private bool _isAuthorised;
         private List<ClaimToThing> _claimToQueries;
+        private string _requestIdHeaderKey;
 
         public ReRouteBuilder()
         {
@@ -96,6 +97,12 @@ namespace Ocelot.Configuration.Builder
             return this;
         }
 
+        public ReRouteBuilder WithRequestIdKey(string input)
+        {
+            _requestIdHeaderKey = input;
+            return this;
+        }
+
         public ReRouteBuilder WithClaimsToHeaders(List<ClaimToThing> input)
         {
             _configHeaderExtractorProperties = input;
@@ -122,7 +129,7 @@ namespace Ocelot.Configuration.Builder
 
         public ReRoute Build()
         {
-            return new ReRoute(_downstreamTemplate, _upstreamTemplate, _upstreamHttpMethod, _upstreamTemplatePattern, _isAuthenticated, new AuthenticationOptions(_authenticationProvider, _authenticationProviderUrl, _scopeName, _requireHttps, _additionalScopes, _scopeSecret), _configHeaderExtractorProperties, _claimToClaims, _routeClaimRequirement, _isAuthorised, _claimToQueries);
+            return new ReRoute(_downstreamTemplate, _upstreamTemplate, _upstreamHttpMethod, _upstreamTemplatePattern, _isAuthenticated, new AuthenticationOptions(_authenticationProvider, _authenticationProviderUrl, _scopeName, _requireHttps, _additionalScopes, _scopeSecret), _configHeaderExtractorProperties, _claimToClaims, _routeClaimRequirement, _isAuthorised, _claimToQueries, _requestIdHeaderKey);
         }
     }
 }

src/Ocelot/Configuration/Creator/YamlOcelotConfigurationCreator.cs

@@ -113,12 +113,17 @@ namespace Ocelot.Configuration.Creator
 
                 return new ReRoute(reRoute.DownstreamTemplate, reRoute.UpstreamTemplate,
                     reRoute.UpstreamHttpMethod, upstreamTemplate, isAuthenticated,
-                    authOptionsForRoute, claimsToHeaders, claimsToClaims, reRoute.RouteClaimsRequirement, isAuthorised, claimsToQueries
+                    authOptionsForRoute, claimsToHeaders, claimsToClaims, 
+                    reRoute.RouteClaimsRequirement, isAuthorised, claimsToQueries,
+                    reRoute.RequestIdKey
                     );
             }
 
-            return new ReRoute(reRoute.DownstreamTemplate, reRoute.UpstreamTemplate, reRoute.UpstreamHttpMethod,
-                upstreamTemplate, isAuthenticated, null, new List<ClaimToThing>(), new List<ClaimToThing>(), reRoute.RouteClaimsRequirement, isAuthorised, new List<ClaimToThing>());
+            return new ReRoute(reRoute.DownstreamTemplate, reRoute.UpstreamTemplate, 
+                reRoute.UpstreamHttpMethod, upstreamTemplate, isAuthenticated, 
+                null, new List<ClaimToThing>(), new List<ClaimToThing>(), 
+                reRoute.RouteClaimsRequirement, isAuthorised, new List<ClaimToThing>(),
+                    reRoute.RequestIdKey);
         }
 
         private List<ClaimToThing> GetAddThingsToRequest(Dictionary<string,string> thingBeingAdded)

src/Ocelot/Configuration/ReRoute.cs

@@ -4,7 +4,7 @@ namespace Ocelot.Configuration
 {
     public class ReRoute
     {
-        public ReRoute(string downstreamTemplate, string upstreamTemplate, string upstreamHttpMethod, string upstreamTemplatePattern, bool isAuthenticated, AuthenticationOptions authenticationOptions, List<ClaimToThing> configurationHeaderExtractorProperties, List<ClaimToThing> claimsToClaims, Dictionary<string, string> routeClaimsRequirement, bool isAuthorised, List<ClaimToThing> claimsToQueries)
+        public ReRoute(string downstreamTemplate, string upstreamTemplate, string upstreamHttpMethod, string upstreamTemplatePattern, bool isAuthenticated, AuthenticationOptions authenticationOptions, List<ClaimToThing> configurationHeaderExtractorProperties, List<ClaimToThing> claimsToClaims, Dictionary<string, string> routeClaimsRequirement, bool isAuthorised, List<ClaimToThing> claimsToQueries, string requestIdKey)
         {
             DownstreamTemplate = downstreamTemplate;
             UpstreamTemplate = upstreamTemplate;
@@ -14,6 +14,7 @@ namespace Ocelot.Configuration
             AuthenticationOptions = authenticationOptions;
             RouteClaimsRequirement = routeClaimsRequirement;
             IsAuthorised = isAuthorised;
+            RequestIdKey = requestIdKey;
             ClaimsToQueries = claimsToQueries
                 ?? new List<ClaimToThing>();
             ClaimsToClaims = claimsToClaims 
@@ -33,6 +34,6 @@ namespace Ocelot.Configuration
         public List<ClaimToThing> ClaimsToHeaders { get; private set; }
         public List<ClaimToThing> ClaimsToClaims { get; private set; }
         public Dictionary<string, string> RouteClaimsRequirement { get; private set; }
-
+        public string RequestIdKey { get; private set; }
     }
 }

src/Ocelot/Configuration/Yaml/YamlReRoute.cs

@@ -19,6 +19,7 @@ namespace Ocelot.Configuration.Yaml
         public Dictionary<string, string> AddHeadersToRequest { get; set; }
         public Dictionary<string, string> AddClaimsToRequest { get; set; }
         public Dictionary<string, string> RouteClaimsRequirement { get; set; }
-        public Dictionary<string, string> AddQueriesToRequest { get; set; } 
+        public Dictionary<string, string> AddQueriesToRequest { get; set; }
+        public string RequestIdKey { get; set; }
     }
 }

src/Ocelot/DependencyInjection/ServiceCollectionExtensions.cs

@@ -1,5 +1,4 @@
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Ocelot.Authentication.Handler.Creator;
@@ -16,6 +15,7 @@ using Ocelot.DownstreamRouteFinder.Finder;
 using Ocelot.DownstreamRouteFinder.UrlMatcher;
 using Ocelot.DownstreamUrlCreator.UrlTemplateReplacer;
 using Ocelot.Headers;
+using Ocelot.Infrastructure.Claims.Parser;
 using Ocelot.Infrastructure.RequestData;
 using Ocelot.QueryStrings;
 using Ocelot.Request.Builder;
@@ -24,8 +24,6 @@ using Ocelot.Responder;
 
 namespace Ocelot.DependencyInjection
 {
-    using Infrastructure.Claims.Parser;
-
     public static class ServiceCollectionExtensions
     {
         public static IServiceCollection AddOcelotYamlConfiguration(this IServiceCollection services, IConfigurationRoot configurationRoot)
@@ -48,6 +46,7 @@ namespace Ocelot.DependencyInjection
             services.AddLogging();
 
             // ocelot services.
+            services.AddSingleton<IRemoveHeaders, RemoveHeaders>();
             services.AddSingleton<IOcelotConfigurationProvider, OcelotConfigurationProvider>();
             services.AddSingleton<IClaimToThingConfigurationParser, ClaimToThingConfigurationParser>();
             services.AddSingleton<IAuthoriser, ClaimsAuthoriser>();

src/Ocelot/DownstreamRouteFinder/Middleware/DownstreamRouteFinderMiddleware.cs

@@ -10,7 +10,6 @@ namespace Ocelot.DownstreamRouteFinder.Middleware
     {
         private readonly RequestDelegate _next;
         private readonly IDownstreamRouteFinder _downstreamRouteFinder;
-        private readonly IRequestScopedDataRepository _requestScopedDataRepository;
 
         public DownstreamRouteFinderMiddleware(RequestDelegate next, 
             IDownstreamRouteFinder downstreamRouteFinder, 
@@ -19,7 +18,6 @@ namespace Ocelot.DownstreamRouteFinder.Middleware
         {
             _next = next;
             _downstreamRouteFinder = downstreamRouteFinder;
-            _requestScopedDataRepository = requestScopedDataRepository;
         }
 
         public async Task Invoke(HttpContext context)

src/Ocelot/Headers/IRemoveHeaders.cs

@@ -0,0 +1,10 @@
+using System.Net.Http.Headers;
+using Ocelot.Responses;
+
+namespace Ocelot.Headers
+{
+    public interface IRemoveHeaders
+    {
+        Response Remove(HttpResponseHeaders headers);
+    }
+}

src/Ocelot/Headers/RemoveHeaders.cs

@@ -0,0 +1,30 @@
+using System.Collections.Generic;
+using System.Net.Http.Headers;
+using Ocelot.Responses;
+
+namespace Ocelot.Headers
+{
+    public class RemoveHeaders : IRemoveHeaders
+    {
+        /// <summary>
+        /// Some webservers return headers that cannot be forwarded to the client
+        /// in a given context such as transfer encoding chunked when ASP.NET is not
+        /// returning the response in this manner
+        /// </summary>
+        private readonly List<string> _unsupportedHeaders = new List<string>
+        {
+            "Transfer-Encoding"
+        };
+
+
+        public Response Remove(HttpResponseHeaders headers)
+        {
+            foreach (var unsupported in _unsupportedHeaders)
+            {
+                headers.Remove(unsupported);
+            }
+
+            return new OkResponse();
+        }
+    }
+}

src/Ocelot/Middleware/ExceptionHandlerMiddleware.cs

@@ -10,7 +10,8 @@ namespace Ocelot.Middleware
         private readonly RequestDelegate _next;
         private readonly ILogger _logger;
 
-        public ExceptionHandlerMiddleware(RequestDelegate next, ILoggerFactory loggerFactory)
+        public ExceptionHandlerMiddleware(RequestDelegate next, 
+            ILoggerFactory loggerFactory)
         {
             _next = next;
             _logger = loggerFactory.CreateLogger<ExceptionHandlerMiddleware>();
@@ -24,20 +25,25 @@ namespace Ocelot.Middleware
             }
             catch (Exception e)
             {
-                var message =
-                    $"Exception caught in global error handler, exception message: {e.Message}, exception stack: {e.StackTrace}";
-
-                if (e.InnerException != null)
-                {
-                    message = $"{message}, inner exception message {e.InnerException.Message}, inner exception stack {e.InnerException.StackTrace}";
-                }
-
+                var message = CreateMessage(context, e);
                 _logger.LogError(new EventId(1, "Ocelot Global Error"), message, e);
-
                 context.Response.StatusCode = 500;
                 context.Response.ContentType = "application/json";
                 await context.Response.WriteAsync("Internal Server Error");
             }
         }
+
+        private static string CreateMessage(HttpContext context, Exception e)
+        {
+            var message =
+                $"RequestId: {context.TraceIdentifier}, Exception caught in global error handler, exception message: {e.Message}, exception stack: {e.StackTrace}";
+
+            if (e.InnerException != null)
+            {
+                message =
+                    $"{message}, inner exception message {e.InnerException.Message}, inner exception stack {e.InnerException.StackTrace}";
+            }
+            return message;
+        }
     }
 }

src/Ocelot/Middleware/OcelotMiddlewareExtensions.cs

@@ -7,6 +7,7 @@ using Ocelot.Headers.Middleware;
 using Ocelot.QueryStrings.Middleware;
 using Ocelot.Request.Middleware;
 using Ocelot.Requester.Middleware;
+using Ocelot.RequestId.Middleware;
 using Ocelot.Responder.Middleware;
 
 namespace Ocelot.Middleware
@@ -49,6 +50,9 @@ namespace Ocelot.Middleware
             // Then we get the downstream route information
             builder.UseDownstreamRouteFinderMiddleware();
 
+            // Now we can look for the requestId
+            builder.UseRequestIdMiddleware();
+
             // Allow pre authentication logic. The idea being people might want to run something custom before what is built in.
             builder.UseIfNotNull(middlewareConfiguration.PreAuthenticationMiddleware);
 

src/Ocelot/Request/Builder/HttpRequestBuilder.cs

@@ -1,7 +1,9 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Net;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Ocelot.Responses;
@@ -10,8 +12,15 @@ namespace Ocelot.Request.Builder
 {
     public class HttpRequestBuilder : IRequestBuilder
     {
-        public async Task<Response<Request>> Build(string httpMethod, string downstreamUrl, Stream content, IHeaderDictionary headers,
-            IRequestCookieCollection cookies, Microsoft.AspNetCore.Http.QueryString queryString, string contentType)
+        public async Task<Response<Request>> Build(
+            string httpMethod, 
+            string downstreamUrl, 
+            Stream content, 
+            IHeaderDictionary headers,
+            IRequestCookieCollection cookies, 
+            QueryString queryString, 
+            string contentType, 
+            RequestId.RequestId requestId)
         {
             var method = new HttpMethod(httpMethod);
 
@@ -21,7 +30,7 @@ namespace Ocelot.Request.Builder
 
             if (content != null)
             {
-                httpRequestMessage.Content = new ByteArrayContent(ToByteArray(content));
+                httpRequestMessage.Content = new ByteArrayContent(await ToByteArray(content));
             }
 
             if (!string.IsNullOrEmpty(contentType))
@@ -45,6 +54,11 @@ namespace Ocelot.Request.Builder
                 }
             }
 
+            if (RequestKeyIsNotNull(requestId) && !RequestIdInHeaders(requestId, httpRequestMessage.Headers))
+            {
+                ForwardRequestIdToDownstreamService(requestId, httpRequestMessage);
+            }
+
             var cookieContainer = new CookieContainer();
 
             //todo get rid of if
@@ -59,13 +73,34 @@ namespace Ocelot.Request.Builder
             return new OkResponse<Request>(new Request(httpRequestMessage, cookieContainer));
         }
 
-        private byte[] ToByteArray(Stream stream)
+        private void ForwardRequestIdToDownstreamService(RequestId.RequestId requestId, HttpRequestMessage httpRequestMessage)
+        {
+            httpRequestMessage.Headers.Add(requestId.RequestIdKey, requestId.RequestIdValue);
+        }
+
+        private bool RequestIdInHeaders(RequestId.RequestId requestId, HttpRequestHeaders headers)
+        {
+            IEnumerable<string> value;
+            if (headers.TryGetValues(requestId.RequestIdKey, out value))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        private bool RequestKeyIsNotNull(RequestId.RequestId requestId)
+        {
+            return !string.IsNullOrEmpty(requestId?.RequestIdKey) && !string.IsNullOrEmpty(requestId.RequestIdValue);
+        }
+
+        private async Task<byte[]> ToByteArray(Stream stream)
         {
             using (stream)
             {
-                using (MemoryStream memStream = new MemoryStream())
+                using (var memStream = new MemoryStream())
                 {
-                    stream.CopyTo(memStream);
+                    await stream.CopyToAsync(memStream);
                     return memStream.ToArray();
                 }
             }

src/Ocelot/Request/Builder/IRequestBuilder.cs

@@ -12,7 +12,8 @@ namespace Ocelot.Request.Builder
             Stream content,
             IHeaderDictionary headers,
             IRequestCookieCollection cookies,
-            Microsoft.AspNetCore.Http.QueryString queryString,
-            string contentType);
+            QueryString queryString,
+            string contentType, 
+            RequestId.RequestId requestId);
     }
 }

src/Ocelot/Request/Middleware/HttpRequestBuilderMiddleware.cs

@@ -22,17 +22,18 @@ namespace Ocelot.Request.Middleware
 
         public async Task Invoke(HttpContext context)
         {
-            var request = await _requestBuilder
-              .Build(context.Request.Method, DownstreamUrl, context.Request.Body,
-              context.Request.Headers, context.Request.Cookies, context.Request.QueryString, context.Request.ContentType);
+            var buildResult = await _requestBuilder
+                .Build(context.Request.Method, DownstreamUrl, context.Request.Body,
+                    context.Request.Headers, context.Request.Cookies, context.Request.QueryString,
+                    context.Request.ContentType, new RequestId.RequestId(DownstreamRoute?.ReRoute?.RequestIdKey, context.TraceIdentifier));
 
-            if (request.IsError)
+            if (buildResult.IsError)
             {
-                SetPipelineError(request.Errors);
+                SetPipelineError(buildResult.Errors);
                 return;
             }
 
-            SetUpstreamRequestForThisRequest(request.Data);
+            SetUpstreamRequestForThisRequest(buildResult.Data);
 
             await _next.Invoke(context);
         }

src/Ocelot/RequestId/DefaultRequestIdKey.cs

@@ -0,0 +1,9 @@
+namespace Ocelot.RequestId
+{
+    public static class DefaultRequestIdKey
+    {
+        // This is set incase anyone isnt doing this specifically with there requests. 
+        // It will not be forwarded on to downstream services unless specfied in the config.
+        public const string Value = "RequestId";
+    }
+}

src/Ocelot/RequestId/Middleware/RequestIdMiddleware.cs

@@ -0,0 +1,45 @@
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Primitives;
+using Ocelot.Infrastructure.RequestData;
+using Ocelot.Middleware;
+
+namespace Ocelot.RequestId.Middleware
+{
+    public class RequestIdMiddleware : OcelotMiddleware
+    {
+        private readonly RequestDelegate _next;
+
+        public RequestIdMiddleware(RequestDelegate next, 
+            IRequestScopedDataRepository requestScopedDataRepository)
+            :base(requestScopedDataRepository)
+        {
+            _next = next;
+        }
+
+        public async Task Invoke(HttpContext context)
+        {
+            SetTraceIdentifier(context);
+
+            await _next.Invoke(context);
+        }
+
+        private void SetTraceIdentifier(HttpContext context)
+        {
+            var key = DefaultRequestIdKey.Value;
+
+            if (DownstreamRoute.ReRoute.RequestIdKey != null)
+            {
+                key = DownstreamRoute.ReRoute.RequestIdKey;
+            }
+
+            StringValues requestId;
+
+            if (context.Request.Headers.TryGetValue(key, out requestId))
+            {
+                context.TraceIdentifier = requestId;
+            }
+        }
+    }
+}

src/Ocelot/RequestId/Middleware/RequestIdMiddlewareExtensions.cs

@@ -0,0 +1,12 @@
+using Microsoft.AspNetCore.Builder;
+
+namespace Ocelot.RequestId.Middleware
+{
+    public static class RequestIdMiddlewareExtensions
+    {
+        public static IApplicationBuilder UseRequestIdMiddleware(this IApplicationBuilder builder)
+        {
+            return builder.UseMiddleware<RequestIdMiddleware>();
+        }
+    }
+}

src/Ocelot/RequestId/RequestId.cs

@@ -0,0 +1,14 @@
+namespace Ocelot.RequestId
+{
+    public class RequestId
+    {
+        public RequestId(string requestIdKey, string requestIdValue)
+        {
+            RequestIdKey = requestIdKey;
+            RequestIdValue = requestIdValue;
+        }
+
+        public string RequestIdKey { get; private set; }
+        public string RequestIdValue { get; private set; }
+    }
+}

src/Ocelot/Requester/Middleware/HttpRequesterMiddleware.cs

@@ -1,5 +1,7 @@
+using System.Diagnostics;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
 using Ocelot.Infrastructure.RequestData;
 using Ocelot.Middleware;
 using Ocelot.Responder;
@@ -25,7 +27,6 @@ namespace Ocelot.Requester.Middleware
 
         public async Task Invoke(HttpContext context)
         {
-
             var response = await _requester.GetResponse(Request);
 
             if (response.IsError)

src/Ocelot/Responder/HttpContextResponder.cs

@@ -1,6 +1,10 @@
-using System.Net.Http;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Primitives;
+using Ocelot.Headers;
 using Ocelot.Responses;
 
 namespace Ocelot.Responder
@@ -11,15 +15,42 @@ namespace Ocelot.Responder
     /// </summary>
     public class HttpContextResponder : IHttpResponder
     {
+        private readonly IRemoveHeaders _removeHeaders;
+
+        public HttpContextResponder(IRemoveHeaders removeHeaders)
+        {
+            _removeHeaders = removeHeaders;
+        }
+
         public async Task<Response> SetResponseOnHttpContext(HttpContext context, HttpResponseMessage response)
         {
-            context.Response.OnStarting(x =>
+            _removeHeaders.Remove(response.Headers);
+
+            foreach (var httpResponseHeader in response.Headers)
             {
-                context.Response.StatusCode = (int)response.StatusCode;
+                context.Response.Headers.Add(httpResponseHeader.Key, new StringValues(httpResponseHeader.Value.ToArray()));
+            }
+
+            var content = await response.Content.ReadAsStreamAsync();
+
+            context.Response.Headers.Add("Content-Length", new[] { content.Length.ToString() });
+
+            context.Response.OnStarting(state =>
+            {
+                var httpContext = (HttpContext)state;
+
+                httpContext.Response.StatusCode = (int)response.StatusCode;
+
                 return Task.CompletedTask;
+
             }, context);
 
-            await context.Response.WriteAsync(await response.Content.ReadAsStringAsync());
+            using (var reader = new StreamReader(content))
+            {
+                var responseContent = reader.ReadToEnd();
+                await context.Response.WriteAsync(responseContent);
+            }
+
             return new OkResponse();       
         }