Feature/hacking consul file config (#157)

* moving things around to see if I can get consul to store fileconfiguration rather than ocelotconfiguration

* more refactoring to see if we can get a test for the feature

* acceptance test passing for updating in consul..need to sort object comparison out

* fixed the failing tests

src/Ocelot/Configuration/Provider/FileConfigurationProvider.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Threading.Tasks;
 using Newtonsoft.Json;
 using Ocelot.Configuration.File;
 using Ocelot.Configuration.Repository;
@@ -16,9 +17,9 @@ namespace Ocelot.Configuration.Provider
             _repo = repo;
         }
         
-        public Response<FileConfiguration> Get()
+        public async Task<Response<FileConfiguration>> Get()
         {
-            var fileConfig = _repo.Get();
+            var fileConfig = await _repo.Get();
             return new OkResponse<FileConfiguration>(fileConfig.Data);
         }
     }

src/Ocelot/Configuration/Provider/IFileConfigurationProvider.cs

@@ -1,3 +1,4 @@
+using System.Threading.Tasks;
 using Ocelot.Configuration.File;
 using Ocelot.Responses;
 
@@ -5,6 +6,6 @@ namespace Ocelot.Configuration.Provider
 {
     public interface IFileConfigurationProvider
     {
-        Response<FileConfiguration> Get();
+        Task<Response<FileConfiguration>> Get();
     }
 }

src/Ocelot/Configuration/Provider/IOcelotConfigurationProvider.cs

@@ -1,4 +1,5 @@
 using System.Threading.Tasks;
+using Ocelot.Configuration.File;
 using Ocelot.Responses;
 
 namespace Ocelot.Configuration.Provider

src/Ocelot/Configuration/Provider/OcelotConfigurationProvider.cs

@@ -1,4 +1,5 @@
 using System.Threading.Tasks;
+using Ocelot.Configuration.File;
 using Ocelot.Configuration.Repository;
 using Ocelot.Responses;
 
@@ -9,16 +10,16 @@ namespace Ocelot.Configuration.Provider
     /// </summary>
     public class OcelotConfigurationProvider : IOcelotConfigurationProvider
     {
-        private readonly IOcelotConfigurationRepository _repo;
+        private readonly IOcelotConfigurationRepository _config;
 
         public OcelotConfigurationProvider(IOcelotConfigurationRepository repo)
         {
-            _repo = repo;
+            _config = repo;
         }
 
         public async Task<Response<IOcelotConfiguration>> Get()
         {
-            var repoConfig = await _repo.Get();
+            var repoConfig = await _config.Get();
 
             if (repoConfig.IsError)
             {

src/Ocelot/Configuration/Repository/ConsulFileConfigurationPoller.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Ocelot.Configuration.File;
+using Ocelot.Configuration.Setter;
+using Ocelot.Logging;
+
+namespace Ocelot.Configuration.Repository
+{
+    public class ConsulFileConfigurationPoller : IDisposable
+    {
+        private IOcelotLogger _logger; 
+        private IFileConfigurationRepository _repo;
+        private IFileConfigurationSetter _setter;
+        private string _previousAsJson;
+        private Timer _timer;
+        private bool _polling;
+
+        public ConsulFileConfigurationPoller(IOcelotLoggerFactory factory, IFileConfigurationRepository repo, IFileConfigurationSetter setter)
+        {
+            _setter = setter;
+            _logger = factory.CreateLogger<ConsulFileConfigurationPoller>();
+            _repo = repo;
+            _previousAsJson = "";
+            _timer = new Timer(async x =>
+            {
+                if(_polling)
+                {
+                    return;
+                }
+
+                _polling = true;
+                await Poll();
+                _polling = false;
+
+            }, null, 0, 1000);
+        }
+        
+        private async Task Poll()
+        {
+            _logger.LogDebug("Started polling consul");
+
+            var fileConfig = await _repo.Get();
+
+            if(fileConfig.IsError)
+            {
+                _logger.LogDebug($"error geting file config, errors are {string.Join(",", fileConfig.Errors.Select(x => x.Message))}");
+                return;
+            }
+
+            var asJson = ToJson(fileConfig.Data);
+
+            if(!fileConfig.IsError && asJson != _previousAsJson)
+            {
+                await _setter.Set(fileConfig.Data);
+                _previousAsJson = asJson;
+            }
+
+            _logger.LogDebug("Finished polling consul");
+        }
+
+        /// <summary>
+        /// We could do object comparison here but performance isnt really a problem. This might be an issue one day!
+        /// </summary>
+        /// <param name="config"></param>
+        /// <returns></returns>
+        private string ToJson(FileConfiguration config)
+        {
+            var currentHash = JsonConvert.SerializeObject(config);
+            return currentHash;
+        }
+
+        public void Dispose()
+        {
+            _timer.Dispose();
+        }
+    }
+}

src/Ocelot/Configuration/Repository/ConsulFileConfigurationRepository.cs

@@ -3,19 +3,20 @@ using System.Text;
 using System.Threading.Tasks;
 using Consul;
 using Newtonsoft.Json;
+using Ocelot.Configuration.File;
 using Ocelot.Responses;
 using Ocelot.ServiceDiscovery;
 
 namespace Ocelot.Configuration.Repository
 {
-    public class ConsulOcelotConfigurationRepository : IOcelotConfigurationRepository
+
+    public class ConsulFileConfigurationRepository : IFileConfigurationRepository
     {
         private readonly ConsulClient _consul;
         private string _ocelotConfiguration = "OcelotConfiguration";
-        private readonly Cache.IOcelotCache<IOcelotConfiguration> _cache;
+        private readonly Cache.IOcelotCache<FileConfiguration> _cache;
 
-
-        public ConsulOcelotConfigurationRepository(Cache.IOcelotCache<IOcelotConfiguration> cache, ServiceProviderConfiguration serviceProviderConfig)
+        public ConsulFileConfigurationRepository(Cache.IOcelotCache<FileConfiguration> cache, ServiceProviderConfiguration serviceProviderConfig)
         {
             var consulHost = string.IsNullOrEmpty(serviceProviderConfig?.ServiceProviderHost) ? "localhost" : serviceProviderConfig?.ServiceProviderHost;
             var consulPort = serviceProviderConfig?.ServiceProviderPort ?? 8500;
@@ -27,32 +28,32 @@ namespace Ocelot.Configuration.Repository
             });
         }
 
-        public async Task<Response<IOcelotConfiguration>> Get()
+        public async Task<Response<FileConfiguration>> Get()
         {
             var config = _cache.Get(_ocelotConfiguration, _ocelotConfiguration);
 
             if (config != null)
             {
-                return new OkResponse<IOcelotConfiguration>(config);
+                return new OkResponse<FileConfiguration>(config);
             }
 
             var queryResult = await _consul.KV.Get(_ocelotConfiguration);
 
             if (queryResult.Response == null)
             {
-                return new OkResponse<IOcelotConfiguration>(null);
+                return new OkResponse<FileConfiguration>(null);
             }
 
             var bytes = queryResult.Response.Value;
 
             var json = Encoding.UTF8.GetString(bytes);
 
-            var consulConfig = JsonConvert.DeserializeObject<OcelotConfiguration>(json);
+            var consulConfig = JsonConvert.DeserializeObject<FileConfiguration>(json);
 
-            return new OkResponse<IOcelotConfiguration>(consulConfig);
+            return new OkResponse<FileConfiguration>(consulConfig);
         }
 
-        public async Task<Response> AddOrReplace(IOcelotConfiguration ocelotConfiguration)
+        public async Task<Response> Set(FileConfiguration ocelotConfiguration)
         {
             var json = JsonConvert.SerializeObject(ocelotConfiguration);
 
@@ -72,7 +73,7 @@ namespace Ocelot.Configuration.Repository
                 return new OkResponse();
             }
 
-            return new ErrorResponse(new UnableToSetConfigInConsulError("Unable to set config in consul"));
+            return new ErrorResponse(new UnableToSetConfigInConsulError($"Unable to set FileConfiguration in consul, response status code from consul was {result.StatusCode}"));
         }
     }
 }

src/Ocelot/Configuration/Repository/FileConfigurationRepository.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Threading.Tasks;
 using Newtonsoft.Json;
 using Ocelot.Configuration.File;
 using Ocelot.Responses;
@@ -8,7 +9,7 @@ namespace Ocelot.Configuration.Repository
     public class FileConfigurationRepository : IFileConfigurationRepository
     {
         private static readonly object _lock = new object();
-        public Response<FileConfiguration> Get()
+        public async Task<Response<FileConfiguration>> Get()
         {
             var configFilePath = $"{AppContext.BaseDirectory}/configuration.json";
             string json = string.Empty;
@@ -20,7 +21,7 @@ namespace Ocelot.Configuration.Repository
             return new OkResponse<FileConfiguration>(fileConfiguration);
         }
 
-        public Response Set(FileConfiguration fileConfiguration)
+        public async Task<Response> Set(FileConfiguration fileConfiguration)
         {
             var configurationPath = $"{AppContext.BaseDirectory}/configuration.json";
 

src/Ocelot/Configuration/Repository/IFileConfigurationRepository.cs

@@ -1,3 +1,4 @@
+using System.Threading.Tasks;
 using Ocelot.Configuration.File;
 using Ocelot.Responses;
 
@@ -5,7 +6,7 @@ namespace Ocelot.Configuration.Repository
 {
     public interface IFileConfigurationRepository
     {
-        Response<FileConfiguration> Get();
-        Response Set(FileConfiguration fileConfiguration);
+        Task<Response<FileConfiguration>> Get();
+        Task<Response> Set(FileConfiguration fileConfiguration);
     }
 }

src/Ocelot/Configuration/Repository/IOcelotConfigurationRepository.cs

@@ -1,4 +1,5 @@
 using System.Threading.Tasks;
+using Ocelot.Configuration.File;
 using Ocelot.Responses;
 
 namespace Ocelot.Configuration.Repository

src/Ocelot/Configuration/Setter/FileConfigurationSetter.cs

@@ -22,7 +22,7 @@ namespace Ocelot.Configuration.Setter
 
         public async Task<Response> Set(FileConfiguration fileConfig)
         {
-            var response = _repo.Set(fileConfig);
+            var response = await _repo.Set(fileConfig);
 
             if(response.IsError)
             {

src/Ocelot/Controllers/FileConfigurationController.cs

@@ -21,9 +21,9 @@ namespace Ocelot.Controllers
         }
 
         [HttpGet]
-        public IActionResult Get()
+        public async Task<IActionResult> Get()
         {
-            var response = _configGetter.Get();
+            var response = await _configGetter.Get();
 
             if(response.IsError)
             {

src/Ocelot/DependencyInjection/ServiceCollectionExtensions.cs

@@ -63,7 +63,8 @@ namespace Ocelot.DependencyInjection
                 .Build();
 
             services.AddSingleton<ServiceProviderConfiguration>(config);
-            services.AddSingleton<IOcelotConfigurationRepository, ConsulOcelotConfigurationRepository>();
+            services.AddSingleton<ConsulFileConfigurationPoller>();
+            services.AddSingleton<IFileConfigurationRepository, ConsulFileConfigurationRepository>();
             return services;
         }
 
@@ -90,6 +91,11 @@ namespace Ocelot.DependencyInjection
             services.TryAddSingleton<ICacheManager<IOcelotConfiguration>>(ocelotConfigCacheManagerOutputCache);
             services.TryAddSingleton<IOcelotCache<IOcelotConfiguration>>(ocelotConfigCacheManager);
 
+              var fileConfigCacheManagerOutputCache = CacheFactory.Build<FileConfiguration>("FileConfigurationCache", settings);
+            var fileConfigCacheManager = new OcelotCacheManagerCache<FileConfiguration>(fileConfigCacheManagerOutputCache);
+            services.TryAddSingleton<ICacheManager<FileConfiguration>>(fileConfigCacheManagerOutputCache);
+            services.TryAddSingleton<IOcelotCache<FileConfiguration>>(fileConfigCacheManager);
+
             services.Configure<FileConfiguration>(configurationRoot);
             services.TryAddSingleton<IOcelotConfigurationCreator, FileOcelotConfigurationCreator>();
             services.TryAddSingleton<IOcelotConfigurationRepository, InMemoryOcelotConfigurationRepository>();

src/Ocelot/Infrastructure/RequestData/HttpDataRepository.cs

@@ -35,6 +35,14 @@ namespace Ocelot.Infrastructure.RequestData
         {
             object obj;
 
+            if(_httpContextAccessor.HttpContext == null || _httpContextAccessor.HttpContext.Items == null)
+            {
+                return new ErrorResponse<T>(new List<Error>
+                {
+                    new CannotFindDataError($"Unable to find data for key: {key} because HttpContext or HttpContext.Items is null")
+                });
+            }
+
             if(_httpContextAccessor.HttpContext.Items.TryGetValue(key, out obj))
             {
                 var data = (T) obj;

src/Ocelot/Middleware/OcelotMiddlewareExtensions.cs

@@ -29,10 +29,13 @@ namespace Ocelot.Middleware
     using Microsoft.AspNetCore.Http;
     using Microsoft.Extensions.Options;
     using Ocelot.Configuration;
+    using Ocelot.Configuration.Creator;
     using Ocelot.Configuration.File;
     using Ocelot.Configuration.Provider;
+    using Ocelot.Configuration.Repository;
     using Ocelot.Configuration.Setter;
     using Ocelot.LoadBalancer.Middleware;
+    using Ocelot.Responses;
 
     public static class OcelotMiddlewareExtensions
     {
@@ -56,7 +59,9 @@ namespace Ocelot.Middleware
         /// <returns></returns>
         public static async Task<IApplicationBuilder> UseOcelot(this IApplicationBuilder builder, OcelotMiddlewareConfiguration middlewareConfiguration)
         {
-            await CreateAdministrationArea(builder);
+            var configuration = await CreateConfiguration(builder);
+            
+            await CreateAdministrationArea(builder, configuration);
 
             ConfigureDiagnosticListener(builder);
 
@@ -155,7 +160,34 @@ namespace Ocelot.Middleware
 
             if (ocelotConfiguration == null || ocelotConfiguration.Data == null || ocelotConfiguration.IsError)
             {
-                var config = await configSetter.Set(fileConfig.Value);
+                Response config = null;
+                var fileConfigRepo = builder.ApplicationServices.GetService(typeof(IFileConfigurationRepository));
+                if (fileConfigRepo.GetType() == typeof(ConsulFileConfigurationRepository))
+                {
+                    var consulFileConfigRepo = (ConsulFileConfigurationRepository) fileConfigRepo;
+                    var ocelotConfigurationRepository =
+                        (IOcelotConfigurationRepository) builder.ApplicationServices.GetService(
+                            typeof(IOcelotConfigurationRepository));
+                    var ocelotConfigurationCreator =
+                        (IOcelotConfigurationCreator) builder.ApplicationServices.GetService(
+                            typeof(IOcelotConfigurationCreator));
+
+                    var fileConfigFromConsul = await consulFileConfigRepo.Get();
+                    if (fileConfigFromConsul.Data == null)
+                    {
+                        config = await configSetter.Set(fileConfig.Value);
+                    }
+                    else
+                    {
+                        var ocelotConfig = await ocelotConfigurationCreator.Create(fileConfigFromConsul.Data);
+                        config = await ocelotConfigurationRepository.AddOrReplace(ocelotConfig.Data);
+                        var hack = builder.ApplicationServices.GetService(typeof(ConsulFileConfigurationPoller));
+                    }
+                }
+                else
+                {
+                    config = await configSetter.Set(fileConfig.Value);
+                }
 
                 if (config == null || config.IsError)
                 {
@@ -173,10 +205,8 @@ namespace Ocelot.Middleware
             return ocelotConfiguration.Data;
         }
 
-        private static async Task CreateAdministrationArea(IApplicationBuilder builder)
+        private static async Task CreateAdministrationArea(IApplicationBuilder builder, IOcelotConfiguration configuration)
         {
-            var configuration = await CreateConfiguration(builder);
-
             var identityServerConfiguration = (IIdentityServerConfiguration)builder.ApplicationServices.GetService(typeof(IIdentityServerConfiguration));
 
             if(!string.IsNullOrEmpty(configuration.AdministrationPath) && identityServerConfiguration != null)