From 347b3c36f4586d804136a6d35c2f1f81b8edb5c2 Mon Sep 17 00:00:00 2001 From: Elias Stepanik <40958815+eliasstepanik@users.noreply.github.com> Date: Fri, 2 Jun 2023 21:54:46 +0200 Subject: [PATCH] Added LoadBalancer and static Functions --- Functions/API/FunctionController.cs | 15 +- Functions/Data/Function.cs | 2 +- Functions/Data/InstanceRuntimeInfo.cs | 6 + Functions/Program.cs | 8 +- Functions/Services/DockerManager.cs | 14 ++ Functions/Services/FunctionManager.cs | 34 ++-- .../Services/Interfaces/IDockerManager.cs | 1 + Functions/Services/Interfaces/ILoadManager.cs | 8 + Functions/Services/LoadManager.cs | 187 ++++++++++++++++++ Functions/Services/TimerManager.cs | 62 ++++++ Functions/appsettings.Production.json | 3 +- TestFunction/Controllers/TestController.cs | 12 +- TestFunction/Test.cs | 22 +++ TestFunction/TestFunction.csproj | 1 + 14 files changed, 341 insertions(+), 34 deletions(-) create mode 100644 Functions/Data/InstanceRuntimeInfo.cs create mode 100644 Functions/Services/Interfaces/ILoadManager.cs create mode 100644 Functions/Services/LoadManager.cs create mode 100644 Functions/Services/TimerManager.cs create mode 100644 TestFunction/Test.cs diff --git a/Functions/API/FunctionController.cs b/Functions/API/FunctionController.cs index f84dcaa..c2faf60 100644 --- a/Functions/API/FunctionController.cs +++ b/Functions/API/FunctionController.cs @@ -12,11 +12,14 @@ public class FunctionController : ControllerBase { private readonly ILogger _logger; private readonly FunctionManager _functionManager; + private readonly ILoadManager _loadManager; - public FunctionController(ILogger logger, FunctionManager functionManager) + + public FunctionController(ILogger logger, FunctionManager functionManager, ILoadManager loadManager) { _logger = logger; _functionManager = functionManager; + _loadManager = loadManager; } [HttpPost("{functionName}/edit")] @@ -30,7 +33,7 @@ public class FunctionController : ControllerBase [HttpPost("{functionName}")] public async Task RunFunctionPost(string functionName,[FromBody] string text) { - var responseContext = await _functionManager.RunInstance(functionName,HttpMethod.Post, text); + var responseContext = await _loadManager.HandleRequest(functionName,HttpMethod.Post, text); if (responseContext.IsSuccessStatusCode) { @@ -47,7 +50,7 @@ public class FunctionController : ControllerBase [HttpGet("{functionName}")] public async Task RunFunctionGet(string functionName) { - var responseContext = await _functionManager.RunInstance(functionName,HttpMethod.Get); + var responseContext = await _loadManager.HandleRequest(functionName,HttpMethod.Get); if (responseContext.IsSuccessStatusCode) { @@ -65,7 +68,7 @@ public class FunctionController : ControllerBase [HttpPatch("{functionName}")] public async Task RunFunctionPatch(string functionName,[FromBody] string text) { - var responseContext = await _functionManager.RunInstance(functionName,HttpMethod.Patch, text); + var responseContext = await _loadManager.HandleRequest(functionName,HttpMethod.Patch, text); if (responseContext.IsSuccessStatusCode) { @@ -83,7 +86,7 @@ public class FunctionController : ControllerBase [HttpPut("{functionName}")] public async Task RunFunctionPut(string functionName,[FromBody] string text) { - var responseContext = await _functionManager.RunInstance(functionName,HttpMethod.Put, text); + var responseContext = await _loadManager.HandleRequest(functionName,HttpMethod.Put, text); if (responseContext.IsSuccessStatusCode) { @@ -101,7 +104,7 @@ public class FunctionController : ControllerBase [HttpDelete("{functionName}")] public async Task RunFunctionDelete(string functionName,[FromBody] string text) { - var responseContext = await _functionManager.RunInstance(functionName,HttpMethod.Delete, text); + var responseContext = await _loadManager.HandleRequest(functionName,HttpMethod.Delete, text); if (responseContext.IsSuccessStatusCode) { diff --git a/Functions/Data/Function.cs b/Functions/Data/Function.cs index 7d6904f..0cc4686 100644 --- a/Functions/Data/Function.cs +++ b/Functions/Data/Function.cs @@ -23,5 +23,5 @@ public class Function public List EnvironmentVariables { get; set; } = null!; - public List Instances { get; set; } = null!; + public List Instances { get; set; } = null!; } \ No newline at end of file diff --git a/Functions/Data/InstanceRuntimeInfo.cs b/Functions/Data/InstanceRuntimeInfo.cs new file mode 100644 index 0000000..539087b --- /dev/null +++ b/Functions/Data/InstanceRuntimeInfo.cs @@ -0,0 +1,6 @@ +namespace Functions.Data; + +public class InstanceRuntimeInfo +{ + public Instance? Instance { get; set; } +} \ No newline at end of file diff --git a/Functions/Program.cs b/Functions/Program.cs index 5998aba..10d7def 100644 --- a/Functions/Program.cs +++ b/Functions/Program.cs @@ -32,11 +32,12 @@ var serverVersion = new MySqlServerVersion(new Version(8, 0, 31)); builder.Services.AddDbContextFactory( dbContextOptions => dbContextOptions .UseMySql(connectionString.ToString(), serverVersion) - .LogTo(Console.WriteLine, LogLevel.Debug) + .LogTo(Console.WriteLine, LogLevel.Error) .EnableSensitiveDataLogging() .EnableDetailedErrors()); - +builder.Services.AddTransient(); +builder.Services.AddSingleton(); builder.Services.AddTransient(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -63,4 +64,7 @@ var dbFactory = app.Services.GetRequiredService(); +loadManager.Start(); + app.Run(); \ No newline at end of file diff --git a/Functions/Services/DockerManager.cs b/Functions/Services/DockerManager.cs index e073b31..05567c4 100644 --- a/Functions/Services/DockerManager.cs +++ b/Functions/Services/DockerManager.cs @@ -117,6 +117,20 @@ public class DockerManager : IDockerManager return false; } + + public async Task GetLoad(string containerId) + { + ContainerStatsParameters parameters = new ContainerStatsParameters() + { + Stream = false, + }; + var response = new ContainerStatsResponse(); + + IProgress progress = new Progress(stats => { response = stats; }); + + await _docker.Containers.GetContainerStatsAsync(containerId,parameters, progress); + return response; + } } public class ContainerResponse diff --git a/Functions/Services/FunctionManager.cs b/Functions/Services/FunctionManager.cs index 75f40ce..15287dd 100644 --- a/Functions/Services/FunctionManager.cs +++ b/Functions/Services/FunctionManager.cs @@ -40,14 +40,14 @@ public class FunctionManager var function = db.Functions.Include(s => s.Instances).Include(s => s.EnvironmentVariables).First(s => s.Name.Equals(functionName)); foreach (var functionInstance in function.Instances) { - _dockerManager.DeleteContainer(functionInstance.InstanceId); + DeleteInstance(functionInstance); } db.Functions.Remove(function); await db.SaveChangesAsync(); await db.DisposeAsync(); } - public async Task RunInstance(string functionName, HttpMethod method, string body = "") + public async Task RunInstance(string functionName) { var db = await _dbContextFactory.CreateDbContextAsync(); var function = db.Functions.Include(s => s.Instances).Include(s => s.EnvironmentVariables).First(s => s.Name.Equals(functionName)); @@ -61,10 +61,16 @@ public class FunctionManager _dockerManager.ConnectNetwork(_configuration["AppConfig:FuctionNetworkName"] ?? throw new InvalidOperationException(), instance.InstanceId); _dockerManager.StartContainer(instance.InstanceId); - - //TODO: If not started delete instance - //Send Request to Container + + return new InstanceRuntimeInfo() + { + Instance = instance + }; + } + + public async Task SendRequest(Instance? instance,string function, HttpMethod method, string body = "") + { if (method.Equals(HttpMethod.Post)) { var message = await _externalEndpointManager.Post(instance.Name, body); @@ -86,39 +92,35 @@ public class FunctionManager { var message = await _externalEndpointManager.Put(instance.Name, body); return await HandleError(message, instance); - - } if (method.Equals(HttpMethod.Delete)) { var message = await _externalEndpointManager.Delete(instance.Name); return await HandleError(message, instance); - - } - return new HttpResponseMessage(HttpStatusCode.BadRequest); } - private async Task HandleError(HttpResponseMessage message, Instance instance) + private async Task HandleError(HttpResponseMessage message, Instance? instance) { var db = await _dbContextFactory.CreateDbContextAsync(); if (!message.IsSuccessStatusCode) { - _dockerManager.DeleteContainer(instance.InstanceId); - var i = db.Instances.First(s => s.InstanceId.Equals(instance.InstanceId)); - db.Instances.Remove(i); - await db.SaveChangesAsync(); + DeleteInstance(instance); return new HttpResponseMessage(HttpStatusCode.BadRequest); } + return message; + } + public async void DeleteInstance(Instance? instance) + { + var db = await _dbContextFactory.CreateDbContextAsync(); _dockerManager.DeleteContainer(instance.InstanceId); var temp_i = db.Instances.First(s => s.InstanceId.Equals(instance.InstanceId)); db.Instances.Remove(temp_i); await db.SaveChangesAsync(); await db.DisposeAsync(); - return message; } diff --git a/Functions/Services/Interfaces/IDockerManager.cs b/Functions/Services/Interfaces/IDockerManager.cs index 917183b..8f6257b 100644 --- a/Functions/Services/Interfaces/IDockerManager.cs +++ b/Functions/Services/Interfaces/IDockerManager.cs @@ -13,4 +13,5 @@ public interface IDockerManager public void DeleteContainer(string containerId); public void CreateNetwork(string name); public Task IsRunning(string containerId); + public Task GetLoad(string containerId); } \ No newline at end of file diff --git a/Functions/Services/Interfaces/ILoadManager.cs b/Functions/Services/Interfaces/ILoadManager.cs new file mode 100644 index 0000000..3be38dd --- /dev/null +++ b/Functions/Services/Interfaces/ILoadManager.cs @@ -0,0 +1,8 @@ +namespace Functions.Services.Interfaces; + +public interface ILoadManager +{ + public Task HandleRequest(string functionName, HttpMethod method, string body = ""); + public void Update(); + public void Start(); +} \ No newline at end of file diff --git a/Functions/Services/LoadManager.cs b/Functions/Services/LoadManager.cs new file mode 100644 index 0000000..799813d --- /dev/null +++ b/Functions/Services/LoadManager.cs @@ -0,0 +1,187 @@ +using Functions.Data; +using Functions.Data.DB; +using Functions.Services.Interfaces; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; + +namespace Functions.Services; + +public class LoadManager : ILoadManager +{ + private readonly FunctionManager _functionManager; + private readonly IDbContextFactory _dbContextFactory; + private readonly IDockerManager _dockerManager; + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public LoadManager(FunctionManager functionManager, IDbContextFactory dbContextFactory, IDockerManager dockerManager,IServiceProvider serviceProvider, ILogger logger) + { + _functionManager = functionManager; + _dbContextFactory = dbContextFactory; + _dockerManager = dockerManager; + _serviceProvider = serviceProvider; + _logger = logger; + } + + public async Task HandleRequest(string functionName, HttpMethod method, string body = "") + { + var db = await _dbContextFactory.CreateDbContextAsync(); + var function = await db.Functions.Include(s => s.Instances).FirstAsync(x => x.Name.Equals(functionName)); + if (function.Instances.Count == 0) + { + await _functionManager.RunInstance(function.Name); + } + foreach (var dbFunction in db.Functions.Include(s => s.Instances)) + { + if (dbFunction.Instances.Count == 0) + { + await _functionManager.RunInstance(dbFunction.Name); + } + } + + var instance = await GetLowestLoadInstance(functionName); + var responseMessage = await _functionManager.SendRequest(instance, functionName, method, body); + return responseMessage; + } + + private async Task IsOverloaded(string containerId) + { + var load = await _dockerManager.GetLoad(containerId); + if (JsonConvert.SerializeObject(load).Equals("{}")) + { + return false; + } + try + { + var usedMemory = load.MemoryStats.Usage; + var availableMemory = load.MemoryStats.Limit; + // ReSharper disable once PossibleLossOfFraction + var memoryUsage = (usedMemory / availableMemory) * 100.0; + var cpuDelta = load.CPUStats.CPUUsage.TotalUsage - load.PreCPUStats.CPUUsage.TotalUsage; + var systemCpuDelta = load.CPUStats.SystemUsage - load.PreCPUStats.CPUUsage.TotalUsage; + var numberCpus = load.CPUStats.OnlineCPUs; + // ReSharper disable once PossibleLossOfFraction + var cpuUsage = (cpuDelta / systemCpuDelta) * numberCpus * 100.0; + if (cpuUsage > 80) + { + return true; + } + if(memoryUsage > 80) + { + return true; + } + } + catch (Exception e) + { + _logger.LogError(e, e.Message); + } + + + return false; + } + + private async Task GetLowestLoadInstance(string functionName) + { + Instance? lowestUsageCpuInstance = null; + double lowestUsageCpu = double.MaxValue; + + Instance? lowestUsageMemoryInstance = null; + double lowestUsageMemory = double.MaxValue; + var db = await _dbContextFactory.CreateDbContextAsync(); + try + { + + foreach (var function in db.Functions.Include(s => s.Instances)) + { + foreach (var functionInstance in function.Instances) + { + var load = await _dockerManager.GetLoad(functionInstance.InstanceId); + var usedMemory = load.MemoryStats.Usage; + var availableMemory = load.MemoryStats.Limit; + var cpuDelta = load.CPUStats.CPUUsage.TotalUsage - load.PreCPUStats.CPUUsage.TotalUsage; + var systemCpuDelta = load.CPUStats.SystemUsage - load.PreCPUStats.CPUUsage.TotalUsage; + var numberCpus = load.CPUStats.OnlineCPUs; + // ReSharper disable once PossibleLossOfFraction + + //TODO: Later + var cpuUsage = (cpuDelta / systemCpuDelta) * numberCpus * 100.0; + // ReSharper disable once PossibleLossOfFraction + var memoryUsage = (usedMemory / availableMemory) * 100.0; + + if (cpuUsage <= lowestUsageCpu) + { + lowestUsageCpu = cpuUsage; + lowestUsageCpuInstance = functionInstance; + } + + if (memoryUsage <= lowestUsageMemory) + { + lowestUsageMemory = memoryUsage; + lowestUsageMemoryInstance = functionInstance; + } + + } + } + + if (lowestUsageCpu <= lowestUsageMemory) + { + return lowestUsageCpuInstance; + } + else + { + return lowestUsageMemoryInstance; + } + } + catch (Exception e) + { + _logger.LogError(e, e.Message); + } + + Random rd = new Random(); + var randomInstance = await db.Functions.Include(s => s.Instances).ToListAsync(); + await db.DisposeAsync(); + return randomInstance.First(s => s.Name.Equals(functionName)).Instances[rd.Next(0,randomInstance.Count)]; + } + + public async void Update() + { + var db = await _dbContextFactory.CreateDbContextAsync(); + foreach (var function in db.Functions.Include(s => s.Instances)) + { + if (function.Instances.Count != 0) + { + foreach (var functionInstance in function.Instances) + { + if (await IsOverloaded(functionInstance!.InstanceId)) + { + await _functionManager.RunInstance(function.Name); + } + else if (!await IsOverloaded(functionInstance.InstanceId) && function.Instances.Count > 1) + { + _functionManager.DeleteInstance(functionInstance); + } + } + } + else + { + await _functionManager.RunInstance(function.Name); + } + + } + + await db.DisposeAsync(); + } + + public async void Start() + { + var db = await _dbContextFactory.CreateDbContextAsync(); + foreach (var function in db.Functions.Include(s => s.Instances)) + { + await _functionManager.RunInstance(function.Name); + } + + await db.DisposeAsync(); + + _serviceProvider.GetRequiredService().StartExecuting(); + } +} \ No newline at end of file diff --git a/Functions/Services/TimerManager.cs b/Functions/Services/TimerManager.cs new file mode 100644 index 0000000..b985e22 --- /dev/null +++ b/Functions/Services/TimerManager.cs @@ -0,0 +1,62 @@ +using System.Net; +using System.Timers; +using Functions.Services.Interfaces; +using Microsoft.EntityFrameworkCore; +using Timer = System.Timers.Timer; + +namespace Functions.Services; +public class JobExecutedEventArgs : EventArgs {} + +public class TimerManager : IDisposable +{ + private readonly ILoadManager _loadManager; + + public TimerManager(ILoadManager loadManager) + { + _loadManager = loadManager; + } + + public event EventHandler JobExecuted; + + void OnJobExecuted() + { + JobExecuted?.Invoke(this, new JobExecutedEventArgs()); + } + + System.Timers.Timer _timer; + bool _running; + + public void StartExecuting() + { + if (!_running) + { + //TimerLogic(); + //initiate a timer + _timer = new Timer(); + _timer.Interval = 1000; + //_Timer.Interval = 60000; + _timer.AutoReset = true; + _timer.Enabled = true; + _timer.Elapsed += HandleTimer; + _running = true; + } + } + + void HandleTimer(object source, ElapsedEventArgs e) + { + _loadManager.Update(); + + //Execute required job + //notify any subscibers to the event + OnJobExecuted(); + } + + public void Dispose() + { + if (_running) + { + //clear up the timer + _timer.Dispose(); + } + } +} \ No newline at end of file diff --git a/Functions/appsettings.Production.json b/Functions/appsettings.Production.json index c290343..237bbaf 100644 --- a/Functions/appsettings.Production.json +++ b/Functions/appsettings.Production.json @@ -2,7 +2,8 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Warning", + "Microsoft.EntityFrameworkCore.Database": "Error" } }, "AllowedHosts": "*", diff --git a/TestFunction/Controllers/TestController.cs b/TestFunction/Controllers/TestController.cs index ed1165e..5133214 100644 --- a/TestFunction/Controllers/TestController.cs +++ b/TestFunction/Controllers/TestController.cs @@ -1,3 +1,5 @@ +using Docker.DotNet; +using Docker.DotNet.Models; using Microsoft.AspNetCore.Mvc; namespace TestFunction.Controllers; @@ -31,15 +33,9 @@ public class TestController : ControllerBase } [HttpPost] - public IEnumerable Post() + public async Task Post(string id) { - return Enumerable.Range(1, 5).Select(index => new WeatherForecast - { - Date = DateOnly.FromDateTime(DateTime.Now.AddDays(index)), - TemperatureC = Random.Shared.Next(-20, 55), - Summary = Summaries[Random.Shared.Next(Summaries.Length)] - }) - .ToArray(); + return await Test.GetLoad(id); } [HttpPatch] diff --git a/TestFunction/Test.cs b/TestFunction/Test.cs new file mode 100644 index 0000000..213e9ed --- /dev/null +++ b/TestFunction/Test.cs @@ -0,0 +1,22 @@ +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace TestFunction; + +public class Test +{ + public static async Task GetLoad(string containerId) + { + ContainerStatsParameters parameters = new ContainerStatsParameters() + { + Stream = false, + }; + var response = new ContainerStatsResponse(); + + IProgress progress = new Progress(stats => { response = stats; }); + + var _docker = new DockerClientConfiguration().CreateClient(); + await _docker.Containers.GetContainerStatsAsync(containerId,parameters, progress); + return response; + } +} \ No newline at end of file diff --git a/TestFunction/TestFunction.csproj b/TestFunction/TestFunction.csproj index 9b6c532..83520c8 100644 --- a/TestFunction/TestFunction.csproj +++ b/TestFunction/TestFunction.csproj @@ -9,6 +9,7 @@ +