From c162be240c16aed553981ea1aa30e6415779160f Mon Sep 17 00:00:00 2001 From: cxxpxr <60411087+cxxpxr@users.noreply.github.com> Date: Mon, 5 Apr 2021 18:25:43 -0400 Subject: [PATCH] Successful load balance test --- .../LRM_LoadBalancer/Endpoint.cs | 34 ++++- .../LRM_LoadBalancer/Program.cs | 34 +++-- .../LRM/Config.cs | 19 +++ .../LRM/Program.cs | 37 ++--- .../LightReflectiveMirrorTransport.cs | 126 +++++++++++++++--- 5 files changed, 186 insertions(+), 64 deletions(-) diff --git a/LoadBalancerProject-DONT-IMPORT-INTO-UNITY/LRM_LoadBalancer/Endpoint.cs b/LoadBalancerProject-DONT-IMPORT-INTO-UNITY/LRM_LoadBalancer/Endpoint.cs index 45ace4e..47392d3 100644 --- a/LoadBalancerProject-DONT-IMPORT-INTO-UNITY/LRM_LoadBalancer/Endpoint.cs +++ b/LoadBalancerProject-DONT-IMPORT-INTO-UNITY/LRM_LoadBalancer/Endpoint.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; @@ -12,25 +13,35 @@ using HttpStatusCode = Grapevine.HttpStatusCode; namespace LightReflectiveMirror.LoadBalancing { + [RestResource] public class Endpoint { + /// + /// Sent from an LRM server node + /// adds it to the list if authenticated. + /// + /// + /// [RestRoute("Get", "/api/auth")] public async Task ReceiveAuthKey(IHttpContext context) { var req = context.Request; string receivedAuthKey = req.Headers["Auth"]; - string port = req.Headers["Port"]; + string endpointPort = req.Headers["EndpointPort"]; + string gamePort = req.Headers["GamePort"]; + string address = context.Request.RemoteEndPoint.Address.ToString(); Console.WriteLine("Received auth req [" + receivedAuthKey + "] == [" + Program.conf.AuthKey+"]"); // if server is authenticated - if (receivedAuthKey != null && address != null && port != null && receivedAuthKey == Program.conf.AuthKey) + if (receivedAuthKey != null && address != null && endpointPort != null && gamePort != null && receivedAuthKey == Program.conf.AuthKey) { - Console.WriteLine($"Server accepted: {address}:{port}"); - - await Program.instance.AddServer($"{address}:{port}"); + Console.WriteLine($"Server accepted: {address}:{gamePort}"); + var _gamePort = Convert.ToUInt16(gamePort); + var _endpointPort = Convert.ToUInt16(endpointPort); + await Program.instance.AddServer(address, _gamePort, _endpointPort); await context.Response.SendResponseAsync(HttpStatusCode.Ok); } @@ -57,7 +68,7 @@ namespace LightReflectiveMirror.LoadBalancing return; } - KeyValuePair lowest = new("Dummy", new RelayStats { ConnectedClients = int.MaxValue }); + KeyValuePair lowest = new(new RelayAddress { Address = "Dummy" }, new RelayStats { ConnectedClients = int.MaxValue }); for (int i = 0; i < servers.Count; i++) { @@ -69,7 +80,16 @@ namespace LightReflectiveMirror.LoadBalancing // respond with the server ip // if the string is still dummy then theres no servers - await context.Response.SendResponseAsync(lowest.Key != "Dummy" ? lowest.Key : HttpStatusCode.InternalServerError); + if (lowest.Key.Address != "Dummy") + { + // ping server to ensure its online. + await Program.instance.ManualPingServer(lowest.Key.Address, lowest.Key.Port); + await context.Response.SendResponseAsync(JsonConvert.SerializeObject(lowest.Key)); + } + else + { + await context.Response.SendResponseAsync(HttpStatusCode.InternalServerError); + } } } diff --git a/LoadBalancerProject-DONT-IMPORT-INTO-UNITY/LRM_LoadBalancer/Program.cs b/LoadBalancerProject-DONT-IMPORT-INTO-UNITY/LRM_LoadBalancer/Program.cs index 80f2a5e..ffadc84 100644 --- a/LoadBalancerProject-DONT-IMPORT-INTO-UNITY/LRM_LoadBalancer/Program.cs +++ b/LoadBalancerProject-DONT-IMPORT-INTO-UNITY/LRM_LoadBalancer/Program.cs @@ -14,7 +14,7 @@ namespace LightReflectiveMirror.LoadBalancing /// Keeps track of all available relays. /// Key is server address, value is CCU. /// - public Dictionary availableRelayServers = new(); + public Dictionary availableRelayServers = new(); private int _pingDelay = 10000; const string API_PATH = "/api/stats"; @@ -56,21 +56,21 @@ namespace LightReflectiveMirror.LoadBalancing } - public async Task AddServer(string serverIP) + public async Task AddServer(string serverIP, ushort port, ushort endpointPort) { - var stats = await ManualPingServer(serverIP); + var stats = await ManualPingServer(serverIP, endpointPort); if(stats.HasValue) - availableRelayServers.Add(serverIP, stats.Value); + availableRelayServers.Add(new RelayAddress { Port = port, EndpointPort = endpointPort, Address = serverIP }, stats.Value); } - async Task ManualPingServer(string serverIP) + public async Task ManualPingServer(string serverIP, ushort port) { using (WebClient wc = new WebClient()) { try { - string receivedStats = await wc.DownloadStringTaskAsync($"http://{serverIP}{API_PATH}"); + string receivedStats = await wc.DownloadStringTaskAsync($"http://{serverIP}:{port}{API_PATH}"); return JsonConvert.DeserializeObject(receivedStats); } @@ -89,26 +89,25 @@ namespace LightReflectiveMirror.LoadBalancing WriteLogMessage("Pinging " + availableRelayServers.Count + " available relays"); // Create a new list so we can modify the collection in our loop. - var keys = new List(availableRelayServers.Keys); + var keys = new List(availableRelayServers.Keys); for(int i = 0; i < keys.Count; i++) { - string url = $"http://{keys[i]}{API_PATH}"; + string url = $"http://{keys[i].Address}:{keys[i].EndpointPort}{API_PATH}"; using (WebClient wc = new WebClient()) { try { var serverStats = wc.DownloadString(url); - Console.WriteLine(serverStats); + var deserializedData = JsonConvert.DeserializeObject(serverStats); - WriteLogMessage("Server " + keys[i] + " still exists, keeping in collection."); + WriteLogMessage("Server " + keys[i].Address + " still exists, keeping in collection."); if (availableRelayServers.ContainsKey(keys[i])) - availableRelayServers[keys[i]] = JsonConvert.DeserializeObject(serverStats); + availableRelayServers[keys[i]] = deserializedData; else - availableRelayServers.Add(keys[i], JsonConvert.DeserializeObject(serverStats)); - + availableRelayServers.Add(keys[i], deserializedData); } catch (Exception ex) { @@ -174,4 +173,13 @@ namespace LightReflectiveMirror.LoadBalancing public int PublicRoomCount; public TimeSpan Uptime; } + + [Serializable] + public struct RelayAddress + { + public ushort Port; + public ushort EndpointPort; + public string Address; + } + } diff --git a/ServerProject-DONT-IMPORT-INTO-UNITY/LRM/Config.cs b/ServerProject-DONT-IMPORT-INTO-UNITY/LRM/Config.cs index 6e07a41..7136368 100644 --- a/ServerProject-DONT-IMPORT-INTO-UNITY/LRM/Config.cs +++ b/ServerProject-DONT-IMPORT-INTO-UNITY/LRM/Config.cs @@ -6,15 +6,34 @@ namespace LightReflectiveMirror { class Config { + //======================== + // Required Settings + //======================== public string TransportDLL = "MultiCompiled.dll"; public string TransportClass = "Mirror.SimpleWebTransport"; public string AuthenticationKey = "Secret Auth Key"; public int UpdateLoopTime = 10; public int UpdateHeartbeatInterval = 100; + + //======================== + // Endpoint REST API Settings + //======================== public bool UseEndpoint = true; public ushort EndpointPort = 8080; public bool EndpointServerList = true; + + //======================== + // Nat Puncher Settings + //======================== public bool EnableNATPunchtroughServer = true; public ushort NATPunchtroughPort = 7776; + + //======================== + // Load Balancer Settings + //======================== + public bool UseLoadBalancer = false; + public string LoadBalancerAuthKey = "AuthKey"; + public string LoadBalancerAddress = "127.0.0.1"; + public ushort LoadBalancerPort = 7070; } } diff --git a/ServerProject-DONT-IMPORT-INTO-UNITY/LRM/Program.cs b/ServerProject-DONT-IMPORT-INTO-UNITY/LRM/Program.cs index 4650955..0991e4b 100644 --- a/ServerProject-DONT-IMPORT-INTO-UNITY/LRM/Program.cs +++ b/ServerProject-DONT-IMPORT-INTO-UNITY/LRM/Program.cs @@ -32,7 +32,6 @@ namespace LightReflectiveMirror private BiDictionary _pendingNATPunches = new BiDictionary(); private int _currentHeartbeatTimer = 0; - private string _externalIp; private byte[] _NATRequest = new byte[500]; private int _NATRequestPosition = 0; @@ -63,7 +62,6 @@ namespace LightReflectiveMirror else { conf = JsonConvert.DeserializeObject(File.ReadAllText(CONFIG_PATH)); - _externalIp = await GetExternalIp(); WriteLogMessage("Loading Assembly... ", ConsoleColor.White, true); try @@ -193,7 +191,8 @@ namespace LightReflectiveMirror Environment.Exit(0); } - await RegisterSelfToLoadBalancer(); + if(conf.UseLoadBalancer) + await RegisterSelfToLoadBalancer(); } while (true) @@ -217,45 +216,33 @@ namespace LightReflectiveMirror } } - - async Task RegisterSelfToLoadBalancer() + private async Task RegisterSelfToLoadBalancer() { - try { // replace hard coded value for config value later - var uri = new Uri("http://localhost:7070/api/auth"); - string externalip = _externalIp.Normalize().Trim(); - string port = conf.EndpointPort.ToString(); - HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create(uri); + var uri = new Uri($"http://{conf.LoadBalancerAddress}:{conf.LoadBalancerPort}/api/auth"); + string endpointPort = conf.EndpointPort.ToString(); + string gamePort = 7777.ToString(); + HttpWebRequest authReq = (HttpWebRequest)WebRequest.Create(uri); - myRequest.Headers.Add("Auth", "AuthKey"); - myRequest.Headers.Add("Port", port); + authReq.Headers.Add("Auth", conf.LoadBalancerAuthKey); + authReq.Headers.Add("EndpointPort", endpointPort); + authReq.Headers.Add("GamePort", gamePort); - WebResponse myResponse = await myRequest.GetResponseAsync(); + var res = await authReq.GetResponseAsync(); return true; } catch { // error adding or load balancer unavailable - WriteLogMessage("Error registering", ConsoleColor.Red); + WriteLogMessage("Error registering - Load Balancer probably timed out.", ConsoleColor.Red); return false; } } - async Task GetExternalIp() - { - HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create("https://ipv4.icanhazip.com/"); - WebResponse myResponse = await myRequest.GetResponseAsync(); - - Stream stream = myResponse.GetResponseStream(); - var ip = new StreamReader(stream).ReadToEnd(); - - return ip; - } - void RunNATPunchLoop() { WriteLogMessage("OK\n", ConsoleColor.Green); diff --git a/UnityTransport/LightReflectiveMirrorTransport.cs b/UnityTransport/LightReflectiveMirrorTransport.cs index 04ebc7b..ca6ea47 100644 --- a/UnityTransport/LightReflectiveMirrorTransport.cs +++ b/UnityTransport/LightReflectiveMirrorTransport.cs @@ -23,18 +23,27 @@ namespace LightReflectiveMirror public bool connectOnAwake = true; public string authenticationKey = "Secret Auth Key"; public UnityEvent diconnectedFromRelay; + [Header("NAT Punchthrough")] [Help("NAT Punchthrough will require the Direct Connect module attached.")] public bool useNATPunch = true; public ushort NATPunchtroughPort = 7776; + + [Header("Load Balancer")] + public bool useLoadBalancer = false; + public ushort loadBalancerPort = 7070; + public string loadBalancerAddress = "127.0.0.1"; + [Header("Server Hosting Data")] public string serverName = "My awesome server!"; public string extraServerData = "Map 1"; public int maxServerPlayers = 10; public bool isPublicServer = true; + [Header("Server List")] public UnityEvent serverListUpdated; public List relayServerList { private set; get; } = new List(); + [Header("Server Information")] public int serverId = -1; @@ -58,7 +67,7 @@ namespace LightReflectiveMirror private BiDictionary _serverProxies = new BiDictionary(); private BiDictionary _connectedRelayClients = new BiDictionary(); private BiDictionary _connectedDirectClients = new BiDictionary(); - + public override bool ClientConnected() => _isClient; private void OnConnectedToRelay() => _connectedToRelay = true; public bool IsAuthenticated() => _isAuthenticated; @@ -66,8 +75,8 @@ namespace LightReflectiveMirror public override bool Available() => _connectedToRelay; public override void ClientConnect(Uri uri) => ClientConnect(uri.Host); public override int GetMaxPacketSize(int channelId = 0) => clientToServerTransport.GetMaxPacketSize(channelId); - - public override string ServerGetClientAddress(int connectionId) + + public override string ServerGetClientAddress(int connectionId) { if (_connectedRelayClients.TryGetBySecond(connectionId, out int relayId)) return relayId.ToString(); @@ -110,7 +119,7 @@ namespace LightReflectiveMirror { if (_isServer) { - if(_serverProxies.TryGetByFirst(newClientEP, out SocketProxy foundProxy)) + if (_serverProxies.TryGetByFirst(newClientEP, out SocketProxy foundProxy)) { if (data.Length > 2) foundProxy.RelayData(data, data.Length); @@ -124,7 +133,7 @@ namespace LightReflectiveMirror if (_isClient) { - if(_clientProxy == null) + if (_clientProxy == null) { _clientProxy = new SocketProxy(_NATIP.Port - 1); _clientProxy.dataReceived += ClientProcessProxyData; @@ -162,7 +171,7 @@ namespace LightReflectiveMirror _directConnectModule = GetComponent(); - if(_directConnectModule != null) + if (_directConnectModule != null) { if (useNATPunch && !_directConnectModule.SupportsNATPunch()) { @@ -199,15 +208,86 @@ namespace LightReflectiveMirror public void ConnectToRelay() { - if (!_connectedToRelay) + if (!useLoadBalancer) { - _clientSendBuffer = new byte[clientToServerTransport.GetMaxPacketSize()]; - - clientToServerTransport.ClientConnect(serverIP); + if (!_connectedToRelay) + { + Connect(serverIP); + } + else + { + Debug.LogWarning("LRM | Already connected to relay!"); + } } else { - Debug.Log("Already connected to relay!"); + if (!_connectedToRelay) + { + StartCoroutine(RelayConnect()); + } + else + { + Debug.LogWarning("LRM | Already connected to relay!"); + } + } + } + + /// + /// Connects to the desired relay + /// + /// + private void Connect(string serverIP, ushort port = 7777) + { + // need to implement custom port + if (clientToServerTransport is LightReflectiveMirrorTransport) + throw new Exception("LRM | Client to Server Transport cannot be LRM."); + + if (clientToServerTransport is kcp2k.KcpTransport kcp) + { + kcp.Port = port; + } + + _clientSendBuffer = new byte[clientToServerTransport.GetMaxPacketSize()]; + clientToServerTransport.ClientConnect(serverIP); + } + + IEnumerator RelayConnect() + { + string url = $"http://{loadBalancerAddress}:{loadBalancerPort}/api/join/"; + + using (UnityWebRequest webRequest = UnityWebRequest.Get(url)) + { + // Request and wait for the desired page. + yield return webRequest.SendWebRequest(); + var result = webRequest.downloadHandler.text; +#if UNITY_2020_1_OR_NEWER + switch (webRequest.result) + { + case UnityWebRequest.Result.ConnectionError: + case UnityWebRequest.Result.DataProcessingError: + case UnityWebRequest.Result.ProtocolError: + Debug.LogWarning("LRM | Network Error while getting a relay to join from Load Balancer."); + break; + + case UnityWebRequest.Result.Success: + var parsedAddress = JsonConvert.DeserializeObject(result); + Connect(parsedAddress.Address, parsedAddress.Port); + endpointServerPort = parsedAddress.EndpointPort; + break; + } +#else + if (webRequest.isNetworkError || webRequest.isHttpError) + { + Debug.LogWarning("LRM | Network Error while getting a relay to join from Load Balancer."); + } + else + { + // join here + var parsedAddress = JsonConvert.DeserializeObject(result); + Connect(parsedAddress.Address, parsedAddress.Port); + endpointServerPort = parsedAddress.EndpointPort; + } +#endif } } @@ -274,7 +354,7 @@ namespace LightReflectiveMirror if (_isServer) { - if(_connectedRelayClients.TryGetByFirst(data.ReadInt(ref pos), out int clientID)) + if (_connectedRelayClients.TryGetByFirst(data.ReadInt(ref pos), out int clientID)) OnServerDataReceived?.Invoke(clientID, new ArraySegment(recvData), channel); } @@ -294,7 +374,7 @@ namespace LightReflectiveMirror int user = data.ReadInt(ref pos); if (_connectedRelayClients.TryGetByFirst(user, out int clientID)) { - OnServerDisconnected?.Invoke(_connectedRelayClients.GetByFirst(clientID)); + OnServerDisconnected?.Invoke(clientID); _connectedRelayClients.Remove(user); } } @@ -358,19 +438,19 @@ namespace LightReflectiveMirror initalData.WriteString(ref sendPos, data.ReadString(ref pos)); // Send 3 to lower chance of it being dropped or corrupted when received on server. - _NATPuncher.Send(initalData, sendPos,_relayPuncherIP); - _NATPuncher.Send(initalData, sendPos,_relayPuncherIP); + _NATPuncher.Send(initalData, sendPos, _relayPuncherIP); + _NATPuncher.Send(initalData, sendPos, _relayPuncherIP); _NATPuncher.Send(initalData, sendPos, _relayPuncherIP); _NATPuncher.BeginReceive(new AsyncCallback(RecvData), _NATPuncher); } break; } } - catch(Exception e) { print(e); } + catch (Exception e) { print(e); } } IEnumerator GetServerList() - { + { string uri = $"http://{serverIP}:{endpointServerPort}/api/compressed/servers"; using (UnityWebRequest webRequest = UnityWebRequest.Get(uri)) @@ -529,7 +609,7 @@ namespace LightReflectiveMirror return true; } - if(_connectedDirectClients.TryGetBySecond(connectionId, out int directId)) + if (_connectedDirectClients.TryGetBySecond(connectionId, out int directId)) return _directConnectModule.KickClient(directId); return false; @@ -573,7 +653,7 @@ namespace LightReflectiveMirror var keys = new List(_serverProxies.GetAllKeys()); - for(int i = 0; i < keys.Count; i++) + for (int i = 0; i < keys.Count; i++) { _serverProxies.GetByFirst(keys[i]).Dispose(); _serverProxies.Remove(keys[i]); @@ -749,4 +829,12 @@ namespace LightReflectiveMirror public int serverId; public string serverData; } + + [Serializable] + public struct RelayAddress + { + public ushort Port; + public ushort EndpointPort; + public string Address; + } }