From 6c2de80be05f4dc439f1ed8d1cf480af11442b3b Mon Sep 17 00:00:00 2001 From: Austin Vandersluis Date: Wed, 9 Oct 2024 00:47:17 -0500 Subject: [PATCH 1/2] Refactor CompilerService to use MessageBroker [skip ci] Significant changes include: - Added new dependencies in CompilerService - Replaced ObjectStreamClient with MessageBroker for handling compiler messages - Updated methods to use the new MessageBroker instance instead of the old client - Modified ArrayPool usage to utilize shared pool - Created new classes, CompilerMessage and MessageType, under a new namespace 'CompilerStream' - Updated package reference version in Oxide.CSharp.csproj file This is not a finished commit! --- src/CompilerService.cs | 21 ++++++++++++++------- src/CompilerStream/CompilerMessage.cs | 14 ++++++++++++++ src/CompilerStream/MessageType.cs | 24 ++++++++++++++++++++++++ src/Oxide.CSharp.csproj | 4 ++-- 4 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 src/CompilerStream/CompilerMessage.cs create mode 100644 src/CompilerStream/MessageType.cs diff --git a/src/CompilerService.cs b/src/CompilerService.cs index c0c069b..3af485c 100644 --- a/src/CompilerService.cs +++ b/src/CompilerService.cs @@ -6,6 +6,7 @@ using Oxide.Core.Logging; using Oxide.Logging; using Oxide.Plugins; +using Oxide.Pooling; using References::Mono.Unix.Native; using System; using System.Collections.Generic; @@ -20,6 +21,8 @@ using System.Text.RegularExpressions; using System.Threading; using Oxide.Core.Extensions; +using Oxide.IO; +using Oxide.IO.TransportMethods; namespace Oxide.CSharp { @@ -33,7 +36,8 @@ internal class CompilerService private volatile int lastId; private volatile bool ready; private Core.Libraries.Timer.TimerInstance idleTimer; - private ObjectStreamClient client; + // private ObjectStreamClient client; + private MessageBroker compilerStream; private string filePath; private string remoteName; private string compilerBasicArguments = "-unsafe true --setting:Force true -ms true"; @@ -79,7 +83,7 @@ private void ExpireFileCache() { lock (CompilerFile.FileCache) { - object[] toRemove = ArrayPool.Get(CompilerFile.FileCache.Count); + object[] toRemove = ArrayPool.Shared.Take(CompilerFile.FileCache.Count); int index = 0; foreach (var file in CompilerFile.FileCache) { @@ -99,7 +103,7 @@ private void ExpireFileCache() CompilerFile.FileCache.Remove(key); } - ArrayPool.Free(toRemove); + ArrayPool.Shared.Return(toRemove); } } @@ -241,6 +245,9 @@ private bool Start() return false; } + ProcessTransportProtocol protocol = new ProcessTransportProtocol(process); + compilerStream = new MessageBroker(protocol, protocol, pool: ArrayPool.Shared); + compilerStream.OnMessageReceived += OnMessage; client = new ObjectStreamClient(process.StandardOutput.BaseStream, process.StandardInput.BaseStream); client.Message += OnMessage; client.Error += OnError; @@ -322,7 +329,7 @@ internal void Stop(bool synchronous, string reason) ExpireFileCache(); } - private void OnMessage(ObjectStreamConnection connection, CompilerMessage message) + private void OnMessage(CompilerMessage message) { if (message == null) { @@ -430,7 +437,7 @@ private void OnMessage(ObjectStreamConnection break; } - connection.PushMessage(message); + compilerStream.SendMessage(message); if (!ready) { @@ -439,7 +446,7 @@ private void OnMessage(ObjectStreamConnection { CompilerMessage msg = messageQueue.Dequeue(); compilations[msg.Id].startedAt = Interface.Oxide.Now; - connection.PushMessage(msg); + compilerStream.SendMessage(msg); } } break; @@ -575,7 +582,7 @@ private void EnqueueCompilation(Compilation compilation) if (ready) { compilation.startedAt = Interface.Oxide.Now; - client.PushMessage(message); + compilerStream.SendMessage(message); } else { diff --git a/src/CompilerStream/CompilerMessage.cs b/src/CompilerStream/CompilerMessage.cs new file mode 100644 index 0000000..0678dcb --- /dev/null +++ b/src/CompilerStream/CompilerMessage.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Oxide.CSharp.CompilerStream +{ + public sealed class CompilerMessage + { + public MessageType Type { get; set; } + + public byte[] Data { get; set; } + } +} diff --git a/src/CompilerStream/MessageType.cs b/src/CompilerStream/MessageType.cs new file mode 100644 index 0000000..f8f0ab9 --- /dev/null +++ b/src/CompilerStream/MessageType.cs @@ -0,0 +1,24 @@ +using System; + +namespace Oxide.CSharp.CompilerStream +{ + [Flags] + public enum MessageType : byte + { + Unknown = 0x00, + + Acknowledge = 0x01, + + Heartbeat = 0x02, + + VersionInfo = 0x04, + + Ready = 0x08, + + Command = 0x16, + + Data = 0x32, + + Shutdown = 0x64 + } +} diff --git a/src/Oxide.CSharp.csproj b/src/Oxide.CSharp.csproj index 101fe04..8356911 100644 --- a/src/Oxide.CSharp.csproj +++ b/src/Oxide.CSharp.csproj @@ -1,4 +1,4 @@ - + @@ -17,8 +17,8 @@ NU1701 + - contentfiles;analyzers;build From b48c6e7d167386bfb702d9f90e5c48b4d4303a31 Mon Sep 17 00:00:00 2001 From: Austin Vandersluis Date: Sun, 3 Nov 2024 01:41:02 -0700 Subject: [PATCH 2/2] Refactor compiler services for Message Broker update --- src/CompilerService.cs | 13 +--- src/CompilerStream/CompilerClient.cs | 98 ++++++++++++++++++++++++ src/CompilerStream/CompilerMessage.cs | 6 +- src/CompilerStream/MessageType.cs | 9 ++- src/ObjectStream/Data/CompilerMessage.cs | 4 +- 5 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 src/CompilerStream/CompilerClient.cs diff --git a/src/CompilerService.cs b/src/CompilerService.cs index 3af485c..3d69b4a 100644 --- a/src/CompilerService.cs +++ b/src/CompilerService.cs @@ -23,6 +23,7 @@ using Oxide.Core.Extensions; using Oxide.IO; using Oxide.IO.TransportMethods; +using Oxide.CSharp.CompilerStream; namespace Oxide.CSharp { @@ -31,13 +32,12 @@ internal class CompilerService private static readonly Regex SymbolEscapeRegex = new Regex(@"[^\w\d]", RegexOptions.Compiled); private const string baseUrl = "https://downloads.oxidemod.com/artifacts/Oxide.Compiler/%7B0%7D/"; private Hash compilations; - private Queue messageQueue; private Process process; private volatile int lastId; private volatile bool ready; private Core.Libraries.Timer.TimerInstance idleTimer; // private ObjectStreamClient client; - private MessageBroker compilerStream; + private CompilerClient client; private string filePath; private string remoteName; private string compilerBasicArguments = "-unsafe true --setting:Force true -ms true"; @@ -49,7 +49,6 @@ internal class CompilerService public CompilerService(Extension extension) { compilations = new Hash(); - messageQueue = new Queue(); string arc = IntPtr.Size == 8 ? "x64" : "x86"; filePath = Path.Combine(Interface.Oxide.RootDirectory, $"Oxide.Compiler"); string downloadUrl = string.Format(baseUrl, extension.Branch); @@ -245,13 +244,7 @@ private bool Start() return false; } - ProcessTransportProtocol protocol = new ProcessTransportProtocol(process); - compilerStream = new MessageBroker(protocol, protocol, pool: ArrayPool.Shared); - compilerStream.OnMessageReceived += OnMessage; - client = new ObjectStreamClient(process.StandardOutput.BaseStream, process.StandardInput.BaseStream); - client.Message += OnMessage; - client.Error += OnError; - client.Start(); + client = new CompilerClient(process); ResetIdleTimer(); Interface.Oxide.LogInfo($"[CSharp] Started Oxide.Compiler v{GetCompilerVersion()} successfully"); return true; diff --git a/src/CompilerStream/CompilerClient.cs b/src/CompilerStream/CompilerClient.cs new file mode 100644 index 0000000..81f481d --- /dev/null +++ b/src/CompilerStream/CompilerClient.cs @@ -0,0 +1,98 @@ +using ObjectStream.Data; +using Oxide.IO; +using Oxide.IO.Serialization; +using Oxide.IO.TransportMethods; +using Oxide.Pooling; +using System; +using System.Diagnostics; +using System.Text; + +namespace Oxide.CSharp.CompilerStream +{ + public class CompilerClient + { + public event Action OnHeatbeat; + + public event Action OnVersionUpdate; + + public event Action OnResult; + + #region Information + + private int MessageId { get; set; } = 0; + + public DateTime LastHeartbeat { get; private set; } + + public Version Version { get; private set; } + + #endregion + + + #region Serializers + + private ISerializer MessageSerializer { get; } + + private ISerializer ResultsSerializer { get; } + + private ISerializer CompilationSerializer { get; } + + #endregion + + private MessageBroker MessageBroker { get; } + + public CompilerClient(Process process) + { + MessageSerializer = new JsonSerializer(Encoding.UTF8, PoolFactory.Default); + ResultsSerializer = new JsonSerializer(Encoding.UTF8, PoolFactory.Default); + CompilationSerializer = new JsonSerializer(Encoding.UTF8, PoolFactory.Default); + ProcessTransportProtocol protocol = new ProcessTransportProtocol(process, killOnDispose: true); + MessageBroker = new MessageBroker(protocol, protocol, MessageSerializer); + MessageBroker.OnMessageReceived += OnMessage; + } + + private void OnMessage(CompilerMessage message) + { + if (HasFlag(message, MessageType.Heartbeat)) + { + LastHeartbeat = DateTime.Now; + OnHeatbeat?.Invoke(); + } + + if (HasFlag(message, MessageType.VersionInfo)) + { + string versionStr = Encoding.UTF8.GetString(message.Data); + Version = new Version(versionStr); + OnVersionUpdate?.Invoke(Version); + } + else if (HasFlag(message, MessageType.Data)) + { + CompilationResult result = ResultsSerializer.Deserialize(message.Data); + OnResult?.Invoke(message.Id, result); + } + else if (HasFlag(message, MessageType.Error)) + { + // TODO: Implement Errors + } + } + + public int Compile(CompilerData project) + { + CompilerMessage message = new CompilerMessage() + { + Id = MessageId++, + Type = MessageType.Data, + Data = CompilationSerializer.Serialize(project) + }; + + MessageBroker.SendMessage(message); + return message.Id; + } + + public void Stop() + { + MessageBroker.Dispose(); + } + + private static bool HasFlag(CompilerMessage message, MessageType type) => ((message.Type & type) == type); + } +} diff --git a/src/CompilerStream/CompilerMessage.cs b/src/CompilerStream/CompilerMessage.cs index 0678dcb..954aeab 100644 --- a/src/CompilerStream/CompilerMessage.cs +++ b/src/CompilerStream/CompilerMessage.cs @@ -1,12 +1,12 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; namespace Oxide.CSharp.CompilerStream { + [Serializable] public sealed class CompilerMessage { + public int Id { get; set; } + public MessageType Type { get; set; } public byte[] Data { get; set; } diff --git a/src/CompilerStream/MessageType.cs b/src/CompilerStream/MessageType.cs index f8f0ab9..bf029d4 100644 --- a/src/CompilerStream/MessageType.cs +++ b/src/CompilerStream/MessageType.cs @@ -3,6 +3,7 @@ namespace Oxide.CSharp.CompilerStream { [Flags] + [Serializable] public enum MessageType : byte { Unknown = 0x00, @@ -15,10 +16,12 @@ public enum MessageType : byte Ready = 0x08, - Command = 0x16, + Command = 0x10, - Data = 0x32, + Data = 0x20, - Shutdown = 0x64 + Error = 0x40, + + Shutdown = 80 } } diff --git a/src/ObjectStream/Data/CompilerMessage.cs b/src/ObjectStream/Data/CompilerMessage.cs index 30a4bc8..302ccad 100644 --- a/src/ObjectStream/Data/CompilerMessage.cs +++ b/src/ObjectStream/Data/CompilerMessage.cs @@ -5,12 +5,12 @@ namespace ObjectStream.Data [Serializable] public class CompilerMessage { + public int Id { get; set; } + public object Data { get; set; } public object ExtraData { get; set; } - public int Id { get; set; } - public CompilerMessageType Type { get; set; } } }