From 723d85cc19516ab1765d80d0e8b4182dcf607231 Mon Sep 17 00:00:00 2001 From: mika Date: Sun, 18 Aug 2024 19:09:53 +0300 Subject: [PATCH 001/110] #BUILD beta --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index dd53f67..32665a4 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,4 @@ Pull requests to improve this converter are welcome! (please create Issue first, + From b2b2ad9d2b2533624598f8354c2def7de4efd84d Mon Sep 17 00:00:00 2001 From: mika Date: Sun, 18 Aug 2024 19:17:50 +0300 Subject: [PATCH 002/110] fix build path #BUILD beta --- ArtifactBuild.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ArtifactBuild.cmd b/ArtifactBuild.cmd index 613c2f9..30115a0 100644 --- a/ArtifactBuild.cmd +++ b/ArtifactBuild.cmd @@ -1,6 +1,6 @@ @echo off pushd "%~dp0" -powershell Compress-7Zip "Bin\Release" -ArchiveFileName "PointCloudConverterX64.zip" -Format Zip +powershell Compress-7Zip "Bin\x64\Release\net8.0-windows10.0.22621.0" -ArchiveFileName "PointCloudConverterX64.zip" -Format Zip :exit popd @echo on From 99e3d2468fca545ce5ec495ecb7013555ae2b34a Mon Sep 17 00:00:00 2001 From: mika Date: Sun, 18 Aug 2024 19:18:56 +0300 Subject: [PATCH 003/110] #BUILD beta --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 32665a4..dd53f67 100644 --- a/README.md +++ b/README.md @@ -37,4 +37,3 @@ Pull requests to improve this converter are welcome! (please create Issue first, - From 1ac39f5afca7a76a4267e592df5da62fc44fdc13 Mon Sep 17 00:00:00 2001 From: mika Date: Mon, 19 Aug 2024 23:08:34 +0300 Subject: [PATCH 004/110] trying to fix releasing writer, better now? --- MainWindow.xaml.cs | 4 +- Structs/ImportSettings.cs | 81 ++++++++++++++++++++++++++----------- Writers/IWriter.cs | 1 + Writers/PCROOT.cs | 84 ++++++++++++++++++--------------------- Writers/UCPC.cs | 5 +++ 5 files changed, 103 insertions(+), 72 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 1f040a5..9d225cf 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -24,7 +24,7 @@ namespace PointCloudConverter { public partial class MainWindow : Window { - static readonly string version = "18.08.2024"; + static readonly string version = "19.08.2024"; static readonly string appname = "PointCloud Converter - " + version; static readonly string rootFolder = AppDomain.CurrentDomain.BaseDirectory; @@ -275,7 +275,7 @@ private static async Task ProcessAllFiles(object workerParamsObject) //// hack to fix progress bar not updating on last file //progressFile++; - // clamp to max of inputfiles (otherwise errors in threading) + // clamp to max of inputfiles-1 (otherwise errors in threading) int maxThreads = Math.Min(importSettings.maxThreads, importSettings.maxFiles - 1); // FIXME: -1 because otherwise keynotfindexception in last file or after it? // clamp to min 1 maxThreads = Math.Max(importSettings.maxThreads, 1); diff --git a/Structs/ImportSettings.cs b/Structs/ImportSettings.cs index 978c142..76bd0d0 100644 --- a/Structs/ImportSettings.cs +++ b/Structs/ImportSettings.cs @@ -24,7 +24,7 @@ public class ImportSettings public ConcurrentDictionary Readers { get; set; } = new ConcurrentDictionary(); public IWriter writer = new UCPC(); //public Dictionary Writers { get; set; } = new Dictionary(); - public ConcurrentDictionary Writers { get; set; } = new ConcurrentDictionary(); + public ConcurrentDictionary> Writers { get; set; } = new ConcurrentDictionary>(); // Method to get or create a reader for a specific task ID @@ -40,17 +40,17 @@ public IReader GetOrCreateReader(int? taskId) return Readers[taskId]; } - public IWriter GetOrCreateWriter(int? taskId) - { - if (!Writers.ContainsKey(taskId)) - { - Writers[taskId] = new PCROOT(taskId); - } + //public IWriter GetOrCreateWriter(int? taskId) + //{ + // if (!Writers.ContainsKey(taskId)) + // { + // Writers[taskId] = new PCROOT(taskId); + // } - //Log.WriteLine(">>>>> Total Writers in dictionary: " + Writers.Count); + // //Log.WriteLine(">>>>> Total Writers in dictionary: " + Writers.Count); - return Writers[taskId]; - } + // return Writers[taskId]; + //} //public void ReleaseReader(int? taskId) //{ @@ -63,35 +63,42 @@ public IWriter GetOrCreateWriter(int? taskId) // } //} - public void ReleaseReader(int? taskId) + public IWriter GetOrCreateWriter(int? taskId) { - // Log the release of the reader for the specified task ID - // Log.WriteLine(">>>>> Releasing reader for task ID: " + taskId); + if (!Writers.TryGetValue(taskId, out var weakWriter) || !weakWriter.TryGetTarget(out var writer)) + { + writer = new PCROOT(taskId); + Writers[taskId] = new WeakReference(writer); + } - if (taskId.HasValue) + return writer; + } + + public void ReleaseWriter(int? taskId) + { + if (taskId.HasValue && Writers.TryRemove(taskId, out var weakWriter)) { - if (Readers.TryRemove(taskId, out var reader)) - { - reader?.Close(); - // reader?.Dispose(); - } - else + if (weakWriter.TryGetTarget(out var writer)) { - Log.WriteLine($"Reader for task ID {taskId} could not be removed because it was not found.", LogEvent.Warning); + Log.WriteLine("ReleaseWriter >>> Memory used: " + GC.GetTotalMemory(false)); + //Log.WriteLine(">>>>> Releasing reader for task ID: " + taskId); + writer?.Cleanup(0); + writer?.Dispose(); + Log.WriteLine("ReleaseWriter <<< Memory used: " + GC.GetTotalMemory(false)); } } } - public void ReleaseWriter(int? taskId) + public void ReleaseReader(int? taskId) { // Log the release of the reader for the specified task ID // Log.WriteLine(">>>>> Releasing reader for task ID: " + taskId); if (taskId.HasValue) { - if (Writers.TryRemove(taskId, out var writer)) + if (Readers.TryRemove(taskId, out var reader)) { - writer?.Cleanup(0); + reader?.Close(); // reader?.Dispose(); } else @@ -101,6 +108,32 @@ public void ReleaseWriter(int? taskId) } } + //public void ReleaseWriter(int? taskId) + //{ + // // Log the release of the reader for the specified task ID + // // Log.WriteLine(">>>>> Releasing reader for task ID: " + taskId); + + // if (taskId.HasValue) + // { + // if (Writers.TryRemove(taskId, out var writer)) + // { + // Log.WriteLine("ReleaseWriter >>> Memory used: " + GC.GetTotalMemory(false)); + // writer?.Cleanup(0); + // writer?.Dispose(); + // //writer = null; + // // clear gc + // System.GC.Collect(); + // //GC.SuppressFinalize(writer); + // System.GC.WaitForPendingFinalizers(); + // Log.WriteLine("ReleaseWriter <<< Memory used: " + GC.GetTotalMemory(false)); + // } + // else + // { + // Log.WriteLine($"Reader for task ID {taskId} could not be removed because it was not found.", LogEvent.Warning); + // } + // } + //} + //public void ReleaseWriter(int? taskId) //{ // //Log.WriteLine(">>>>> Releasing writer for task ID: " + taskId); diff --git a/Writers/IWriter.cs b/Writers/IWriter.cs index a84a294..2c3c1aa 100644 --- a/Writers/IWriter.cs +++ b/Writers/IWriter.cs @@ -22,6 +22,7 @@ public interface IWriter void Cleanup(int fileIndex); // close filestream void Close(); + void Dispose(); } } diff --git a/Writers/PCROOT.cs b/Writers/PCROOT.cs index a78b840..9966d96 100644 --- a/Writers/PCROOT.cs +++ b/Writers/PCROOT.cs @@ -15,11 +15,17 @@ public class PCROOT : IWriter, IDisposable const string tileExtension = ".pct"; const string sep = "|"; - static ImportSettings importSettings; BufferedStream bsPoints = null; BinaryWriter writerPoints = null; + ImportSettings importSettings; static List nodeBounds = new List(); // for all tiles + static float cloudMinX = float.PositiveInfinity; + static float cloudMinY = float.PositiveInfinity; + static float cloudMinZ = float.PositiveInfinity; + static float cloudMaxX = float.NegativeInfinity; + static float cloudMaxY = float.NegativeInfinity; + static float cloudMaxZ = float.NegativeInfinity; Dictionary keyCache = new Dictionary(); @@ -35,23 +41,17 @@ public class PCROOT : IWriter, IDisposable Dictionary> nodeIntensity = new Dictionary>(); Dictionary> nodeTime = new Dictionary>(); - static float cloudMinX = float.PositiveInfinity; - static float cloudMinY = float.PositiveInfinity; - static float cloudMinZ = float.PositiveInfinity; - static float cloudMaxX = float.NegativeInfinity; - static float cloudMaxY = float.NegativeInfinity; - static float cloudMaxZ = float.NegativeInfinity; int? taskID; public void Dispose() { - //Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); - //Log.WriteLine("*** PCROOT writer disposed for task: " + taskID); + Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); + Log.WriteLine("*** PCROOT writer disposed for task: " + taskID); Dispose(true); GC.SuppressFinalize(this); GC.Collect(); - //Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); + Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); } @@ -59,23 +59,25 @@ private void ClearDictionary(Dictionary> dictionary) { if (dictionary != null) { - foreach (var key in dictionary.Keys) + foreach (var list in dictionary.Values) { - dictionary[key]?.Clear(); + list.Clear(); // Clear the list to free up memory } - dictionary.Clear(); + dictionary.Clear(); // Clear the dictionary itself + dictionary = null; // Help GC by removing reference } - } - + } + private void ClearDictionary(Dictionary> dictionary) { if (dictionary != null) { - foreach (var key in dictionary.Keys) + foreach (var list in dictionary.Values) { - dictionary[key]?.Clear(); + list.Clear(); // Clear the list to free up memory } - dictionary.Clear(); + dictionary.Clear(); // Clear the dictionary itself + dictionary = null; // Help GC by removing reference } } @@ -83,40 +85,30 @@ protected virtual void Dispose(bool disposing) { if (disposing) { + // Dispose managed resources here bsPoints?.Dispose(); writerPoints?.Dispose(); - keyCache.Clear(); - keyCache = null; - + // Clear and dispose instance dictionaries ClearDictionary(nodeX); - nodeX = null; - ClearDictionary(nodeY); - nodeY = null; - ClearDictionary(nodeZ); - nodeZ = null; - ClearDictionary(nodeR); - nodeR = null; - ClearDictionary(nodeG); - nodeG = null; - ClearDictionary(nodeB); - nodeB = null; - ClearDictionary(nodeIntensity); - nodeIntensity = null; - ClearDictionary(nodeTime); - nodeTime = null; + + keyCache.Clear(); + keyCache = null; } + + // If there were unmanaged resources, you'd clean them up here } ~PCROOT() { + Log.WriteLine("pcroot writer finalized for task: " + taskID); Dispose(false); } @@ -313,15 +305,15 @@ void IWriter.Close() // } // if last file // clear all lists - keyCache.Clear(); - nodeX.Clear(); - nodeY.Clear(); - nodeZ.Clear(); - nodeR.Clear(); - nodeG.Clear(); - nodeB.Clear(); - nodeIntensity.Clear(); - nodeTime.Clear(); + //keyCache.Clear(); + //nodeX.Clear(); + //nodeY.Clear(); + //nodeZ.Clear(); + //nodeR.Clear(); + //nodeG.Clear(); + //nodeB.Clear(); + //nodeIntensity.Clear(); + //nodeTime.Clear(); // dispose bsPoints?.Dispose(); @@ -331,7 +323,7 @@ void IWriter.Close() void IWriter.Cleanup(int fileIndex) { - Dispose(); + //Dispose(); } void IWriter.Randomize() diff --git a/Writers/UCPC.cs b/Writers/UCPC.cs index 24285db..7a41f51 100644 --- a/Writers/UCPC.cs +++ b/Writers/UCPC.cs @@ -343,5 +343,10 @@ void IWriter.Close() writerColorsV2.Close(); bsColorsV2.Dispose(); } + + void IWriter.Dispose() + { + // TODO ? + } } } From 4858b92254f1b850b94f2a3b756ca571aee51e84 Mon Sep 17 00:00:00 2001 From: mika Date: Tue, 20 Aug 2024 21:51:45 +0300 Subject: [PATCH 005/110] fix pcroot header save (missed init for main writer), fix UCPC to work with new task loop (but only single threaded for now), add check for UCPC and output file, --- MainWindow.xaml.cs | 21 +++++++++++++++++++-- Structs/ImportSettings.cs | 17 ++++++++++++++++- Tools/ArgParser.cs | 14 ++++++++++---- Writers/UCPC.cs | 34 ++++++++++++++++++++++++++-------- 4 files changed, 71 insertions(+), 15 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 9d225cf..1958e6b 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -350,7 +350,13 @@ private static async Task ProcessAllFiles(object workerParamsObject) //Trace.WriteLine(" ---------------------- all finished -------------------- "); // now write header for for pcroot (using main writer) - importSettings.writer.Close(); + Log.WriteLine("close writer"); + if (importSettings.exportFormat == ExportFormat.PCROOT) + { + Log.WriteLine(importSettings.writer.ToString()); + importSettings.writer.Close(); + // UCPC calls close in Save() itself + } // if this was last file //if (fileIndex == (importSettings.maxFiles - 1)) @@ -553,7 +559,15 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) var taskWriter = importSettings.GetOrCreateWriter(taskId); - //var writerRes = importSettings.writer.InitWriter(importSettings, pointCount); + //// for saving pcroot header, we need this writer + var mainWriterRes = importSettings.writer.InitWriter(importSettings, pointCount); + if (mainWriterRes == false) + { + Log.WriteLine("Error> Failed to initialize main Writer, fileindex: " + fileIndex + " taskid:" + taskId); + return false; + } + Log.WriteLine("*************"+importSettings.writer.ToString()); + var writerRes = taskWriter.InitWriter(importSettings, pointCount); if (writerRes == false) { @@ -583,6 +597,8 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) // stop at limit count if (importSettings.useLimit == true && i > pointCount) break; + // FIXME: need to add skip and keep point skipper here, to make skipping faster! + // get point XYZ Float3 point = taskReader.GetXYZ(); if (point.hasError == true) break; @@ -657,6 +673,7 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) // collect this point XYZ and RGB into node, optionally intensity also //importSettings.writer.AddPoint(i, (float)point.x, (float)point.y, (float)point.z, rgb.r, rgb.g, rgb.b, importSettings.importIntensity, intensity.r, importSettings.averageTimestamp, time); + // TODO can remove importsettings, its already passed on init taskWriter.AddPoint(i, (float)point.x, (float)point.y, (float)point.z, rgb.r, rgb.g, rgb.b, importSettings.importIntensity, intensity.r, importSettings.averageTimestamp, time); progressPoint = i; } // for all points diff --git a/Structs/ImportSettings.cs b/Structs/ImportSettings.cs index 76bd0d0..d2ee616 100644 --- a/Structs/ImportSettings.cs +++ b/Structs/ImportSettings.cs @@ -67,7 +67,22 @@ public IWriter GetOrCreateWriter(int? taskId) { if (!Writers.TryGetValue(taskId, out var weakWriter) || !weakWriter.TryGetTarget(out var writer)) { - writer = new PCROOT(taskId); + if (exportFormat == ExportFormat.UCPC) + { + Log.WriteLine("Creating UCPC writer for task ID: " + taskId); + writer = new UCPC(); + } + else if (exportFormat == ExportFormat.PCROOT) + { + Log.WriteLine("Creating PCROOT writer for task ID: " + taskId); + writer = new PCROOT(taskId); + } + else + { + Log.WriteLine("Writer format not supported: " + exportFormat, LogEvent.Error); + writer = null; + } + Writers[taskId] = new WeakReference(writer); } diff --git a/Tools/ArgParser.cs b/Tools/ArgParser.cs index d849242..d3f2dde 100644 --- a/Tools/ArgParser.cs +++ b/Tools/ArgParser.cs @@ -771,16 +771,22 @@ public static ImportSettings Parse(string[] args, string rootFolder) importSettings.errors.Add("Must have -rgb OR -intensity enabled"); } + if (importSettings.exportFormat == ExportFormat.UCPC && importSettings.maxThreads > 1) + { + importSettings.errors.Add("UCPC format doesnt support multi-threading yet, use 1 thread only (or remove -maxthreads param)"); + } + //// check mismatching settings for v2 vs v3 //if (importSettings.exportFormat == ExportFormat.UCPC) //{ // //if (importSettings.gridSize) //} - //if (importSettings.batch == true && importSettings.exportFormat != ExportFormat.PCROOT) - //{ - // importSettings.errors.Add("Folder batch is only supported for PCROOT (v3) version: -exportformat=pcroot"); - //} + Log.WriteLine(importSettings.outputFile); + if (importSettings.batch == true && importSettings.exportFormat == ExportFormat.UCPC && Path.GetExtension(importSettings.outputFile).ToLower() == ".ucpc") + { + importSettings.errors.Add("With UCPC batching, do not set output filename - set ONLY output folder (each ucpc file will be saved separately)"); + } if (importSettings.skipPoints == true && importSettings.keepPoints == true) { diff --git a/Writers/UCPC.cs b/Writers/UCPC.cs index 7a41f51..a72ee24 100644 --- a/Writers/UCPC.cs +++ b/Writers/UCPC.cs @@ -13,7 +13,17 @@ namespace PointCloudConverter.Writers { public class UCPC : IWriter { - ImportSettings importSettings; + //ImportSettings importSettings; + private ImportSettings __importSettings; + + public ImportSettings importSettings + { + get { return __importSettings; } + set { __importSettings = value; + Log.WriteLine("set importsettings"); + } + } + int pointCount; BufferedStream bsPoints = null; @@ -36,6 +46,7 @@ public class UCPC : IWriter bool IWriter.InitWriter(ImportSettings _importSettings, int _pointCount) { + Log.WriteLine("Initializing UCPC writer.."); importSettings = _importSettings; pointCount = _pointCount; @@ -65,7 +76,7 @@ bool IWriter.InitWriter(ImportSettings _importSettings, int _pointCount) } catch (Exception e) { - Console.WriteLine(e.Message); + Log.WriteLine(e.Message); return false; } @@ -252,16 +263,23 @@ void IWriter.AddPoint(int index, float x, float y, float z, float r, float g, fl if (z < cloudMinZ) cloudMinZ = z; if (z > cloudMaxZ) cloudMaxZ = z; - importSettings.writer.WriteXYZ(x, y, z); - importSettings.writer.WriteRGB(r, g, b); + //importSettings.writer.WriteXYZ(x, y, z); + //importSettings.writer.WriteRGB(r, g, b); + ((IWriter)this).WriteXYZ(x, y, z); + ((IWriter)this).WriteRGB(r, g, b); } void IWriter.Save(int fileIndex) { - importSettings.writer.CreateHeader(pointCount); - if (importSettings.randomize == true) importSettings.writer.Randomize(); - importSettings.writer.Close(); - importSettings.writer.Cleanup(fileIndex); + //importSettings.writer.CreateHeader(pointCount); + ((IWriter)this).CreateHeader(pointCount); + //if (importSettings.randomize == true) importSettings.writer.Randomize(); + if (importSettings.randomize == true) ((IWriter)this).Randomize(); + //importSettings.writer.Close(); + Log.WriteLine("----------*-*-*-*-*-*-*-*-*- in UCPC Save"); + ((IWriter)this).Close(); + //importSettings.writer.Cleanup(fileIndex); + ((IWriter)this).Cleanup(fileIndex); } void IWriter.Cleanup(int fileIndex) From 7db7dff743f3c376045ab5e344647bbb7d126dab Mon Sep 17 00:00:00 2001 From: mika Date: Tue, 20 Aug 2024 22:34:18 +0300 Subject: [PATCH 006/110] disable debug logs --- MainWindow.xaml.cs | 14 +++++++------- Structs/ImportSettings.cs | 6 ++---- Tools/ArgParser.cs | 1 - Writers/PCROOT.cs | 7 +++---- Writers/UCPC.cs | 6 ++---- 5 files changed, 14 insertions(+), 20 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 1958e6b..84dd5cc 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -350,10 +350,8 @@ private static async Task ProcessAllFiles(object workerParamsObject) //Trace.WriteLine(" ---------------------- all finished -------------------- "); // now write header for for pcroot (using main writer) - Log.WriteLine("close writer"); if (importSettings.exportFormat == ExportFormat.PCROOT) { - Log.WriteLine(importSettings.writer.ToString()); importSettings.writer.Close(); // UCPC calls close in Save() itself } @@ -560,13 +558,15 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) var taskWriter = importSettings.GetOrCreateWriter(taskId); //// for saving pcroot header, we need this writer - var mainWriterRes = importSettings.writer.InitWriter(importSettings, pointCount); - if (mainWriterRes == false) + if (importSettings.exportFormat == ExportFormat.PCROOT) { - Log.WriteLine("Error> Failed to initialize main Writer, fileindex: " + fileIndex + " taskid:" + taskId); - return false; + var mainWriterRes = importSettings.writer.InitWriter(importSettings, pointCount); + if (mainWriterRes == false) + { + Log.WriteLine("Error> Failed to initialize main Writer, fileindex: " + fileIndex + " taskid:" + taskId); + return false; + } } - Log.WriteLine("*************"+importSettings.writer.ToString()); var writerRes = taskWriter.InitWriter(importSettings, pointCount); if (writerRes == false) diff --git a/Structs/ImportSettings.cs b/Structs/ImportSettings.cs index d2ee616..399eb20 100644 --- a/Structs/ImportSettings.cs +++ b/Structs/ImportSettings.cs @@ -69,12 +69,10 @@ public IWriter GetOrCreateWriter(int? taskId) { if (exportFormat == ExportFormat.UCPC) { - Log.WriteLine("Creating UCPC writer for task ID: " + taskId); writer = new UCPC(); } else if (exportFormat == ExportFormat.PCROOT) { - Log.WriteLine("Creating PCROOT writer for task ID: " + taskId); writer = new PCROOT(taskId); } else @@ -95,11 +93,11 @@ public void ReleaseWriter(int? taskId) { if (weakWriter.TryGetTarget(out var writer)) { - Log.WriteLine("ReleaseWriter >>> Memory used: " + GC.GetTotalMemory(false)); + //Log.WriteLine("ReleaseWriter >>> Memory used: " + GC.GetTotalMemory(false)); //Log.WriteLine(">>>>> Releasing reader for task ID: " + taskId); writer?.Cleanup(0); writer?.Dispose(); - Log.WriteLine("ReleaseWriter <<< Memory used: " + GC.GetTotalMemory(false)); + //Log.WriteLine("ReleaseWriter <<< Memory used: " + GC.GetTotalMemory(false)); } } } diff --git a/Tools/ArgParser.cs b/Tools/ArgParser.cs index d3f2dde..0d89221 100644 --- a/Tools/ArgParser.cs +++ b/Tools/ArgParser.cs @@ -782,7 +782,6 @@ public static ImportSettings Parse(string[] args, string rootFolder) // //if (importSettings.gridSize) //} - Log.WriteLine(importSettings.outputFile); if (importSettings.batch == true && importSettings.exportFormat == ExportFormat.UCPC && Path.GetExtension(importSettings.outputFile).ToLower() == ".ucpc") { importSettings.errors.Add("With UCPC batching, do not set output filename - set ONLY output folder (each ucpc file will be saved separately)"); diff --git a/Writers/PCROOT.cs b/Writers/PCROOT.cs index 9966d96..6fef400 100644 --- a/Writers/PCROOT.cs +++ b/Writers/PCROOT.cs @@ -46,12 +46,11 @@ public class PCROOT : IWriter, IDisposable public void Dispose() { - Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); - Log.WriteLine("*** PCROOT writer disposed for task: " + taskID); + //Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); Dispose(true); GC.SuppressFinalize(this); GC.Collect(); - Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); + //Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); } @@ -108,7 +107,7 @@ protected virtual void Dispose(bool disposing) ~PCROOT() { - Log.WriteLine("pcroot writer finalized for task: " + taskID); + //Log.WriteLine("pcroot writer finalized for task: " + taskID); Dispose(false); } diff --git a/Writers/UCPC.cs b/Writers/UCPC.cs index a72ee24..cad8ba3 100644 --- a/Writers/UCPC.cs +++ b/Writers/UCPC.cs @@ -20,7 +20,7 @@ public ImportSettings importSettings { get { return __importSettings; } set { __importSettings = value; - Log.WriteLine("set importsettings"); + //Log.WriteLine("set importsettings"); } } @@ -46,7 +46,6 @@ public ImportSettings importSettings bool IWriter.InitWriter(ImportSettings _importSettings, int _pointCount) { - Log.WriteLine("Initializing UCPC writer.."); importSettings = _importSettings; pointCount = _pointCount; @@ -76,7 +75,7 @@ bool IWriter.InitWriter(ImportSettings _importSettings, int _pointCount) } catch (Exception e) { - Log.WriteLine(e.Message); + Log.WriteLine(e.Message, LogEvent.Error); return false; } @@ -276,7 +275,6 @@ void IWriter.Save(int fileIndex) //if (importSettings.randomize == true) importSettings.writer.Randomize(); if (importSettings.randomize == true) ((IWriter)this).Randomize(); //importSettings.writer.Close(); - Log.WriteLine("----------*-*-*-*-*-*-*-*-*- in UCPC Save"); ((IWriter)this).Close(); //importSettings.writer.Cleanup(fileIndex); ((IWriter)this).Cleanup(fileIndex); From 24020c2293c265cebf7fabc9f31c5388e2d45ba5 Mon Sep 17 00:00:00 2001 From: mika Date: Tue, 20 Aug 2024 22:36:05 +0300 Subject: [PATCH 007/110] #BUILD beta --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index dd53f67..32665a4 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,4 @@ Pull requests to improve this converter are welcome! (please create Issue first, + From 178550938dea45a40d08ebd04f30d89c9d866f06 Mon Sep 17 00:00:00 2001 From: mika Date: Sat, 31 Aug 2024 22:35:46 +0300 Subject: [PATCH 008/110] remove WinForms references, remove Sendkey() hack in console app: replaced with native sendkey, add pcroot writer pool (to try to reduce memory usage), --- MainWindow.xaml.cs | 20 ++++- PointCloudConverter.csproj | 2 +- Structs/ImportSettings.cs | 141 +++++++++++++++++++++++--------- Structs/Metadata/GeoTiffKeys.cs | 2 +- Writers/PCROOT.cs | 25 +++++- 5 files changed, 147 insertions(+), 43 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 84dd5cc..4a8c75c 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -37,6 +37,16 @@ public partial class MainWindow : Window [DllImport("kernel32.dll", SetLastError = true)] static extern bool FreeConsole(); + const uint WM_CHAR = 0x0102; + const int VK_ENTER = 0x0D; + + [DllImport("kernel32.dll")] + static extern IntPtr GetConsoleWindow(); + + [DllImport("user32.dll")] + static extern int SendMessage(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam); + + Thread workerThread; static bool abort = false; public static MainWindow mainWindowStatic; @@ -96,6 +106,7 @@ private async void Main() Log.WriteLine("\n::: " + appname + " :::\n"); //Console.WriteLine("\n::: " + appname + " :::\n"); Console.ForegroundColor = ConsoleColor.White; + IntPtr cw = GetConsoleWindow(); // check args, null here because we get the args later var importSettings = ArgParser.Parse(null, rootFolder); @@ -135,8 +146,10 @@ private async void Main() { Log.WriteLine("{\"event\": \"" + LogEvent.End + "\", \"elapsed\": \"" + elapsedString + "\",\"version\":\"" + version + ",\"errors\":" + errorCounter + "}", LogEvent.End); } - // hack for console exit https://stackoverflow.com/a/67940480/5452781 - SendKeys.SendWait("{ENTER}"); + + // https://stackoverflow.com/a/45620138/5452781 + SendMessage(cw, WM_CHAR, (IntPtr)VK_ENTER, IntPtr.Zero); + FreeConsole(); Environment.Exit(Environment.ExitCode); } @@ -151,6 +164,7 @@ private async void Main() } + // main processing loop private static async Task ProcessAllFiles(object workerParamsObject) @@ -281,6 +295,8 @@ private static async Task ProcessAllFiles(object workerParamsObject) maxThreads = Math.Max(importSettings.maxThreads, 1); Log.WriteLine("Using MaxThreads: " + maxThreads); + // init pool + importSettings.InitWriterPool(maxThreads,importSettings.exportFormat); var semaphore = new SemaphoreSlim(importSettings.maxThreads); diff --git a/PointCloudConverter.csproj b/PointCloudConverter.csproj index 670281c..dde3b2a 100644 --- a/PointCloudConverter.csproj +++ b/PointCloudConverter.csproj @@ -10,7 +10,7 @@ PointCloudConverter true 10.0.17763.0 - True + False PointCloudConverter PointCloudConverter.App False diff --git a/Structs/ImportSettings.cs b/Structs/ImportSettings.cs index 399eb20..80a19d4 100644 --- a/Structs/ImportSettings.cs +++ b/Structs/ImportSettings.cs @@ -24,8 +24,21 @@ public class ImportSettings public ConcurrentDictionary Readers { get; set; } = new ConcurrentDictionary(); public IWriter writer = new UCPC(); //public Dictionary Writers { get; set; } = new Dictionary(); - public ConcurrentDictionary> Writers { get; set; } = new ConcurrentDictionary>(); + //public ConcurrentDictionary> Writers { get; set; } = new ConcurrentDictionary>(); + private readonly ConcurrentBag _writerPool = new ConcurrentBag(); + private readonly ConcurrentDictionary _allocatedWriters = new ConcurrentDictionary(); + private int _maxWriters = 16; + public void InitWriterPool(int maxThreads, ExportFormat export) + { + //exportFormat = export; + _maxWriters = maxThreads; + // Initialize the pool with the maximum number of writers + for (int i = 0; i < _maxWriters; i++) + { + _writerPool.Add(CreateNewWriter()); // Create and add writers to the pool + } + } // Method to get or create a reader for a specific task ID public IReader GetOrCreateReader(int? taskId) @@ -40,6 +53,60 @@ public IReader GetOrCreateReader(int? taskId) return Readers[taskId]; } + private IWriter CreateNewWriter() + { + Log.WriteLine(">>>>> Creating new writer: "+exportFormat); + if (exportFormat == ExportFormat.UCPC) + { + return new UCPC(); + } + else if (exportFormat == ExportFormat.PCROOT) + { + return new PCROOT(null); // No taskId when creating the pool, it's assigned later + } + else + { + Log.WriteLine("Writer format not supported: " + exportFormat, LogEvent.Error); + return null; + } + } + + public IWriter GetOrCreateWriter(int? taskId) + { + if (!_allocatedWriters.TryGetValue(taskId, out var writer)) + { + // Try to get a writer from the pool + if (_writerPool.TryTake(out writer)) + { + // Assign the writer to the task + _allocatedWriters[taskId] = writer; + } + else + { + // If no writers are available, create a new one (this should rarely happen if the pool is well-sized) + writer = CreateNewWriter(); + _allocatedWriters[taskId] = writer; + } + } + + return writer; + } + + public void ReleaseWriter(int? taskId) + { + if (taskId.HasValue && _allocatedWriters.TryRemove(taskId, out var writer)) + { + // Log.WriteLine("ReleaseWriter >>> Memory used: " + GC.GetTotalMemory(false)); + // Clean up the writer if necessary + writer?.Cleanup(0); + //writer?.Dispose(); + // Return the writer to the pool for reuse + _writerPool.Add(writer); + // Log.WriteLine("ReleaseWriter >>> Memory used: " + GC.GetTotalMemory(false)); + + } + } + //public IWriter GetOrCreateWriter(int? taskId) //{ // if (!Writers.ContainsKey(taskId)) @@ -63,44 +130,44 @@ public IReader GetOrCreateReader(int? taskId) // } //} - public IWriter GetOrCreateWriter(int? taskId) - { - if (!Writers.TryGetValue(taskId, out var weakWriter) || !weakWriter.TryGetTarget(out var writer)) - { - if (exportFormat == ExportFormat.UCPC) - { - writer = new UCPC(); - } - else if (exportFormat == ExportFormat.PCROOT) - { - writer = new PCROOT(taskId); - } - else - { - Log.WriteLine("Writer format not supported: " + exportFormat, LogEvent.Error); - writer = null; - } + //public IWriter GetOrCreateWriter(int? taskId) + //{ + // if (!Writers.TryGetValue(taskId, out var weakWriter) || !weakWriter.TryGetTarget(out var writer)) + // { + // if (exportFormat == ExportFormat.UCPC) + // { + // writer = new UCPC(); + // } + // else if (exportFormat == ExportFormat.PCROOT) + // { + // writer = new PCROOT(taskId); + // } + // else + // { + // Log.WriteLine("Writer format not supported: " + exportFormat, LogEvent.Error); + // writer = null; + // } - Writers[taskId] = new WeakReference(writer); - } + // Writers[taskId] = new WeakReference(writer); + // } - return writer; - } + // return writer; + //} - public void ReleaseWriter(int? taskId) - { - if (taskId.HasValue && Writers.TryRemove(taskId, out var weakWriter)) - { - if (weakWriter.TryGetTarget(out var writer)) - { - //Log.WriteLine("ReleaseWriter >>> Memory used: " + GC.GetTotalMemory(false)); - //Log.WriteLine(">>>>> Releasing reader for task ID: " + taskId); - writer?.Cleanup(0); - writer?.Dispose(); - //Log.WriteLine("ReleaseWriter <<< Memory used: " + GC.GetTotalMemory(false)); - } - } - } + //public void ReleaseWriter(int? taskId) + //{ + // if (taskId.HasValue && Writers.TryRemove(taskId, out var weakWriter)) + // { + // if (weakWriter.TryGetTarget(out var writer)) + // { + // Log.WriteLine("ReleaseWriter >>> Memory used: " + GC.GetTotalMemory(false)); + // //Log.WriteLine(">>>>> Releasing reader for task ID: " + taskId); + // writer?.Cleanup(0); + // writer?.Dispose(); + // Log.WriteLine("ReleaseWriter <<< Memory used: " + GC.GetTotalMemory(false)); + // } + // } + //} public void ReleaseReader(int? taskId) { @@ -170,7 +237,7 @@ public void ReleaseReader(int? taskId) [JsonConverter(typeof(JsonStringEnumConverter))] public ImportFormat importFormat { get; set; } = ImportFormat.LAS; //default to las for now [JsonConverter(typeof(JsonStringEnumConverter))] - public ExportFormat exportFormat { get; set; } = ExportFormat.UCPC; // defaults to UCPC (v2) + public ExportFormat exportFormat { get; set; } = ExportFormat.PCROOT; // defaults to PCROOT (v3) now public List inputFiles { get; set; } = new List(); public string outputFile { get; set; } = null; diff --git a/Structs/Metadata/GeoTiffKeys.cs b/Structs/Metadata/GeoTiffKeys.cs index 8055b6a..5653729 100644 --- a/Structs/Metadata/GeoTiffKeys.cs +++ b/Structs/Metadata/GeoTiffKeys.cs @@ -16,7 +16,7 @@ using System; using System.Collections.Generic; -using static System.Windows.Forms.VisualStyles.VisualStyleElement.TextBox; +//using static System.Windows.Forms.VisualStyles.VisualStyleElement.TextBox; namespace Free.Ports.LibGeoTiff { diff --git a/Writers/PCROOT.cs b/Writers/PCROOT.cs index 6fef400..4e851b8 100644 --- a/Writers/PCROOT.cs +++ b/Writers/PCROOT.cs @@ -48,8 +48,12 @@ public void Dispose() { //Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); Dispose(true); - GC.SuppressFinalize(this); GC.Collect(); +// GC.SuppressFinalize(this); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + //GC.Collect(); //Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); } @@ -76,7 +80,7 @@ private void ClearDictionary(Dictionary> dictionary) list.Clear(); // Clear the list to free up memory } dictionary.Clear(); // Clear the dictionary itself - dictionary = null; // Help GC by removing reference + //dictionary = null; // Help GC by removing reference } } @@ -322,7 +326,24 @@ void IWriter.Close() void IWriter.Cleanup(int fileIndex) { + //Log.WriteLine("Cleanup: this doesnt do anything yet.."); //Dispose(); + bsPoints?.Dispose(); + writerPoints?.Dispose(); + + // Clear and dispose instance dictionaries + ClearDictionary(nodeX); + ClearDictionary(nodeY); + ClearDictionary(nodeZ); + ClearDictionary(nodeR); + ClearDictionary(nodeG); + ClearDictionary(nodeB); + ClearDictionary(nodeIntensity); + ClearDictionary(nodeTime); + + keyCache.Clear(); + + } void IWriter.Randomize() From c40c415a25ce2810baa29c2306e1610886c9c75b Mon Sep 17 00:00:00 2001 From: mika Date: Sun, 1 Sep 2024 19:57:31 +0300 Subject: [PATCH 009/110] json: print list of saved files per file, --- MainWindow.xaml.cs | 17 +++++++++++++++-- Structs/ImportSettings.cs | 2 +- Writers/PCROOT.cs | 36 ++++++++++++++++++++---------------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 4a8c75c..038c7a6 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -296,7 +296,7 @@ private static async Task ProcessAllFiles(object workerParamsObject) Log.WriteLine("Using MaxThreads: " + maxThreads); // init pool - importSettings.InitWriterPool(maxThreads,importSettings.exportFormat); + importSettings.InitWriterPool(maxThreads, importSettings.exportFormat); var semaphore = new SemaphoreSlim(importSettings.maxThreads); @@ -490,7 +490,7 @@ static void ProgressTick(object sender, EventArgs e) // process single file static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) { - //Log.WriteLine("parsefile, taskid: " + taskId + " fileindex: " + fileIndex); + Log.WriteLine("Started processing file: " + importSettings.inputFiles[fileIndex]); // each thread needs its own reader bool res; @@ -705,8 +705,21 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) //taskReader.Dispose(); importSettings.ReleaseWriter(taskId); //Log.WriteLine("------------ reader and writer released ------------"); + + // TODO add event for finished writing this file, and return list of output files + //jsonString = "{" + + // "\"event\": \"" + LogEvent.File + "\"," + + // "\"path\": " + System.Text.Json.JsonSerializer.Serialize(importSettings.inputFiles[fileIndex]) + "," + + // //"\"size\": " + new FileInfo(importSettings.inputFiles[fileIndex]).Length + "," + + // //"\"points\": " + pointCount + "," + + // "\"status\": \"" + LogStatus.Complete + "\"" + + // "}"; + + //Log.WriteLine(jsonString, LogEvent.File); + } // if importMetadataOnly == false + //Log.WriteLine("taskid: " + taskId + " done"); return true; } // ParseFile diff --git a/Structs/ImportSettings.cs b/Structs/ImportSettings.cs index 80a19d4..e13b70e 100644 --- a/Structs/ImportSettings.cs +++ b/Structs/ImportSettings.cs @@ -55,7 +55,7 @@ public IReader GetOrCreateReader(int? taskId) private IWriter CreateNewWriter() { - Log.WriteLine(">>>>> Creating new writer: "+exportFormat); + ///Log.WriteLine(">>>>> Creating new writer: "+exportFormat); if (exportFormat == ExportFormat.UCPC) { return new UCPC(); diff --git a/Writers/PCROOT.cs b/Writers/PCROOT.cs index 4e851b8..e6824ec 100644 --- a/Writers/PCROOT.cs +++ b/Writers/PCROOT.cs @@ -49,7 +49,7 @@ public void Dispose() //Log.WriteLine("Memory used: " + GC.GetTotalMemory(false)); Dispose(true); GC.Collect(); -// GC.SuppressFinalize(this); + // GC.SuppressFinalize(this); GC.WaitForPendingFinalizers(); GC.Collect(); @@ -444,6 +444,7 @@ unsafe void IntToBytes(int value, byte[] buffer, int offset) static int skippedPointsCounter = 0; static bool useLossyFiltering = false; //not used, for testing only + // returns list of saved files void IWriter.Save(int fileIndex) { if (useLossyFiltering == true) @@ -451,24 +452,12 @@ void IWriter.Save(int fileIndex) Console.WriteLine("************* useLossyFiltering ****************"); } - string fileOnly = Path.GetFileNameWithoutExtension(importSettings.outputFile); string baseFolder = Path.GetDirectoryName(importSettings.outputFile); // TODO no need colors for json.. could move this inside custom logger, so that it doesnt do anything, if json Console.ForegroundColor = ConsoleColor.Blue; - // TODO add enum for status - - string jsonString = "{" + - "\"event\": \"" + LogEvent.File + "\"," + - "\"status\": \"" + LogStatus.Complete + "\"," + - "\"path\": " + JsonSerializer.Serialize(importSettings.inputFiles[fileIndex]) + "," + - "\"tiles\": " + nodeX.Count + "," + - "\"folder\": " + JsonSerializer.Serialize(baseFolder) + "}"; - - // TODO combine 2 outputs.. only other one shows up now Log.WriteLine("Saving " + nodeX.Count + " tiles into: " + baseFolder); - Log.WriteLine(jsonString, LogEvent.End); Console.ForegroundColor = ConsoleColor.White; @@ -483,6 +472,8 @@ void IWriter.Save(int fileIndex) List nodeTempIntensity = null; List nodeTempTime = null; + List outputFiles = new List(); + // process all tiles //foreach (KeyValuePair> nodeData in nodeX) foreach (KeyValuePair> nodeData in nodeX) @@ -542,7 +533,7 @@ void IWriter.Save(int fileIndex) Tools.Shuffle(ref nodeTempX, ref nodeTempY, ref nodeTempZ, ref nodeTempR, ref nodeTempG, ref nodeTempB); } } - } + } // randomize // get this node bounds, TODO but we know node(grid cell) x,y,z values? float minX = float.PositiveInfinity; @@ -565,10 +556,14 @@ void IWriter.Save(int fileIndex) fullpathFileOnly = fileOnly + "_" + fileIndex + "_" + key + tileExtension; } - // prepare file + // save this tile + //Log.WriteLine("*** Saving tile: " + fullpathFileOnly + " (" + nodeTempX.Count + " points)"); bsPoints = new BufferedStream(new FileStream(fullpath, FileMode.Create)); writerPoints = new BinaryWriter(bsPoints); + // collect list of saved files + outputFiles.Add(fullpath); + int cellX = 0; int cellY = 0; int cellZ = 0; @@ -773,7 +768,6 @@ void IWriter.Save(int fileIndex) totalPointsWritten++; } // loop all points in tile (node) - // close tile file writerPoints.Close(); bsPoints.Dispose(); @@ -880,6 +874,16 @@ void IWriter.Save(int fileIndex) nodeBounds.Add(cb); } // loop all nodes/tiles foreach + // finished this file + string jsonString = "{" + + "\"event\": \"" + LogEvent.File + "\"," + + "\"status\": \"" + LogStatus.Complete + "\"," + + "\"path\": " + JsonSerializer.Serialize(importSettings.inputFiles[fileIndex]) + "," + + "\"tiles\": " + nodeX.Count + "," + + "\"folder\": " + JsonSerializer.Serialize(baseFolder) + "}" + + "\"filenames\": " + JsonSerializer.Serialize(outputFiles); + Log.WriteLine(jsonString, LogEvent.End); + } // Save() From 5bded6c3f98078af1b24ba28cdc0339cc98ffa38 Mon Sep 17 00:00:00 2001 From: mika Date: Sun, 1 Sep 2024 20:30:35 +0300 Subject: [PATCH 010/110] locked progressfile increment --- MainWindow.xaml.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 038c7a6..dde4d69 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -312,7 +312,9 @@ private static async Task ProcessAllFiles(object workerParamsObject) await semaphore.WaitAsync(cancellationToken); //int? taskId = Task.CurrentId; // Get the current task ID - progressFile = i; + //progressFile = i; + Interlocked.Increment(ref progressFile); + //bool isLastTask = (i == len - 1); // Check if this is the last task From 7ed8f79a8f8c0697b01d1b143903e5032f521369 Mon Sep 17 00:00:00 2001 From: mika Date: Sun, 1 Sep 2024 22:04:58 +0300 Subject: [PATCH 011/110] add initial multithreading progressbars for GUI --- MainWindow.xaml | 10 ++- MainWindow.xaml.cs | 155 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 145 insertions(+), 20 deletions(-) diff --git a/MainWindow.xaml b/MainWindow.xaml index 78456d2..bef1259 100644 --- a/MainWindow.xaml +++ b/MainWindow.xaml @@ -146,8 +146,16 @@ - + + + + + + + diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index dde4d69..1218275 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -19,6 +19,7 @@ using Brushes = System.Windows.Media.Brushes; using System.Threading.Tasks; using PointCloudConverter.Readers; +using System.Collections.Concurrent; namespace PointCloudConverter { @@ -302,6 +303,7 @@ private static async Task ProcessAllFiles(object workerParamsObject) var tasks = new List(); + for (int i = 0, len = importSettings.maxFiles; i < len; i++) { if (cancellationToken.IsCancellationRequested) @@ -318,7 +320,7 @@ private static async Task ProcessAllFiles(object workerParamsObject) //bool isLastTask = (i == len - 1); // Check if this is the last task - int index = i; // Capture the current index in the loop + int index = i; // Capture the current file index in the loop int len2 = len; tasks.Add(Task.Run(async () => { @@ -429,7 +431,7 @@ private static async Task ProcessAllFiles(object workerParamsObject) // clear timer progressTimerThread.Stop(); mainWindowStatic.progressBarFiles.Foreground = Brushes.Green; - mainWindowStatic.progressBarPoints.Foreground = Brushes.Green; + //mainWindowStatic.progressBarPoints.Foreground = Brushes.Green; })); } // ProcessAllFiles @@ -441,6 +443,7 @@ void HideProcessingPanel() static void StartProgressTimer() { + Log.WriteLine("Starting progress timer..*-*************************"); progressTimerThread = new DispatcherTimer(DispatcherPriority.Background, Application.Current.Dispatcher); progressTimerThread.Tick += ProgressTick; progressTimerThread.Interval = TimeSpan.FromSeconds(1); @@ -449,29 +452,130 @@ static void StartProgressTimer() Application.Current.Dispatcher.Invoke(new Action(() => { mainWindowStatic.progressBarFiles.Foreground = Brushes.Red; - mainWindowStatic.progressBarPoints.Foreground = Brushes.Red; + //mainWindowStatic.progressBarPoints.Foreground = Brushes.Red; mainWindowStatic.lblStatus.Content = ""; })); } - static void ProgressTick(object sender, EventArgs e) + private static List progressInfos = new List(); + private static object lockObject = new object(); + + public class ProgressInfo { - if (progressTotalPoints > 0) + public int Index { get; set; } // Index of the ProgressBar in the UI + public int CurrentValue { get; set; } // Current progress value + public int MaxValue { get; set; } // Maximum value for the progress + } + + static void InitProgressBars(int threadCount) + { + ClearProgressBars(); + + Log.WriteLine("Creating progress bars: " + threadCount); + progressInfos.Clear(); + + for (int i = 0; i < threadCount; i++) { - //mainWindowStatic.progressBarFiles.Value = ((float)((progressFile+1) / (float)(progressTotalFiles+1))); - mainWindowStatic.progressBarFiles.Value = progressFile; - mainWindowStatic.progressBarFiles.Maximum = progressTotalFiles + 1; - mainWindowStatic.progressBarPoints.Value = progressPoint / (float)progressTotalPoints; - mainWindowStatic.lblStatus.Content = lastStatusMessage; + ProgressBar newProgressBar = new ProgressBar + { + Height = 10, + Width = 490 / threadCount, + Value = 0, + Maximum = 100, // TODO set value in parsefile? + HorizontalAlignment = HorizontalAlignment.Left, + Margin = new Thickness(1, 0, 1, 0), + Foreground = Brushes.Lime, + Background = null, + //BorderBrush = Brushes.Red, + //ToolTip = $"Thread {i}" + }; + + // Initialize ProgressInfo for each ProgressBar + var progressInfo = new ProgressInfo + { + Index = i, // Index in the StackPanel + CurrentValue = 0, // Initial value + MaxValue = 100 // Example max value + }; + + progressInfos.Add(progressInfo); + + mainWindowStatic.ProgressBarsContainer.Children.Add(newProgressBar); } - else + } + + static void ClearProgressBars() + { + mainWindowStatic.ProgressBarsContainer.Children.Clear(); + } + + //static void ProgressTick(object sender, EventArgs e) + //{ + // if (progressTotalPoints > 0) + // { + // //mainWindowStatic.progressBarFiles.Value = ((float)((progressFile+1) / (float)(progressTotalFiles+1))); + // mainWindowStatic.progressBarFiles.Value = progressFile; + // mainWindowStatic.progressBarFiles.Maximum = progressTotalFiles + 1; + // //mainWindowStatic.progressBarPoints.Value = progressPoint / (float)progressTotalPoints; + // mainWindowStatic.lblStatus.Content = lastStatusMessage; + // } + // else + // { + // mainWindowStatic.progressBarFiles.Value = 0; + // //mainWindowStatic.progressBarPoints.Value = 0; + // mainWindowStatic.lblStatus.Content = ""; + // } + //} + + static void ProgressTick(object sender, EventArgs e) + { + Application.Current.Dispatcher.Invoke(() => { - mainWindowStatic.progressBarFiles.Value = 0; - mainWindowStatic.progressBarPoints.Value = 0; - mainWindowStatic.lblStatus.Content = ""; - } + if (progressTotalPoints > 0) + { + mainWindowStatic.progressBarFiles.Value = progressFile; + mainWindowStatic.progressBarFiles.Maximum = progressTotalFiles + 1; + mainWindowStatic.lblStatus.Content = lastStatusMessage; + + // Update all progress bars based on the current values in the List + lock (lockObject) // Lock to safely read progressInfos + { + foreach (var progressInfo in progressInfos) + { + int index = progressInfo.Index; + int currentValue = progressInfo.CurrentValue; + int maxValue = progressInfo.MaxValue; + + // Access ProgressBar directly from the StackPanel.Children using its index + if (index >= 0 && index < mainWindowStatic.ProgressBarsContainer.Children.Count) + { + if (mainWindowStatic.ProgressBarsContainer.Children[index] is ProgressBar progressBar) + { + progressBar.Maximum = maxValue; + progressBar.Value = currentValue; // Update ProgressBar value + //progressBar.ToolTip = $"Thread {index} - {currentValue} / {maxValue}"; // not visible, because modal dialog + } + } + } + } + } + else + { + mainWindowStatic.progressBarFiles.Value = 0; + mainWindowStatic.lblStatus.Content = ""; + + foreach (UIElement element in mainWindowStatic.ProgressBarsContainer.Children) + { + if (element is ProgressBar progressBar) + { + progressBar.Value = 0; + } + } + } + }); } + static (bool, float, float, float) GetBounds(ImportSettings importSettings, int fileIndex) { var res = importSettings.reader.InitReader(importSettings, fileIndex); @@ -492,6 +596,8 @@ static void ProgressTick(object sender, EventArgs e) // process single file static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) { + progressTotalPoints = 1; // FIXME dummy for progress bar + Log.WriteLine("Started processing file: " + importSettings.inputFiles[fileIndex]); // each thread needs its own reader @@ -500,6 +606,12 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) //importSettings.reader = new LAZ(taskId); IReader taskReader = importSettings.GetOrCreateReader(taskId); + ProgressInfo progressInfo = null; + lock (lockObject) + { + progressInfo = progressInfos[fileIndex % progressInfos.Count]; // Example of cyclic assignment + } + try { res = taskReader.InitReader(importSettings, fileIndex); @@ -593,8 +705,10 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) return false; } - progressPoint = 0; - progressTotalPoints = importSettings.useLimit ? pointCount : fullPointCount; + //progressPoint = 0; + progressInfo.CurrentValue = 0; + progressInfo.MaxValue = importSettings.useLimit ? pointCount : fullPointCount; + //progressTotalPoints = importSettings.useLimit ? pointCount : fullPointCount; lastStatusMessage = "Processing points.."; @@ -693,7 +807,8 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) //importSettings.writer.AddPoint(i, (float)point.x, (float)point.y, (float)point.z, rgb.r, rgb.g, rgb.b, importSettings.importIntensity, intensity.r, importSettings.averageTimestamp, time); // TODO can remove importsettings, its already passed on init taskWriter.AddPoint(i, (float)point.x, (float)point.y, (float)point.z, rgb.r, rgb.g, rgb.b, importSettings.importIntensity, intensity.r, importSettings.averageTimestamp, time); - progressPoint = i; + //progressPoint = i; + progressInfo.CurrentValue = i; } // for all points lastStatusMessage = "Saving files.."; @@ -721,7 +836,6 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) } // if importMetadataOnly == false - //Log.WriteLine("taskid: " + taskId + " done"); return true; } // ParseFile @@ -863,6 +977,9 @@ void StartProcess(bool doProcess = true) CancellationToken = _cancellationTokenSource.Token }; + InitProgressBars(importSettings.maxThreads); + + Task.Run(() => ProcessAllFiles(workerParams)); } } From 53fdd062de137da9399f9f9bad8caae9396dcf98 Mon Sep 17 00:00:00 2001 From: mika Date: Mon, 2 Sep 2024 23:02:35 +0300 Subject: [PATCH 012/110] add "Progress" log event, print json progress for each file, BUG: if use more threads than files, prints null for extra threads --- MainWindow.xaml.cs | 56 ++++++++++++++++++++++++---------------------- Tools/LogText.cs | 1 + 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 1218275..dcaf3f0 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -133,6 +133,8 @@ private async void Main() CancellationToken = _cancellationTokenSource.Token }; + InitProgressBars(importSettings.maxThreads, importSettings.useJSONLog); + await Task.Run(() => ProcessAllFiles(workerParams)); } @@ -173,7 +175,6 @@ private static async Task ProcessAllFiles(object workerParamsObject) var workerParams = (WorkerParams)workerParamsObject; var importSettings = workerParams.ImportSettings; var cancellationToken = workerParams.CancellationToken; - // Use cancellationToken to check for cancellation if (cancellationToken.IsCancellationRequested) { @@ -462,12 +463,14 @@ static void StartProgressTimer() public class ProgressInfo { - public int Index { get; set; } // Index of the ProgressBar in the UI - public int CurrentValue { get; set; } // Current progress value - public int MaxValue { get; set; } // Maximum value for the progress + public int Index { get; internal set; } // Index of the ProgressBar in the UI + public int CurrentValue { get; internal set; } // Current progress value + public int MaxValue { get; internal set; } // Maximum value for the progress + public string FilePath { get; internal set; } + public bool UseJsonLog { get; internal set; } } - static void InitProgressBars(int threadCount) + static void InitProgressBars(int threadCount, bool useJsonLog) { ClearProgressBars(); @@ -495,7 +498,8 @@ static void InitProgressBars(int threadCount) { Index = i, // Index in the StackPanel CurrentValue = 0, // Initial value - MaxValue = 100 // Example max value + MaxValue = 100, + UseJsonLog = useJsonLog }; progressInfos.Add(progressInfo); @@ -509,24 +513,6 @@ static void ClearProgressBars() mainWindowStatic.ProgressBarsContainer.Children.Clear(); } - //static void ProgressTick(object sender, EventArgs e) - //{ - // if (progressTotalPoints > 0) - // { - // //mainWindowStatic.progressBarFiles.Value = ((float)((progressFile+1) / (float)(progressTotalFiles+1))); - // mainWindowStatic.progressBarFiles.Value = progressFile; - // mainWindowStatic.progressBarFiles.Maximum = progressTotalFiles + 1; - // //mainWindowStatic.progressBarPoints.Value = progressPoint / (float)progressTotalPoints; - // mainWindowStatic.lblStatus.Content = lastStatusMessage; - // } - // else - // { - // mainWindowStatic.progressBarFiles.Value = 0; - // //mainWindowStatic.progressBarPoints.Value = 0; - // mainWindowStatic.lblStatus.Content = ""; - // } - //} - static void ProgressTick(object sender, EventArgs e) { Application.Current.Dispatcher.Invoke(() => @@ -554,6 +540,20 @@ static void ProgressTick(object sender, EventArgs e) progressBar.Maximum = maxValue; progressBar.Value = currentValue; // Update ProgressBar value //progressBar.ToolTip = $"Thread {index} - {currentValue} / {maxValue}"; // not visible, because modal dialog + + // print json progress + if (progressInfo.UseJsonLog) // TODO now same bool value is for each progressinfo.. + { + string jsonString = "{" + + "\"event\": \"" + LogEvent.Progress + "\"," + + "\"thread\": " + index + "," + + "\"currentPoint\": " + currentValue + "," + + "\"totalPoints\": " + maxValue + "," + + "\"percentage\": " + (int)((currentValue / (float)maxValue) * 100) + "%," + + "\"file\": " + System.Text.Json.JsonSerializer.Serialize(progressInfo.FilePath) + + "}"; + Log.WriteLine(jsonString, LogEvent.Progress); + } } } } @@ -609,7 +609,8 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) ProgressInfo progressInfo = null; lock (lockObject) { - progressInfo = progressInfos[fileIndex % progressInfos.Count]; // Example of cyclic assignment + Log.WriteLine(progressInfos.Count + " : " + fileIndex, LogEvent.Info); + progressInfo = progressInfos[fileIndex % progressInfos.Count]; } try @@ -710,6 +711,8 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) progressInfo.MaxValue = importSettings.useLimit ? pointCount : fullPointCount; //progressTotalPoints = importSettings.useLimit ? pointCount : fullPointCount; + progressInfo.FilePath = importSettings.inputFiles[fileIndex]; + lastStatusMessage = "Processing points.."; string jsonString = "{" + @@ -977,8 +980,7 @@ void StartProcess(bool doProcess = true) CancellationToken = _cancellationTokenSource.Token }; - InitProgressBars(importSettings.maxThreads); - + InitProgressBars(importSettings.maxThreads, importSettings.useJSONLog); Task.Run(() => ProcessAllFiles(workerParams)); } diff --git a/Tools/LogText.cs b/Tools/LogText.cs index 760977b..9da3a9f 100644 --- a/Tools/LogText.cs +++ b/Tools/LogText.cs @@ -13,6 +13,7 @@ public enum LogEvent Error, Warning, Info, + Progress, Debug } From 02c9b62568d39b6d8f36eb230099e9480a538788 Mon Sep 17 00:00:00 2001 From: mika Date: Mon, 2 Sep 2024 23:41:50 +0300 Subject: [PATCH 013/110] fix maxthreads (clamp to available threads and maxfiles) --- MainWindow.xaml.cs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index dcaf3f0..5cdd258 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -25,7 +25,7 @@ namespace PointCloudConverter { public partial class MainWindow : Window { - static readonly string version = "19.08.2024"; + static readonly string version = "02.09.2024"; static readonly string appname = "PointCloud Converter - " + version; static readonly string rootFolder = AppDomain.CurrentDomain.BaseDirectory; @@ -133,7 +133,7 @@ private async void Main() CancellationToken = _cancellationTokenSource.Token }; - InitProgressBars(importSettings.maxThreads, importSettings.useJSONLog); + InitProgressBars(importSettings); await Task.Run(() => ProcessAllFiles(workerParams)); } @@ -291,10 +291,9 @@ private static async Task ProcessAllFiles(object workerParamsObject) //// hack to fix progress bar not updating on last file //progressFile++; - // clamp to max of inputfiles-1 (otherwise errors in threading) - int maxThreads = Math.Min(importSettings.maxThreads, importSettings.maxFiles - 1); // FIXME: -1 because otherwise keynotfindexception in last file or after it? + int maxThreads = Math.Min(importSettings.maxThreads, importSettings.maxFiles); // clamp to min 1 - maxThreads = Math.Max(importSettings.maxThreads, 1); + maxThreads = Math.Max(maxThreads, 1); Log.WriteLine("Using MaxThreads: " + maxThreads); // init pool @@ -470,10 +469,19 @@ public class ProgressInfo public bool UseJsonLog { get; internal set; } } - static void InitProgressBars(int threadCount, bool useJsonLog) + static void InitProgressBars(ImportSettings importSettings) { ClearProgressBars(); + int threadCount = importSettings.maxThreads; + // clamp to max files + threadCount = Math.Min(threadCount, importSettings.maxFiles); + threadCount = Math.Max(threadCount, 1); + + Log.WriteLine("Creating progress bars: " + threadCount); + + bool useJsonLog = importSettings.useJSONLog; + Log.WriteLine("Creating progress bars: " + threadCount); progressInfos.Clear(); @@ -980,7 +988,7 @@ void StartProcess(bool doProcess = true) CancellationToken = _cancellationTokenSource.Token }; - InitProgressBars(importSettings.maxThreads, importSettings.useJSONLog); + InitProgressBars(importSettings); Task.Run(() => ProcessAllFiles(workerParams)); } From 3400374813759748973692307b2fd7fd3a51f701 Mon Sep 17 00:00:00 2001 From: unitycoder Date: Tue, 3 Sep 2024 22:29:11 +0300 Subject: [PATCH 014/110] GUI: make cancellation faster, change thread progressbars green when finished, fix: reset cancel token after start process again --- MainWindow.xaml.cs | 143 ++++++++++++++++++++++++--------------------- 1 file changed, 75 insertions(+), 68 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 5cdd258..0ac343b 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -48,7 +48,6 @@ public partial class MainWindow : Window static extern int SendMessage(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam); - Thread workerThread; static bool abort = false; public static MainWindow mainWindowStatic; bool isInitialiazing = true; @@ -331,7 +330,7 @@ private static async Task ProcessAllFiles(object workerParamsObject) try { // Do actual point cloud parsing for this file and pass taskId - var res = ParseFile(importSettings, index, taskId); + var res = ParseFile(importSettings, index, taskId, cancellationToken); if (!res) { Interlocked.Increment(ref errorCounter); // thread-safe error counter increment @@ -495,7 +494,7 @@ static void InitProgressBars(ImportSettings importSettings) Maximum = 100, // TODO set value in parsefile? HorizontalAlignment = HorizontalAlignment.Left, Margin = new Thickness(1, 0, 1, 0), - Foreground = Brushes.Lime, + Foreground = Brushes.Red, Background = null, //BorderBrush = Brushes.Red, //ToolTip = $"Thread {i}" @@ -525,63 +524,67 @@ static void ProgressTick(object sender, EventArgs e) { Application.Current.Dispatcher.Invoke(() => { - if (progressTotalPoints > 0) + //if (progressTotalPoints > 0) + //{ + mainWindowStatic.progressBarFiles.Value = progressFile; + mainWindowStatic.progressBarFiles.Maximum = progressTotalFiles + 1; + mainWindowStatic.lblStatus.Content = lastStatusMessage; + + // Update all progress bars based on the current values in the List + lock (lockObject) // Lock to safely read progressInfos { - mainWindowStatic.progressBarFiles.Value = progressFile; - mainWindowStatic.progressBarFiles.Maximum = progressTotalFiles + 1; - mainWindowStatic.lblStatus.Content = lastStatusMessage; - - // Update all progress bars based on the current values in the List - lock (lockObject) // Lock to safely read progressInfos + foreach (var progressInfo in progressInfos) { - foreach (var progressInfo in progressInfos) - { - int index = progressInfo.Index; - int currentValue = progressInfo.CurrentValue; - int maxValue = progressInfo.MaxValue; + int index = progressInfo.Index; + int currentValue = progressInfo.CurrentValue; + int maxValue = progressInfo.MaxValue; - // Access ProgressBar directly from the StackPanel.Children using its index - if (index >= 0 && index < mainWindowStatic.ProgressBarsContainer.Children.Count) + // Access ProgressBar directly from the StackPanel.Children using its index + if (index >= 0 && index < mainWindowStatic.ProgressBarsContainer.Children.Count) + { + if (mainWindowStatic.ProgressBarsContainer.Children[index] is ProgressBar progressBar) { - if (mainWindowStatic.ProgressBarsContainer.Children[index] is ProgressBar progressBar) + progressBar.Maximum = maxValue; + progressBar.Value = currentValue; + progressBar.Foreground = (currentValue + 1 >= maxValue ? Brushes.Lime : Brushes.Red); //+1 hack fix + //progressBar.ToolTip = $"Thread {index} - {currentValue} / {maxValue}"; // not visible, because modal dialog + //Log.WriteLine("ProgressTick: " + index + " " + currentValue + " / " + maxValue); + + // print json progress + if (progressInfo.UseJsonLog) // TODO now same bool value is for each progressinfo.. { - progressBar.Maximum = maxValue; - progressBar.Value = currentValue; // Update ProgressBar value - //progressBar.ToolTip = $"Thread {index} - {currentValue} / {maxValue}"; // not visible, because modal dialog - - // print json progress - if (progressInfo.UseJsonLog) // TODO now same bool value is for each progressinfo.. - { - string jsonString = "{" + - "\"event\": \"" + LogEvent.Progress + "\"," + - "\"thread\": " + index + "," + - "\"currentPoint\": " + currentValue + "," + - "\"totalPoints\": " + maxValue + "," + - "\"percentage\": " + (int)((currentValue / (float)maxValue) * 100) + "%," + - "\"file\": " + System.Text.Json.JsonSerializer.Serialize(progressInfo.FilePath) + - "}"; - Log.WriteLine(jsonString, LogEvent.Progress); - } + string jsonString = "{" + + "\"event\": \"" + LogEvent.Progress + "\"," + + "\"thread\": " + index + "," + + "\"currentPoint\": " + currentValue + "," + + "\"totalPoints\": " + maxValue + "," + + "\"percentage\": " + (int)((currentValue / (float)maxValue) * 100) + "%," + + "\"file\": " + System.Text.Json.JsonSerializer.Serialize(progressInfo.FilePath) + + "}"; + Log.WriteLine(jsonString, LogEvent.Progress); } } } - } - } - else - { - mainWindowStatic.progressBarFiles.Value = 0; - mainWindowStatic.lblStatus.Content = ""; - - foreach (UIElement element in mainWindowStatic.ProgressBarsContainer.Children) - { - if (element is ProgressBar progressBar) - { - progressBar.Value = 0; - } - } - } + } // foreach progressinfo + } // lock + //} + //else // finished ? + //{ + // Log.WriteLine("*************** ProgressTick: progressTotalPoints is 0, finishing.."); + // mainWindowStatic.progressBarFiles.Value = 0; + // mainWindowStatic.lblStatus.Content = ""; + + // foreach (UIElement element in mainWindowStatic.ProgressBarsContainer.Children) + // { + // if (element is ProgressBar progressBar) + // { + // progressBar.Value = 0; + // progressBar.Foreground = Brushes.Lime; + // } + // } + //} }); - } + } // ProgressTick() static (bool, float, float, float) GetBounds(ImportSettings importSettings, int fileIndex) @@ -602,7 +605,7 @@ static void ProgressTick(object sender, EventArgs e) } // process single file - static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) + static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId, CancellationToken cancellationToken) { progressTotalPoints = 1; // FIXME dummy for progress bar @@ -733,6 +736,8 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) Log.WriteLine(jsonString, LogEvent.File); + int checkCancelEvery = fullPointCount / 100; + // Loop all points for (int i = 0; i < fullPointCount; i++) //for (int i = 0; i < 1000; i++) @@ -740,6 +745,16 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId) // stop at limit count if (importSettings.useLimit == true && i > pointCount) break; + // check for cancel every 1% of points + if (i % checkCancelEvery == 0) + { + if (cancellationToken.IsCancellationRequested) + { + Log.WriteLine("Parse task was canceled."); + return false; + } + } + // FIXME: need to add skip and keep point skipper here, to make skipping faster! // get point XYZ @@ -856,6 +871,12 @@ private void btnConvert_Click(object sender, RoutedEventArgs e) // reset progress progressTotalFiles = 0; progressTotalPoints = 0; + + // reset cancel token + _cancellationTokenSource = new CancellationTokenSource(); + abort = false; + + if (ValidateSettings() == true) { ProgressTick(null, null); @@ -1008,15 +1029,6 @@ private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs // Signal the cancellation to the worker thread _cancellationTokenSource.Cancel(); - - if (workerThread != null) - { - // Wait for the worker thread to finish - workerThread.Join(); - - // Optionally exit the application - Environment.Exit((int)ExitCode.Cancelled); - } } private void btnBrowseInput_Click(object sender, RoutedEventArgs e) @@ -1236,14 +1248,9 @@ private void Window_Loaded(object sender, RoutedEventArgs e) private void BtnCancel_Click(object sender, RoutedEventArgs e) { - abort = true; + Log.WriteLine("Aborting - Please wait.."); _cancellationTokenSource.Cancel(); - - if (workerThread != null) - { - workerThread.Join(); - Environment.Exit((int)ExitCode.Cancelled); - } + abort = true; } private void cmbExportFormat_SelectionChanged(object sender, SelectionChangedEventArgs e) From dd853d2562d42d782fe394b7461de870dbb8cb5c Mon Sep 17 00:00:00 2001 From: mika Date: Tue, 3 Sep 2024 22:36:10 +0300 Subject: [PATCH 015/110] #BUILD beta --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 32665a4..a4ca901 100644 --- a/README.md +++ b/README.md @@ -38,3 +38,4 @@ Pull requests to improve this converter are welcome! (please create Issue first, + From 6f468473b9cf74cd63e52893333234ad7aa0d48e Mon Sep 17 00:00:00 2001 From: unitycoder Date: Thu, 5 Sep 2024 22:53:59 +0300 Subject: [PATCH 016/110] test external iwriter dll --- {Writers => Interfaces}/IWriter.cs | 6 ++--- Interfaces/Shared.csproj | 13 ++++++++++ MainWindow.xaml.cs | 4 ++++ PointCloudConverter.csproj | 24 ++++++++++--------- PointCloudConverter.sln | 16 +++++++++++++ Tools/PluginLoader.cs | 38 ++++++++++++++++++++++++++++++ Writers/PCROOT.cs | 5 ++-- Writers/UCPC.cs | 4 ++-- 8 files changed, 91 insertions(+), 19 deletions(-) rename {Writers => Interfaces}/IWriter.cs (86%) create mode 100644 Interfaces/Shared.csproj create mode 100644 Tools/PluginLoader.cs diff --git a/Writers/IWriter.cs b/Interfaces/IWriter.cs similarity index 86% rename from Writers/IWriter.cs rename to Interfaces/IWriter.cs index 2c3c1aa..25892cd 100644 --- a/Writers/IWriter.cs +++ b/Interfaces/IWriter.cs @@ -1,11 +1,9 @@ -using PointCloudConverter.Structs; - -namespace PointCloudConverter.Writers +namespace PointCloudConverter.Writers { public interface IWriter { // create output filestream, called before looping through points - bool InitWriter(ImportSettings importSettings, int pointCount); + bool InitWriter(TSettings importSettings, int pointCount); // optional: if need to create special file header void CreateHeader(int pointCount); // output point X,Y,Z values to file diff --git a/Interfaces/Shared.csproj b/Interfaces/Shared.csproj new file mode 100644 index 0000000..41d202f --- /dev/null +++ b/Interfaces/Shared.csproj @@ -0,0 +1,13 @@ + + + + net8.0 + enable + enable + AnyCPU;x64 + + + + false + + diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 0ac343b..7869ccb 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -85,6 +85,10 @@ private async void Main() // default code Environment.ExitCode = (int)ExitCode.Success; + // load plugins + //var testwriter = PointCloudConverter.Plugins.PluginLoader.LoadWriter("plugins/GLTFWriter.dll"); + //testwriter.Close(); + // using from commandline if (args.Length > 1) { diff --git a/PointCloudConverter.csproj b/PointCloudConverter.csproj index dde3b2a..3f43361 100644 --- a/PointCloudConverter.csproj +++ b/PointCloudConverter.csproj @@ -1,8 +1,8 @@  WinExe - net8.0-windows10.0.22621.0 - true + net8.0-windows10.0.22621.0 + true enable enable Icons\app.ico @@ -16,24 +16,22 @@ False False False - AnyCPU;x64 - - - 4 - full + x64 4 full - - 4 - full - 4 full + + + + + + @@ -50,6 +48,10 @@ + + + + diff --git a/PointCloudConverter.sln b/PointCloudConverter.sln index 51091f9..09242a5 100644 --- a/PointCloudConverter.sln +++ b/PointCloudConverter.sln @@ -5,16 +5,32 @@ VisualStudioVersion = 17.10.35122.118 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PointCloudConverter", "PointCloudConverter.csproj", "{B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Interfaces\Shared.csproj", "{692B05A5-DEB5-4B3B-8171-1C126783003B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU Debug|x64 = Debug|x64 + Release|Any CPU = Release|Any CPU Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}.Debug|Any CPU.Build.0 = Debug|Any CPU {B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}.Debug|x64.ActiveCfg = Debug|Any CPU {B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}.Debug|x64.Build.0 = Debug|Any CPU + {B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}.Release|Any CPU.Build.0 = Release|Any CPU {B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}.Release|x64.ActiveCfg = Release|x64 {B348688B-00FB-45A8-8BBD-D2D64FD7E8D8}.Release|x64.Build.0 = Release|x64 + {692B05A5-DEB5-4B3B-8171-1C126783003B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {692B05A5-DEB5-4B3B-8171-1C126783003B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {692B05A5-DEB5-4B3B-8171-1C126783003B}.Debug|x64.ActiveCfg = Debug|Any CPU + {692B05A5-DEB5-4B3B-8171-1C126783003B}.Debug|x64.Build.0 = Debug|Any CPU + {692B05A5-DEB5-4B3B-8171-1C126783003B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {692B05A5-DEB5-4B3B-8171-1C126783003B}.Release|Any CPU.Build.0 = Release|Any CPU + {692B05A5-DEB5-4B3B-8171-1C126783003B}.Release|x64.ActiveCfg = Release|x64 + {692B05A5-DEB5-4B3B-8171-1C126783003B}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Tools/PluginLoader.cs b/Tools/PluginLoader.cs new file mode 100644 index 0000000..d030b3e --- /dev/null +++ b/Tools/PluginLoader.cs @@ -0,0 +1,38 @@ +using PointCloudConverter.Writers; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; + +namespace PointCloudConverter.Plugins +{ + public static class PluginLoader + { + + public static IWriter LoadWriter(string pluginPath) + { + if (!File.Exists(pluginPath)) + throw new FileNotFoundException($"The plugin at {pluginPath} could not be found."); + + // Load the plugin assembly + var pluginAssembly = Assembly.LoadFrom(pluginPath); + + // Find the specific type 'PointCloudConverter.Writers.GLTF' + var writerType = pluginAssembly.GetType("PointCloudConverter.Writers.GLTF"); + + if (writerType == null) + throw new InvalidOperationException($"No valid implementation of IWriter found in {pluginPath}"); + + // Check if the type implements IWriter + if (!typeof(IWriter).IsAssignableFrom(writerType)) + throw new InvalidOperationException($"{writerType.FullName} does not implement IWriter"); + + // Create an instance of the IWriter implementation + return (IWriter)Activator.CreateInstance(writerType); + } + } +} diff --git a/Writers/PCROOT.cs b/Writers/PCROOT.cs index e6824ec..82cb196 100644 --- a/Writers/PCROOT.cs +++ b/Writers/PCROOT.cs @@ -122,10 +122,11 @@ public PCROOT(int? _taskID) taskID = _taskID; } - bool IWriter.InitWriter(ImportSettings _importSettings, int _pointCount) + public bool InitWriter(TSettings _importSettings, int pointCount) { //Log.WriteLine("--------------------- initwriter for taskID: " + taskID); + var res = true; // clear old nodes @@ -141,7 +142,7 @@ bool IWriter.InitWriter(ImportSettings _importSettings, int _pointCount) bsPoints = null; writerPoints = null; - importSettings = _importSettings; + importSettings = (ImportSettings)(object)_importSettings; return res; } diff --git a/Writers/UCPC.cs b/Writers/UCPC.cs index cad8ba3..95115ce 100644 --- a/Writers/UCPC.cs +++ b/Writers/UCPC.cs @@ -44,9 +44,9 @@ public ImportSettings importSettings static float cloudMaxY = float.NegativeInfinity; static float cloudMaxZ = float.NegativeInfinity; - bool IWriter.InitWriter(ImportSettings _importSettings, int _pointCount) + bool IWriter.InitWriter(TSettings _importSettings, int _pointCount) { - importSettings = _importSettings; + importSettings = (ImportSettings)(object)_importSettings; pointCount = _pointCount; pointsTempFile = importSettings.outputFile + "_PointsTemp"; From 18c8cd2c93ea334f6bd1da9cee1367dc96dee18a Mon Sep 17 00:00:00 2001 From: unitycoder Date: Thu, 5 Sep 2024 23:08:46 +0300 Subject: [PATCH 017/110] fix cancellation exception --- MainWindow.xaml.cs | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 7869ccb..d6fc366 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -25,7 +25,7 @@ namespace PointCloudConverter { public partial class MainWindow : Window { - static readonly string version = "02.09.2024"; + static readonly string version = "05.09.2024"; static readonly string appname = "PointCloud Converter - " + version; static readonly string rootFolder = AppDomain.CurrentDomain.BaseDirectory; @@ -47,8 +47,6 @@ public partial class MainWindow : Window [DllImport("user32.dll")] static extern int SendMessage(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam); - - static bool abort = false; public static MainWindow mainWindowStatic; bool isInitialiazing = true; @@ -314,7 +312,24 @@ private static async Task ProcessAllFiles(object workerParamsObject) return; } - await semaphore.WaitAsync(cancellationToken); + //await semaphore.WaitAsync(cancellationToken); + try + { + await semaphore.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + // Handle the cancellation scenario here + Log.WriteLine("Wait was canceled."); + } + finally + { + // Ensure the semaphore is released, if needed + if (semaphore.CurrentCount == 0) // Make sure we don't release more times than we acquire + { + semaphore.Release(); + } + } //int? taskId = Task.CurrentId; // Get the current task ID //progressFile = i; @@ -356,10 +371,14 @@ private static async Task ProcessAllFiles(object workerParamsObject) { Log.WriteLine("Timeout occurred: " + ex.Message, LogEvent.Error); } + catch (OperationCanceledException) + { + MessageBox.Show("Operation was canceled."); + } catch (Exception ex) { Log.WriteLine("Exception> " + ex.Message, LogEvent.Error); - throw; // Rethrow to ensure Task.WhenAll sees the exception + //throw; // Rethrow to ensure Task.WhenAll sees the exception } finally { @@ -754,7 +773,7 @@ static bool ParseFile(ImportSettings importSettings, int fileIndex, int? taskId, { if (cancellationToken.IsCancellationRequested) { - Log.WriteLine("Parse task was canceled."); + //Log.WriteLine("Parse task (" + taskId + ") was canceled for: " + importSettings.inputFiles[fileIndex]); return false; } } @@ -878,8 +897,6 @@ private void btnConvert_Click(object sender, RoutedEventArgs e) // reset cancel token _cancellationTokenSource = new CancellationTokenSource(); - abort = false; - if (ValidateSettings() == true) { @@ -1027,6 +1044,7 @@ void StartProcess(bool doProcess = true) } private CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource(); + private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e) { SaveSettings(); @@ -1254,7 +1272,6 @@ private void BtnCancel_Click(object sender, RoutedEventArgs e) { Log.WriteLine("Aborting - Please wait.."); _cancellationTokenSource.Cancel(); - abort = true; } private void cmbExportFormat_SelectionChanged(object sender, SelectionChangedEventArgs e) From 52b1c318efe8a983e8180adec65752e46c2c19d7 Mon Sep 17 00:00:00 2001 From: unitycoder Date: Sun, 8 Sep 2024 20:38:03 +0300 Subject: [PATCH 018/110] fix help button for net 8.0 --- MainWindow.xaml.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index d6fc366..acf59eb 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -1346,7 +1346,19 @@ private void txtInputFile_Drop(object sender, DragEventArgs e) private void btnHelp_Click(object sender, RoutedEventArgs e) { - Process.Start("https://github.com/unitycoder/PointCloudConverter/wiki"); + try + { + var processStartInfo = new ProcessStartInfo + { + FileName = "https://github.com/unitycoder/PointCloudConverter/wiki", + UseShellExecute = true + }; + Process.Start(processStartInfo); + } + catch (Exception ex) + { + MessageBox.Show($"Unable to open the link. Error: {ex.Message}"); + } } private void chkAutoOffset_Checked(object sender, RoutedEventArgs e) From 08af2e303f9541f946f020406d68b5844be46066 Mon Sep 17 00:00:00 2001 From: unitycoder Date: Sun, 8 Sep 2024 20:58:00 +0300 Subject: [PATCH 019/110] update version --- MainWindow.xaml.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index acf59eb..e092d1d 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -25,7 +25,7 @@ namespace PointCloudConverter { public partial class MainWindow : Window { - static readonly string version = "05.09.2024"; + static readonly string version = "08.09.2024"; static readonly string appname = "PointCloud Converter - " + version; static readonly string rootFolder = AppDomain.CurrentDomain.BaseDirectory; From 812f182d57ea24b5b4a18af396dd3032722903f8 Mon Sep 17 00:00:00 2001 From: unitycoder Date: Mon, 9 Sep 2024 00:07:14 +0300 Subject: [PATCH 020/110] GUI: add import/export settings from text file --- MainWindow.xaml | 5 +- MainWindow.xaml.cs | 163 ++++++++++++++++++++++++++++++++++++++++++++- Tools/ArgParser.cs | 16 ++++- 3 files changed, 180 insertions(+), 4 deletions(-) diff --git a/MainWindow.xaml b/MainWindow.xaml index bef1259..2fcd10f 100644 --- a/MainWindow.xaml +++ b/MainWindow.xaml @@ -22,8 +22,11 @@