using HalconDotNet; using LB_SmartVisionCommon; using LB_VisionControls; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using OpenCvSharp; using SmartMore.ViMo; using System; using System.Collections.Generic; using System.Diagnostics; using System.Drawing.Imaging; using System.Linq; using System.Reflection; using Point = OpenCvSharp.Point; namespace LB_VisionProcesses.Alogrithms.BigModel.Segment { public class SegmentTool : TAlgorithm { public Dictionary dicSegmentParam; /// /// 模型路径 /// private string modelPath = string.Empty; /// /// 模型ID /// private string modelID = "0"; /// /// 是否使用GPU /// private bool isUseGPU = false; /// /// GPUID /// private int deviceGPUID = 0; /// /// BatchSize /// private int batchSize = 1; /// /// DP解决方案 /// private Solution solution; /// /// 分割模型 /// private ISegmentationModule module; /// /// 分割参数 /// private SegmentationParams segParams; public List SegFeatures = new List(); public SegmentTool() { strProcessClass = "LB_VisionProcesses.Alogrithms.BigModel.Segment.SegmentTool"; strProcessName = "语义分割工具"; //模型输入 Params.Inputs.Add("ModelPath", ""); Params.Inputs.Add("ModuleId", "0"); Params.Inputs.Add("UseGpu", false); Params.Inputs.Add("DeviceId", 0); Params.Inputs.Add("BatchSize", 1); Params.Inputs.Add("MaxTimeOut", 5000); //面积阈值 Params.Inputs.Add("AreaThresholds", new Dictionary()); //长边过滤阈值 Params.Inputs.Add("LongSideThresholds", new Dictionary()); //短边过滤阈值 Params.Inputs.Add("ShortSideThresholds", new Dictionary()); //平均置信度阈值 //Params.Inputs.Add("MeanScoreThresholds", 0.0f); //类别置信度阈值 Params.Inputs.Add("ClassesScoreThresholds", new Dictionary()); //ROI //Params.Inputs.Add("ROIx1", 0); //Params.Inputs.Add("ROIy1", 0); //Params.Inputs.Add("ROIx2", 0); //Params.Inputs.Add("ROIy2", 0); Params.Inputs.Add("ROI", new List()); //筛选后输出输出 Params.Outputs.Add("Count", 0); Params.Outputs.Add("ClassID", new List()); Params.Outputs.Add("Labels", new List()); Params.Outputs.Add("Area", new List()); Params.Outputs.Add("LongSide", new List()); Params.Outputs.Add("ShortSide", new List()); Params.Outputs.Add("MeanScore", new List()); //bbox Params.Outputs.Add("X", new List()); Params.Outputs.Add("Y", new List()); Params.Outputs.Add("Height", new List()); Params.Outputs.Add("Width", new List()); //轮廓线 Params.Outputs.Add("OuterContours", new List>>()); } private void VisualizeSegmentationResult(SegmentationResponse rsp, Mat image, Dictionary AreaThresholds, Dictionary LongSideThresholds, Dictionary ShortSideThresholds, Dictionary ClassesScoreThresholds) { // 预定义类别颜色映射表(支持最多20个类别) var classColors = new List { new Scalar(0, 0, 255), // 0: 红色 new Scalar(0, 255, 0), // 1: 绿色 new Scalar(255, 0, 0), // 2: 蓝色 new Scalar(255, 255, 0), // 3: 青色 new Scalar(255, 0, 255), // 4: 品红 new Scalar(0, 255, 255), // 5: 黄色 new Scalar(128, 0, 0), // 6: 深红 new Scalar(0, 128, 0), // 7: 深绿 new Scalar(0, 0, 128), // 8: 深蓝 new Scalar(128, 128, 0), // 9: 橄榄色 new Scalar(128, 0, 128), // 10: 紫色 new Scalar(0, 128, 128), // 11: 靛蓝 new Scalar(255, 165, 0), // 12: 橙色 new Scalar(255, 192, 203),// 13: 粉红色 new Scalar(147, 112, 219),// 14: 紫罗兰 new Scalar(255, 99, 71), // 15: 番茄红 new Scalar(255, 20, 147), // 16: 紫色 new Scalar(30, 144, 255), // 17: 道奇蓝 new Scalar(255, 69, 0), // 18: 橙红色 new Scalar(0, 206, 209) // 19: 碧绿色 }; Record = new ObjectRecord(); int Count = 0; var ClassIDList = new List(); var LabelsList = new List(); var AreaList = new List(); var LongSideList = new List(); var ShortSideList = new List(); var MeanScoreList = new List(); var XList = new List(); var YList = new List(); var HeightList = new List(); var WidthList = new List(); var OuterContourList = new List>(); rsp.LabelMap = null; // 使用 RegionInfo 信息可视化 foreach (var info in rsp.RegionInfos) { var r_area = info.Area; var r_label = info.Label; var r_shortSide = info.ShortSide; var r_longSide = info.LongSide; var r_Id = info.Id; var r_MeanScore = info.MeanScore; var bbox = info.BoundingRect; // 提取轮廓 var contours = new List> { info.OuterContour.Select(x => new Point(x.X, x.Y)).ToList() }; List contour = info.OuterContour.Select(x => new Point((int)x.X, (int)x.Y)).ToList(); // 浮点→整数 if (r_area >= AreaThresholds[r_label] && r_longSide >= LongSideThresholds[r_label] && r_shortSide >= ShortSideThresholds[r_label] /*&& r_MeanScore >= ClassesScoreThresholds[r_label]*/) { AreaList.Add(r_area); ClassIDList.Add(r_Id); LongSideList.Add(r_longSide); ShortSideList.Add(r_shortSide); MeanScoreList.Add(r_MeanScore); LabelsList.Add(r_label); XList.Add(bbox.X); YList.Add(bbox.Y); HeightList.Add(bbox.Height); WidthList.Add(bbox.Width); OuterContourList.Add(contour); // 获取类别颜色(自动循环使用预定义颜色) int colorIndex = info.Id % classColors.Count; var color = classColors[colorIndex]; //Cv2.DrawContours(image, contours, -1, color, 2); //Cv2.PutText(image, $"{info.Label}, area = {info.Area}", new Point(bbox.X, bbox.Y), HersheyFonts.HersheySimplex, 1, color, 2); if (contour == null || contour.Count == 0) { throw new ArgumentException("轮廓点集合不能为空或无数据"); } HTuple rowTuple = new HTuple(); // Halcon中row对应Y坐标 HTuple colTuple = new HTuple(); // Halcon中column对应X坐标 foreach (Point p in contour) { rowTuple.Append(p.Y); colTuple.Append(p.X); } HOperatorSet.GenContourPolygonXld(out HObject contour1, rowTuple, colTuple); ((ObjectRecord)Record).AddXld(contour1); } } Count = AreaList.Count(); if (Count > 0) { Msg = "检测到缺陷,NG:" + Count; Result = false; } else { Msg = "运行成功"; Result = true; } Params.Outputs["Count"] = Count; Params.Outputs["ClassID"] = ClassIDList; Params.Outputs["Area"] = AreaList; Params.Outputs["LongSideList"] = LongSideList; Params.Outputs["ShortSideList"] = ShortSideList; Params.Outputs["MeanScore"] = MeanScoreList; Params.Outputs["X"] = XList; Params.Outputs["Y"] = YList; Params.Outputs["Height"] = HeightList; Params.Outputs["Width"] = WidthList; Params.Outputs["OuterContours"] = OuterContourList; } private void VisualizeSegmentationResult(SegmentationResponse rsp, Dictionary AreaThresholds, Dictionary LongSideThresholds, Dictionary ShortSideThresholds, Dictionary ClassesScoreThresholds) { // 预定义类别颜色映射表(支持最多20个类别) var classColors = new List { new Scalar(0, 0, 255), // 0: 红色 new Scalar(0, 255, 0), // 1: 绿色 new Scalar(255, 0, 0), // 2: 蓝色 new Scalar(255, 255, 0), // 3: 青色 new Scalar(255, 0, 255), // 4: 品红 new Scalar(0, 255, 255), // 5: 黄色 new Scalar(128, 0, 0), // 6: 深红 new Scalar(0, 128, 0), // 7: 深绿 new Scalar(0, 0, 128), // 8: 深蓝 new Scalar(128, 128, 0), // 9: 橄榄色 new Scalar(128, 0, 128), // 10: 紫色 new Scalar(0, 128, 128), // 11: 靛蓝 new Scalar(255, 165, 0), // 12: 橙色 new Scalar(255, 192, 203),// 13: 粉红色 new Scalar(147, 112, 219),// 14: 紫罗兰 new Scalar(255, 99, 71), // 15: 番茄红 new Scalar(255, 20, 147), // 16: 紫色 new Scalar(30, 144, 255), // 17: 道奇蓝 new Scalar(255, 69, 0), // 18: 橙红色 new Scalar(0, 206, 209) // 19: 碧绿色 }; Record = new ObjectRecord(); int Count = 0; var ClassIDList = new List(); var LabelsList = new List(); var AreaList = new List(); var LongSideList = new List(); var ShortSideList = new List(); var MeanScoreList = new List(); var XList = new List(); var YList = new List(); var HeightList = new List(); var WidthList = new List(); var OuterContourList = new List>(); rsp.LabelMap = null; // 使用 RegionInfo 信息可视化 foreach (var info in rsp.RegionInfos) { var r_area = info.Area; var r_label = info.Label; var r_shortSide = info.ShortSide; var r_longSide = info.LongSide; var r_Id = info.Id; var r_MeanScore = info.MeanScore; var bbox = info.BoundingRect; // 提取轮廓 var contours = new List> { info.OuterContour.Select(x => new Point(x.X, x.Y)).ToList() }; List contour = info.OuterContour.Select(x => new Point((int)x.X, (int)x.Y)).ToList(); // 浮点→整数 if (r_area >= AreaThresholds[r_label] && r_longSide >= LongSideThresholds[r_label] && r_shortSide >= ShortSideThresholds[r_label] /*&& r_MeanScore >= ClassesScoreThresholds[r_label]*/) { AreaList.Add(r_area); ClassIDList.Add(r_Id); LongSideList.Add(r_longSide); ShortSideList.Add(r_shortSide); MeanScoreList.Add(r_MeanScore); LabelsList.Add(r_label); XList.Add(bbox.X); YList.Add(bbox.Y); HeightList.Add(bbox.Height); WidthList.Add(bbox.Width); OuterContourList.Add(contour); // 获取类别颜色(自动循环使用预定义颜色) int colorIndex = info.Id % classColors.Count; var color = classColors[colorIndex]; //Cv2.DrawContours(image, contours, -1, color, 2); //Cv2.PutText(image, $"{info.Label}, area = {info.Area}", new Point(bbox.X, bbox.Y), HersheyFonts.HersheySimplex, 1, color, 2); if (contour == null || contour.Count == 0) { throw new ArgumentException("轮廓点集合不能为空或无数据"); } HTuple rowTuple = new HTuple(); // Halcon中row对应Y坐标 HTuple colTuple = new HTuple(); // Halcon中column对应X坐标 foreach (Point p in contour) { rowTuple.Append(p.Y); colTuple.Append(p.X); } HOperatorSet.GenContourPolygonXld(out HObject contour1, rowTuple, colTuple); ((ObjectRecord)Record).AddXld(contour1); } } Count = AreaList.Count(); if (Count > 0) { Msg = "检测到缺陷,NG:" + Count; Result = false; } else { Msg = "运行成功"; Result = true; } Params.Outputs["Count"] = Count; Params.Outputs["ClassID"] = ClassIDList; Params.Outputs["Area"] = AreaList; Params.Outputs["LongSideList"] = LongSideList; Params.Outputs["ShortSideList"] = ShortSideList; Params.Outputs["MeanScore"] = MeanScoreList; Params.Outputs["X"] = XList; Params.Outputs["Y"] = YList; Params.Outputs["Height"] = HeightList; Params.Outputs["Width"] = WidthList; Params.Outputs["OuterContours"] = OuterContourList; } public override bool Run() { DateTime StartTime = DateTime.Now; InitRunParams(); HOperatorSet.GenEmptyObj(out HObject EmptyObj); OutputImage = EmptyObj; // 创建并启动任务 TAlgorithmMain(); RunTime = (DateTime.Now - StartTime).TotalMilliseconds; return Result; } /// /// 算子逻辑 /// public override void TAlgorithmMain() { //InitRunParams(); #region 初始化变量 HObject ho_Regions, ho_ConnectedRegions; HOperatorSet.GenEmptyObj(out ho_Regions); HOperatorSet.GenEmptyObj(out ho_ConnectedRegions); #endregion try { if (InputImage == null) { Msg = "输入图片为空"; Result = false; return; } if (!(InputImage is Mat)) { Msg = "输入图片格式不为Mat"; Result = false; return; } #region 裁剪区域 object DomainImage = null; if (!ReduceDomainImage(InputImage, ref DomainImage)) { Msg = "裁剪区域失败"; Result = false; return; } Mat hoDomainImage = DomainImage as Mat; #endregion if (module == null) { LoadModel(); Debug.WriteLine("运行逻辑加载模型"); } #region 算子逻辑 //var ModelPath = ProcessParams.ConvertToString(Params.Inputs["ModelPath"]); //var ModuleId = ProcessParams.ConvertToString(Params.Inputs["ModuleId"]); //var UseGpu = Convert.ToBoolean(Params.Inputs["UseGpu"]); //var DeviceId = Convert.ToInt32(Params.Inputs["DeviceId"]); //var BatchSize = Convert.ToInt32(Params.Inputs["BatchSize"]); //Dictionary AreaThresholds = Params.Inputs["AreaThresholds"] as Dictionary ?? new Dictionary(); //Dictionary LongSideThresholds = Params.Inputs["LongSideThresholds"] as Dictionary ?? new Dictionary(); //Dictionary ShortSideThresholds = Params.Inputs["ShortSideThresholds"] as Dictionary ?? new Dictionary(); //Dictionary ClassesScoreThresholds = Params.Inputs["ClassesScoreThresholds"] as Dictionary ?? new Dictionary(); Dictionary AreaThresholds = null; Dictionary LongSideThresholds = null; Dictionary ShortSideThresholds = null; Dictionary ClassesScoreThresholds = null; // 获取SegmentTool中的阈值字典 var areaThresholdsObj = Params.Inputs["AreaThresholds"]; if (areaThresholdsObj is JObject jObject) { // 最终转换为目标字典类型 AreaThresholds = jObject.ToObject>(); } else if (areaThresholdsObj is Object) { AreaThresholds = areaThresholdsObj as Dictionary; if (AreaThresholds == null) { AreaThresholds = new Dictionary(); foreach (var kvp in areaThresholdsObj as Dictionary) { try { // 尝试将object转为double double value = Convert.ToDouble(kvp.Value); AreaThresholds.Add(kvp.Key, value); } catch (Exception ex) { // 处理转换失败的情况(可选,比如打印日志、抛自定义异常) Debug.WriteLine($"转换键 {kvp.Key} 失败:{ex.Message}"); } } } } else { //// 处理类型不符的异常情况(可选,增强健壮性) //throw new System.Exception("AreaThresholds的实际类型不是JObject,转换失败"); AreaThresholds = new Dictionary(); } var longSideThresholdsObj = Params.Inputs["LongSideThresholds"]; if (longSideThresholdsObj is JObject jObjectL) { // 最终转换为目标字典类型 LongSideThresholds = jObjectL.ToObject>(); } else if (longSideThresholdsObj is Object) { LongSideThresholds = longSideThresholdsObj as Dictionary; if (LongSideThresholds == null) { LongSideThresholds = new Dictionary(); foreach (var kvp in longSideThresholdsObj as Dictionary) { try { // 尝试将object转为double double value = Convert.ToDouble(kvp.Value); LongSideThresholds.Add(kvp.Key, value); } catch (Exception ex) { // 处理转换失败的情况(可选,比如打印日志、抛自定义异常) Debug.WriteLine($"转换键 {kvp.Key} 失败:{ex.Message}"); } } } } else { //// 处理类型不符的异常情况(可选,增强健壮性) //throw new System.Exception("LongSideThresholds的实际类型不是JObject,转换失败"); LongSideThresholds = new Dictionary(); } var shortSideThresholdsObj = Params.Inputs["ShortSideThresholds"]; if (shortSideThresholdsObj is JObject jObjectS) { // 最终转换为目标字典类型 ShortSideThresholds = jObjectS.ToObject>(); } else if (shortSideThresholdsObj is Object) { ShortSideThresholds = shortSideThresholdsObj as Dictionary; if (ShortSideThresholds == null) { ShortSideThresholds = new Dictionary(); foreach (var kvp in shortSideThresholdsObj as Dictionary) { try { // 尝试将object转为double double value = Convert.ToDouble(kvp.Value); ShortSideThresholds.Add(kvp.Key, value); } catch (Exception ex) { // 处理转换失败的情况(可选,比如打印日志、抛自定义异常) Debug.WriteLine($"转换键 {kvp.Key} 失败:{ex.Message}"); } } } } else { //// 处理类型不符的异常情况(可选,增强健壮性) //throw new System.Exception("shortSideThresholds的实际类型不是JObject,转换失败"); ShortSideThresholds = new Dictionary(); } var classesScoreThresholdsObj = Params.Inputs["ClassesScoreThresholds"]; if (classesScoreThresholdsObj is JObject jObjectC) { // 最终转换为目标字典类型 ClassesScoreThresholds = jObjectC.ToObject>(); } else if (classesScoreThresholdsObj is Object) { ClassesScoreThresholds = classesScoreThresholdsObj as Dictionary; if (ClassesScoreThresholds == null) { ClassesScoreThresholds = new Dictionary(); foreach (var kvp in classesScoreThresholdsObj as Dictionary) { try { // 尝试将object转为double double value = Convert.ToDouble(kvp.Value); ClassesScoreThresholds.Add(kvp.Key, value); } catch (Exception ex) { // 处理转换失败的情况(可选,比如打印日志、抛自定义异常) Debug.WriteLine($"转换键 {kvp.Key} 失败:{ex.Message}"); } } } } else { //// 处理类型不符的异常情况(可选,增强健壮性) //throw new System.Exception("classesScoreThresholds的实际类型不是JObject,转换失败"); ClassesScoreThresholds = new Dictionary(); } //判断和加载模型 try { //// create an empty solution //{ // AsyncLogHelper.Info($"load solution from: {Params.Inputs["MinThreshold"]}"); // solution.LoadFromFile(modelPath); // load solution from model.vimosln // AsyncLogHelper.Info($"create Module: {modelID}, use gpu: {isUseGPU}, device id: {deviceGPUID}"); // module = solution.CreateModule(modelID, isUseGPU, deviceGPUID, batchSize); // create module // { // var segParams = module.Params; // 读取模型参数 // segParams.WithMask = false; // 可选,关闭 mask 输出 // segParams.WithRegionInfo = true; // 开启 RegionInfo 输出,以便可视化 // segParams.RegionInfoFlag = SegRegionInfoFlag.Area // | SegRegionInfoFlag.InnerContours // | SegRegionInfoFlag.OuterContour // | SegRegionInfoFlag.LongSide // | SegRegionInfoFlag.ShortSide; // module.Params = segParams; // 设置模型参数 // AsyncLogHelper.Info($"create request from image: " + "imagePath"); // using (Mat visImage = hoDomainImage.Clone()) // clone request image for visualize // { // var req = new Request(hoDomainImage); // create request from image // AsyncLogHelper.Info("run pipelines"); // module.Run(req, out SegmentationResponse rsp); // AsyncLogHelper.Info("inference done"); // AsyncLogHelper.Info("do visualize"); // VisualizeSegmentationResult(rsp, visImage, AreaThresholds, LongSideThresholds, ShortSideThresholds, ClassesScoreThresholds); // visualize // OutputImage = visImage; // //Console.WriteLine("save vis image to: vis_image.png"); // //Cv2.ImWrite("vis_image.png", visImage); // save visualize image // //Console.WriteLine("save vis image done"); // } // } //} //using (Mat visImage = hoDomainImage.Clone()) // clone request image for visualize //{ // Request req = new Request(hoDomainImage); // create request from image // AsyncLogHelper.Info("run pipelines"); // module.Run(req, out SegmentationResponse rsp); // AsyncLogHelper.Info("inference done"); // AsyncLogHelper.Info("do visualize"); // //VisualizeSegmentationResult(rsp, visImage, AreaThresholds, LongSideThresholds, ShortSideThresholds, ClassesScoreThresholds); // visualize // VisualizeSegmentationResult(rsp, AreaThresholds, LongSideThresholds, ShortSideThresholds, ClassesScoreThresholds); // visualize // OutputImage = DomainImage; // //Console.WriteLine("save vis image to: vis_image.png"); // //Cv2.ImWrite("vis_image.png", visImage); // save visualize image // //Console.WriteLine("save vis image done"); //} Request req = new Request(hoDomainImage); // create request from image AsyncLogHelper.Info("run pipelines"); module.Run(req, out SegmentationResponse rsp); AsyncLogHelper.Info("inference done"); AsyncLogHelper.Info("do visualize"); //VisualizeSegmentationResult(rsp, visImage, AreaThresholds, LongSideThresholds, ShortSideThresholds, ClassesScoreThresholds); // visualize VisualizeSegmentationResult(rsp, AreaThresholds, LongSideThresholds, ShortSideThresholds, ClassesScoreThresholds); // visualize OutputImage = DomainImage; //hoDomainImage.Dispose(); } catch (Exception ex) { Msg = "生成OutputImage失败,原因是:" + ex.ToString(); Result = false; return; } #endregion //Msg = "运行成功"; //Result = true; return; } catch (Exception ex) { Msg = "运行失败,原因是:" + ex.ToString().TrimEnd(); OutputImage = null; Result = false; return; } finally { bCompleted = true; #region 内存释放 ho_Regions.Dispose(); ho_ConnectedRegions.Dispose(); #endregion } } public override bool Save(string filePath = "") { return base.Save(filePath); } public override bool Load(string fullPath = "") { if (base.Load(fullPath)) { return LoadModel(); } else { return false; } } public bool LoadModel(string fullPath = "") { if (!string.IsNullOrEmpty(fullPath) && fullPath.Trim() != "") { this.modelPath = fullPath; Params.Inputs.Add("ModelPath", fullPath); } SegFeatures.Clear(); this.modelID = Params.Inputs["ModuleId"]?.ToString();//(JArray.FromObject(Params.Inputs["标签"]))?.ToObject>(); this.isUseGPU = Convert.ToBoolean(Params.Inputs["UseGpu"]?.ToString()); this.deviceGPUID = Convert.ToInt32(Params.Inputs["DeviceId"]?.ToString()); this.batchSize = Convert.ToInt32(Params.Inputs["BatchSize"]?.ToString()); this.modelPath = Params.Inputs["ModelPath"]?.ToString(); if (System.IO.File.Exists(modelPath)) { try { if (module != null) { module.Dispose(); module = null; } if (solution != null) { solution.Dispose(); solution = null; } if (solution == null) { solution = new Solution(); AsyncLogHelper.Info($"load solution from: {Params.Inputs["ModelPath"]}"); solution.LoadFromFile(modelPath); // load solution from model.vimosln } else { solution.Dispose(); AsyncLogHelper.Info($"load solution from: {Params.Inputs["ModelPath"]}"); solution.LoadFromFile(modelPath); // load solution from model.vimosln } if (module == null) { AsyncLogHelper.Info($"create Module: {modelID}, use gpu: {isUseGPU}, device id: {deviceGPUID}"); module = solution.CreateModule(modelID, isUseGPU, deviceGPUID, batchSize); // create module } else { module.Dispose(); AsyncLogHelper.Info($"create Module: {modelID}, use gpu: {isUseGPU}, device id: {deviceGPUID}"); module = solution.CreateModule(modelID, isUseGPU, deviceGPUID, batchSize); // create module } segParams = module.Params; // 读取模型参数 segParams.WithMask = false; // 可选,关闭 mask 输出 segParams.WithRegionInfo = true; // 开启 RegionInfo 输出,以便可视化 segParams.RegionInfoFlag = SegRegionInfoFlag.Area | SegRegionInfoFlag.InnerContours | SegRegionInfoFlag.OuterContour | SegRegionInfoFlag.LongSide | SegRegionInfoFlag.ShortSide; module.Params = segParams; // 设置模型参数 foreach (var item in solution.ModuleInfoList) { foreach (var item1 in item.Features.ToList()) { SegFeatures.Add(item1); } } // 加载模型后先运行下空图片进行缓存 下次运行时提速[无效] //image_predict(new Mat(1920, 1080, MatType.CV_16UC3)); } catch (Exception ex) { Msg = $"创建分割模型接口失败,请检查参数,{ex.Message}"; return false; } } else { Msg = "模型路径不存在"; return false; } return true; } } }