算法调整提交

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/ExecutionCenter.java

@@ -2,10 +2,7 @@ package com.njcn.algorithm;
 
 import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.collection.CollectionUtil;
-import cn.hutool.core.date.DatePattern;
+import cn.hutool.core.date.*;
-import cn.hutool.core.date.DateTime;
-import cn.hutool.core.date.DateUnit;
-import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.text.StrPool;
 import cn.hutool.core.util.StrUtil;
 import com.njcn.algorithm.pojo.bo.BaseParam;
@@ -54,6 +51,8 @@ import java.util.stream.Collectors;
 @RequiredArgsConstructor
 public class ExecutionCenter extends BaseController {
 
+    private Long calTime = System.currentTimeMillis();
+
     private static final Logger logger = LoggerFactory.getLogger(ExecutionCenter.class);
 
     @Resource
@@ -95,7 +94,23 @@ public class ExecutionCenter extends BaseController {
      */
     private void dealResponse(CalculatedParam calculatedParam, LiteflowResponse liteflowResponse, String methodDescribe) {
         if (liteflowResponse.isSuccess()) {
+            Runtime runtime = Runtime.getRuntime();
+            // 获取 JVM 最大可用内存（以字节为单位）
+            long maxMemory = runtime.maxMemory();
+            // 获取 JVM 当前已分配的内存（以字节为单位）
+            long totalMemory = runtime.totalMemory();
+            // 获取 JVM 当前空闲内存（以字节为单位）
+            long freeMemory = runtime.freeMemory();
+            // 计算已使用的内存
+            long usedMemory = totalMemory - freeMemory;
+
+            System.out.println("最大可用内存: " + maxMemory / (1024 * 1024) + " MB");
+            System.out.println("当前已分配的内存: " + totalMemory / (1024 * 1024) + " MB");
+            System.out.println("当前空闲内存: " + freeMemory / (1024 * 1024) + " MB");
+            System.out.println("已使用的内存: " + usedMemory / (1024 * 1024) + " MB");
+            System.out.println("算法执行成功-----------------------------------------");
             logger.info("日期{}，{}执行{}成功", calculatedParam.getDataDate(), methodDescribe, calculatedParam.isFullChain() ? "全链" : "指定节点：".concat(String.join(StrPool.COMMA, calculatedParam.getTagNames())));
+            logger.info("当前算法执行总时长为-------------------------------------{}",(System.currentTimeMillis()-calTime)/1000/60);
         } else {
             Map<String, List<CmpStep>> executeSteps = liteflowResponse.getExecuteSteps();
             CmpStep failStep = null;
@@ -148,6 +163,7 @@ public class ExecutionCenter extends BaseController {
             liteflowResponse = flowExecutor.execute2Resp("measurement_point", calculatedParam);
             dealResponse(calculatedParam, liteflowResponse, methodDescribe);
         }
+        calTime = System.currentTimeMillis();
     }
 
     @OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -182,6 +198,7 @@ public class ExecutionCenter extends BaseController {
             liteflowResponse = flowExecutor.execute2Resp("special_analysis", calculatedParam);
             dealResponse(calculatedParam, liteflowResponse, methodDescribe);
         }
+        calTime = System.currentTimeMillis();
     }
 
     @OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -225,6 +242,7 @@ public class ExecutionCenter extends BaseController {
             liteflowResponse = flowExecutor.execute2Resp("measurement_point_hour", calculatedParam);
             dealResponse(calculatedParam, liteflowResponse, methodDescribe);
         }
+        calTime = System.currentTimeMillis();
     }
 
     @OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -265,6 +283,7 @@ public class ExecutionCenter extends BaseController {
             liteflowResponse = flowExecutor.execute2Resp("device", calculatedParam);
             dealResponse(calculatedParam, liteflowResponse, methodDescribe);
         }
+        calTime = System.currentTimeMillis();
     }
 
     @OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -303,7 +322,7 @@ public class ExecutionCenter extends BaseController {
             liteflowResponse = flowExecutor.execute2Resp("org_point", calculatedParam);
             dealResponse(calculatedParam, liteflowResponse, methodDescribe);
         }
-
+        calTime = System.currentTimeMillis();
     }
 
     //
@@ -391,7 +410,7 @@ public class ExecutionCenter extends BaseController {
             liteflowResponse = flowExecutor.execute2Resp("sub_station", calculatedParam);
             dealResponse(calculatedParam, liteflowResponse, methodDescribe);
         }
-
+        calTime = System.currentTimeMillis();
     }
 //
 //    /**

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataCleanServiceImpl.java

@@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.util.ObjectUtil;
 import com.njcn.algorithm.pojo.bo.CalculatedParam;
 import com.njcn.algorithm.service.line.IDataCleanService;
+import com.njcn.algorithm.utils.MemorySizeUtil;
 import com.njcn.dataProcess.api.*;
 import com.njcn.dataProcess.enums.DataCleanEnum;
 import com.njcn.dataProcess.param.DataCleanParam;
@@ -108,6 +109,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
             lineParam.setLineId(list);
             List<DataVDto> partList = dataVFeignClient.getRawData(lineParam).getData();
             if (CollUtil.isNotEmpty(partList)) {
+                logger.info("{}数据清洗dataV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
                 partList.forEach(item->{
                     //数据清洗
                     List<PqDataVerify> pqDataVerifies = judgeDataV(map, lineMap.get(item.getLineId()),item);
@@ -120,6 +122,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
     @Override
@@ -145,6 +149,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
             lineParam.setLineId(list);
             List<DataIDto> partList = dataIFeignClient.getRawData(lineParam).getData();
             if (CollUtil.isNotEmpty(partList)) {
+                logger.info("{}数据清洗dataI集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
                 partList.forEach(item->{
                     //数据清洗
                     List<PqDataVerify> pqDataVerifies = judgeDataI(map, lineMap.get(item.getLineId()),item);
@@ -157,6 +162,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
     @Override
@@ -188,6 +195,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
     @Override
@@ -207,6 +216,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             lineParam.setLineId(list);
             List<DataHarmDto> partList = dataInharmVFeignClient.getRawData(lineParam).getData();
             if (CollUtil.isNotEmpty(partList)) {
+                logger.info("{}数据清洗dataInharmV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
+
                 partList.forEach(item->{
                     //数据清洗
                     List<PqDataVerify> pqDataVerifies = judgeDataInHarmV(map,item);
@@ -219,6 +230,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
     @Override
@@ -250,6 +263,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
     @Override
@@ -281,6 +296,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
     @Override
@@ -312,6 +329,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
     @Override
@@ -343,6 +362,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
     @Override
@@ -374,6 +395,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
             pqDataVerifyFeignClient.insertData(result);
             result.clear();
         }
+
+        System.gc();
     }
 
 
@@ -704,7 +727,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
         pqReasonableRangeDto = map.get(DataCleanEnum.RmsI.getCode());
         phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
         if (phaseList.contains(dto.getPhasicType())) {
-            if (dto.getRms() <= line.getCT1()) {
+            if (dto.getRms() > line.getCT1()) {
                 PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
                         ,dto.getMinTime()
                         ,dto.getValueType()

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataComAssServiceImpl.java

@@ -108,7 +108,7 @@ public class DataComAssServiceImpl implements IDataComAssService {
         if (CollUtil.isNotEmpty(list)) {
             dataComAssFeignClient.batchInsertion(list);
         }
-
+        System.gc();
     }
 
     /**

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataVCvtServiceImpl.java

@@ -84,6 +84,7 @@ public class DataVCvtServiceImpl implements IDataHarmRateVCvtService {
         });
         dataHarmRateVFeignClient.addList(result);
 
+        System.gc();
     }
 
     private List<DataHarmRateVCvtDto> cvtChange(List<DataHarmRateVCvtDto> v, CvtHarmonicCorrectionFactorsDTO cvtHarmonicCorrectionFactorsDTO) {

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DayDataServiceImpl.java

@@ -3,6 +3,7 @@ package com.njcn.algorithm.serviceimpl.line;
 import cn.hutool.core.collection.CollUtil;
 import com.njcn.algorithm.pojo.bo.CalculatedParam;
 import com.njcn.algorithm.service.line.IDayDataService;
+import com.njcn.algorithm.utils.MemorySizeUtil;
 import com.njcn.dataProcess.api.*;
 import com.njcn.dataProcess.enums.DataCleanEnum;
 import com.njcn.dataProcess.param.LineCountEvaluateParam;
@@ -84,6 +85,7 @@ public class DayDataServiceImpl implements IDayDataService {
             //获取原始数据
             List<CommonMinuteDto> partList = dataVFeignClient.getBaseData(lineParam).getData();
             if (CollUtil.isNotEmpty(partList)) {
+                logger.info("{}dataV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(),MemorySizeUtil.getObjectSize(partList));
                 partList.forEach(item->{
                     //相别
                     List<CommonMinuteDto.PhasicType> phasicTypeList = item.getPhasicTypeList();
@@ -112,6 +114,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataVFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -159,6 +162,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataIFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -207,6 +211,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataFlickerFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -255,6 +260,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataFlucFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -302,6 +308,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataHarmphasicIFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -349,6 +356,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataHarmphasicVFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -396,6 +404,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataHarmpowerPFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -443,6 +452,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataHarmpowerQFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -490,6 +500,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataHarmpowerSFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -537,6 +548,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataHarmRateIFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -584,6 +596,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataHarmRateVFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -631,6 +644,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataInharmIFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -678,6 +692,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataInharmVFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     @Override
@@ -726,6 +741,7 @@ public class DayDataServiceImpl implements IDayDataService {
             dataPltFeignClient.addList(result);
             result.clear();
         }
+        System.gc();
     }
 
     //获取异常数据集合
@@ -740,7 +756,10 @@ public class DayDataServiceImpl implements IDayDataService {
                             Collectors.mapping(item->TimeUtils.LocalDateTimeToString(item.getTime()), Collectors.toList())
                     ));
             lineParam.setAbnormalTime(timeMap);
+
+            pqDataVerifies.clear();
         }
+
     }
 
     //指标处理

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataCrossingServiceImpl.java

@@ -236,6 +236,8 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
 
         }
 
+        System.gc();
+
     }
 
     @Override
@@ -269,6 +271,8 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
         if (CollUtil.isNotEmpty(list)) {
             dataLimitTargetFeignClient.batchInsertion(list);
         }
+
+        System.gc();
     }
 
     @Override

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataIntegrityServiceImpl.java

@@ -95,7 +95,7 @@ public class IDataIntegrityServiceImpl implements IDataIntegrityService {
         if (CollUtil.isNotEmpty(poList)) {
             dataIntegrityFeignClient.batchInsertion(poList);
         }
-
+        System.gc();
     }
 
     @Override

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataOnlineRateServiceImpl.java

@@ -125,6 +125,7 @@ public class IDataOnlineRateServiceImpl implements IDataOnlineRateService {
         if (CollectionUtil.isNotEmpty(list)) {
             onlineRateFeignClient.batchInsertion(list);
         }
+        System.gc();
     }
 
     @Override

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataOrgPointServiceImpl.java

@@ -55,6 +55,7 @@ public class IDataOrgPointServiceImpl implements IDataOrgPointService {
         if(CollUtil.isNotEmpty(info)){
             dataOrgIntegrityFeignClient.batchInsertion(info);
         }
+        System.gc();
     }
 
     @Override

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IEventDetailServiceImpl.java

@@ -45,7 +45,7 @@ import java.util.stream.Collectors;
 public class IEventDetailServiceImpl implements IEventDetailService {
     private static final Logger logger = LoggerFactory.getLogger(IDataOnlineRateServiceImpl.class);
     @Value("${line.num}")
-    private Integer NUM = 100;
+    private Integer NUM = 10;
     @Resource
     private RmpEventDetailFeignClient eventDetailFeignClient;
     @Resource
@@ -88,7 +88,7 @@ public class IEventDetailServiceImpl implements IEventDetailService {
         if(CollUtil.isNotEmpty(result)){
             eventDetailCountFeignClient.batchInsertionD(result);
         }
-
+        System.gc();
     }
 
     public Boolean isData(RmpEventDetailDTO dto, DictData dictData) {

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/PollutionServiceImpl.java

@@ -65,6 +65,7 @@ public class PollutionServiceImpl implements IPollutionService {
 
     @Override
     public void handleDay(CalculatedParam<String> calculatedParam) {
+        System.out.println("当前执行污区监测点算法++++++++++++++++++++++++++++++++++");
         List<DataPollutionD> pollutionList;
         List<DictData> dictDataList = dicDataFeignClient.getDicDataByTypeCode(DicDataTypeEnum.POLLUTION_STATIS.getCode()).getData();
         Map<String, DictData> dictData = dictDataList.stream().collect(Collectors.toMap(DictData::getCode, Function.identity()));

algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/utils/MemorySizeUtil.java

@@ -0,0 +1,22 @@
+package com.njcn.algorithm.utils;
+
+import org.apache.commons.lang.SerializationUtils;
+import org.springframework.stereotype.Component;
+
+import java.lang.instrument.Instrumentation;
+import java.util.List;
+
+/**
+ * @Author: cdf
+ * @CreateTime: 2025-04-08
+ * @Description:
+ */
+
+public class MemorySizeUtil {
+
+    public static double getObjectSize(List<?> list) {
+        byte[] serialized = SerializationUtils.serialize((java.io.Serializable) list);
+        return (double)serialized.length/1024/1024;
+    }
+
+}

algorithm/algorithm-boot/src/main/resources/bootstrap.yml

@@ -31,8 +31,6 @@ spring:
             refresh: true
           - data-Id: share-config-datasource-db.yaml
             refresh: true
-          - data-Id: data-platform.yaml
-            refresh: true
   main:
     allow-bean-definition-overriding: true
 
@@ -58,7 +56,7 @@ mybatis-plus:
   #别名扫描
   type-aliases-package: com.njcn.harmonic.pojo
 line:
-  num: 100
+  num: 10
 
 mqtt:
   client-id: @artifactId@${random.value}

data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/pojo/dto/CommonMinuteDto.java

@@ -14,6 +14,7 @@ import java.util.List;
 @Data
 public class CommonMinuteDto implements Serializable {
 
+
     @JsonFormat(pattern = "yyyy-MM-dd")
     private String time;
 
@@ -26,7 +27,7 @@ public class CommonMinuteDto implements Serializable {
     private List<PhasicType> phasicTypeList;
 
     @Data
-    public static class PhasicType {
+    public static class PhasicType implements Serializable{
 
         @ApiModelProperty("相别,'A'表示A相，'B'表示B相,'C'表示C相,'T'表示总, 'M'表示无相别")
         private String phasicType;
@@ -36,7 +37,7 @@ public class CommonMinuteDto implements Serializable {
 
 
     @Data
-    public static class ValueType {
+    public static class ValueType implements Serializable{
 
         @ApiModelProperty("数据类型 最大值：max、最小值：min、平均值：avg、95值：cp95")
         private String valueType;

data-processing/data-processing-boot/src/main/resources/bootstrap.yml

@@ -32,8 +32,6 @@ spring:
             refresh: true
           - data-Id: share-config-datasource-db.yaml
             refresh: true
-          - data-Id: data-platform.yaml
-            refresh: true
   main:
     allow-bean-definition-overriding: true
   servlet: