Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

算法调整提交

Browse Source
This commit is contained in:
chendaofei
2025-04-09 17:53:43 +08:00
parent 2c12f4966a
commit be83c07215
15 changed files with 106 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/ExecutionCenter.java
View File

@@ -2,10 +2,7 @@ package com.njcn.algorithm;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.*;
import cn.hutool.core.text.StrPool;
import cn.hutool.core.util.StrUtil;
import com.njcn.algorithm.pojo.bo.BaseParam;
@@ -54,6 +51,8 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
public class ExecutionCenter extends BaseController {
private Long calTime = System.currentTimeMillis();
private static final Logger logger = LoggerFactory.getLogger(ExecutionCenter.class);
@Resource
@@ -95,7 +94,23 @@ public class ExecutionCenter extends BaseController {
*/
private void dealResponse(CalculatedParam calculatedParam, LiteflowResponse liteflowResponse, String methodDescribe) {
if (liteflowResponse.isSuccess()) {
Runtime runtime = Runtime.getRuntime();
// 获取 JVM 最大可用内存（以字节为单位）
long maxMemory = runtime.maxMemory();
// 获取 JVM 当前已分配的内存（以字节为单位）
long totalMemory = runtime.totalMemory();
// 获取 JVM 当前空闲内存（以字节为单位）
long freeMemory = runtime.freeMemory();
// 计算已使用的内存
long usedMemory = totalMemory - freeMemory;
System.out.println("最大可用内存: " + maxMemory / (1024 * 1024) + " MB");
System.out.println("当前已分配的内存: " + totalMemory / (1024 * 1024) + " MB");
System.out.println("当前空闲内存: " + freeMemory / (1024 * 1024) + " MB");
System.out.println("已使用的内存: " + usedMemory / (1024 * 1024) + " MB");
System.out.println("算法执行成功-----------------------------------------");
logger.info("日期{}，{}执行{}成功", calculatedParam.getDataDate(), methodDescribe, calculatedParam.isFullChain() ? "全链" : "指定节点：".concat(String.join(StrPool.COMMA, calculatedParam.getTagNames())));
logger.info("当前算法执行总时长为-------------------------------------{}",(System.currentTimeMillis()-calTime)/1000/60);
} else {
Map<String, List<CmpStep>> executeSteps = liteflowResponse.getExecuteSteps();
CmpStep failStep = null;
@@ -148,6 +163,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("measurement_point", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -182,6 +198,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("special_analysis", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -225,6 +242,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("measurement_point_hour", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -265,6 +283,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("device", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -303,7 +322,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("org_point", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
//
@@ -391,7 +410,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("sub_station", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
//
// /**

25
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataCleanServiceImpl.java
View File

@@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.ObjectUtil;
import com.njcn.algorithm.pojo.bo.CalculatedParam;
import com.njcn.algorithm.service.line.IDataCleanService;
import com.njcn.algorithm.utils.MemorySizeUtil;
import com.njcn.dataProcess.api.*;
import com.njcn.dataProcess.enums.DataCleanEnum;
import com.njcn.dataProcess.param.DataCleanParam;
@@ -108,6 +109,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataVDto> partList = dataVFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataV(map, lineMap.get(item.getLineId()),item);
@@ -120,6 +122,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@Override
@@ -145,6 +149,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataIDto> partList = dataIFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataI集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataI(map, lineMap.get(item.getLineId()),item);
@@ -157,6 +162,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@Override
@@ -188,6 +195,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@Override
@@ -207,6 +216,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataHarmDto> partList = dataInharmVFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataInharmV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataInHarmV(map,item);
@@ -219,6 +230,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@Override
@@ -250,6 +263,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@Override
@@ -281,6 +296,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@Override
@@ -312,6 +329,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@Override
@@ -343,6 +362,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@Override
@@ -374,6 +395,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
System.gc();
}
@@ -704,7 +727,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqReasonableRangeDto = map.get(DataCleanEnum.RmsI.getCode());
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (dto.getRms() <= line.getCT1()) {
if (dto.getRms() > line.getCT1()) {
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
,dto.getMinTime()
,dto.getValueType()

2
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataComAssServiceImpl.java
View File

@@ -108,7 +108,7 @@ public class DataComAssServiceImpl implements IDataComAssService {
if (CollUtil.isNotEmpty(list)) {
dataComAssFeignClient.batchInsertion(list);
}
System.gc();
}
/**

1
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataVCvtServiceImpl.java
View File

@@ -84,6 +84,7 @@ public class DataVCvtServiceImpl implements IDataHarmRateVCvtService {
});
dataHarmRateVFeignClient.addList(result);
System.gc();
}
private List<DataHarmRateVCvtDto> cvtChange(List<DataHarmRateVCvtDto> v, CvtHarmonicCorrectionFactorsDTO cvtHarmonicCorrectionFactorsDTO) {

19
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DayDataServiceImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.algorithm.serviceimpl.line;
import cn.hutool.core.collection.CollUtil;
import com.njcn.algorithm.pojo.bo.CalculatedParam;
import com.njcn.algorithm.service.line.IDayDataService;
import com.njcn.algorithm.utils.MemorySizeUtil;
import com.njcn.dataProcess.api.*;
import com.njcn.dataProcess.enums.DataCleanEnum;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
@@ -84,6 +85,7 @@ public class DayDataServiceImpl implements IDayDataService {
//获取原始数据
List<CommonMinuteDto> partList = dataVFeignClient.getBaseData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}dataV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(),MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//相别
List<CommonMinuteDto.PhasicType> phasicTypeList = item.getPhasicTypeList();
@@ -112,6 +114,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataVFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -159,6 +162,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataIFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -207,6 +211,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataFlickerFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -255,6 +260,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataFlucFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -302,6 +308,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataHarmphasicIFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -349,6 +356,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataHarmphasicVFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -396,6 +404,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataHarmpowerPFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -443,6 +452,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataHarmpowerQFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -490,6 +500,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataHarmpowerSFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -537,6 +548,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataHarmRateIFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -584,6 +596,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataHarmRateVFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -631,6 +644,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataInharmIFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -678,6 +692,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataInharmVFeignClient.addList(result);
result.clear();
}
System.gc();
}
@Override
@@ -726,6 +741,7 @@ public class DayDataServiceImpl implements IDayDataService {
dataPltFeignClient.addList(result);
result.clear();
}
System.gc();
}
//获取异常数据集合
@@ -740,7 +756,10 @@ public class DayDataServiceImpl implements IDayDataService {
Collectors.mapping(item->TimeUtils.LocalDateTimeToString(item.getTime()), Collectors.toList())
));
lineParam.setAbnormalTime(timeMap);
pqDataVerifies.clear();
}
}
//指标处理

4
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataCrossingServiceImpl.java
View File

@@ -236,6 +236,8 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
}
System.gc();
}
@Override
@@ -269,6 +271,8 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
if (CollUtil.isNotEmpty(list)) {
dataLimitTargetFeignClient.batchInsertion(list);
}
System.gc();
}
@Override

2
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataIntegrityServiceImpl.java
View File

@@ -95,7 +95,7 @@ public class IDataIntegrityServiceImpl implements IDataIntegrityService {
if (CollUtil.isNotEmpty(poList)) {
dataIntegrityFeignClient.batchInsertion(poList);
}
System.gc();
}
@Override

1
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataOnlineRateServiceImpl.java
View File

@@ -125,6 +125,7 @@ public class IDataOnlineRateServiceImpl implements IDataOnlineRateService {
if (CollectionUtil.isNotEmpty(list)) {
onlineRateFeignClient.batchInsertion(list);
}
System.gc();
}
@Override

1
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataOrgPointServiceImpl.java
View File

@@ -55,6 +55,7 @@ public class IDataOrgPointServiceImpl implements IDataOrgPointService {
if(CollUtil.isNotEmpty(info)){
dataOrgIntegrityFeignClient.batchInsertion(info);
}
System.gc();
}
@Override

4
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IEventDetailServiceImpl.java
View File

@@ -45,7 +45,7 @@ import java.util.stream.Collectors;
public class IEventDetailServiceImpl implements IEventDetailService {
private static final Logger logger = LoggerFactory.getLogger(IDataOnlineRateServiceImpl.class);
@Value("${line.num}")
private Integer NUM = 100;
private Integer NUM = 10;
@Resource
private RmpEventDetailFeignClient eventDetailFeignClient;
@Resource
@@ -88,7 +88,7 @@ public class IEventDetailServiceImpl implements IEventDetailService {
if(CollUtil.isNotEmpty(result)){
eventDetailCountFeignClient.batchInsertionD(result);
}
System.gc();
}
public Boolean isData(RmpEventDetailDTO dto, DictData dictData) {

1
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/PollutionServiceImpl.java
View File

@@ -65,6 +65,7 @@ public class PollutionServiceImpl implements IPollutionService {
@Override
public void handleDay(CalculatedParam<String> calculatedParam) {
System.out.println("当前执行污区监测点算法++++++++++++++++++++++++++++++++++");
List<DataPollutionD> pollutionList;
List<DictData> dictDataList = dicDataFeignClient.getDicDataByTypeCode(DicDataTypeEnum.POLLUTION_STATIS.getCode()).getData();
Map<String, DictData> dictData = dictDataList.stream().collect(Collectors.toMap(DictData::getCode, Function.identity()));

22
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/utils/MemorySizeUtil.java Normal file
View File

@@ -0,0 +1,22 @@
package com.njcn.algorithm.utils;
import org.apache.commons.lang.SerializationUtils;
import org.springframework.stereotype.Component;
import java.lang.instrument.Instrumentation;
import java.util.List;
/**
* @Author: cdf
* @CreateTime: 2025-04-08
* @Description:
*/
public class MemorySizeUtil {
public static double getObjectSize(List<?> list) {
byte[] serialized = SerializationUtils.serialize((java.io.Serializable) list);
return (double)serialized.length/1024/1024;
}
}

4
algorithm/algorithm-boot/src/main/resources/bootstrap.yml
View File

@@ -31,8 +31,6 @@ spring:
refresh: true
- data-Id: share-config-datasource-db.yaml
refresh: true
- data-Id: data-platform.yaml
refresh: true
main:
allow-bean-definition-overriding: true
@@ -58,7 +56,7 @@ mybatis-plus:
#别名扫描
type-aliases-package: com.njcn.harmonic.pojo
line:
num: 100
num: 10
mqtt:
client-id: @artifactId@${random.value}

5
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/pojo/dto/CommonMinuteDto.java
View File

@@ -14,6 +14,7 @@ import java.util.List;
@Data
public class CommonMinuteDto implements Serializable {
@JsonFormat(pattern = "yyyy-MM-dd")
private String time;
@@ -26,7 +27,7 @@ public class CommonMinuteDto implements Serializable {
private List<PhasicType> phasicTypeList;
@Data
public static class PhasicType {
public static class PhasicType implements Serializable{
@ApiModelProperty("相别,'A'表示A相，'B'表示B相,'C'表示C相,'T'表示总, 'M'表示无相别")
private String phasicType;
@@ -36,7 +37,7 @@ public class CommonMinuteDto implements Serializable {
@Data
public static class ValueType {
public static class ValueType implements Serializable{
@ApiModelProperty("数据类型 最大值：max、最小值：min、平均值：avg、95值：cp95")
private String valueType;

2
data-processing/data-processing-boot/src/main/resources/bootstrap.yml
View File

@@ -32,8 +32,6 @@ spring:
refresh: true
- data-Id: share-config-datasource-db.yaml
refresh: true
- data-Id: data-platform.yaml
refresh: true
main:
allow-bean-definition-overriding: true
servlet: