Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

现场算法测试优化

Browse Source
This commit is contained in:
xy
2025-05-19 09:52:02 +08:00
parent 272c723775
commit 228b694322
46 changed files with 166 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataCleanServiceImpl.java
View File

@@ -72,8 +72,6 @@ public class DataCleanServiceImpl implements IDataCleanService {
@Override
public void dataVCleanHandler(CalculatedParam calculatedParam) {
MemorySizeUtil.getNowMemory();
logger.info("{},dataV表异常数据算法执行=====》", LocalDateTime.now());
List<PqDataVerify> result = new ArrayList<>();
//获取标准
@@ -105,6 +103,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList = null;
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -135,6 +134,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataIDto> partList = dataIFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataI集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataI(map, lineMap.get(item.getLineId()),item);
@@ -144,6 +144,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList = null;
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -168,6 +169,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataPltDto> partList = dataPltFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataPlt集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataPlt(map,item);
@@ -177,6 +179,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList = null;
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -201,6 +204,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataHarmDto> partList = dataInharmVFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataInHarmV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataInHarmV(map,item);
@@ -210,6 +214,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList = null;
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -234,6 +239,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataHarmDto> partList = dataHarmRateVFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataHarmRateV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataHarmRateV(map,item);
@@ -243,6 +249,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList = null;
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -267,6 +274,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataPowerPDto> partList = dataHarmpowerPFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataHarmPowerP集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataHarmPowerP(map,item);
@@ -276,6 +284,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList = null;
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -300,6 +309,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataHarmDto> partList = dataHarmphasicVFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataHarmPhasicV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataHarmPhasicV(map,item);
@@ -309,6 +319,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList.clear();
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -333,6 +344,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataFlucDto> partList = dataFlucFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataFluc集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataFluc(map,item);
@@ -342,6 +354,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList = null;
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -366,6 +379,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataFlickerDto> partList = dataFlickerFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataFlicker集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataFlicker(map,item);
@@ -375,6 +389,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
partList = null;
});
if (CollUtil.isNotEmpty(result)) {
logger.info("本次清洗数据量为>>>>>>>>>>>>:{}", result.size());
pqDataVerifyFeignClient.insertData(result);
result.clear();
}
@@ -469,25 +484,25 @@ public class DataCleanServiceImpl implements IDataCleanService {
}
//正序电压
pqReasonableRangeDto = map.get(DataCleanEnum.VPos.getCode());
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (dto.getVPos() < (pqReasonableRangeDto.getMinValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getVPos() > (pqReasonableRangeDto.getMaxValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
//log.info("dataV-正序电压数据异常,已清洗!数据值:{},数据时间:{}", dto.getVPos(), dto.getMinTime());
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
,dto.getMinTime()
,dto.getValueType()
,dto.getPhasicType()
,pqReasonableRangeDto.getIndexCode()
,pqReasonableRangeDto.getIndexName()
,pqReasonableRangeDto.getInfluxdbTableName()
,dto.getVPos()
,pqReasonableRangeDto.getMinValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel())
,pqReasonableRangeDto.getMaxValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel()));
list.add(pqDataVerify);
}
}
// pqReasonableRangeDto = map.get(DataCleanEnum.VPos.getCode());
// phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
// if (phaseList.contains(dto.getPhasicType())) {
// if (dto.getVPos() < (pqReasonableRangeDto.getMinValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel()))
// || dto.getVPos() > (pqReasonableRangeDto.getMaxValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
// //log.info("dataV-正序电压数据异常,已清洗!数据值:{},数据时间:{}", dto.getVPos(), dto.getMinTime());
// PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
// ,dto.getMinTime()
// ,dto.getValueType()
// ,dto.getPhasicType()
// ,pqReasonableRangeDto.getIndexCode()
// ,pqReasonableRangeDto.getIndexName()
// ,pqReasonableRangeDto.getInfluxdbTableName()
// ,dto.getVPos()
// ,pqReasonableRangeDto.getMinValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel())
// ,pqReasonableRangeDto.getMaxValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel()));
// list.add(pqDataVerify);
// }
// }
//负序电压
pqReasonableRangeDto = map.get(DataCleanEnum.VNeg.getCode());
@@ -554,7 +569,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
//线电压有效值
pqReasonableRangeDto = map.get(DataCleanEnum.RmsLvr.getCode());
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (phaseList.contains(dto.getPhasicType()) && ObjectUtil.isNotNull(dto.getRmsLvr())) {
if (dto.getRmsLvr() < (pqReasonableRangeDto.getMinValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getRmsLvr() > (pqReasonableRangeDto.getMaxValue()*DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
//log.info("dataV-线电压有效值数据异常,已清洗!数据值:{},数据时间:{}", dto.getRmsLvr(), dto.getMinTime());
@@ -706,11 +721,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
List<PqDataVerify> list = new ArrayList<>();
PqReasonableRangeDto pqReasonableRangeDto;
List<String> phaseList;
//线电压有效值
//电流有效值
pqReasonableRangeDto = map.get(DataCleanEnum.RmsI.getCode());
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (dto.getRms() > line.getCT1()) {
if (dto.getRms() >= line.getCT1() * 1.2) {
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
,dto.getMinTime()
,dto.getValueType()

5
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DayDataServiceImpl.java
View File

@@ -1597,6 +1597,11 @@ public class DayDataServiceImpl implements IDayDataService {
break;
}
}
//这里对数据进行判断，如果值过大，都超出数据库存储范围会报错，无法插入，所以这里设置门槛 上下限都为1千万
if (Objects.nonNull(result)) {
result = Math.min(Math.max(result, -9999999.0), 9999999.0);
}
return result;
}

17
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataCrossingServiceImpl.java
View File

@@ -82,6 +82,23 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
@Override
public void limitRateHandler(CalculatedParam calculatedParam) {
MemorySizeUtil.getNowMemory();
Runtime runtime = Runtime.getRuntime();
// 获取 JVM 最大可用内存（以字节为单位）
long maxMemory = runtime.maxMemory();
// 获取 JVM 当前已分配的内存（以字节为单位）
long totalMemory = runtime.totalMemory();
// 获取 JVM 当前空闲内存（以字节为单位）
long freeMemory = runtime.freeMemory();
// 计算已使用的内存
long usedMemory = totalMemory - freeMemory;
System.out.println("最大可用内存: " + maxMemory / (1024 * 1024) + " MB");
System.out.println("当前已分配的内存: " + totalMemory / (1024 * 1024) + " MB");
System.out.println("当前空闲内存: " + freeMemory / (1024 * 1024) + " MB");
System.out.println("已使用的内存: " + usedMemory / (1024 * 1024) + " MB");
System.out.println("第一次分析结束-----------------------------------------");
logger.info("{},limitRate表转r_stat_limit_rate_d算法开始=====》", LocalDateTime.now());
List<DataLimitDetailDto> result = new ArrayList<>();
//远程接口获取分钟数据

10
algorithm/algorithm-boot/src/main/resources/bootstrap.yml
View File

@@ -13,6 +13,16 @@ server:
feign:
sentinel:
enabled: true
compression:
request:
enabled: true
# 配置压缩支持的MIME TYPE
mime-types: text/xml,application/xml,application/json
# 配置压缩数据大小的下限
min-request-size: 1024
response:
# 配置响应GZIP压缩
enabled: true
spring:
application:
name: @artifactId@