算法调整

This commit is contained in:
xy
2025-12-15 19:32:19 +08:00
parent d8d516f63d
commit 386fd1ed1e
2 changed files with 166 additions and 166 deletions

View File

@@ -1357,8 +1357,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (ObjectUtil.isNotNull(dto.getRms())) {
if (dto.getRms() < (pqReasonableRangeDto.getMinValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getRms() > (pqReasonableRangeDto.getMaxValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
if (dto.getRms() < (pqReasonableRangeDto.getMinValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getRms() > (pqReasonableRangeDto.getMaxValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
//log.info("dataV-相电压有效值数据异常,已清洗!数据值:{},数据时间:{}", dto.getRms(), dto.getMinTime());
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
, dto.getMinTime()
@@ -1382,8 +1382,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (ObjectUtil.isNotNull(dto.getVPos())) {
if (dto.getVPos() < (pqReasonableRangeDto.getMinValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getVPos() > (pqReasonableRangeDto.getMaxValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
if (dto.getVPos() < (pqReasonableRangeDto.getMinValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getVPos() > (pqReasonableRangeDto.getMaxValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
//log.info("dataV-正序电压数据异常,已清洗!数据值:{},数据时间:{}", dto.getVPos(), dto.getMinTime());
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
, dto.getMinTime()
@@ -1407,8 +1407,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (ObjectUtil.isNotNull(dto.getVNeg())) {
if (dto.getVNeg() < (pqReasonableRangeDto.getMinValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getVNeg() > (pqReasonableRangeDto.getMaxValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
if (dto.getVNeg() < (pqReasonableRangeDto.getMinValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getVNeg() > (pqReasonableRangeDto.getMaxValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
//log.info("dataV-负序电压数据异常,已清洗!数据值:{},数据时间:{}", dto.getVNeg(), dto.getMinTime());
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
, dto.getMinTime()
@@ -1432,8 +1432,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (ObjectUtil.isNotNull(dto.getVZero())) {
if (dto.getVZero() < (pqReasonableRangeDto.getMinValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getVZero() > (pqReasonableRangeDto.getMaxValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
if (dto.getVZero() < (pqReasonableRangeDto.getMinValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getVZero() > (pqReasonableRangeDto.getMaxValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
//log.info("dataV-零序电压数据异常,已清洗!数据值:{},数据时间:{}", dto.getVZero(), dto.getMinTime());
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
, dto.getMinTime()
@@ -1480,8 +1480,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
pqReasonableRangeDto = map.get(DataCleanEnum.RmsLvr.getCode());
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType()) && ObjectUtil.isNotNull(dto.getRmsLvr())) {
if (dto.getRmsLvr() < (pqReasonableRangeDto.getMinValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getRmsLvr() > (pqReasonableRangeDto.getMaxValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
if (dto.getRmsLvr() < (pqReasonableRangeDto.getMinValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getRmsLvr() > (pqReasonableRangeDto.getMaxValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
//log.info("dataV-线电压有效值数据异常,已清洗!数据值:{},数据时间:{}", dto.getRmsLvr(), dto.getMinTime());
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
, dto.getMinTime()
@@ -1612,8 +1612,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
phaseList = Arrays.asList(pqReasonableRangeDto.getPhaseType().split(","));
if (phaseList.contains(dto.getPhasicType())) {
if (ObjectUtil.isNotNull(dto.getV1())) {
if (dto.getV1() < (pqReasonableRangeDto.getMinValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getV1() > (pqReasonableRangeDto.getMaxValue() * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
if (dto.getV1() < (pqReasonableRangeDto.getMinValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))
|| dto.getV1() > (pqReasonableRangeDto.getMaxValue() / 100.0 * DataCommonUtils.getVoltageData(line.getVoltageLevel()))) {
//log.info("dataV-基波电压数据异常,已清洗!数据值:{},数据时间:{}", dto.getV1(), dto.getMinTime());
PqDataVerify pqDataVerify = getPqDataVerify(dto.getLineId()
, dto.getMinTime()

View File

@@ -106,161 +106,161 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
System.out.println("第一次分析结束-----------------------------------------");
logger.info("limitRate表转r_stat_limit_rate_d算法开始执行日期为{}=====》", calculatedParam.getDataDate());
// List<DataLimitDetailDto> result = new ArrayList<>();
// //远程接口获取分钟数据
// LineCountEvaluateParam lineParam = new LineCountEvaluateParam();
// lineParam.setStartTime(TimeUtils.getBeginOfDay(calculatedParam.getDataDate()));
// lineParam.setEndTime(TimeUtils.getEndOfDay(calculatedParam.getDataDate()));
// lineParam.setType(calculatedParam.getType());
// List<String> lineIds = calculatedParam.getIdList();
// //获取所有监测点的限值
// List<Overlimit> overLimitList = commTerminalGeneralClient.getOverLimitDataByIds(lineIds).getData();
// Map<String, Overlimit> overLimitMap = overLimitList.stream().collect(Collectors.toMap(Overlimit::getId, Function.identity()));
// //添加异常数据时间点
// getAbnormalData(lineParam);
// //以100个监测点分片处理
// List<List<String>> pendingIds = ListUtils.partition(lineIds, NUM);
// ArrayList<String> phase = ListUtil.toList(PhaseType.PHASE_A, PhaseType.PHASE_B, PhaseType.PHASE_C);
// MemorySizeUtil.getNowMemory();
// pendingIds.forEach(list -> {
// lineParam.setLineId(list);
// //获取电压数据
// List<DataVDto> dataVAllTime = dataVFeignClient.getRawData(lineParam).getData();
// //闪变数据
// List<DataPltDto> dataPltAllTime = dataPltFeignClient.getRawData(lineParam).getData();
// //谐波数据
// List<DataHarmDto> dataVHarmList = dataHarmRateVFeignClient.getRawData(lineParam).getData();
// //间谐波数据
// List<DataHarmDto> dataVInHarmList = dataInharmVFeignClient.getRawData(lineParam).getData();
// //电流数据
// List<DataIDto> dataIList = dataIFeignClient.getRawData(lineParam).getData();
// /**
// * 功能描述:获取influxDB -> data_v ->
// * 总计算次数(用data_v中phasic_type=A,value_type=avg,quality_flag=0来参与统计)
// */
// Map<String, List<DataVDto>> allTime = dataVAllTime.stream()
// .filter(x -> PhaseType.PHASE_A.equals(x.getPhasicType()))
List<DataLimitDetailDto> result = new ArrayList<>();
//远程接口获取分钟数据
LineCountEvaluateParam lineParam = new LineCountEvaluateParam();
lineParam.setStartTime(TimeUtils.getBeginOfDay(calculatedParam.getDataDate()));
lineParam.setEndTime(TimeUtils.getEndOfDay(calculatedParam.getDataDate()));
lineParam.setType(calculatedParam.getType());
List<String> lineIds = calculatedParam.getIdList();
//获取所有监测点的限值
List<Overlimit> overLimitList = commTerminalGeneralClient.getOverLimitDataByIds(lineIds).getData();
Map<String, Overlimit> overLimitMap = overLimitList.stream().collect(Collectors.toMap(Overlimit::getId, Function.identity()));
//添加异常数据时间点
getAbnormalData(lineParam);
//以100个监测点分片处理
List<List<String>> pendingIds = ListUtils.partition(lineIds, NUM);
ArrayList<String> phase = ListUtil.toList(PhaseType.PHASE_A, PhaseType.PHASE_B, PhaseType.PHASE_C);
MemorySizeUtil.getNowMemory();
pendingIds.forEach(list -> {
lineParam.setLineId(list);
//获取电压数据
List<DataVDto> dataVAllTime = dataVFeignClient.getRawData(lineParam).getData();
//闪变数据
List<DataPltDto> dataPltAllTime = dataPltFeignClient.getRawData(lineParam).getData();
//谐波数据
List<DataHarmDto> dataVHarmList = dataHarmRateVFeignClient.getRawData(lineParam).getData();
//间谐波数据
List<DataHarmDto> dataVInHarmList = dataInharmVFeignClient.getRawData(lineParam).getData();
//电流数据
List<DataIDto> dataIList = dataIFeignClient.getRawData(lineParam).getData();
/**
* 功能描述:获取influxDB -> data_v ->
* 总计算次数(用data_v中phasic_type=A,value_type=avg,quality_flag=0来参与统计)
*/
Map<String, List<DataVDto>> allTime = dataVAllTime.stream()
.filter(x -> PhaseType.PHASE_A.equals(x.getPhasicType()))
.filter(x -> InfluxDbSqlConstant.AVG_WEB.equalsIgnoreCase(x.getValueType()))
.collect(Collectors.groupingBy(DataVDto::getLineId));
/**
* 功能描述:获取influxDB -> data_plt ->
* 闪变总计算次数(用data_plt中phasic_type=A,value_type=avg,quality_flag=0来参与统计)
*/
//fixme 冀北现场 闪变原始表没有 value_type 这个参数
Map<String, List<DataPltDto>> pltAllTime = dataPltAllTime.stream()
.filter(x -> PhaseType.PHASE_A.equals(x.getPhasicType()))
// .filter(x -> InfluxDbSqlConstant.AVG_WEB.equalsIgnoreCase(x.getValueType()))
// .collect(Collectors.groupingBy(DataVDto::getLineId));
//
// /**
// * 功能描述:获取influxDB -> data_plt ->
// * 闪变总计算次数(用data_plt中phasic_type=A,value_type=avg,quality_flag=0来参与统计)
// */
// //fixme 冀北现场 闪变原始表没有 value_type 这个参数
// Map<String, List<DataPltDto>> pltAllTime = dataPltAllTime.stream()
// .filter(x -> PhaseType.PHASE_A.equals(x.getPhasicType()))
//// .filter(x -> InfluxDbSqlConstant.AVG_WEB.equalsIgnoreCase(x.getValueType()))
// .collect(Collectors.groupingBy(DataPltDto::getLineId));
//
// /**
// *功能描述:获取influxDB -> data_harmrate_v ->
// * 2-25次谐波电压含有率 -> A相||B相||C相的日95%概率值
// */
// Map<String, List<DataHarmDto>> harmRateV = dataVHarmList.stream()
// .filter(x -> phase.contains(x.getPhasicType()))
// .filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()))
// .collect(Collectors.groupingBy(DataHarmDto::getLineId));
//
// /**
// * 功能描述:获取influxDB -> data_i -> 2-25次谐波电流 -> 日95%概率值
// */
// Map<String, List<DataIDto>> dataI = dataIList.stream()
// .filter(x -> phase.contains(x.getPhasicType()))
// .filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()))
// .collect(Collectors.groupingBy(DataIDto::getLineId));
//
//
// /**
// * 功能描述:获取influxDB -> data_inharm_v -> 0.5-15.5次间谐波电压含有率 -> 日95%概率值
// */
// Map<String, List<DataHarmDto>> inHarmV = dataVInHarmList.stream()
// .filter(x -> phase.contains(x.getPhasicType()))
// .filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()))
// .collect(Collectors.groupingBy(DataHarmDto::getLineId));
//
// /**
// * 功能描述:获取influxDB -> data_v -> 电压总谐波畸变率 -> 日95%概率值
// */
// Map<String, List<DataVDto>> dataVThd = dataVAllTime.stream()
// .filter(x -> phase.contains(x.getPhasicType()))
// .filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()))
// .collect(Collectors.groupingBy(DataVDto::getLineId));
//
// /**
// * 功能描述:获取influxDB -> data_v -> 负序电压不平衡度 -> 最大值 && 日95%概率值
// */
// Map<String, List<DataVDto>> dataVUnbalance = dataVAllTime.stream()
// .filter(x -> InfluxDBTableConstant.PHASE_TYPE_T.equals(x.getPhasicType()))
// .filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()) ||
// InfluxDBTableConstant.MAX.equals(x.getValueType()))
// .collect(Collectors.groupingBy(DataVDto::getLineId));
//
// /**
// * 功能描述:获取influxDB -> data_i -> 负序电流 -> 最大值 && 日95%概率值
// */
// Map<String, List<DataIDto>> dataINeg = dataIList.stream()
// .filter(x -> InfluxDBTableConstant.PHASE_TYPE_T.equals(x.getPhasicType()))
// .filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()) ||
// InfluxDBTableConstant.MAX.equals(x.getValueType()))
// .collect(Collectors.groupingBy(DataIDto::getLineId));
//
// /**
// * 功能描述:获取influxDB -> data_v -> 频率偏差 -> 最大值 && 最小值
// */
// Map<String, List<DataVDto>> dataVFreq = dataVAllTime.stream()
// .filter(x -> InfluxDBTableConstant.PHASE_TYPE_T.equals(x.getPhasicType()))
// .filter(x -> InfluxDBTableConstant.MIN.equals(x.getValueType()) ||
// InfluxDBTableConstant.MAX.equals(x.getValueType()))
// .collect(Collectors.groupingBy(DataVDto::getLineId));
// /**
// * 功能描述:获取influxDB -> data_v -> 电压偏差 -> 最大值
// */
// Map<String, List<DataVDto>> dataVDev = dataVAllTime.stream()
// .filter(x -> phase.contains(x.getPhasicType()))
// .filter(x -> InfluxDBTableConstant.MAX.equals(x.getValueType()))
// .collect(Collectors.groupingBy(DataVDto::getLineId));
//
// /**
// * 功能描述:获取influxDB -> data_plt -> 长时间闪变 -> 注(取最大值原始算法去掉了,现没有根据最大值比较)
// */
// Map<String, List<DataPltDto>> dataPlt = dataPltAllTime.stream()
// .filter(x -> phase.contains(x.getPhasicType()))
// .collect(Collectors.groupingBy(DataPltDto::getLineId));
//
// for (String item : list) {
// if(ObjectUtil.isNotNull(overLimitMap.get(item))){
// result.addAll(getData(calculatedParam.getDataDate(),
// overLimitMap.get(item),
// allTime.get(item),
// pltAllTime.get(item),
// harmRateV.get(item),
// dataI.get(item),
// inHarmV.get(item),
// dataVThd.get(item),
// dataVUnbalance.get(item),
// dataINeg.get(item),
// dataVFreq.get(item),
// dataVDev.get(item),
// dataPlt.get(item)));
// }
// }
// });
// MemorySizeUtil.getNowMemory();
// if (CollUtil.isNotEmpty(result)) {
// //存储数据
// List<DataLimitRateDto> dataLimitRate = result.stream().map(DataLimitDetailDto::getDataLimitRate).filter(ObjectUtil::isNotNull).collect(Collectors.toList());
// if(CollUtil.isNotEmpty(dataLimitRate)){
// dataLimitRateFeignClient.batchInsertion(dataLimitRate);
// }
// }
// if (CollUtil.isNotEmpty(result)) {
// //存储数据
// List<DataLimitRateDetailDto> detail = result.stream().map(DataLimitDetailDto::getDataLimitRateDetail).filter(x -> ObjectUtil.isNotNull(x)).collect(Collectors.toList());
// if(CollUtil.isNotEmpty(detail)) {
// dataLimitRateDetailFeignClient.batchInsertion(detail);
// }
// }
.collect(Collectors.groupingBy(DataPltDto::getLineId));
/**
*功能描述:获取influxDB -> data_harmrate_v ->
* 2-25次谐波电压含有率 -> A相||B相||C相的日95%概率值
*/
Map<String, List<DataHarmDto>> harmRateV = dataVHarmList.stream()
.filter(x -> phase.contains(x.getPhasicType()))
.filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()))
.collect(Collectors.groupingBy(DataHarmDto::getLineId));
/**
* 功能描述:获取influxDB -> data_i -> 2-25次谐波电流 -> 日95%概率值
*/
Map<String, List<DataIDto>> dataI = dataIList.stream()
.filter(x -> phase.contains(x.getPhasicType()))
.filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()))
.collect(Collectors.groupingBy(DataIDto::getLineId));
/**
* 功能描述:获取influxDB -> data_inharm_v -> 0.5-15.5次间谐波电压含有率 -> 日95%概率值
*/
Map<String, List<DataHarmDto>> inHarmV = dataVInHarmList.stream()
.filter(x -> phase.contains(x.getPhasicType()))
.filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()))
.collect(Collectors.groupingBy(DataHarmDto::getLineId));
/**
* 功能描述:获取influxDB -> data_v -> 电压总谐波畸变率 -> 日95%概率值
*/
Map<String, List<DataVDto>> dataVThd = dataVAllTime.stream()
.filter(x -> phase.contains(x.getPhasicType()))
.filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()))
.collect(Collectors.groupingBy(DataVDto::getLineId));
/**
* 功能描述:获取influxDB -> data_v -> 负序电压不平衡度 -> 最大值 && 日95%概率值
*/
Map<String, List<DataVDto>> dataVUnbalance = dataVAllTime.stream()
.filter(x -> InfluxDBTableConstant.PHASE_TYPE_T.equals(x.getPhasicType()))
.filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()) ||
InfluxDBTableConstant.MAX.equals(x.getValueType()))
.collect(Collectors.groupingBy(DataVDto::getLineId));
/**
* 功能描述:获取influxDB -> data_i -> 负序电流 -> 最大值 && 日95%概率值
*/
Map<String, List<DataIDto>> dataINeg = dataIList.stream()
.filter(x -> InfluxDBTableConstant.PHASE_TYPE_T.equals(x.getPhasicType()))
.filter(x -> InfluxDBTableConstant.CP95.equals(x.getValueType()) ||
InfluxDBTableConstant.MAX.equals(x.getValueType()))
.collect(Collectors.groupingBy(DataIDto::getLineId));
/**
* 功能描述:获取influxDB -> data_v -> 频率偏差 -> 最大值 && 最小值
*/
Map<String, List<DataVDto>> dataVFreq = dataVAllTime.stream()
.filter(x -> InfluxDBTableConstant.PHASE_TYPE_T.equals(x.getPhasicType()))
.filter(x -> InfluxDBTableConstant.MIN.equals(x.getValueType()) ||
InfluxDBTableConstant.MAX.equals(x.getValueType()))
.collect(Collectors.groupingBy(DataVDto::getLineId));
/**
* 功能描述:获取influxDB -> data_v -> 电压偏差 -> 最大值
*/
Map<String, List<DataVDto>> dataVDev = dataVAllTime.stream()
.filter(x -> phase.contains(x.getPhasicType()))
.filter(x -> InfluxDBTableConstant.MAX.equals(x.getValueType()))
.collect(Collectors.groupingBy(DataVDto::getLineId));
/**
* 功能描述:获取influxDB -> data_plt -> 长时间闪变 -> 注(取最大值原始算法去掉了,现没有根据最大值比较)
*/
Map<String, List<DataPltDto>> dataPlt = dataPltAllTime.stream()
.filter(x -> phase.contains(x.getPhasicType()))
.collect(Collectors.groupingBy(DataPltDto::getLineId));
for (String item : list) {
if(ObjectUtil.isNotNull(overLimitMap.get(item))){
result.addAll(getData(calculatedParam.getDataDate(),
overLimitMap.get(item),
allTime.get(item),
pltAllTime.get(item),
harmRateV.get(item),
dataI.get(item),
inHarmV.get(item),
dataVThd.get(item),
dataVUnbalance.get(item),
dataINeg.get(item),
dataVFreq.get(item),
dataVDev.get(item),
dataPlt.get(item)));
}
}
});
MemorySizeUtil.getNowMemory();
if (CollUtil.isNotEmpty(result)) {
//存储数据
List<DataLimitRateDto> dataLimitRate = result.stream().map(DataLimitDetailDto::getDataLimitRate).filter(ObjectUtil::isNotNull).collect(Collectors.toList());
if(CollUtil.isNotEmpty(dataLimitRate)){
dataLimitRateFeignClient.batchInsertion(dataLimitRate);
}
}
if (CollUtil.isNotEmpty(result)) {
//存储数据
List<DataLimitRateDetailDto> detail = result.stream().map(DataLimitDetailDto::getDataLimitRateDetail).filter(x -> ObjectUtil.isNotNull(x)).collect(Collectors.toList());
if(CollUtil.isNotEmpty(detail)) {
dataLimitRateDetailFeignClient.batchInsertion(detail);
}
}
System.gc();
}