Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

算法优化

Browse Source
This commit is contained in:
xy
2025-07-07 16:12:47 +08:00
parent 922ea002cd
commit 85d6b843ac
1 changed files with 33 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataCleanServiceImpl.java
View File

@@ -51,7 +51,6 @@ import java.util.stream.Collectors;
/** /**
* @author xy * @author xy
*/ */
@Slf4j
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class DataCleanServiceImpl implements IDataCleanService { public class DataCleanServiceImpl implements IDataCleanService {
@@ -59,7 +58,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
private static final Logger logger = LoggerFactory.getLogger(DataCleanServiceImpl.class); private static final Logger logger = LoggerFactory.getLogger(DataCleanServiceImpl.class);
private final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault()); private final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault());
@Value("${line.num}") @Value("${line.num}")
private Integer NUM = 100; private Integer NUM = 100;
@@ -148,7 +147,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
param.setEndTime(endTime); param.setEndTime(endTime);
param.setDataType(false); param.setDataType(false);
log.info("监测点：{},数据剔除初始时间:{},结束时间:{}", item.getMeasurementPointId(),startTime,endTime); logger.info("监测点：{},数据剔除初始时间:{},结束时间:{}", item.getMeasurementPointId(),startTime,endTime);
//修改dataV数据 //修改dataV数据
List<DataVDto> dataV = dataVFeignClient.getRawData(param).getData(); List<DataVDto> dataV = dataVFeignClient.getRawData(param).getData();
@@ -157,7 +156,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataVFeignClient.addInfluxDbList(dataV); dataVFeignClient.addInfluxDbList(dataV);
log.info("修改dataV数据量:{}", dataV.size()); logger.info("修改dataV数据量:{}", dataV.size());
} }
//修改dataI数据 //修改dataI数据
@@ -167,7 +166,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataIFeignClient.addInfluxDbList(dataI); dataIFeignClient.addInfluxDbList(dataI);
log.info("修改dataI数据量:{}", dataI.size()); logger.info("修改dataI数据量:{}", dataI.size());
} }
//修改dataFlicker数据 //修改dataFlicker数据
@@ -177,7 +176,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataFlickerFeignClient.addInfluxDbList(dataFlicker); dataFlickerFeignClient.addInfluxDbList(dataFlicker);
log.info("修改dataFlicker数据量:{}", dataFlicker.size()); logger.info("修改dataFlicker数据量:{}", dataFlicker.size());
} }
//修改dataFluc数据 //修改dataFluc数据
@@ -187,7 +186,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataFlucFeignClient.addInfluxDbList(dataFluc); dataFlucFeignClient.addInfluxDbList(dataFluc);
log.info("修改dataFluc数据量:{}", dataFluc.size()); logger.info("修改dataFluc数据量:{}", dataFluc.size());
} }
//修改dataHarmPhasicI数据 //修改dataHarmPhasicI数据
@@ -197,7 +196,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataHarmphasicIFeignClient.addInfluxDbList(dataHarmI); dataHarmphasicIFeignClient.addInfluxDbList(dataHarmI);
log.info("修改dataHarmI数据量:{}", dataHarmI.size()); logger.info("修改dataHarmI数据量:{}", dataHarmI.size());
} }
//修改dataHarmPhasicV数据 //修改dataHarmPhasicV数据
@@ -207,7 +206,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataHarmphasicVFeignClient.addInfluxDbList(DataHarmV); dataHarmphasicVFeignClient.addInfluxDbList(DataHarmV);
log.info("修改dataHarmPhasicV数据量:{}", DataHarmV.size()); logger.info("修改dataHarmPhasicV数据量:{}", DataHarmV.size());
} }
//修改dataHarmPowerP数据 //修改dataHarmPowerP数据
@@ -217,7 +216,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataHarmpowerPFeignClient.addInfluxDbList(dataPowerP); dataHarmpowerPFeignClient.addInfluxDbList(dataPowerP);
log.info("dataHarmPowerP数据量:{}", dataPowerP.size()); logger.info("dataHarmPowerP数据量:{}", dataPowerP.size());
} }
//修改dataHarmPowerQ数据 //修改dataHarmPowerQ数据
@@ -227,7 +226,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataHarmpowerQFeignClient.addInfluxDbList(dataHarmPowerQ); dataHarmpowerQFeignClient.addInfluxDbList(dataHarmPowerQ);
log.info("修改dataHarmPowerQ数据量:{}", dataHarmPowerQ.size()); logger.info("修改dataHarmPowerQ数据量:{}", dataHarmPowerQ.size());
} }
//修改dataHarmPowerS数据 //修改dataHarmPowerS数据
@@ -237,7 +236,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataHarmpowerSFeignClient.addInfluxDbList(dataHarmPowerS); dataHarmpowerSFeignClient.addInfluxDbList(dataHarmPowerS);
log.info("修改dataHarmPowerS数据量:{}", dataHarmPowerS.size()); logger.info("修改dataHarmPowerS数据量:{}", dataHarmPowerS.size());
} }
//修改dataHarmRateI数据 //修改dataHarmRateI数据
@@ -247,7 +246,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataHarmRateIFeignClient.addInfluxDbList(dataHarmRateI); dataHarmRateIFeignClient.addInfluxDbList(dataHarmRateI);
log.info("修改dataHarmRateI数据量:{}", dataHarmRateI.size()); logger.info("修改dataHarmRateI数据量:{}", dataHarmRateI.size());
} }
//修改dataHarmRateV数据 //修改dataHarmRateV数据
@@ -257,7 +256,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataHarmRateVFeignClient.addInfluxDbList(DataHarmRateV); dataHarmRateVFeignClient.addInfluxDbList(DataHarmRateV);
log.info("修改dataHarmRateV数据量:{}", DataHarmRateV.size()); logger.info("修改dataHarmRateV数据量:{}", DataHarmRateV.size());
} }
//修改dataInHarmI数据 //修改dataInHarmI数据
@@ -267,7 +266,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataInharmIFeignClient.addInfluxDbList(dataInHarmI); dataInharmIFeignClient.addInfluxDbList(dataInHarmI);
log.info("修改dataInHarmI数据量:{}", dataInHarmI.size()); logger.info("修改dataInHarmI数据量:{}", dataInHarmI.size());
} }
//修改dataInHarmV数据 //修改dataInHarmV数据
@@ -277,7 +276,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataInharmVFeignClient.addInfluxDbList(dataInHarmV); dataInharmVFeignClient.addInfluxDbList(dataInHarmV);
log.info("修改dataInHarmV数据量:{}", dataInHarmV.size()); logger.info("修改dataInHarmV数据量:{}", dataInHarmV.size());
} }
//修改dataPlt数据 //修改dataPlt数据
@@ -287,7 +286,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
item1.setAbnormalFlag(1); item1.setAbnormalFlag(1);
}); });
dataPltFeignClient.addInfluxDbList(dataPlt); dataPltFeignClient.addInfluxDbList(dataPlt);
log.info("修改dataPlt数据量:{}", dataPlt.size()); logger.info("修改dataPlt数据量:{}", dataPlt.size());
} }
}); });
} }
@@ -312,7 +311,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineDetail.forEach(item->{ lineDetail.forEach(item->{
List<Map<String,Object>> resultData = new ArrayList<>(); List<Map<String,Object>> resultData = new ArrayList<>();
Set<String> allTimeSet = new HashSet<>();
PqDataVerifyBak bak = new PqDataVerifyBak(); PqDataVerifyBak bak = new PqDataVerifyBak();
bak.setLineId(item.getLineId()); bak.setLineId(item.getLineId());
bak.setTime(LocalDate.parse(calculatedParam.getDataDate(), DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN))); bak.setTime(LocalDate.parse(calculatedParam.getDataDate(), DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
@@ -349,6 +348,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleDataV(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataV(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -368,6 +368,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleDataI(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataI(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -387,6 +388,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleDataPlt(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataPlt(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -406,6 +408,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleInHarmV(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleInHarmV(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -425,6 +428,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleHarmRateV(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleHarmRateV(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -444,6 +448,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleHarmPowerP(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleHarmPowerP(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -463,6 +468,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleHarmPhasicV(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleHarmPhasicV(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -482,6 +488,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleDataFluc(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataFluc(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -501,11 +508,19 @@ public class DataCleanServiceImpl implements IDataCleanService {
Map<String,Object> dtoMap = handleDataFlicker(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataFlicker(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet())); Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval()); dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
allTimeSet.addAll(timeSet);
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
} }
} }
if (CollUtil.isNotEmpty(allTimeSet)) {
Map<String,Object> dtoMap = new HashMap<>();
dtoMap.put("lineErrorTimes",allTimeSet.size() * item.getTimeInterval());
resultData.add(dtoMap);
}
if (CollUtil.isNotEmpty(resultData)) { if (CollUtil.isNotEmpty(resultData)) {
bak.setState(1); bak.setState(1);
//存储文件 //存储文件