Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

算法优化

Browse Source
This commit is contained in:
xy
2025-07-04 09:47:13 +08:00
parent ae083f9c3f
commit 764c327558
5 changed files with 98 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataCleanServiceImpl.java
View File

@@ -44,6 +44,7 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -344,7 +345,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataV(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataV(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
@@ -361,7 +362,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataI(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataI(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
@@ -378,7 +379,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataPlt(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataPlt(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
@@ -395,7 +396,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleInHarmV(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleInHarmV(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
@@ -412,7 +413,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmRateV(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmRateV(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
@@ -429,7 +430,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmPowerP(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmPowerP(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
@@ -446,7 +447,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmPhasicV(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmPhasicV(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
@@ -463,7 +464,7 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataFluc(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataFluc(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
@@ -480,13 +481,16 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies);
});
if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataFlicker(result,bak);
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataFlicker(result,bak,item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap);
}
}
}
if (CollUtil.isNotEmpty(resultData)) {
bak.setState(1);
//存储文件
InputStream reportStream = IoUtil.toStream(new Gson().toJson(resultData), CharsetUtil.UTF_8);
@@ -502,77 +506,77 @@ public class DataCleanServiceImpl implements IDataCleanService {
System.gc();
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataV(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataV(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//频率
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list1 = handleData(codeMap.get(DataCleanEnum.Freq.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list1 = handleData(codeMap.get(DataCleanEnum.Freq.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list1)) {
bak.setFreq(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.Freq.getCode(),list1);
}
//频率偏差
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list2 = handleData(codeMap.get(DataCleanEnum.FreqDev.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list2 = handleData(codeMap.get(DataCleanEnum.FreqDev.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list2)) {
bak.setFreqDev(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.FreqDev.getCode(),list2);
}
//相电压有效值
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list3 = handleData(codeMap.get(DataCleanEnum.RmsV.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list3 = handleData(codeMap.get(DataCleanEnum.RmsV.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list3)) {
bak.setVRms(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.RmsV.getCode(),list3);
}
//正序电压
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list4 = handleData(codeMap.get(DataCleanEnum.VPos.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list4 = handleData(codeMap.get(DataCleanEnum.VPos.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list4)) {
bak.setVPos(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.VPos.getCode(),list4);
}
//负序电压
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list5 = handleData(codeMap.get(DataCleanEnum.VNeg.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list5 = handleData(codeMap.get(DataCleanEnum.VNeg.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list5)) {
bak.setVNeg(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.VNeg.getCode(),list5);
}
//零序电压
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list6 = handleData(codeMap.get(DataCleanEnum.VZero.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list6 = handleData(codeMap.get(DataCleanEnum.VZero.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list6)) {
bak.setVZero(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.VZero.getCode(),list6);
}
//电压不平衡度
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list7 = handleData(codeMap.get(DataCleanEnum.VUnbalance.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list7 = handleData(codeMap.get(DataCleanEnum.VUnbalance.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list7)) {
bak.setVUnbalance(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.VUnbalance.getCode(),list7);
}
//线电压有效值
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list8 = handleData(codeMap.get(DataCleanEnum.RmsLvr.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list8 = handleData(codeMap.get(DataCleanEnum.RmsLvr.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list8)) {
bak.setRmsLvr(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.RmsLvr.getCode(),list8);
}
//电压正偏差
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list9 = handleData(codeMap.get(DataCleanEnum.VuDev.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list9 = handleData(codeMap.get(DataCleanEnum.VuDev.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list9)) {
bak.setVuDev(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.VuDev.getCode(),list9);
}
//电压负偏差
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list10 = handleData(codeMap.get(DataCleanEnum.VlDev.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list10 = handleData(codeMap.get(DataCleanEnum.VlDev.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list10)) {
bak.setVlDev(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.VlDev.getCode(),list10);
}
//电压总谐波畸变率
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list11 = handleData(codeMap.get(DataCleanEnum.VThd.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list11 = handleData(codeMap.get(DataCleanEnum.VThd.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list11)) {
bak.setVThd(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.VThd.getCode(),list11);
}
//相（线）电压基波有效值
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list12 = handleData(codeMap.get(DataCleanEnum.V_Data.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list12 = handleData(codeMap.get(DataCleanEnum.V_Data.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list12)) {
bak.setV(1);
map.put(DataCleanEnum.DataV.getCode() + "-" + DataCleanEnum.V_Data.getCode(),list12);
@@ -580,11 +584,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataI(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataI(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//电流有效值
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.RmsI.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.RmsI.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list)) {
bak.setIRms(1);
map.put(DataCleanEnum.DataI.getCode() + "-" + DataCleanEnum.RmsI.getCode(),list);
@@ -592,11 +596,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataPlt(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataPlt(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//长时闪变
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Plt.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Plt.getCode()), timeInterval);
if (CollUtil.isNotEmpty(list)) {
bak.setPlt(1);
map.put(DataCleanEnum.DataPlt.getCode() + "-" + DataCleanEnum.Plt.getCode(),list);
@@ -604,11 +608,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleInHarmV(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleInHarmV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//间谐波电压含有率
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V_InHarm.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V_InHarm.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list)) {
bak.setVInharm(1);
map.put(DataCleanEnum.DataInHarmV.getCode() + "-" + DataCleanEnum.V_InHarm.getCode(),list);
@@ -616,11 +620,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmRateV(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmRateV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//谐波电压含有率
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V_Rate.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V_Rate.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list)) {
bak.setVHarm(1);
map.put(DataCleanEnum.DataHarmRateV.getCode() + "-" + DataCleanEnum.V_Rate.getCode(),list);
@@ -628,11 +632,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmPowerP(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmPowerP(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//功率因素
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Pf.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Pf.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list)) {
bak.setPf(1);
map.put(DataCleanEnum.DataHarmPowerP.getCode() + "-" + DataCleanEnum.Pf.getCode(),list);
@@ -640,17 +644,17 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmPhasicV(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmPhasicV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//谐波电压相角
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list)) {
bak.setVPhasic(1);
map.put(DataCleanEnum.DataHarmPhasicV.getCode() + "-" + DataCleanEnum.V.getCode(),list);
}
//谐波电压基波相角
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list2 = handleData(codeMap.get(DataCleanEnum.V1.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list2 = handleData(codeMap.get(DataCleanEnum.V1.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list2)) {
bak.setV1Phasic(1);
map.put(DataCleanEnum.DataHarmPhasicV.getCode() + "-" + DataCleanEnum.V1.getCode(),list2);
@@ -658,11 +662,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataFluc(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataFluc(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//电压波动
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Fluc.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Fluc.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list)) {
bak.setFluc(1);
map.put(DataCleanEnum.DataFluc.getCode() + "-" + DataCleanEnum.Fluc.getCode(),list);
@@ -670,11 +674,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataFlicker(List<PqDataVerify> result, PqDataVerifyBak bak) {
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataFlicker(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//短时闪变
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Pst.getCode()));
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Pst.getCode()),timeInterval);
if (CollUtil.isNotEmpty(list)) {
bak.setPst(1);
map.put(DataCleanEnum.DataFlicker.getCode() + "-" + DataCleanEnum.Pst.getCode(),list);
@@ -682,9 +686,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map;
}
public List<DataCleanJsonDTO.DataHarmCleanJsonDTO> handleData(List<PqDataVerify> list) {
public List<DataCleanJsonDTO.DataHarmCleanJsonDTO> handleData(List<PqDataVerify> list, Integer timeInterval) {
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> dataList = new ArrayList<>();
if (CollUtil.isNotEmpty(list)) {
AtomicInteger allCounts = new AtomicInteger();
Set<String> timeSet = new HashSet<>();
DataCleanJsonDTO.DataHarmCleanJsonDTO dto = new DataCleanJsonDTO.DataHarmCleanJsonDTO();
dto.setTargetName(list.get(0).getIndexName());
List<DataCleanJsonDTO.DataHarmCleanNormalJsonDTO> list1 = new ArrayList<>();
@@ -695,15 +701,29 @@ public class DataCleanServiceImpl implements IDataCleanService {
DataCleanJsonDTO.DataHarmCleanNormalJsonDTO dataCleanJsonDTO = new DataCleanJsonDTO.DataHarmCleanNormalJsonDTO();
dataCleanJsonDTO.setPhasic(k);
dataCleanJsonDTO.setValueType(k2);
dataCleanJsonDTO.setTime(v2.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toList()));
dataCleanJsonDTO.setValue(v2.stream()
timeSet.addAll(v2.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
//只取值最大的10组数据
List<PqDataVerify> topTen = v2.stream()
//过滤掉null元素
.filter(Objects::nonNull)
.sorted(Comparator.comparing(
PqDataVerify::getAbnormalValue,
Comparator.nullsLast(Comparator.reverseOrder())))
.limit(10)
.collect(Collectors.toList());
dataCleanJsonDTO.setTime(topTen.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toList()));
dataCleanJsonDTO.setValue(topTen.stream()
.map(pq -> {
Double value = pq.getAbnormalValue();
return value != null ? Math.round(value * 100) / 100.0 : null;
})
.collect(Collectors.toList()));
dataCleanJsonDTO.setErrorCounts(v2.size());
allCounts.set(allCounts.get() + v2.size());
list1.add(dataCleanJsonDTO);
});
dto.setErrorCounts(allCounts.get());
dto.setErrorTimes(timeSet.size()*timeInterval);
dto.setList(list1);
});
dataList.add(dto);
@@ -711,9 +731,11 @@ public class DataCleanServiceImpl implements IDataCleanService {
return dataList;
}
public List<DataCleanJsonDTO.DataHarmCleanJsonDTO> handleHarmData(List<PqDataVerify> list) {
public List<DataCleanJsonDTO.DataHarmCleanJsonDTO> handleHarmData(List<PqDataVerify> list, Integer timeInterval) {
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> dataList = new ArrayList<>();
if (CollUtil.isNotEmpty(list)) {
AtomicInteger allCounts = new AtomicInteger();
Set<String> timeSet = new HashSet<>();
Map<String,List<PqDataVerify>> indexNameMap = list.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexName));
indexNameMap.forEach((k,v)->{
DataCleanJsonDTO.DataHarmCleanJsonDTO dto = new DataCleanJsonDTO.DataHarmCleanJsonDTO();
@@ -726,15 +748,29 @@ public class DataCleanServiceImpl implements IDataCleanService {
DataCleanJsonDTO.DataHarmCleanNormalJsonDTO dto1 = new DataCleanJsonDTO.DataHarmCleanNormalJsonDTO();
dto1.setPhasic(k2);
dto1.setValueType(k3);
dto1.setTime(v3.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toList()));
dto1.setValue(v3.stream()
timeSet.addAll(v3.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
//只取值最大的10组数据
List<PqDataVerify> topTen = v3.stream()
//过滤掉null元素
.filter(Objects::nonNull)
.sorted(Comparator.comparing(
PqDataVerify::getAbnormalValue,
Comparator.nullsLast(Comparator.reverseOrder())))
.limit(10)
.collect(Collectors.toList());
dto1.setTime(topTen.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toList()));
dto1.setValue(topTen.stream()
.map(pq -> {
Double value = pq.getAbnormalValue();
return value != null ? Math.round(value * 100) / 100.0 : null;
})
.collect(Collectors.toList()));
dto1.setErrorCounts(v3.size());
allCounts.set(allCounts.get() + v3.size());
dataCleanJsonDTOList.add(dto1);
});
dto.setErrorCounts(allCounts.get());
dto.setErrorTimes(timeSet.size()*timeInterval);
dto.setList(dataCleanJsonDTOList);
});
dataList.add(dto);
@@ -1561,6 +1597,12 @@ public class DataCleanServiceImpl implements IDataCleanService {
for (int i = 1; i <= 50; i++) {
// 通过反射获取dto.getV(i)的值
double vValue = getVValue(dto, i);
if (vValue < -180) {
vValue = vValue + 360;
}
if (vValue > 180) {
vValue = vValue - 360;
}
if (vValue < pqReasonableRangeDto.getMinValue() || vValue > pqReasonableRangeDto.getMaxValue()) {
PqDataVerify pqDataVerify = getPqDataVerify(
dto.getLineId(),

10
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/dto/DataCleanJsonDTO.java
View File

@@ -21,6 +21,13 @@ public class DataCleanJsonDTO {
private String targetName;
private List<DataHarmCleanNormalJsonDTO> list;
@ApiModelProperty("异常时间")
private Integer errorTimes;
@ApiModelProperty("异常数据量")
private Integer errorCounts;
}
@Data
@@ -37,6 +44,9 @@ public class DataCleanJsonDTO {
@ApiModelProperty("数据类型")
private String valueType;
@ApiModelProperty("异常数据量")
private Integer errorCounts;
}
}

4
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/influxdb/InfluxdbPqsCommunicateImpl.java
View File

@@ -132,10 +132,6 @@ public class InfluxdbPqsCommunicateImpl implements IPqsCommunicate {
deviceFeignClient.updateDevComFlag(devComFlagDTO);
}
}

3
message/message-boot/src/main/java/com/njcn/message/consumer/TopicLogsConsumer.java
View File

@@ -4,13 +4,11 @@ import com.alibaba.fastjson.JSONObject;
import com.njcn.message.constant.MessageStatus;
import com.njcn.message.constant.RedisKeyPrefix;
import com.njcn.message.messagedto.FrontLogslMessage;
import com.njcn.message.messagedto.TopicReplyDTO;
import com.njcn.middle.rocket.constant.EnhanceMessageConstant;
import com.njcn.middle.rocket.handler.EnhanceConsumerMessageHandler;
import com.njcn.redis.pojo.enums.AppRedisKey;
import com.njcn.redis.pojo.enums.RedisKeyEnum;
import com.njcn.redis.utils.RedisUtil;
import com.njcn.stat.api.MessAnalysisFeignClient;
import com.njcn.system.api.FrontLogsFeignClient;
import com.njcn.system.api.RocketMqLogFeignClient;
import com.njcn.system.pojo.dto.PqFrontLogsDTO;
@@ -19,7 +17,6 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;

2
migration-influxdb/migration-influxdb-read-boot/src/main/java/com/njcn/migration/read/service/impl/MigrationServiceImpl.java
View File

@@ -95,7 +95,7 @@ public class MigrationServiceImpl implements MigrationService {
migrationInsertFeignClient.insertDataInharmV(dataInharmV.listDataInharmV(evaluateParam));
migrationInsertFeignClient.insertDataPlt(dataPlt.listDataPlt(evaluateParam));
migrationInsertFeignClient.batchInsertion(eventDetail.getRawData(evaluateParam));
if (!param.getIsManual() && StrUtil.isNotBlank(format)) {
TimeUtil.putLineTime(lineId, format);
}