Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

算法优化

Browse Source
This commit is contained in:
xy
2025-07-07 11:04:16 +08:00
parent f6e69e13bd
commit 327901ef10
2 changed files with 47 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataCleanServiceImpl.java
View File

@@ -310,7 +310,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setStartTime(TimeUtils.getBeginOfDay(calculatedParam.getDataDate())); lineParam.setStartTime(TimeUtils.getBeginOfDay(calculatedParam.getDataDate()));
lineParam.setEndTime(TimeUtils.getEndOfDay(calculatedParam.getDataDate())); lineParam.setEndTime(TimeUtils.getEndOfDay(calculatedParam.getDataDate()));
lineDetail.forEach(item->{ lineDetail.forEach(item->{
List<Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>>> resultData = new ArrayList<>();
List<Map<String,Object>> resultData = new ArrayList<>();
PqDataVerifyBak bak = new PqDataVerifyBak(); PqDataVerifyBak bak = new PqDataVerifyBak();
bak.setLineId(item.getLineId()); bak.setLineId(item.getLineId());
@@ -345,7 +346,9 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataV(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataV(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -362,7 +365,9 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataI(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataI(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -379,7 +384,9 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataPlt(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataPlt(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -396,7 +403,9 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleInHarmV(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleInHarmV(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -413,7 +422,9 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmRateV(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleHarmRateV(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -430,7 +441,9 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmPowerP(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleHarmPowerP(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -447,7 +460,9 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleHarmPhasicV(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleHarmPhasicV(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -464,7 +479,9 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataFluc(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataFluc(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
@@ -481,16 +498,15 @@ public class DataCleanServiceImpl implements IDataCleanService {
result.addAll(pqDataVerifies); result.addAll(pqDataVerifies);
}); });
if (CollUtil.isNotEmpty(result)) { if (CollUtil.isNotEmpty(result)) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> dtoMap = handleDataFlicker(result,bak,item.getTimeInterval()); Map<String,Object> dtoMap = handleDataFlicker(result,bak,item.getTimeInterval());
Set<String> timeSet = new HashSet<>(result.stream().map(dt -> dt.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_TIME_PATTERN))).collect(Collectors.toSet()));
dtoMap.put("errorTimes",timeSet.size() * item.getTimeInterval());
if (CollUtil.isNotEmpty(dtoMap)) { if (CollUtil.isNotEmpty(dtoMap)) {
resultData.add(dtoMap); resultData.add(dtoMap);
} }
} }
} }
if (CollUtil.isNotEmpty(resultData)) { if (CollUtil.isNotEmpty(resultData)) {
bak.setState(1); bak.setState(1);
//存储文件 //存储文件
InputStream reportStream = IoUtil.toStream(new Gson().toJson(resultData), CharsetUtil.UTF_8); InputStream reportStream = IoUtil.toStream(new Gson().toJson(resultData), CharsetUtil.UTF_8);
@@ -506,8 +522,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
System.gc(); System.gc();
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataV(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) { public Map<String,Object> handleDataV(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//频率 //频率
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list1 = handleData(codeMap.get(DataCleanEnum.Freq.getCode()),timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list1 = handleData(codeMap.get(DataCleanEnum.Freq.getCode()),timeInterval);
@@ -584,8 +600,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map; return map;
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataI(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) { public Map<String,Object> handleDataI(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//电流有效值 //电流有效值
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.RmsI.getCode()),timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.RmsI.getCode()),timeInterval);
@@ -596,8 +612,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map; return map;
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataPlt(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) { public Map<String,Object> handleDataPlt(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//长时闪变 //长时闪变
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Plt.getCode()), timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Plt.getCode()), timeInterval);
@@ -608,8 +624,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map; return map;
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleInHarmV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) { public Map<String,Object> handleInHarmV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//间谐波电压含有率 //间谐波电压含有率
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V_InHarm.getCode()),timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V_InHarm.getCode()),timeInterval);
@@ -620,8 +636,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map; return map;
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmRateV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) { public Map<String,Object> handleHarmRateV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//谐波电压含有率 //谐波电压含有率
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V_Rate.getCode()),timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V_Rate.getCode()),timeInterval);
@@ -632,8 +648,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map; return map;
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmPowerP(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) { public Map<String,Object> handleHarmPowerP(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//功率因素 //功率因素
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Pf.getCode()),timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Pf.getCode()),timeInterval);
@@ -644,8 +660,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map; return map;
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleHarmPhasicV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) { public Map<String,Object> handleHarmPhasicV(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//谐波电压相角 //谐波电压相角
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V.getCode()),timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleHarmData(codeMap.get(DataCleanEnum.V.getCode()),timeInterval);
@@ -662,8 +678,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map; return map;
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataFluc(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) { public Map<String,Object> handleDataFluc(List<PqDataVerify> result, PqDataVerifyBak bak, Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//电压波动 //电压波动
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Fluc.getCode()),timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Fluc.getCode()),timeInterval);
@@ -674,8 +690,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
return map; return map;
} }
public Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> handleDataFlicker(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) { public Map<String,Object> handleDataFlicker(List<PqDataVerify> result, PqDataVerifyBak bak,Integer timeInterval) {
Map<String,List<DataCleanJsonDTO.DataHarmCleanJsonDTO>> map = new HashMap<>(); Map<String,Object> map = new HashMap<>();
Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode)); Map<String,List<PqDataVerify>> codeMap = result.stream().collect(Collectors.groupingBy(PqDataVerify::getIndexCode));
//短时闪变 //短时闪变
List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Pst.getCode()),timeInterval); List<DataCleanJsonDTO.DataHarmCleanJsonDTO> list = handleData(codeMap.get(DataCleanEnum.Pst.getCode()),timeInterval);

3
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/dto/DataCleanJsonDTO.java
View File

@@ -4,7 +4,6 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* @author xy * @author xy
@@ -12,8 +11,6 @@ import java.util.Map;
@Data @Data
public class DataCleanJsonDTO { public class DataCleanJsonDTO {
Map<String,List<DataHarmCleanJsonDTO>> map;
@Data @Data
public static class DataHarmCleanJsonDTO { public static class DataHarmCleanJsonDTO {