Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据清除 修正索引错误与空值判断逻辑

Browse Source
This commit is contained in:
wr
2025-03-24 10:54:35 +08:00
parent 65f4066dd4
commit 90a5a27db0
10 changed files with 113 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/ExecutionCenter.java
View File

@@ -125,6 +125,8 @@ public class ExecutionCenter extends BaseController {
CalculatedParam calculatedParam = judgeExecuteParam(baseParam);
// 测点索引
if (CollectionUtils.isEmpty(calculatedParam.getIdList())) {
// calculatedParam.setIdList(Arrays.asList("c5f4925dbe333230810fe1a6afc51dfb","293178e56cd207e6e33586090f5034be","2df0cb6a87ef523b726c0a9ff6cfd288","43a1391b503bcd7f1b9a2f8f90e53e0f"
// ));
calculatedParam.setIdList(commTerminalGeneralClient.getRunMonitorIds().getData());
}
LiteflowResponse liteflowResponse;

8
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DayDataServiceImpl.java
View File

@@ -1289,7 +1289,7 @@ public class DayDataServiceImpl implements IDayDataService {
dto.setI47(getData(type,valueType.getValueList().get(46),scheme));
dto.setI48(getData(type,valueType.getValueList().get(47),scheme));
dto.setI49(getData(type,valueType.getValueList().get(48),scheme));
dto.setI50(getData(type,valueType.getValueList().get(50),scheme));
dto.setI50(getData(type,valueType.getValueList().get(49),scheme));
}
public void channelDataHarmRateVHandler(CommonMinuteDto.ValueType pojo1, CommonMinuteDto.ValueType pojo2, DataHarmRateVDto dto, boolean scheme) {
@@ -1356,7 +1356,7 @@ public class DayDataServiceImpl implements IDayDataService {
dto.setV47(getData(type,valueType.getValueList().get(46),scheme));
dto.setV48(getData(type,valueType.getValueList().get(47),scheme));
dto.setV49(getData(type,valueType.getValueList().get(48),scheme));
dto.setV50(getData(type,valueType.getValueList().get(50),scheme));
dto.setV50(getData(type,valueType.getValueList().get(49),scheme));
}
public void channelDataInHarmIHandler(CommonMinuteDto.ValueType pojo1, CommonMinuteDto.ValueType pojo2, DataInHarmIDto dto, boolean scheme) {
@@ -1423,7 +1423,7 @@ public class DayDataServiceImpl implements IDayDataService {
dto.setI47(getData(type,valueType.getValueList().get(46),scheme));
dto.setI48(getData(type,valueType.getValueList().get(47),scheme));
dto.setI49(getData(type,valueType.getValueList().get(48),scheme));
dto.setI50(getData(type,valueType.getValueList().get(50),scheme));
dto.setI50(getData(type,valueType.getValueList().get(49),scheme));
}
public void channelDataInHarmVHandler(CommonMinuteDto.ValueType pojo1, CommonMinuteDto.ValueType pojo2, DataInHarmVDto dto, boolean scheme) {
@@ -1490,7 +1490,7 @@ public class DayDataServiceImpl implements IDayDataService {
dto.setV47(getData(type,valueType.getValueList().get(46),scheme));
dto.setV48(getData(type,valueType.getValueList().get(47),scheme));
dto.setV49(getData(type,valueType.getValueList().get(48),scheme));
dto.setV50(getData(type,valueType.getValueList().get(50),scheme));
dto.setV50(getData(type,valueType.getValueList().get(49),scheme));
}
public void channelDataPltHandler(CommonMinuteDto.ValueType pojo1, DataPltDto dto, String valueType, boolean scheme) {

3
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataCrossingServiceImpl.java
View File

@@ -201,6 +201,7 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
.collect(Collectors.groupingBy(DataPltDto::getLineId));
for (String item : list) {
if(ObjectUtil.isNotNull(overLimitMap.get(item))){
result.addAll(getData(calculatedParam.getDataDate(),
overLimitMap.get(item),
allTime.get(item),
@@ -214,7 +215,7 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
dataVFreq.get(item),
dataVDev.get(item),
dataPlt.get(item)));
}
}
});

1
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataOrgPointServiceImpl.java
View File

@@ -12,7 +12,6 @@ import com.njcn.dataProcess.util.TimeUtils;
import com.njcn.device.biz.pojo.dto.DeptGetChildrenMoreDTO;
import com.njcn.device.biz.pojo.dto.LineDevGetDTO;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.ListUtils;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;

13
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/influxdb/InfluxdbDataFlickerImpl.java
View File

@@ -144,12 +144,21 @@ public class InfluxdbDataFlickerImpl extends MppServiceImpl<RStatDataFlickerRela
influxQueryWrapper.lt(lineParam.getColumnName(), Double.parseDouble(lineParam.getLt()));
}
List<DataFlicker> result = new ArrayList<>();
quality(result, influxQueryWrapper, lineParam);
List<DataFlicker> list = dataFlickerMapper.selectByQueryWrapper(influxQueryWrapper);
if (CollUtil.isNotEmpty(list)) {
Map<String, List<String>> abnormalTime = lineParam.getAbnormalTime();
if (abnormalTime.containsKey(lineParam.getLineId().get(0))) {
List<String> timeList = abnormalTime.get(lineParam.getLineId().get(0));
//有异常数据,当前监测点自身的异常数据
if (CollectionUtil.isNotEmpty(timeList)) {
result.addAll(list.stream().filter(item -> !timeList.contains(DATE_TIME_FORMATTER.format(item.getTime()))).collect(Collectors.toList()));
}
}
}
return result.size();
}
private void quality(List<DataFlicker> result, InfluxQueryWrapper influxQueryWrapper, LineCountEvaluateParam lineParam) {
List<DataFlicker> list = dataFlickerMapper.selectByQueryWrapper(influxQueryWrapper);
Map<String, List<DataFlicker>> lineMap = list.stream().collect(Collectors.groupingBy(DataFlicker::getLineId));
//有异常数据

20
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/influxdb/InfluxdbDataVImpl.java
View File

@@ -182,7 +182,17 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
influxQueryWrapper.lt(lineParam.getColumnName(), Double.parseDouble(lineParam.getLt()));
}
List<DataV> result = new ArrayList<>();
quality(result, influxQueryWrapper,lineParam);
List<DataV> list = dataVMapper.selectByQueryWrapper(influxQueryWrapper);
if (CollUtil.isNotEmpty(list)) {
Map<String, List<String>> abnormalTime = lineParam.getAbnormalTime();
if (abnormalTime.containsKey(lineParam.getLineId().get(0))) {
List<String> timeList = abnormalTime.get(lineParam.getLineId().get(0));
//有异常数据,当前监测点自身的异常数据
if (CollectionUtil.isNotEmpty(timeList)) {
result.addAll(list.stream().filter(item -> !timeList.contains(DATE_TIME_FORMATTER.format(item.getTime()))).collect(Collectors.toList()));
}
}
}
return result.size();
}
@@ -339,7 +349,8 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
}
@Override
public List<MeasurementCountDTO> getMeasurementCount(List<String> lineIndex, String startTime, String endTime) {
public List<MeasurementCountDTO> getMeasurementCount(List<String> lineIndex, String startTime, String
endTime) {
InfluxQueryWrapper influxQueryWrapper = new InfluxQueryWrapper(DataV.class, MeasurementCountDTO.class);
influxQueryWrapper.regular(DataV::getLineId, lineIndex)
.eq(DataV::getValueType, InfluxDbSqlConstant.MAX)
@@ -392,8 +403,10 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
return result;
}
private void quality(List<DataV> result,InfluxQueryWrapper influxQueryWrapper, LineCountEvaluateParam lineParam) {
private void quality(List<DataV> result, InfluxQueryWrapper influxQueryWrapper, LineCountEvaluateParam
lineParam) {
List<DataV> list = dataVMapper.selectByQueryWrapper(influxQueryWrapper);
if (CollUtil.isNotEmpty(list)) {
Map<String, List<DataV>> lineMap = list.stream().collect(Collectors.groupingBy(DataV::getLineId));
//有异常数据
Map<String, List<String>> timeMap = lineParam.getAbnormalTime();
@@ -425,3 +438,4 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
}
}
}
}

4
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataHarmRateVImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.dataProcess.service.impl.relation;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.util.ObjectUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
import com.njcn.dataProcess.dao.relation.mapper.DataHarmrateVRelationMapper;
@@ -52,7 +53,6 @@ public class RelationDataHarmRateVImpl extends MppServiceImpl<RStatDataHarmRateV
}
@Override
public void batchInsertion(List<DataHarmrateVDTO> dataHarmrateVDTOList) {
int totalCount = dataHarmrateVDTOList.size();
@@ -115,7 +115,9 @@ public class RelationDataHarmRateVImpl extends MppServiceImpl<RStatDataHarmRateV
);
for (RStatDataHarmRateVD rStatDataHarmRateVD : rStatDataHarmRateVDList) {
DataHarmDto dto = BeanUtil.copyProperties(rStatDataHarmRateVD, DataHarmDto.class);
if (ObjectUtil.isNotNull(rStatDataHarmRateVD.getTime())) {
dto.setTime(rStatDataHarmRateVD.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
}
result.add(dto);
}
return quality(result, lineParam);

3
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataIImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.dataProcess.service.impl.relation;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@@ -113,7 +114,9 @@ public class RelationDataIImpl extends MppServiceImpl<RStatDataIRelationMapper,
for (RStatDataID rStatDataID : list) {
DataIDto dto = BeanUtil.copyProperties(rStatDataID, DataIDto.class);
if (ObjectUtil.isNotNull(rStatDataID.getTime())) {
dto.setTime(rStatDataID.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
}
result.add(dto);
}
return quality(result, lineParam);

3
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataPltImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.dataProcess.service.impl.relation;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
@@ -109,7 +110,9 @@ public class RelationDataPltImpl extends MppServiceImpl<RStatDataPltRelationMapp
List<RStatDataPltD> rStatDataVDList = this.list(queryWrapper);
for (RStatDataPltD rStatDataPltD : rStatDataVDList) {
DataPltDto dto = BeanUtil.copyProperties(rStatDataPltD, DataPltDto.class);
if (ObjectUtil.isNotNull(rStatDataPltD.getTime())) {
dto.setTime(rStatDataPltD.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
}
result.add(dto);
}
return quality(result, lineParam);

3
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataVImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.dataProcess.service.impl.relation;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
@@ -153,7 +154,9 @@ public class RelationDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
List<RStatDataVD> rStatDataVDList = this.list(queryWrapper);
for (RStatDataVD rStatDataVD : rStatDataVDList) {
DataVDto dto = BeanUtil.copyProperties(rStatDataVD, DataVDto.class);
if (ObjectUtil.isNotNull(rStatDataVD.getTime())) {
dto.setTime(rStatDataVD.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
}
info.add(dto);
}
return quality(info, lineParam);