Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据清除 修正索引错误与空值判断逻辑

Browse Source
This commit is contained in:
wr
2025-03-24 10:54:35 +08:00
parent 65f4066dd4
commit 90a5a27db0
10 changed files with 113 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/ExecutionCenter.java
View File

@@ -125,6 +125,8 @@ public class ExecutionCenter extends BaseController {
CalculatedParam calculatedParam = judgeExecuteParam(baseParam);
// 测点索引
if (CollectionUtils.isEmpty(calculatedParam.getIdList())) {
// calculatedParam.setIdList(Arrays.asList("c5f4925dbe333230810fe1a6afc51dfb","293178e56cd207e6e33586090f5034be","2df0cb6a87ef523b726c0a9ff6cfd288","43a1391b503bcd7f1b9a2f8f90e53e0f"
// ));
calculatedParam.setIdList(commTerminalGeneralClient.getRunMonitorIds().getData());
}
LiteflowResponse liteflowResponse;

8
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DayDataServiceImpl.java
View File

@@ -1289,7 +1289,7 @@ public class DayDataServiceImpl implements IDayDataService {
dto.setI47(getData(type,valueType.getValueList().get(46),scheme));
dto.setI48(getData(type,valueType.getValueList().get(47),scheme));
dto.setI49(getData(type,valueType.getValueList().get(48),scheme));
dto.setI50(getData(type,valueType.getValueList().get(50),scheme));
dto.setI50(getData(type,valueType.getValueList().get(49),scheme));
}
public void channelDataHarmRateVHandler(CommonMinuteDto.ValueType pojo1, CommonMinuteDto.ValueType pojo2, DataHarmRateVDto dto, boolean scheme) {
@@ -1356,7 +1356,7 @@ public class DayDataServiceImpl implements IDayDataService {
dto.setV47(getData(type,valueType.getValueList().get(46),scheme));
dto.setV48(getData(type,valueType.getValueList().get(47),scheme));
dto.setV49(getData(type,valueType.getValueList().get(48),scheme));
dto.setV50(getData(type,valueType.getValueList().get(50),scheme));
dto.setV50(getData(type,valueType.getValueList().get(49),scheme));
}
public void channelDataInHarmIHandler(CommonMinuteDto.ValueType pojo1, CommonMinuteDto.ValueType pojo2, DataInHarmIDto dto, boolean scheme) {
@@ -1423,7 +1423,7 @@ public class DayDataServiceImpl implements IDayDataService {
dto.setI47(getData(type,valueType.getValueList().get(46),scheme));
dto.setI48(getData(type,valueType.getValueList().get(47),scheme));
dto.setI49(getData(type,valueType.getValueList().get(48),scheme));
dto.setI50(getData(type,valueType.getValueList().get(50),scheme));
dto.setI50(getData(type,valueType.getValueList().get(49),scheme));
}
public void channelDataInHarmVHandler(CommonMinuteDto.ValueType pojo1, CommonMinuteDto.ValueType pojo2, DataInHarmVDto dto, boolean scheme) {
@@ -1490,7 +1490,7 @@ public class DayDataServiceImpl implements IDayDataService {
dto.setV47(getData(type,valueType.getValueList().get(46),scheme));
dto.setV48(getData(type,valueType.getValueList().get(47),scheme));
dto.setV49(getData(type,valueType.getValueList().get(48),scheme));
dto.setV50(getData(type,valueType.getValueList().get(50),scheme));
dto.setV50(getData(type,valueType.getValueList().get(49),scheme));
}
public void channelDataPltHandler(CommonMinuteDto.ValueType pojo1, DataPltDto dto, String valueType, boolean scheme) {

3
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataCrossingServiceImpl.java
View File

@@ -201,6 +201,7 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
.collect(Collectors.groupingBy(DataPltDto::getLineId));
for (String item : list) {
if(ObjectUtil.isNotNull(overLimitMap.get(item))){
result.addAll(getData(calculatedParam.getDataDate(),
overLimitMap.get(item),
allTime.get(item),
@@ -214,7 +215,7 @@ public class IDataCrossingServiceImpl implements IDataCrossingService {
dataVFreq.get(item),
dataVDev.get(item),
dataPlt.get(item)));
}
}
});

1
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/IDataOrgPointServiceImpl.java
View File

@@ -12,7 +12,6 @@ import com.njcn.dataProcess.util.TimeUtils;
import com.njcn.device.biz.pojo.dto.DeptGetChildrenMoreDTO;
import com.njcn.device.biz.pojo.dto.LineDevGetDTO;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.ListUtils;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;

13
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/influxdb/InfluxdbDataFlickerImpl.java
View File

@@ -144,12 +144,21 @@ public class InfluxdbDataFlickerImpl extends MppServiceImpl<RStatDataFlickerRela
influxQueryWrapper.lt(lineParam.getColumnName(), Double.parseDouble(lineParam.getLt()));
}
List<DataFlicker> result = new ArrayList<>();
quality(result, influxQueryWrapper, lineParam);
List<DataFlicker> list = dataFlickerMapper.selectByQueryWrapper(influxQueryWrapper);
if (CollUtil.isNotEmpty(list)) {
Map<String, List<String>> abnormalTime = lineParam.getAbnormalTime();
if (abnormalTime.containsKey(lineParam.getLineId().get(0))) {
List<String> timeList = abnormalTime.get(lineParam.getLineId().get(0));
//有异常数据,当前监测点自身的异常数据
if (CollectionUtil.isNotEmpty(timeList)) {
result.addAll(list.stream().filter(item -> !timeList.contains(DATE_TIME_FORMATTER.format(item.getTime()))).collect(Collectors.toList()));
}
}
}
return result.size();
}
private void quality(List<DataFlicker> result, InfluxQueryWrapper influxQueryWrapper, LineCountEvaluateParam lineParam) {
List<DataFlicker> list = dataFlickerMapper.selectByQueryWrapper(influxQueryWrapper);
Map<String, List<DataFlicker>> lineMap = list.stream().collect(Collectors.groupingBy(DataFlicker::getLineId));
//有异常数据

24
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/influxdb/InfluxdbDataVImpl.java
View File

@@ -181,8 +181,18 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
if (ObjectUtil.isNotEmpty(lineParam.getLt())) {
influxQueryWrapper.lt(lineParam.getColumnName(), Double.parseDouble(lineParam.getLt()));
}
List<DataV> result=new ArrayList<>();
quality(result, influxQueryWrapper,lineParam);
List<DataV> result = new ArrayList<>();
List<DataV> list = dataVMapper.selectByQueryWrapper(influxQueryWrapper);
if (CollUtil.isNotEmpty(list)) {
Map<String, List<String>> abnormalTime = lineParam.getAbnormalTime();
if (abnormalTime.containsKey(lineParam.getLineId().get(0))) {
List<String> timeList = abnormalTime.get(lineParam.getLineId().get(0));
//有异常数据,当前监测点自身的异常数据
if (CollectionUtil.isNotEmpty(timeList)) {
result.addAll(list.stream().filter(item -> !timeList.contains(DATE_TIME_FORMATTER.format(item.getTime()))).collect(Collectors.toList()));
}
}
}
return result.size();
}
@@ -339,7 +349,8 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
}
@Override
public List<MeasurementCountDTO> getMeasurementCount(List<String> lineIndex, String startTime, String endTime) {
public List<MeasurementCountDTO> getMeasurementCount(List<String> lineIndex, String startTime, String
endTime) {
InfluxQueryWrapper influxQueryWrapper = new InfluxQueryWrapper(DataV.class, MeasurementCountDTO.class);
influxQueryWrapper.regular(DataV::getLineId, lineIndex)
.eq(DataV::getValueType, InfluxDbSqlConstant.MAX)
@@ -388,12 +399,14 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
if (CollUtil.isNotEmpty(lineParam.getPhasicType())) {
influxQueryWrapper.regular(DataV::getPhasicType, lineParam.getPhasicType());
}
quality(result, influxQueryWrapper,lineParam);
quality(result, influxQueryWrapper, lineParam);
return result;
}
private void quality(List<DataV> result,InfluxQueryWrapper influxQueryWrapper, LineCountEvaluateParam lineParam) {
private void quality(List<DataV> result, InfluxQueryWrapper influxQueryWrapper, LineCountEvaluateParam
lineParam) {
List<DataV> list = dataVMapper.selectByQueryWrapper(influxQueryWrapper);
if (CollUtil.isNotEmpty(list)) {
Map<String, List<DataV>> lineMap = list.stream().collect(Collectors.groupingBy(DataV::getLineId));
//有异常数据
Map<String, List<String>> timeMap = lineParam.getAbnormalTime();
@@ -424,4 +437,5 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
result.addAll(list);
}
}
}
}

16
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataHarmRateVImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.dataProcess.service.impl.relation;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.util.ObjectUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
import com.njcn.dataProcess.dao.relation.mapper.DataHarmrateVRelationMapper;
@@ -52,13 +53,12 @@ public class RelationDataHarmRateVImpl extends MppServiceImpl<RStatDataHarmRateV
}
@Override
public void batchInsertion(List<DataHarmrateVDTO> dataHarmrateVDTOList) {
int totalCount = dataHarmrateVDTOList.size();
int minSize = Math.min(120, totalCount);
if(totalCount<=0){
if (totalCount <= 0) {
return;
}
List<DataHarmrateV> collect = dataHarmrateVDTOList.stream().map(temp -> {
@@ -87,7 +87,7 @@ public class RelationDataHarmRateVImpl extends MppServiceImpl<RStatDataHarmRateV
@Override
public void addList(List<DataHarmRateVDto> list) {
List<RStatDataHarmRateVD> result = new ArrayList<>();
list.forEach(item->{
list.forEach(item -> {
RStatDataHarmRateVD data = new RStatDataHarmRateVD();
data.setTime(TimeUtils.LocalDataTimeToLocalDate2(item.getTime()));
data.setPhasicType(item.getPhasicType());
@@ -107,18 +107,20 @@ public class RelationDataHarmRateVImpl extends MppServiceImpl<RStatDataHarmRateV
public List<DataHarmDto> getHarmRateVData(LineCountEvaluateParam lineParam) {
List<DataHarmDto> result = new ArrayList<>();
List<RStatDataHarmRateVD> rStatDataHarmRateVDList = dataHarmRateV.list(new LambdaQueryWrapper<RStatDataHarmRateVD>()
.in(RStatDataHarmRateVD::getLineId,lineParam.getLineId())
.in(RStatDataHarmRateVD::getLineId, lineParam.getLineId())
.in(RStatDataHarmRateVD::getValueType, lineParam.getValueType())
.in(RStatDataHarmRateVD::getPhasicType, lineParam.getPhasicType())
.ge(RStatDataHarmRateVD::getTime, lineParam.getStartTime())
.le(RStatDataHarmRateVD::getTime, lineParam.getEndTime())
);
for(RStatDataHarmRateVD rStatDataHarmRateVD : rStatDataHarmRateVDList){
DataHarmDto dto = BeanUtil.copyProperties(rStatDataHarmRateVD,DataHarmDto.class);
for (RStatDataHarmRateVD rStatDataHarmRateVD : rStatDataHarmRateVDList) {
DataHarmDto dto = BeanUtil.copyProperties(rStatDataHarmRateVD, DataHarmDto.class);
if (ObjectUtil.isNotNull(rStatDataHarmRateVD.getTime())) {
dto.setTime(rStatDataHarmRateVD.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
}
result.add(dto);
}
return quality(result,lineParam);
return quality(result, lineParam);
}
private List<DataHarmDto> quality(List<DataHarmDto> list, LineCountEvaluateParam lineParam) {

17
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataIImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.dataProcess.service.impl.relation;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@@ -51,7 +52,7 @@ public class RelationDataIImpl extends MppServiceImpl<RStatDataIRelationMapper,
int totalCount = dataIDTOList.size();
int minSize = Math.min(120, totalCount);
if(totalCount<=0){
if (totalCount <= 0) {
return;
}
List<DataI> collect = dataIDTOList.stream().map(temp -> {
@@ -84,7 +85,7 @@ public class RelationDataIImpl extends MppServiceImpl<RStatDataIRelationMapper,
@Override
public void addList(List<DataIDto> dataIDtoList) {
List<RStatDataID> result = new ArrayList<>();
dataIDtoList.forEach(item->{
dataIDtoList.forEach(item -> {
RStatDataID dataI = new RStatDataID();
dataI.setTime(TimeUtils.LocalDataTimeToLocalDate2(item.getTime()));
dataI.setPhasicType(item.getPhasicType());
@@ -100,7 +101,7 @@ public class RelationDataIImpl extends MppServiceImpl<RStatDataIRelationMapper,
List<DataIDto> result = new ArrayList<>();
QueryWrapper<RStatDataID> queryWrapper = new QueryWrapper<>();
if(StrUtil.isNotBlank(lineParam.getColumnName())){
if (StrUtil.isNotBlank(lineParam.getColumnName())) {
queryWrapper.select(lineParam.getColumnName());
}
queryWrapper.lambda()
@@ -108,15 +109,17 @@ public class RelationDataIImpl extends MppServiceImpl<RStatDataIRelationMapper,
.in(RStatDataID::getPhasicType, lineParam.getPhasicType())
.ge(RStatDataID::getTime, lineParam.getStartTime())
.le(RStatDataID::getTime, lineParam.getEndTime())
.in(RStatDataID::getLineId,lineParam.getLineId());
.in(RStatDataID::getLineId, lineParam.getLineId());
List<RStatDataID> list = this.list(queryWrapper);
for(RStatDataID rStatDataID:list){
DataIDto dto =BeanUtil.copyProperties(rStatDataID,DataIDto.class);
for (RStatDataID rStatDataID : list) {
DataIDto dto = BeanUtil.copyProperties(rStatDataID, DataIDto.class);
if (ObjectUtil.isNotNull(rStatDataID.getTime())) {
dto.setTime(rStatDataID.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
}
result.add(dto);
}
return quality(result,lineParam);
return quality(result, lineParam);
}
private List<DataIDto> quality(List<DataIDto> list, LineCountEvaluateParam lineParam) {

15
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataPltImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.dataProcess.service.impl.relation;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
@@ -48,7 +49,7 @@ public class RelationDataPltImpl extends MppServiceImpl<RStatDataPltRelationMapp
int totalCount = dataPltDTOList.size();
int minSize = Math.min(120, totalCount);
if(totalCount<=0){
if (totalCount <= 0) {
return;
}
List<DataPlt> collect = dataPltDTOList.stream().map(temp -> {
@@ -82,7 +83,7 @@ public class RelationDataPltImpl extends MppServiceImpl<RStatDataPltRelationMapp
@Override
public void addList(List<DataPltDto> list) {
List<RStatDataPltD> result = new ArrayList<>();
list.forEach(item->{
list.forEach(item -> {
RStatDataPltD data = new RStatDataPltD();
data.setTime(TimeUtils.LocalDataTimeToLocalDate2(item.getTime()));
data.setPhasicType(item.getPhasicType());
@@ -97,7 +98,7 @@ public class RelationDataPltImpl extends MppServiceImpl<RStatDataPltRelationMapp
public List<DataPltDto> getDataPlt(LineCountEvaluateParam lineParam) {
List<DataPltDto> result = new ArrayList<>();
QueryWrapper<RStatDataPltD> queryWrapper = new QueryWrapper<>();
if(StrUtil.isNotBlank(lineParam.getColumnName())){
if (StrUtil.isNotBlank(lineParam.getColumnName())) {
queryWrapper.select(lineParam.getColumnName());
}
queryWrapper.lambda()
@@ -107,12 +108,14 @@ public class RelationDataPltImpl extends MppServiceImpl<RStatDataPltRelationMapp
.le(RStatDataPltD::getTime, lineParam.getEndTime());
List<RStatDataPltD> rStatDataVDList = this.list(queryWrapper);
for(RStatDataPltD rStatDataPltD : rStatDataVDList){
DataPltDto dto = BeanUtil.copyProperties(rStatDataPltD,DataPltDto.class);
for (RStatDataPltD rStatDataPltD : rStatDataVDList) {
DataPltDto dto = BeanUtil.copyProperties(rStatDataPltD, DataPltDto.class);
if (ObjectUtil.isNotNull(rStatDataPltD.getTime())) {
dto.setTime(rStatDataPltD.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
}
result.add(dto);
}
return quality(result,lineParam);
return quality(result, lineParam);
}
private List<DataPltDto> quality(List<DataPltDto> list, LineCountEvaluateParam lineParam) {

5
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataVImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.dataProcess.service.impl.relation;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
@@ -153,10 +154,12 @@ public class RelationDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
List<RStatDataVD> rStatDataVDList = this.list(queryWrapper);
for (RStatDataVD rStatDataVD : rStatDataVDList) {
DataVDto dto = BeanUtil.copyProperties(rStatDataVD, DataVDto.class);
if (ObjectUtil.isNotNull(rStatDataVD.getTime())) {
dto.setTime(rStatDataVD.getTime().format(DateTimeFormatter.ofPattern(DatePattern.NORM_DATE_PATTERN)));
}
info.add(dto);
}
return quality(info,lineParam);
return quality(info, lineParam);
}
private List<DataVDto> quality(List<DataVDto> list, LineCountEvaluateParam lineParam) {