Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dataV原始数据数据清洗

Browse Source
This commit is contained in:
xy
2025-02-12 15:21:37 +08:00
parent 1f9c7e0b95
commit 935df88f3f
14 changed files with 246 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/executor/MeasurementExecutor.java
View File

@@ -1,6 +1,7 @@
package com.njcn.algorithm.executor;
import com.njcn.algorithm.service.line.DayDataService;
import com.njcn.algorithm.service.line.IDataCleanService;
import com.njcn.algorithm.service.line.IDayDataService;
import com.yomahub.liteflow.annotation.LiteflowComponent;
import com.yomahub.liteflow.annotation.LiteflowMethod;
import com.yomahub.liteflow.core.NodeComponent;
@@ -23,12 +24,26 @@ import javax.annotation.Resource;
public class MeasurementExecutor extends BaseExecutor {
@Resource
private DayDataService dayDataService;
private IDayDataService dayDataService;
@Resource
private IDataCleanService dataCleanService;
/********************************************算法负责人:xy***********************************************************/
/**
* 算法名: 3.4.1.1-----监测点报表_日表(r_stat_data_*_d)
* @author xuyang
* 数据清洗（dataV）
* @author xy
*/
@LiteflowMethod(value = LiteFlowMethodEnum.IS_ACCESS, nodeId = "dataVClean", nodeType = NodeTypeEnum.COMMON)
public boolean dataVCleanAccess(NodeComponent bindCmp) {
return isAccess(bindCmp);
}
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "dataVClean", nodeType = NodeTypeEnum.COMMON)
public void dataVCleanProcess(NodeComponent bindCmp) {
dataCleanService.dataVCleanHandler(bindCmp.getRequestData());
}
/**
* 监测点报表_日表(r_stat_data_*_d)
* @author xy
*/
@LiteflowMethod(value = LiteFlowMethodEnum.IS_ACCESS, nodeId = "dataV", nodeType = NodeTypeEnum.COMMON)
public boolean dataVToDayAccess(NodeComponent bindCmp) {
@@ -39,7 +54,4 @@ public class MeasurementExecutor extends BaseExecutor {
dayDataService.dataVHandler(bindCmp.getRequestData());
}
/********************************************算法负责人:xy结束***********************************************************/
}

17
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/service/line/IDataCleanService.java Normal file
View File

@@ -0,0 +1,17 @@
package com.njcn.algorithm.service.line;
import com.njcn.algorithm.pojo.bo.CalculatedParam;
/**
* @author xy
*/
public interface IDataCleanService {
/***
* dataV数据清洗
* @author xy
* @param calculatedParam 查询条件
*/
void dataVCleanHandler(CalculatedParam calculatedParam);
}

2
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/service/line/DayDataService.java → algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/service/line/IDayDataService.java
View File

@@ -5,7 +5,7 @@ import com.njcn.algorithm.pojo.bo.CalculatedParam;
/**
* @author xy
*/
public interface DayDataService {
public interface IDayDataService {
/***

56
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataCleanServiceImpl.java Normal file
View File

@@ -0,0 +1,56 @@
package com.njcn.algorithm.serviceimpl.line;
import cn.hutool.core.collection.CollUtil;
import com.njcn.algorithm.pojo.bo.CalculatedParam;
import com.njcn.algorithm.service.line.IDataCleanService;
import com.njcn.dataProcess.api.DataVFeignClient;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.DataVDto;
import com.njcn.dataProcess.util.TimeUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.ListUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
/**
* @author xy
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class DataCleanServiceImpl implements IDataCleanService {
private static final Logger logger = LoggerFactory.getLogger(DataCleanServiceImpl.class);
private final static Integer NUM = 100;
@Resource
private DataVFeignClient dataVFeignClient;
@Override
public void dataVCleanHandler(CalculatedParam calculatedParam) {
List<DataVDto> result = new ArrayList<>();
LineCountEvaluateParam lineParam = new LineCountEvaluateParam();
lineParam.setStartTime(TimeUtils.getBeginOfDay(calculatedParam.getDataDate()));
lineParam.setEndTime(TimeUtils.getEndOfDay(calculatedParam.getDataDate()));
List<List<String>> pendingIds = ListUtils.partition(calculatedParam.getIdList(),NUM);
pendingIds.forEach(list->{
lineParam.setLineId(list);
List<DataVDto> partList = dataVFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
partList.forEach(item->{
item.setAbnormalFlag(0);
});
result.addAll(partList);
}
});
if (CollUtil.isNotEmpty(result)) {
dataVFeignClient.addInfluxDbList(result);
}
}
}

11
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DayDataServiceImpl.java
View File

@@ -2,7 +2,7 @@ package com.njcn.algorithm.serviceimpl.line;
import cn.hutool.core.collection.CollUtil;
import com.njcn.algorithm.pojo.bo.CalculatedParam;
import com.njcn.algorithm.service.line.DayDataService;
import com.njcn.algorithm.service.line.IDayDataService;
import com.njcn.dataProcess.api.DataVFeignClient;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
@@ -26,7 +26,7 @@ import java.util.*;
@Slf4j
@Component
@RequiredArgsConstructor
public class DayDataServiceImpl implements DayDataService {
public class DayDataServiceImpl implements IDayDataService {
private static final Logger logger = LoggerFactory.getLogger(DayDataServiceImpl.class);
private final static Integer NUM = 100;
@@ -54,15 +54,14 @@ public class DayDataServiceImpl implements DayDataService {
//数据类型
List<CommonMinuteDto.ValueType> valueTypeList = item2.getValueTypeList();
//获取平均值集合
CommonMinuteDto.ValueType valueTypes = valueTypeList.stream().filter(type-> type.getValueType().equalsIgnoreCase(InfluxDbSqlConstant.AVG_WEB)).findFirst().orElse(null);
CommonMinuteDto.ValueType valueTypes = valueTypeList.stream().filter(type-> type.getValueType().equalsIgnoreCase(InfluxDbSqlConstant.AVG_WEB)).findFirst().orElse(null);
valueTypeList.forEach(item3->{
DataVDto dto = new DataVDto();
dto.setTime(item.getTime());
dto.setLineId(item.getLineId());
dto.setPhasicType(item2.getPhasicType());
dto.setValueType(item3.getValueType());
//todo 数据清洗的功能需要讨论在哪完成
dto.setQualityFlag(0);
dto.setQualityFlag("0");
channelDataVHandler(item3,valueTypes,dto,true);
result.add(dto);
});
@@ -87,7 +86,7 @@ public class DayDataServiceImpl implements DayDataService {
} else {
valueType = pojo1;
}
//todo 按照集合排列顺序取值
//按照指标集合排列顺序取值
dto.setFreq(getData(valueType.getValueType(),valueType.getValueList().get(0),scheme));
dto.setFreqDev(getData(valueType.getValueType(),valueType.getValueList().get(1),scheme));
dto.setRms(getData(valueType.getValueType(),valueType.getValueList().get(2),scheme));