dataV原始数据数据清洗

2025-02-12 15:21:37 +08:00
parent 1f9c7e0b95
commit 935df88f3f
14 changed files with 246 additions and 76 deletions
--- a/data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/controller/DataVController.java
+++ b/data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/controller/DataVController.java
@@ -6,10 +6,10 @@ import com.njcn.common.pojo.enums.common.LogEnum;
 import com.njcn.common.pojo.enums.response.CommonResponseEnum;
 import com.njcn.common.pojo.response.HttpResult;
 import com.njcn.common.utils.HttpResultUtil;
-import com.njcn.dataProcess.dto.DataVDTO;
-import com.njcn.dataProcess.dto.DataVFiveItemDTO;
 import com.njcn.dataProcess.annotation.InsertBean;
 import com.njcn.dataProcess.annotation.QueryBean;
+import com.njcn.dataProcess.dto.DataVDTO;
+import com.njcn.dataProcess.dto.DataVFiveItemDTO;
 import com.njcn.dataProcess.param.LineCountEvaluateParam;
 import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
 import com.njcn.dataProcess.pojo.dto.DataVDto;
@@ -75,6 +75,15 @@ public class DataVController extends BaseController {
        return HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, localDateTimeList, methodDescribe);
    }

+    @OperateInfo(info = LogEnum.BUSINESS_COMMON,operateType = OperateType.QUERY)
+    @PostMapping("/getRawData")
+    @ApiOperation("获取原始数据")
+    public HttpResult<List<DataVDto>> getRawData(@RequestBody LineCountEvaluateParam lineParam) {
+        String methodDescribe = getMethodDescribe("getRawData");
+        List<DataVDto> data = dataVQuery.getRawData(lineParam);
+        return HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, data, methodDescribe);
+    }
+
    @OperateInfo(info = LogEnum.BUSINESS_COMMON,operateType = OperateType.QUERY)
    @PostMapping("/getBaseData")
    @ApiOperation("获取算法基础数据")
@@ -86,12 +95,19 @@ public class DataVController extends BaseController {

    @OperateInfo(info = LogEnum.BUSINESS_COMMON,operateType = OperateType.ADD)
    @PostMapping("/addList")
-    @ApiOperation("批量插入数据")
+    @ApiOperation("关系型数据库插入数据")
    public HttpResult<String> addList(@RequestBody List<DataVDto> dataVDtoList) {
        String methodDescribe = getMethodDescribe("addList");
        dataVInsert.addList(dataVDtoList);
        return HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, "", methodDescribe);
    }

-
+    @OperateInfo(info = LogEnum.BUSINESS_COMMON,operateType = OperateType.ADD)
+    @PostMapping("/addInfluxDbList")
+    @ApiOperation("时序数据库插入数据")
+    public HttpResult<String> addInfluxDbList(@RequestBody List<DataVDto> dataVList) {
+        String methodDescribe = getMethodDescribe("addInfluxDbList");
+        dataVInsert.addInfluxDbList(dataVList);
+        return HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, "", methodDescribe);
+    }
 }
--- a/data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/IDataV.java
+++ b/data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/IDataV.java
@@ -4,6 +4,7 @@ import com.github.jeffreyning.mybatisplus.service.IMppService;
 import com.njcn.dataProcess.dto.DataVDTO;
 import com.njcn.dataProcess.dto.DataVFiveItemDTO;
 import com.njcn.dataProcess.param.LineCountEvaluateParam;
+import com.njcn.dataProcess.po.influx.DataV;
 import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
 import com.njcn.dataProcess.pojo.dto.DataVDto;
 import com.njcn.dataProcess.pojo.po.RStatDataVD;
@@ -30,6 +31,13 @@ public interface IDataV extends IMppService<RStatDataVD> {

    List<LocalDateTime> monitoringTime(String lineId, String localData);

+    /**
+     * 获取原始数据
+     * @param lineParam
+     * @return
+     */
+    List<DataVDto> getRawData(LineCountEvaluateParam lineParam);
+
    /**
     * 获取监测点原始数据
     * @param lineParam 监测点参数
@@ -40,4 +48,6 @@ public interface IDataV extends IMppService<RStatDataVD> {
     * 批量插入数据
     */
    void addList(List<DataVDto> dataVDtoList);
+
+    void addInfluxDbList(List<DataVDto> dataVList);
 }
--- a/data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/influxdb/InfluxdbDataVImpl.java
+++ b/data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/influxdb/InfluxdbDataVImpl.java
@@ -2,17 +2,15 @@ package com.njcn.dataProcess.service.impl.influxdb;

 import cn.hutool.core.collection.CollectionUtil;
 import cn.hutool.core.collection.ListUtil;
-
 import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
 import com.njcn.common.utils.HarmonicTimesUtil;
 import com.njcn.dataProcess.constant.InfluxDBTableConstant;
 import com.njcn.dataProcess.constant.PhaseType;
+import com.njcn.dataProcess.dao.imapper.DataVMapper;
 import com.njcn.dataProcess.dao.relation.mapper.RStatDataVRelationMapper;
 import com.njcn.dataProcess.dto.DataVDTO;
 import com.njcn.dataProcess.dto.DataVFiveItemDTO;
 import com.njcn.dataProcess.dto.LineDataVFiveItemDTO;
-
-import com.njcn.dataProcess.dao.imapper.DataVMapper;
 import com.njcn.dataProcess.param.LineCountEvaluateParam;
 import com.njcn.dataProcess.po.influx.DataV;
 import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
@@ -23,10 +21,13 @@ import com.njcn.dataProcess.util.TimeUtils;
 import com.njcn.influx.constant.InfluxDbSqlConstant;
 import com.njcn.influx.query.InfluxQueryWrapper;
 import org.apache.commons.collections4.ListUtils;
+import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;

 import javax.annotation.Resource;
 import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 import java.util.*;
 import java.util.stream.Collectors;

@@ -38,6 +39,8 @@ import java.util.stream.Collectors;
@Service("InfluxdbDataVImpl")
 public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper, RStatDataVD> implements IDataV {

+    private final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.systemDefault());;
+
    @Resource
    private DataVMapper dataVMapper;

@@ -113,10 +116,23 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
        return null;
    }

+    @Override
+    public List<DataVDto> getRawData(LineCountEvaluateParam lineParam) {
+        List<DataVDto> result = new ArrayList<>();
+        List<DataV> list = getMinuteDataV(lineParam.getLineId(), lineParam.getStartTime(), lineParam.getEndTime(), false);
+        list.forEach(item->{
+            DataVDto dto = new DataVDto();
+            BeanUtils.copyProperties(item,dto);
+            dto.setMinTime(DATE_TIME_FORMATTER.format(item.getTime()));
+            result.add(dto);
+        });
+        return result;
+    }
+
    @Override
    public List<CommonMinuteDto> getBaseData(LineCountEvaluateParam lineParam) {
        List<CommonMinuteDto> result = new ArrayList<>();
-        List<DataV> dataVList = getMinuteDataV(lineParam.getLineId(), lineParam.getStartTime(), lineParam.getEndTime());
+        List<DataV> dataVList = getMinuteDataV(lineParam.getLineId(), lineParam.getStartTime(), lineParam.getEndTime(),true);
        if (CollectionUtil.isNotEmpty(dataVList)) {
            String time = TimeUtils.StringTimeToString(lineParam.getStartTime());
            //以监测点分组
@@ -137,7 +153,7 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
                    valueTypeMap.forEach((valueType,valueTypeList)->{
                        CommonMinuteDto.ValueType value = new CommonMinuteDto.ValueType();
                        value.setValueType(valueType);
-                        //todo 规定好集合指标参数
+                        //规定好集合指标参数
                        List<Double> data1 = valueTypeList.stream().map(DataV::getFreq).collect(Collectors.toList());
                        List<Double> data2 = valueTypeList.stream().map(DataV::getFreqDev).collect(Collectors.toList());
                        List<Double> data3 = valueTypeList.stream().map(DataV::getRms).collect(Collectors.toList());
@@ -226,10 +242,22 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,

    }

+    @Override
+    public void addInfluxDbList(List<DataVDto> dataVList) {
+        List<DataV> result = new ArrayList<>();
+        dataVList.forEach(item->{
+            DataV dataV = new DataV();
+            BeanUtils.copyProperties(item, dataV);
+            dataV.setTime(LocalDateTime.parse(item.getMinTime(), DATE_TIME_FORMATTER).atZone(ZoneId.systemDefault()).toInstant());
+            result.add(dataV);
+        });
+        dataVMapper.insertBatch(result);
+    }
+
    /**
     * 按监测点集合、时间条件获取dataV分钟数据
     */
-    public List<DataV> getMinuteDataV(List<String> lineList, String startTime, String endTime) {
+    public List<DataV> getMinuteDataV(List<String> lineList, String startTime, String endTime, boolean clean) {
        InfluxQueryWrapper influxQueryWrapper = new InfluxQueryWrapper(DataV.class);
        influxQueryWrapper.samePrefixAndSuffix(InfluxDbSqlConstant.V, "", HarmonicTimesUtil.harmonicTimesList(1, 50, 1));
        influxQueryWrapper.regular(DataV::getLineId, lineList)
@@ -247,8 +275,11 @@ public class InfluxdbDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
                .select(DataV::getVZero)
                .select(DataV::getVlDev)
                .select(DataV::getVuDev)
+                .select(DataV::getQualityFlag)
                .between(DataV::getTime, startTime, endTime);
-        System.out.println(influxQueryWrapper.generateSql());
+        if (clean) {
+            influxQueryWrapper.eq(DataV::getAbnormalFlag,0);
+        }
        return dataVMapper.selectByQueryWrapper(influxQueryWrapper);
    }

--- a/data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataVImpl.java
+++ b/data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/RelationDataVImpl.java
@@ -2,11 +2,10 @@ package com.njcn.dataProcess.service.impl.relation;

 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
+import com.njcn.dataProcess.dao.relation.mapper.DataVRelationMapper;
 import com.njcn.dataProcess.dao.relation.mapper.RStatDataVRelationMapper;
 import com.njcn.dataProcess.dto.DataVDTO;
 import com.njcn.dataProcess.dto.DataVFiveItemDTO;
-
-import com.njcn.dataProcess.dao.relation.mapper.DataVRelationMapper;
 import com.njcn.dataProcess.param.LineCountEvaluateParam;
 import com.njcn.dataProcess.po.relation.DataV;
 import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
@@ -84,6 +83,11 @@ public class RelationDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
        return result;
    }

+    @Override
+    public List<DataVDto> getRawData(LineCountEvaluateParam lineParam) {
+        return Collections.emptyList();
+    }
+
    @Override
    public List<CommonMinuteDto> getBaseData(LineCountEvaluateParam lineParam) {
        return Collections.emptyList();
@@ -99,6 +103,13 @@ public class RelationDataVImpl extends MppServiceImpl<RStatDataVRelationMapper,
            BeanUtils.copyProperties(item, dataV);
            result.add(dataV);
        });
-        iDataV.saveOrUpdateBatchByMultiId(result);
+        iDataV.saveOrUpdateBatch(result);
    }
+
+    @Override
+    public void addInfluxDbList(List<DataVDto> dataVList) {
+
+    }
+
+
 }
--- a/data-processing/data-processing-boot/src/main/resources/bootstrap.yml
+++ b/data-processing/data-processing-boot/src/main/resources/bootstrap.yml
@@ -55,6 +55,7 @@ mqtt:
 data:
  source:
    query: Influxdb
+#    insert: Influxdb
    insert: Relation
 #mybatis配置信息
 mybatis-plus: