Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dataV原始数据数据清洗

Browse Source
This commit is contained in:
xy
2025-02-12 15:21:37 +08:00
parent 1f9c7e0b95
commit 935df88f3f
14 changed files with 246 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataVFeignClient.java
View File

@@ -5,6 +5,7 @@ import com.njcn.common.pojo.response.HttpResult;
import com.njcn.dataProcess.api.fallback.DataVFeignClientFallbackFactory;
import com.njcn.dataProcess.dto.DataVDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.po.influx.DataV;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataVDto;
import org.springframework.cloud.openfeign.FeignClient;
@@ -25,13 +26,24 @@ public interface DataVFeignClient {
@PostMapping("/batchInsertion")
HttpResult<String> batchInsertion(@RequestBody List<DataVDTO> dataVDTOList);
@PostMapping("/monitoringTime")
HttpResult<List<LocalDateTime>> monitoringTime(@RequestParam("lineId") String lineId, @RequestParam("localData") String localData) ;
//获取原始数据
@PostMapping("/getRawData")
HttpResult<List<DataVDto>> getRawData(@RequestBody LineCountEvaluateParam lineParam);
//获取算法基础数据
@PostMapping("/getBaseData")
HttpResult<List<CommonMinuteDto>> getBaseData(@RequestBody LineCountEvaluateParam lineParam);
//批量插入数据
@PostMapping("/addList")
HttpResult<String> addList(@RequestBody List<DataVDto> dataVDtoList);
HttpResult<String> addList(@RequestBody List<DataVDto> list);
//批量插入时序数据
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataVDto> list);
}

15
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataVFeignClientFallbackFactory.java
View File

@@ -6,10 +6,10 @@ import com.njcn.common.pojo.response.HttpResult;
import com.njcn.dataProcess.api.DataVFeignClient;
import com.njcn.dataProcess.dto.DataVDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.po.influx.DataV;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataVDto;
import com.njcn.dataProcess.util.DataProcessingEnumUtil;
import com.njcn.system.utils.SystemEnumUtil;
import feign.hystrix.FallbackFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -54,6 +54,12 @@ public class DataVFeignClientFallbackFactory implements FallbackFactory<DataVFei
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<List<DataVDto>> getRawData(LineCountEvaluateParam lineParam) {
log.error("{}异常，降级处理，异常为：{}","获取原始数据",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<List<CommonMinuteDto>> getBaseData(LineCountEvaluateParam lineParam) {
log.error("{}异常，降级处理，异常为：{}","获取算法基础数据",cause.toString());
@@ -61,11 +67,16 @@ public class DataVFeignClientFallbackFactory implements FallbackFactory<DataVFei
}
@Override
public HttpResult<String> addList(List<DataVDto> dataVDtoList) {
public HttpResult<String> addList(List<DataVDto> list) {
log.error("{}异常，降级处理，异常为：{}","批量插入数据",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataVDto> list) {
log.error("{}异常，降级处理，异常为：{}","时序数据库插入数据",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

74
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/po/influx/DataV.java
View File

@@ -33,15 +33,28 @@ public class DataV {
@JsonSerialize(using = InstantDateSerializer.class)
private Instant time;
@Column(name = "line_id", tag = true)
private String lineId;
@Column(name = "phasic_type", tag = true)
private String phasicType;
@Column(name = "value_type", tag = true)
private String valueType;
@Column(name = "quality_flag", tag = true)
private String qualityFlag;
//是否是异常指标数据，0否1是
@Column(name = "abnormal_flag")
private Integer abnormalFlag;
@Column(name = "freq")
private Double freq;
@Column(name = "freq_dev")
private Double freqDev;
@Column(name = "quality_flag", tag = true)
private String qualityFlag;
@Column(name = "rms")
private Double rms;
@@ -54,6 +67,21 @@ public class DataV {
@Column(name = "vu_dev")
private Double vuDev;
@Column(name = "v_neg")
private Double vNeg;
@Column(name = "v_pos")
private Double vPos;
@Column(name = "v_thd")
private Double vThd;
@Column(name = "v_unbalance")
private Double vUnbalance;
@Column(name = "v_zero")
private Double vZero;
@Column(name = "v_1")
private Double v1;
@@ -204,46 +232,6 @@ public class DataV {
@Column(name = "v_50")
private Double v50;
@Column(name = "v_neg")
private Double vNeg;
@Column(name = "v_pos")
private Double vPos;
@Column(name = "v_thd")
private Double vThd;
@Column(name = "v_unbalance")
private Double vUnbalance;
@Column(name = "v_zero")
private Double vZero;
@Column(name = "line_id", tag = true)
private String lineId;
@Column(name = "phasic_type", tag = true)
private String phasicType;
@Column(name = "value_type", tag = true)
private String valueType;
//自定义字段-总计算次数
@Column(name = "all_time")
private Integer allTime;
//自定义字段
@Column(name = "mean")
private Double mean;
//自定义字段
@Column(name = "count")
private Integer count;
//是否是异常指标数据，0否1是
@Column(name = "abnormal_flag")
private Integer abnormalFlag;
public static List<DataV> relationToInfluxDB(DataVDTO dataV) {
if (dataV == null) {
return null;

8
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/pojo/dto/DataVDto.java
View File

@@ -15,6 +15,9 @@ public class DataVDto implements Serializable {
@JsonFormat(pattern = "yyyy-MM-dd")
private String time;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private String minTime;
@ApiModelProperty("监测点Id")
private String lineId;
@@ -25,7 +28,10 @@ public class DataVDto implements Serializable {
private String valueType;
@ApiModelProperty("数据质量标志（0-表示是正常数据、1-表示是错误数据、2-表示是有事件的数据）数据库默认是0，污染数据不参与报表统计")
private Integer qualityFlag;
private String qualityFlag;
@ApiModelProperty("数据清洗标识 0:正常 1:异常")
private Integer abnormalFlag;
@ApiModelProperty("频率")
private Double freq;