Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增数据清洗算法(文件方式存储)

Browse Source
This commit is contained in:
xy
2025-06-05 17:41:13 +08:00
parent 56aa40211d
commit 6179e7f434
12 changed files with 941 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/PqDataVerifyNewFeignClient.java Normal file
View File

@@ -0,0 +1,27 @@
package com.njcn.dataProcess.api;
import com.njcn.common.pojo.constant.ServerInfo;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.dataProcess.api.fallback.PqDataVerifyNewFeignClientFallbackFactory;
import com.njcn.dataProcess.pojo.po.PqDataVerifyBak;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import java.util.List;
/**
* @author xy
* @version 1.0.0
* @date 2025年02月13日 20:11
*/
@FeignClient(value = ServerInfo.PLATFORM_DATA_PROCESSING_BOOT, path = "/pqDataVerifyNew", fallbackFactory = PqDataVerifyNewFeignClientFallbackFactory.class, contextId = "pqDataVerifyNew")
public interface PqDataVerifyNewFeignClient {
@PostMapping("/insertDataBatch")
HttpResult<List<String>> insertDataBatch(@RequestBody List<PqDataVerifyBak> list);
@PostMapping("/insertData")
HttpResult<List<String>> insertData(@RequestBody PqDataVerifyBak pqDataVerifyBak);
}

52
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/PqDataVerifyNewFeignClientFallbackFactory.java Normal file
View File

@@ -0,0 +1,52 @@
package com.njcn.dataProcess.api.fallback;
import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.exception.BusinessException;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.dataProcess.api.PqDataVerifyNewFeignClient;
import com.njcn.dataProcess.pojo.po.PqDataVerifyBak;
import com.njcn.dataProcess.util.DataProcessingEnumUtil;
import feign.hystrix.FallbackFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* @author xy
* @version 1.0.0
* @date 2025年02月13日 20:13
*/
@Slf4j
@Component
public class PqDataVerifyNewFeignClientFallbackFactory implements FallbackFactory<PqDataVerifyNewFeignClient> {
/**
* 输出远程请求接口异常日志
* @param cause RPC请求异常
*/
@Override
public PqDataVerifyNewFeignClient create(Throwable cause) {
//判断抛出异常是否为解码器抛出的业务异常
Enum<?> exceptionEnum = CommonResponseEnum.SERVICE_FALLBACK;
if(cause.getCause() instanceof BusinessException){
BusinessException businessException = (BusinessException) cause.getCause();
exceptionEnum = DataProcessingEnumUtil.getExceptionEnum(businessException.getResult());
}
Enum<?> finalExceptionEnum = exceptionEnum;
return new PqDataVerifyNewFeignClient() {
@Override
public HttpResult<List<String>> insertDataBatch(List<PqDataVerifyBak> list) {
log.error("{}异常，降级处理，异常为：{}","批量存储清洗的异常数据",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<List<String>> insertData(PqDataVerifyBak pqDataVerifyBak) {
log.error("{}异常，降级处理，异常为：{}","单监测点存储清洗的异常数据",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

42
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/dto/DataCleanJsonDTO.java Normal file
View File

@@ -0,0 +1,42 @@
package com.njcn.dataProcess.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* @author xy
*/
@Data
public class DataCleanJsonDTO {
Map<String,List<DataHarmCleanJsonDTO>> map;
@Data
public static class DataHarmCleanJsonDTO {
@ApiModelProperty("指标名称")
private String targetName;
private List<DataHarmCleanNormalJsonDTO> list;
}
@Data
public static class DataHarmCleanNormalJsonDTO {
@ApiModelProperty("时间集合")
private List<String> time;
@ApiModelProperty("数据集合")
private List<Double> value;
@ApiModelProperty("相别")
private String phasic;
@ApiModelProperty("数据类型")
private String valueType;
}
}

5
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/enums/DataCleanEnum.java
View File

@@ -48,7 +48,8 @@ public enum DataCleanEnum {
Fluc("fluc","电压波动"),
//DataHarmPhasicV
V("v","次谐波电压基波相角"),
V("v","次谐波电压相角"),
V1("v_1","谐波电压基波相角"),
//DataHarmRateV
V_Rate("v","次谐波电压含有率"),
@@ -66,8 +67,8 @@ public enum DataCleanEnum {
Plt("plt","长时闪变"),
//DataV
FreqDev("freq_dev","频率偏差"),
Freq("freq","频率"),
FreqDev("freq_dev","频率偏差"),
RmsV("rms","相电压有效值"),
VPos("v_pos","正序电压"),
VNeg("v_neg","负序电压"),

175
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/pojo/po/PqDataVerifyBak.java Normal file
View File

@@ -0,0 +1,175 @@
package com.njcn.dataProcess.pojo.po;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.github.jeffreyning.mybatisplus.anno.MppMultiId;
import lombok.Getter;
import lombok.Setter;
import java.io.Serializable;
import java.time.LocalDate;
/**
* <p>
*
* </p>
*
* @author xy
* @since 2025-02-17
*/
@Getter
@Setter
@TableName("pq_data_verify_bak")
public class PqDataVerifyBak implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 监测点id
*/
@MppMultiId
@TableField(value = "line_id")
private String lineId;
/**
* 异常数据时间
*/
@MppMultiId
@TableField(value = "time_id")
@JsonFormat(pattern = "yyyy-MM-dd",timezone="GMT+8")
private LocalDate time;
/**
* 频率指标
*/
@TableField(value = "freq")
private Integer freq = 0;
/**
* 频率偏差指标
*/
@TableField(value = "freq_dev")
private Integer freqDev = 0;
/**
* 相电压有效值
*/
@TableField(value = "v_rms")
private Integer vRms = 0;
/**
* 正序电压
*/
@TableField(value = "v_pos")
private Integer vPos = 0;
/**
* 负序电压
*/
@TableField(value = "v_neg")
private Integer vNeg = 0;
/**
* 零序电压
*/
@TableField(value = "v_zero")
private Integer vZero = 0;
/**
* 电压不平衡度
*/
@TableField(value = "v_unbalance")
private Integer vUnbalance = 0;
/**
* 线电压有效值
*/
@TableField(value = "rms_lvr")
private Integer rmsLvr = 0;
/**
* 电压正偏差
*/
@TableField(value = "vu_dev")
private Integer vuDev = 0;
/**
* 电压负偏差
*/
@TableField(value = "vl_Dev")
private Integer vlDev = 0;
/**
* 电压总谐波畸变率
*/
@TableField(value = "v_thd")
private Integer vThd = 0;
/**
* 相电压基波有效值
*/
@TableField(value = "v")
private Integer v = 0;
/**
* 电流有效值
*/
@TableField(value = "i_rms")
private Integer iRms = 0;
/**
* 长时闪变
*/
@TableField(value = "plt")
private Integer plt = 0;
/**
* 间谐波电压含有率
*/
@TableField(value = "v_inharm")
private Integer vInharm = 0;
/**
* 谐波电压含有率
*/
@TableField(value = "v_harm")
private Integer vHarm = 0;
/**
* 功率因数
*/
@TableField(value = "pf")
private Integer pf = 0;
/**
* 谐波电压相角
*/
@TableField(value = "v_phasic")
private Integer vPhasic = 0;
/**
* 谐波电压基波相角
*/
@TableField(value = "v1_phasic")
private Integer v1Phasic = 0;
/**
* 电压波动
*/
@TableField(value = "fluc")
private Integer fluc = 0;
/**
* 短时闪变
*/
@TableField(value = "pst")
private Integer pst = 0;
/**
* 文件路径
*/
@TableField(value = "path")
private String filePath;
}

56
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/controller/PqDataVerifyNewController.java Normal file
View File

@@ -0,0 +1,56 @@
package com.njcn.dataProcess.controller;
import com.njcn.common.pojo.annotation.OperateInfo;
import com.njcn.common.pojo.constant.OperateType;
import com.njcn.common.pojo.enums.common.LogEnum;
import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.common.utils.HttpResultUtil;
import com.njcn.dataProcess.pojo.po.PqDataVerifyBak;
import com.njcn.dataProcess.service.IPqDataVerifyNewService;
import com.njcn.web.controller.BaseController;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
import java.util.List;
/**
* <p>
* 前端控制器
* </p>
*
* @author xy
* @since 2025-02-17
*/
@RestController
@RequestMapping("/pqDataVerifyNew")
public class PqDataVerifyNewController extends BaseController {
@Resource
private IPqDataVerifyNewService pqDataVerifyNewService;
@OperateInfo(info = LogEnum.BUSINESS_COMMON,operateType = OperateType.ADD)
@PostMapping("/insertDataBatch")
@ApiOperation("存储清洗的异常数据")
public HttpResult<List<String>> insertDataBatch(@RequestBody List<PqDataVerifyBak> list) {
String methodDescribe = getMethodDescribe("insertDataBatch");
pqDataVerifyNewService.insertDataBatch(list);
return HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, null, methodDescribe);
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON,operateType = OperateType.ADD)
@PostMapping("/insertData")
@ApiOperation("存储清洗的异常数据")
public HttpResult<List<String>> insertData(@RequestBody PqDataVerifyBak pqDataVerifyBak) {
String methodDescribe = getMethodDescribe("insertData");
pqDataVerifyNewService.insertData(pqDataVerifyBak);
return HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, null, methodDescribe);
}
}

16
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/dao/relation/mapper/PqDataVerifyNewMapper.java Normal file
View File

@@ -0,0 +1,16 @@
package com.njcn.dataProcess.dao.relation.mapper;
import com.github.jeffreyning.mybatisplus.base.MppBaseMapper;
import com.njcn.dataProcess.pojo.po.PqDataVerifyBak;
/**
* <p>
* Mapper 接口
* </p>
*
* @author xy
* @since 2025-02-17
*/
public interface PqDataVerifyNewMapper extends MppBaseMapper<PqDataVerifyBak> {
}

28
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/IPqDataVerifyNewService.java Normal file
View File

@@ -0,0 +1,28 @@
package com.njcn.dataProcess.service;
import com.github.jeffreyning.mybatisplus.service.IMppService;
import com.njcn.dataProcess.pojo.po.PqDataVerifyBak;
import java.util.List;
/**
* <p>
* 服务类
* </p>
*
* @author xy
* @since 2025-02-17
*/
public interface IPqDataVerifyNewService extends IMppService<PqDataVerifyBak> {
/**
* 异常数据插入
* @param list
*/
void insertDataBatch(List<PqDataVerifyBak> list);
/**
* 异常数据插入
*/
void insertData(PqDataVerifyBak pqDataVerifyBak);
}

31
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/relation/PqDataVerifyNewServiceImpl.java Normal file
View File

@@ -0,0 +1,31 @@
package com.njcn.dataProcess.service.impl.relation;
import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
import com.njcn.dataProcess.dao.relation.mapper.PqDataVerifyNewMapper;
import com.njcn.dataProcess.pojo.po.PqDataVerifyBak;
import com.njcn.dataProcess.service.IPqDataVerifyNewService;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* <p>
* 服务实现类
* </p>
*
* @author xy
* @since 2025-02-17
*/
@Service
public class PqDataVerifyNewServiceImpl extends MppServiceImpl<PqDataVerifyNewMapper, PqDataVerifyBak> implements IPqDataVerifyNewService {
@Override
public void insertDataBatch(List<PqDataVerifyBak> list) {
this.saveOrUpdateBatchByMultiId(list,1000);
}
@Override
public void insertData(PqDataVerifyBak pqDataVerifyBak) {
this.saveOrUpdateByMultiId(pqDataVerifyBak);
}
}