Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据质量清洗算法优化

Browse Source
This commit is contained in:
xy
2025-05-23 16:18:58 +08:00
parent 412a67f6fd
commit 56aa40211d
68 changed files with 471 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataFlickerFeignClient.java
View File

@@ -1,13 +1,19 @@
package com.njcn.dataProcess.api;
import com.njcn.common.pojo.annotation.OperateInfo;
import com.njcn.common.pojo.constant.OperateType;
import com.njcn.common.pojo.constant.ServerInfo;
import com.njcn.common.pojo.enums.common.LogEnum;
import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.common.utils.HttpResultUtil;
import com.njcn.dataProcess.api.fallback.DataFlickerFeignClientFallbackFactory;
import com.njcn.dataProcess.dto.DataFlickerDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataFlickerDto;
import com.njcn.dataProcess.pojo.dto.DataIDto;
import io.swagger.annotations.ApiOperation;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@@ -36,4 +42,7 @@ public interface DataFlickerFeignClient {
@PostMapping("/getColumnNameCountRawData")
HttpResult<Integer> getColumnNameCountRawData(@RequestBody LineCountEvaluateParam lineParam);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataFlickerDto> list);
}

3
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataFlucFeignClient.java
View File

@@ -35,4 +35,7 @@ public interface DataFlucFeignClient {
@PostMapping("/addList")
HttpResult<String> addList(@RequestBody List<DataFlucDto> data);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataFlucDto> list);
}

9
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataHarmRateVFeignClient.java
View File

@@ -1,19 +1,13 @@
package com.njcn.dataProcess.api;
import com.njcn.common.pojo.annotation.OperateInfo;
import com.njcn.common.pojo.constant.OperateType;
import com.njcn.common.pojo.constant.ServerInfo;
import com.njcn.common.pojo.enums.common.LogEnum;
import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.common.utils.HttpResultUtil;
import com.njcn.dataProcess.api.fallback.DataHarmRateVFeignClientFallbackFactory;
import com.njcn.dataProcess.dto.DataHarmrateVDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataHarmDto;
import com.njcn.dataProcess.pojo.dto.DataHarmRateVDto;
import io.swagger.annotations.ApiOperation;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@@ -43,4 +37,7 @@ public interface DataHarmRateVFeignClient {
@PostMapping("/batchInsertion")
HttpResult<String> batchInsertion(@RequestBody List<DataHarmrateVDTO> dataHarmrateVDTOList);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataHarmDto> list);
}

11
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataHarmphasicIFeignClient.java
View File

@@ -1,20 +1,12 @@
package com.njcn.dataProcess.api;
import com.njcn.common.pojo.annotation.OperateInfo;
import com.njcn.common.pojo.constant.OperateType;
import com.njcn.common.pojo.constant.ServerInfo;
import com.njcn.common.pojo.enums.common.LogEnum;
import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.common.utils.HttpResultUtil;
import com.njcn.dataProcess.api.fallback.DataHarmphasicIFeignClientFallbackFactory;
import com.njcn.dataProcess.dto.DataHarmphasicIDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataFlucDto;
import com.njcn.dataProcess.pojo.dto.DataHarmPhasicIDto;
import io.swagger.annotations.ApiOperation;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@@ -41,4 +33,7 @@ public interface DataHarmphasicIFeignClient {
@PostMapping("/getRawData")
HttpResult<List<DataHarmPhasicIDto>> getRawData(@RequestBody LineCountEvaluateParam lineParam);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataHarmPhasicIDto> list);
}

9
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataHarmphasicVFeignClient.java
View File

@@ -1,13 +1,19 @@
package com.njcn.dataProcess.api;
import com.njcn.common.pojo.annotation.OperateInfo;
import com.njcn.common.pojo.constant.OperateType;
import com.njcn.common.pojo.constant.ServerInfo;
import com.njcn.common.pojo.enums.common.LogEnum;
import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.common.utils.HttpResultUtil;
import com.njcn.dataProcess.api.fallback.DataHarmphasicVFeignClientFallbackFactory;
import com.njcn.dataProcess.dto.DataHarmphasicVDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataHarmDto;
import com.njcn.dataProcess.pojo.dto.DataHarmPhasicVDto;
import io.swagger.annotations.ApiOperation;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@@ -36,4 +42,7 @@ public interface DataHarmphasicVFeignClient {
@PostMapping("/addList")
HttpResult<String> addList(@RequestBody List<DataHarmPhasicVDto> data);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataHarmDto> list);
}

9
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataHarmpowerPFeignClient.java
View File

@@ -1,7 +1,12 @@
package com.njcn.dataProcess.api;
import com.njcn.common.pojo.annotation.OperateInfo;
import com.njcn.common.pojo.constant.OperateType;
import com.njcn.common.pojo.constant.ServerInfo;
import com.njcn.common.pojo.enums.common.LogEnum;
import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.common.utils.HttpResultUtil;
import com.njcn.dataProcess.api.fallback.DataHarmpowerPFeignClientFallbackFactory;
import com.njcn.dataProcess.dto.DataHarmpowerPDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
@@ -9,6 +14,7 @@ import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataHarmPhasicVDto;
import com.njcn.dataProcess.pojo.dto.DataHarmPowerPDto;
import com.njcn.dataProcess.pojo.dto.DataPowerPDto;
import io.swagger.annotations.ApiOperation;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@@ -34,4 +40,7 @@ public interface DataHarmpowerPFeignClient {
@PostMapping("/addList")
HttpResult<String> addList(@RequestBody List<DataHarmPowerPDto> data);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataPowerPDto> list);
}

9
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataHarmpowerQFeignClient.java
View File

@@ -1,18 +1,12 @@
package com.njcn.dataProcess.api;
import com.njcn.common.pojo.annotation.OperateInfo;
import com.njcn.common.pojo.constant.OperateType;
import com.njcn.common.pojo.constant.ServerInfo;
import com.njcn.common.pojo.enums.common.LogEnum;
import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.common.utils.HttpResultUtil;
import com.njcn.dataProcess.api.fallback.DataHarmpowerQFeignClientFallbackFactory;
import com.njcn.dataProcess.dto.DataHarmpowerQDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataHarmPowerQDto;
import io.swagger.annotations.ApiOperation;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@@ -41,4 +35,7 @@ public interface DataHarmpowerQFeignClient {
@PostMapping("/getRawData")
HttpResult<List<DataHarmPowerQDto>> getRawData(@RequestBody LineCountEvaluateParam lineParam);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataHarmPowerQDto> list);
}

3
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataHarmpowerSFeignClient.java
View File

@@ -41,4 +41,7 @@ public interface DataHarmpowerSFeignClient {
@PostMapping("/getRawData")
HttpResult<List<DataHarmPowerSDto>> getRawData(@RequestBody LineCountEvaluateParam lineParam);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataHarmPowerSDto> list);
}

3
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataInharmIFeignClient.java
View File

@@ -33,4 +33,7 @@ public interface DataInharmIFeignClient {
@PostMapping("/getRawData")
HttpResult<List<DataInHarmIDto>> getRawData(@RequestBody LineCountEvaluateParam lineParam);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataInHarmIDto> list);
}

5
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataInharmVFeignClient.java
View File

@@ -2,13 +2,11 @@ package com.njcn.dataProcess.api;
import com.njcn.common.pojo.constant.ServerInfo;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.dataProcess.api.fallback.DataInharmVFeignClientFallbackFactory;
import com.njcn.dataProcess.dto.DataInharmVDTO;
import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataHarmDto;
import com.njcn.dataProcess.pojo.dto.DataInHarmIDto;
import com.njcn.dataProcess.pojo.dto.DataInHarmVDto;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
@@ -36,4 +34,7 @@ public interface DataInharmVFeignClient {
@PostMapping("/addList")
HttpResult<String> addList(@RequestBody List<DataInHarmVDto> list);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataHarmDto> list);
}

3
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/DataPltFeignClient.java
View File

@@ -37,4 +37,7 @@ public interface DataPltFeignClient {
@PostMapping("/getDataPlt")
HttpResult<List<DataPltDto>> getDataPlt(@RequestBody LineCountEvaluateParam lineParam);
@PostMapping("/addInfluxDbList")
HttpResult<String> addInfluxDbList(@RequestBody List<DataPltDto> list);
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataFlickerFeignClientFallbackFactory.java
View File

@@ -71,6 +71,12 @@ public class DataFlickerFeignClientFallbackFactory implements FallbackFactory<Da
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataFlickerDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataFlicker原始数据录入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataFlucFeignClientFallbackFactory.java
View File

@@ -65,6 +65,12 @@ public class DataFlucFeignClientFallbackFactory implements FallbackFactory<DataF
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataFlucDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataFluc原始数据录入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataHarmRateVFeignClientFallbackFactory.java
View File

@@ -74,6 +74,12 @@ public class DataHarmRateVFeignClientFallbackFactory implements FallbackFactory<
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataHarmDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataHarmRateV原始数据插入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataHarmphasicIFeignClientFallbackFactory.java
View File

@@ -65,6 +65,12 @@ public class DataHarmphasicIFeignClientFallbackFactory implements FallbackFactor
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataHarmPhasicIDto> list) {
log.error("{}异常，降级处理，异常为：{}","时序数据库插入数据",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataHarmphasicVFeignClientFallbackFactory.java
View File

@@ -65,6 +65,12 @@ public class DataHarmphasicVFeignClientFallbackFactory implements FallbackFactor
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataHarmDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataHarmPhasicV原始数据录入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataHarmpowerPFeignClientFallbackFactory.java
View File

@@ -65,6 +65,12 @@ public class DataHarmpowerPFeignClientFallbackFactory implements FallbackFactory
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataPowerPDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataPowerP原始数据插入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataHarmpowerQFeignClientFallbackFactory.java
View File

@@ -64,6 +64,12 @@ public class DataHarmpowerQFeignClientFallbackFactory implements FallbackFactory
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataHarmPowerQDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataHarmPowerQ原始数据插入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

7
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataHarmpowerSFeignClientFallbackFactory.java
View File

@@ -9,7 +9,6 @@ import com.njcn.dataProcess.param.LineCountEvaluateParam;
import com.njcn.dataProcess.pojo.dto.CommonMinuteDto;
import com.njcn.dataProcess.pojo.dto.DataHarmPowerSDto;
import com.njcn.dataProcess.util.DataProcessingEnumUtil;
import com.njcn.system.utils.SystemEnumUtil;
import feign.hystrix.FallbackFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -65,6 +64,12 @@ public class DataHarmpowerSFeignClientFallbackFactory implements FallbackFactory
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataHarmPowerSDto> list) {
log.error("{}异常，降级处理，异常为：{}","harmPowerS插入原始数据",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataInharmIFeignClientFallbackFactory.java
View File

@@ -65,6 +65,12 @@ public class DataInharmIFeignClientFallbackFactory implements FallbackFactory<Da
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataInHarmIDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataInharmI原始数据插入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataInharmVFeignClientFallbackFactory.java
View File

@@ -65,6 +65,12 @@ public class DataInharmVFeignClientFallbackFactory implements FallbackFactory<Da
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataHarmDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataInharmV原始数据插入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

6
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/api/fallback/DataPltFeignClientFallbackFactory.java
View File

@@ -70,6 +70,12 @@ public class DataPltFeignClientFallbackFactory implements FallbackFactory<DataPl
throw new BusinessException(finalExceptionEnum);
}
@Override
public HttpResult<String> addInfluxDbList(List<DataPltDto> list) {
log.error("{}异常，降级处理，异常为：{}","dataPlt原始数据插入",cause.toString());
throw new BusinessException(finalExceptionEnum);
}
};
}
}

2
data-processing/data-processing-api/src/main/java/com/njcn/dataProcess/dto/DataHarmphasicIDTO.java
View File

@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
@@ -22,6 +23,7 @@ import java.time.LocalDateTime;
*/
@Getter
@Setter
@Data
public class DataHarmphasicIDTO implements Serializable{
private static final long serialVersionUID = 1L;