Middleware/cn-data-platform
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

算法调整提交

Browse Source
This commit is contained in:
chendaofei
2025-04-11 10:40:40 +08:00
parent ba8d04c27a
commit fecc53e36a
5 changed files with 98 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/ExecutionCenter.java
View File

@@ -51,8 +51,6 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
public class ExecutionCenter extends BaseController {
private Long calTime = System.currentTimeMillis();
private static final Logger logger = LoggerFactory.getLogger(ExecutionCenter.class);
@Resource
@@ -108,9 +106,16 @@ public class ExecutionCenter extends BaseController {
System.out.println("当前已分配的内存: " + totalMemory / (1024 * 1024) + " MB");
System.out.println("当前空闲内存: " + freeMemory / (1024 * 1024) + " MB");
System.out.println("已使用的内存: " + usedMemory / (1024 * 1024) + " MB");
System.out.println("算法执行成功-----------------------------------------");
logger.info("日期{}，{}执行{}成功", calculatedParam.getDataDate(), methodDescribe, calculatedParam.isFullChain() ? "全链" : "指定节点：".concat(String.join(StrPool.COMMA, calculatedParam.getTagNames())));
logger.info("当前算法执行总时长为-------------------------------------{}",(System.currentTimeMillis()-calTime)/1000/60);
// 获取执行步骤列表
long allTime = 0;
Map<String, List<CmpStep>> executeSteps = liteflowResponse.getExecuteSteps();
for (String key : executeSteps.keySet()) {
List<CmpStep> cmpSteps = executeSteps.get(key);
long timeSum = cmpSteps.stream().mapToLong(CmpStep::getTimeSpent).sum();
allTime+=timeSum;
}
logger.info("日期{}，{}执行{}成功,执行总时长{}分钟", calculatedParam.getDataDate(), methodDescribe, calculatedParam.isFullChain() ? "全链" : "指定节点：".concat(String.join(StrPool.COMMA, calculatedParam.getTagNames())),allTime/1000/60);
} else {
Map<String, List<CmpStep>> executeSteps = liteflowResponse.getExecuteSteps();
CmpStep failStep = null;
@@ -163,7 +168,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("measurement_point", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -198,7 +203,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("special_analysis", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -242,7 +247,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("measurement_point_hour", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -283,7 +288,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("device", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@@ -322,7 +327,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("org_point", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
//
@@ -410,7 +415,7 @@ public class ExecutionCenter extends BaseController {
liteflowResponse = flowExecutor.execute2Resp("sub_station", calculatedParam);
dealResponse(calculatedParam, liteflowResponse, methodDescribe);
}
calTime = System.currentTimeMillis();
}
//
// /**

8
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/DataCleanServiceImpl.java
View File

@@ -251,6 +251,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataHarmDto> partList = dataHarmRateVFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataHarmRateV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataHarmRateV(map,item);
@@ -284,6 +286,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataPowerPDto> partList = dataHarmpowerPFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataHarmPowerP集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataHarmPowerP(map,item);
@@ -317,6 +321,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataHarmDto> partList = dataHarmphasicVFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataHarmPhasicV集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataHarmPhasicV(map,item);
@@ -350,6 +356,8 @@ public class DataCleanServiceImpl implements IDataCleanService {
lineParam.setLineId(list);
List<DataFlucDto> partList = dataFlucFeignClient.getRawData(lineParam).getData();
if (CollUtil.isNotEmpty(partList)) {
logger.info("{}数据清洗dataFluc集合大小为>>>>>>>>>>>>{}",lineParam.getStartTime(), MemorySizeUtil.getObjectSize(partList));
partList.forEach(item->{
//数据清洗
List<PqDataVerify> pqDataVerifies = judgeDataFluc(map,item);

1
algorithm/algorithm-boot/src/main/java/com/njcn/algorithm/serviceimpl/line/SpecialAnalysisServiceImpl.java
View File

@@ -184,6 +184,7 @@ public class SpecialAnalysisServiceImpl implements ISpecialAnalysisService {
spThroughFeignClient.batchInsertionThrough(info);
}
System.gc();
}
@Override

2
algorithm/algorithm-boot/src/main/resources/bootstrap.yml
View File

@@ -56,7 +56,7 @@ mybatis-plus:
#别名扫描
type-aliases-package: com.njcn.harmonic.pojo
line:
num: 10
num: 20
mqtt:
client-id: @artifactId@${random.value}

72
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/controller/DataVController.java
View File

@@ -26,6 +26,7 @@ import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -162,4 +163,75 @@ public class DataVController extends BaseController {
return HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, dataV, methodDescribe);
}
@OperateInfo(info = LogEnum.BUSINESS_COMMON)
@GetMapping("/memoryTest")
@ApiOperation("n内存测试")
public HttpResult<List<DataVDto>> memoryTest(@RequestParam("sum")Integer sum,@RequestParam("flag")Integer flag) {
String methodDescribe = getMethodDescribe("memoryTest");
Runtime runtime = Runtime.getRuntime();
// 获取 JVM 最大可用内存（以字节为单位）
long maxMemory = runtime.maxMemory();
// 获取 JVM 当前已分配的内存（以字节为单位）
long totalMemory = runtime.totalMemory();
// 获取 JVM 当前空闲内存（以字节为单位）
long freeMemory = runtime.freeMemory();
// 计算已使用的内存
long usedMemory = totalMemory - freeMemory;
System.out.println("最大可用内存: " + maxMemory / (1024 * 1024) + " MB");
System.out.println("当前已分配的内存: " + totalMemory / (1024 * 1024) + " MB");
System.out.println("当前空闲内存: " + freeMemory / (1024 * 1024) + " MB");
System.out.println("已使用的内存: " + usedMemory / (1024 * 1024) + " MB");
System.out.println("第一次分析结束-----------------------------------------");
if(flag == 1) {
try {
if (sum <= 0) {
System.out.println("传入的参数必须是正整数。");
return null;
}
// 计算总字节数
long totalSize = (long) sum * 1024 * 1024 * 1024;
// 每个小块数组的大小，这里设为 1GB
int chunkSize = 1024 * 1024 * 1024;
List<byte[]> arrays = new ArrayList<>();
long remainingSize = totalSize;
while (remainingSize > 0) {
int currentSize = (int) Math.min(remainingSize, chunkSize);
byte[] chunk = new byte[currentSize];
arrays.add(chunk);
remainingSize -= currentSize;
}
System.out.printf("成功生成 %dGB 的 byte 数组（分块存储）。%n", sum);
} catch (NumberFormatException e) {
System.out.println("传入的参数必须是有效的整数。");
} catch (OutOfMemoryError e) {
System.out.println("内存不足，无法生成指定大小的数组。请确保系统有足够的可用内存，或者调整 JVM 堆内存大小。");
e.printStackTrace();
}
}
// 获取 JVM 最大可用内存（以字节为单位）
maxMemory = runtime.maxMemory();
// 获取 JVM 当前已分配的内存（以字节为单位）
totalMemory = runtime.totalMemory();
// 获取 JVM 当前空闲内存（以字节为单位）
freeMemory = runtime.freeMemory();
// 计算已使用的内存
usedMemory = totalMemory - freeMemory;
System.out.println("最大可用内存: " + maxMemory / (1024 * 1024) + " MB");
System.out.println("当前已分配的内存: " + totalMemory / (1024 * 1024) + " MB");
System.out.println("当前空闲内存: " + freeMemory / (1024 * 1024) + " MB");
System.out.println("已使用的内存: " + usedMemory / (1024 * 1024) + " MB");
System.out.println("第二次分析结束-----------------------------------------");
return HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, null, methodDescribe);
}
}